From 3b15e6c63a26217f3ff4a11818f74d181e0d6b4b Mon Sep 17 00:00:00 2001 From: Varshith Puli Date: Sat, 14 Nov 2026 12:11:21 +0000 Subject: [PATCH 01/12] fix(agent): fold large unchanged subtrees in the full-screen fallback (#1788) When an action changes most of the screen, the presenter sends the whole tree again, repeating subtrees the model already holds. A persistent sidebar can repeat hundreds of unchanged nodes on every page transition. The fallback now folds a subtree of 10 or more nodes to its root line with the node count and the revision it was last seen at, when every node in it reads as before with the same node ids. The first screen still goes out whole, observe is untouched, and a truncated listing never folds. A subtree holding the focused node stays listed. --- .changeset/mcp-fold-unchanged-subtrees.md | 4 + packages/core/src/agent/screen-update.ts | 125 +++++++++++++++++- .../tests/unit/agent-screen-update.test.ts | 121 +++++++++++++++++ skills/e2e/references/agent.md | 4 +- 4 files changed, 248 insertions(+), 6 deletions(-) create mode 100644 .changeset/mcp-fold-unchanged-subtrees.md diff --git a/.changeset/mcp-fold-unchanged-subtrees.md b/.changeset/mcp-fold-unchanged-subtrees.md new file mode 100644 index 0000000000..e39814cbf0 --- /dev/null +++ b/.changeset/mcp-fold-unchanged-subtrees.md @@ -0,0 +1,4 @@ +"e2e": patch +--- + +When an action changes most of the screen, the whole-screen fallback now folds large unchanged subtrees (10 or more nodes) to a one-line summary naming the node count and the revision they were last seen at, instead of repeating them. `observe` still returns the complete tree. diff --git a/packages/core/src/agent/screen-update.ts b/packages/core/src/agent/screen-update.ts index d46cc95504..73d73ea428 100644 --- a/packages/core/src/agent/screen-update.ts +++ b/packages/core/src/agent/screen-update.ts @@ -6,7 +6,9 @@ * the lines that appeared, the lines that changed, the lines that went away. * Node ids are stable for as long as an element exists, so a line the model * saw earlier still names the same element, and a diff is a complete update - * rather than a hint. A screen that changed mostly goes out whole again. + * rather than a hint. A screen that changed mostly goes out whole again, + * with large subtrees the model already holds folded to one line each, so a + * persistent sidebar is not repeated on every page transition. * * A screen may carry its pixels too. The presenter is where a screenshot * joins the text: it notes the image's size and coordinate space under the @@ -29,6 +31,9 @@ const MAX_DIFF_LINES = 60; /** A diff touching more than this share of the new screen goes out whole. */ const MAX_DIFF_SHARE = 0.5; +/** A subtree folds only when it holds at least this many nodes below its root. */ +const MIN_FOLD_NODES = 10; + /** Every full screen is introduced by this phrase; the transcript elision looks for it. */ export const FULL_SCREEN_PATTERN = /Current screen \(revision /; @@ -38,6 +43,8 @@ interface ShownScreen { /** Node lines in document order, without indentation or focus; the truncation marker is not a node. */ readonly order: readonly string[]; readonly byId: ReadonlyMap; + /** Leading spaces of each node line, so a subtree can be found in `order`. */ + readonly indent: ReadonlyMap; readonly nodes: number; /** The listing is cut short (byte limit or engine cap): nodes beyond it exist but are not listed. */ readonly truncated: boolean; @@ -165,7 +172,7 @@ export class ScreenPresenter { return `${lead}${renderUnchanged(previous.revision, observation, expectChange, pixels)}${closed}`; } if (changes >= MAX_DIFF_LINES || changes > MAX_DIFF_SHARE * next.order.length) { - return `${lead}The screen changed substantially since revision ${previous.revision}. ${renderFull(observation)}${closed}`; + return `${lead}The screen changed substantially since revision ${previous.revision}. ${renderFull(observation, previous)}${closed}`; } const assurance = observation.truncated ? 'The screen listing is truncated: nodes past the cut are not listed and none is reported removed; every listed node keeps the id you have.' @@ -266,13 +273,118 @@ function describeCounts(diff: readonly string[]): string { return counts === '' ? 'no listed node changed' : counts; } -/** One full screen, headed so the elision can find it later. */ -function renderFull(observation: ExecutorObservation): string { +/** One full screen, headed so the elision can find it later. A screen that + * follows one the model holds folds its large unchanged subtrees: the root + * line stays, with the count of the nodes inside and the revision they were + * last seen at, and the nodes inside are left out. The first screen of a + * step has nothing to fold against, and a truncated listing never folds, + * since the nodes past the cut are unknown. */ +function renderFull(observation: ExecutorObservation, previous?: ShownScreen): string { if (observation.treeUnavailable === true) { return `Current screen (revision ${observation.revision}${describeLocation(observation)}, semantic capture unavailable):\n${observation.text}`; } const nodes = indexScreen(observation).nodes; - return `Current screen (revision ${observation.revision}${describeLocation(observation)}, ${String(nodes)} nodes):\n${observation.text}`; + const text = previous !== undefined && !observation.truncated + ? foldUnchangedSubtrees(observation, previous) + : observation.text; + return `Current screen (revision ${observation.revision}${describeLocation(observation)}, ${String(nodes)} nodes):\n${text}`; +} + +/** A node line of a screen listing, with its depth and id. */ +interface TreeEntry { + readonly raw: string; + readonly indent: number; + readonly id: string | undefined; +} + +/** Leading spaces of one listing line: its depth in the tree. */ +function indentOf(line: string): number { + return line.length - line.trimStart().length; +} + +/** True when the line's state bracket names the focused node. */ +function hasFocus(line: string): boolean { + const states = / \[([^\]]*)\]$/.exec(line)?.[1]; + return states !== undefined && states.split(' ').includes('focused'); +} + +/** + * The screen listing with every large unchanged subtree replaced by its root + * line plus a summary. A subtree folds only when it is not the document + * root, holds enough nodes to be worth the hiding, carries no focused node, + * and reads exactly as the model last saw it, with the same node ids: a node + * added, changed, or removed inside it keeps the whole subtree listed, and + * its children are then considered on their own. + */ +function foldUnchangedSubtrees(observation: ExecutorObservation, previous: ShownScreen): string { + const entries: TreeEntry[] = observation.text.split('\n').map((raw) => ({ + raw, + indent: indentOf(raw), + id: lineId(raw), + })); + const out: string[] = []; + let index = 0; + while (index < entries.length) { + const entry = entries[index]!; + if (entry.id === undefined) { + out.push(entry.raw); + index += 1; + continue; + } + let end = index + 1; + while (end < entries.length) { + const next = entries[end]!; + if (next.id === undefined || next.indent <= entry.indent) break; + end += 1; + } + const inside = end - index - 1; + if (index > 0 && entry.indent > 0 && inside >= MIN_FOLD_NODES && subtreeUnchanged(entries, index, end, previous)) { + out.push(`${entry.raw} (${String(inside)} nodes inside, unchanged since revision ${previous.revision})`); + index = end; + } else { + out.push(entry.raw); + index += 1; + } + } + return out.join('\n'); +} + +/** + * True when every node line in the span reads as the model last saw it and + * the span holds the same node ids it held then. Focus alone never counts + * as a change, but a subtree holding the focused node never folds: the fold + * would hide where focus is. + */ +function subtreeUnchanged(entries: readonly TreeEntry[], start: number, end: number, previous: ShownScreen): boolean { + const current = new Set(); + for (let index = start; index < end; index += 1) { + const entry = entries[index]!; + if (entry.id === undefined || hasFocus(entry.raw)) return false; + if (previous.byId.get(entry.id) !== withoutFocus(entry.raw).trimStart()) return false; + current.add(entry.id); + } + const before = previousSubtreeIds(previous, entries[start]!.id!); + if (before === undefined || before.size !== current.size) return false; + for (const id of current) if (!before.has(id)) return false; + return true; +} + +/** The node ids of the subtree rooted at `rootId` in the shown screen. */ +function previousSubtreeIds(previous: ShownScreen, rootId: string): Set | undefined { + const rootIndex = previous.order.findIndex((line) => lineId(line) === rootId); + if (rootIndex === -1) return undefined; + const rootIndent = previous.indent.get(rootId); + if (rootIndent === undefined) return undefined; + const ids = new Set(); + for (let index = rootIndex; index < previous.order.length; index += 1) { + const id = lineId(previous.order[index]!); + if (id === undefined) return undefined; + const indent = previous.indent.get(id); + if (indent === undefined) return undefined; + if (index > rootIndex && indent <= rootIndent) break; + ids.add(id); + } + return ids; } /** @@ -363,6 +475,7 @@ function withoutFocus(line: string): string { function indexScreen(observation: ExecutorObservation): ShownScreen { const order: string[] = []; const byId = new Map(); + const indent = new Map(); for (const raw of observation.text.split('\n')) { const id = lineId(raw); if (id === undefined) continue; @@ -371,11 +484,13 @@ function indexScreen(observation: ExecutorObservation): ShownScreen { const line = withoutFocus(raw).trimStart(); order.push(line); byId.set(id, line); + indent.set(id, indentOf(raw)); } return { revision: observation.revision, order, byId, + indent, nodes: byId.size, truncated: observation.truncated, keyboard: order.some((line) => KEYBOARD_LINE.test(line)), diff --git a/packages/e2e/tests/unit/agent-screen-update.test.ts b/packages/e2e/tests/unit/agent-screen-update.test.ts index 8038b7b815..847f704fdb 100644 --- a/packages/e2e/tests/unit/agent-screen-update.test.ts +++ b/packages/e2e/tests/unit/agent-screen-update.test.ts @@ -236,6 +236,127 @@ describe('ScreenPresenter', () => { }); }); +describe('ScreenPresenter folds large unchanged subtrees in the full-screen fallback', () => { + /** A sidebar of `count` links under a navigation node, ids #n3 and up. */ + function sidebar(count: number): string[] { + return [ + ' #n2 navigation "Sidebar"', + ...Array.from({ length: count }, (_, i) => ` #n${String(i + 3)} link "Item ${String(i + 1)}"`), + ]; + } + + const MAIN_LIST = [ + ' #n20 main "Order list"', + ' #n21 heading "Orders"', + ' #n22 button "Order 1"', + ' #n23 button "Order 2"', + ]; + + const MAIN_DETAILS = [ + ' #n20 main "Order details"', + ' #n30 heading "Order 42"', + ' #n31 text "Total $12"', + ' #n32 text "Placed yesterday"', + ' #n33 button "Refund"', + ' #n34 button "Ship"', + ' #n35 button "Print"', + ' #n36 button "Archive"', + ' #n37 link "Back to orders"', + ]; + + function orderScreen(main: readonly string[], sidebarCount: number): string[] { + return ['#n1 document "Orders"', ...sidebar(sidebarCount), ...main]; + } + + it('folds a large unchanged sidebar, and lists the changed main in full', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const text = presenter.update(screen('b2', orderScreen(MAIN_DETAILS, 12), { path: '/orders' }), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).toContain('Current screen (revision b2, path /orders, 23 nodes):'); + expect(text).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + expect(text).not.toContain('#n3 link "Item 1"'); + expect(text).not.toContain('#n14 link "Item 12"'); + expect(text).toContain(' #n30 heading "Order 42"'); + expect(text).toContain(' #n37 link "Back to orders"'); + }); + + it('keeps a small unchanged subtree listed in full', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 3))); + const text = presenter.update(screen('b2', orderScreen(MAIN_DETAILS, 3)), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain(' #n3 link "Item 1"'); + expect(text).toContain(' #n30 heading "Order 42"'); + }); + + it('folds an unchanged group inside a changed sidebar, and lists the changed sibling', () => { + const presenter = new ScreenPresenter(); + const before = [ + '#n1 document "Orders"', + ' #n2 navigation "Sidebar"', + ' #n3 list "Sections"', + ...Array.from({ length: 12 }, (_, i) => ` #n${String(i + 40)} link "Section ${String(i + 1)}"`), + ' #n60 link "Settings"', + ...MAIN_LIST, + ]; + presenter.initial(screen('b1', before)); + const after = before.map((line) => (line.includes('#n60 ') ? ' #n60 link "Settings (2)"' : line)); + after.splice(after.indexOf(' #n20 main "Order list"'), 4, ...MAIN_DETAILS); + const text = presenter.update(screen('b2', after), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).toContain(' #n3 list "Sections" (12 nodes inside, unchanged since revision b1)'); + expect(text).not.toContain('#n40 link "Section 1"'); + expect(text).toContain(' #n60 link "Settings (2)"'); + expect(text).not.toContain('navigation "Sidebar" ('); + }); + + it('does not fold a subtree that lost a node, even when the rest is unchanged', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const after = orderScreen(MAIN_DETAILS, 12).filter((line) => !line.includes('#n5 ')); + const text = presenter.update(screen('b2', after), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain(' #n3 link "Item 1"'); + }); + + it('does not fold a subtree holding the focused node', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const after = orderScreen(MAIN_DETAILS, 12).map((line) => + (line.includes('#n8 ') ? ' #n8 link "Item 6" [focused]' : line), + ); + const text = presenter.update(screen('b2', after), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain('#n8 link "Item 6" [focused]'); + }); + + it('does not fold on a truncated listing, whose cut nodes are unknown', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + // Twelve more added nodes, so the change still trips the whole-screen fallback + // even though a truncated listing reports no removals. + const extra = Array.from({ length: 12 }, (_, i) => ` #n${String(i + 40)} button "Extra ${String(i + 1)}"`); + const after = [...orderScreen(MAIN_DETAILS, 12), ...extra]; + const text = presenter.update(screen('b2', after, { truncated: true }), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain(' #n3 link "Item 1"'); + }); + + it('sends the first screen whole, with no folding', () => { + const presenter = new ScreenPresenter(); + const text = presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + expect(text).toContain('Current screen (revision b1, 18 nodes):'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain(' #n3 link "Item 1"'); + expect(text).toContain(' #n14 link "Item 12"'); + }); +}); + describe('ScreenPresenter and the on-screen keyboard', () => { const KEYBOARD = [ ' #n20 keyboard "Padding-Left"', diff --git a/skills/e2e/references/agent.md b/skills/e2e/references/agent.md index c564dbb333..29e3266da4 100644 --- a/skills/e2e/references/agent.md +++ b/skills/e2e/references/agent.md @@ -355,7 +355,9 @@ summaries, and your context; never raw HTML, cookies, headers, environment values, or a `Secret`'s value; password fields masked. The first screen of a step arrives whole; later action results report what changed, keyed by node ids stable while an element exists, or the whole screen when -most changed. Pixels arrive through `vision` on a judgment or the act loop's +most changed, with large subtrees that have not changed since the last screen +folded to one line each naming the node count and the revision they were last +seen at. `observe` still returns the complete tree. Pixels arrive through `vision` on a judgment or the act loop's `screenshot` and point tools, with secure fields masked. When the browser engine's tree capture times out, the model gets a screenshot and a warning, a judgment needs `vision: true`, and no control may be inferred From 81681e4762b3b42cee4129f607f2a259ab886cc9 Mon Sep 17 00:00:00 2001 From: Varshith Puli Date: Sat, 14 Nov 2026 13:02:19 +0000 Subject: [PATCH 02/12] fix(changeset): restore missing opening frontmatter delimiter Signed-off-by: Varshith Puli --- .changeset/mcp-fold-unchanged-subtrees.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.changeset/mcp-fold-unchanged-subtrees.md b/.changeset/mcp-fold-unchanged-subtrees.md index e39814cbf0..d07bad1f39 100644 --- a/.changeset/mcp-fold-unchanged-subtrees.md +++ b/.changeset/mcp-fold-unchanged-subtrees.md @@ -1,3 +1,4 @@ +--- "e2e": patch --- From 91ed3b91b4ef38ec6725e1c2684ffe08cd8c1827 Mon Sep 17 00:00:00 2001 From: Varshith Puli Date: Sat, 14 Nov 2026 13:29:39 +0000 Subject: [PATCH 03/12] fix(agent): name the last full listing in fold lines and keep it through elision A folded subtree used to name the last observation's revision, but that observation may have been rendered as a diff that never listed the subtree, or may itself have folded it. The presenter now remembers, per subtree root, the revision that last listed it in full and a digest of what it read then. A fold line names that revision, and only when the subtree still reads the same: a subtree that changed in between is listed in full again, which puts it on record at the new revision. compactScreenHistory preserves every screen a surviving folded screen's fold lines depend on, following the dependency transitively, so the elision notice stays true. The stale-bytes budget now counts only the screens that may be elided. Signed-off-by: Varshith Puli --- .changeset/mcp-fold-unchanged-subtrees.md | 2 +- packages/core/src/agent/screen-update.ts | 227 +++++++++++++++--- .../core/src/agent/transcript-compaction.ts | 83 +++++-- .../tests/unit/agent-screen-update.test.ts | 71 ++++++ .../tests/unit/transcript-compaction.test.ts | 65 +++++ skills/e2e/references/agent.md | 4 +- 6 files changed, 399 insertions(+), 53 deletions(-) diff --git a/.changeset/mcp-fold-unchanged-subtrees.md b/.changeset/mcp-fold-unchanged-subtrees.md index d07bad1f39..1be3551116 100644 --- a/.changeset/mcp-fold-unchanged-subtrees.md +++ b/.changeset/mcp-fold-unchanged-subtrees.md @@ -2,4 +2,4 @@ "e2e": patch --- -When an action changes most of the screen, the whole-screen fallback now folds large unchanged subtrees (10 or more nodes) to a one-line summary naming the node count and the revision they were last seen at, instead of repeating them. `observe` still returns the complete tree. +When an action changes most of the screen, the whole-screen fallback now folds large unchanged subtrees (10 or more nodes) to a one-line summary naming the node count and the revision that last listed them in full, instead of repeating them. A fold line never names a revision that only reported the subtree as a change, or that itself folded it. Transcript elision keeps the screens those fold lines depend on, so a folded line never names a screen the model no longer holds. `observe` still returns the complete tree. diff --git a/packages/core/src/agent/screen-update.ts b/packages/core/src/agent/screen-update.ts index 73d73ea428..1a2e776050 100644 --- a/packages/core/src/agent/screen-update.ts +++ b/packages/core/src/agent/screen-update.ts @@ -7,8 +7,12 @@ * Node ids are stable for as long as an element exists, so a line the model * saw earlier still names the same element, and a diff is a complete update * rather than a hint. A screen that changed mostly goes out whole again, - * with large subtrees the model already holds folded to one line each, so a - * persistent sidebar is not repeated on every page transition. + * with large subtrees the model already holds folded to one line each, + * naming the revision that last listed them in full, so a persistent sidebar + * is not repeated on every page transition. A fold line never names a + * revision that only reported the subtree as a diff, or that itself folded + * it: the presenter remembers, per subtree, the revision that last listed + * it whole and what it read then. * * A screen may carry its pixels too. The presenter is where a screenshot * joins the text: it notes the image's size and coordinate space under the @@ -22,6 +26,7 @@ import type { ToolResultPart } from 'ai'; import type { ExecutorObservation, ExecutorPixels } from './executor.ts'; +import { sha256Hex } from '../internal/ids.ts'; import { interactiveNodeCount, pixelDigest } from './observation.ts'; import type { VisionDegradation } from '../run/steps.ts'; @@ -57,6 +62,13 @@ interface ShownScreen { /** A node line whose role is the on-screen keyboard, as device trees list it. */ const KEYBOARD_LINE = /^#\S+ keyboard\b/; +/** A subtree as the model last saw it whole: the revision that listed it in full, and what it read then. */ +interface FullyShownSubtree { + readonly revision: string; + /** SHA-256 of the subtree's node lines, focus stripped, in document order. */ + readonly digest: string; +} + /** * What the model needs to hear when an action closed the keyboard: on a * touch screen the tap that closes it is often spent on closing it, so the @@ -123,6 +135,8 @@ export class ScreenPresenter { private screenshotDigest: string | undefined; /** True once the engine said it captures no screenshots at all. */ private pixelsDenied = false; + /** The revision that last listed each subtree in full, by its root node id, and what it read then. */ + private fullyShown = new Map(); /** True once a screenshot went to the model in this step. */ get showingPixels(): boolean { @@ -137,7 +151,40 @@ export class ScreenPresenter { /** The step's first screen, whole. */ initial(observation: ExecutorObservation): string { this.shown = observation.treeUnavailable === true ? undefined : indexScreen(observation); - return renderFull(observation); + return this.renderWhole(observation); + } + + /** + * A whole screen with nothing to fold against, and the record that it + * listed every node in full at its revision. A truncated listing records + * nothing: nodes past the cut were never shown. + */ + private renderWhole(observation: ExecutorObservation): string { + const rendered = renderFull(observation); + this.noteListed(rendered.listed); + return rendered.text; + } + + /** + * Records the subtrees `listed` names as last listed in full at the newest + * revision, and drops the record for nodes no longer on screen. Called with + * no map by a render that listed nothing in full, a diff or an unchanged + * screen, which only prunes: a subtree the model last saw whole stays on + * record until a node in it leaves the screen. + */ + private noteListed(listed?: ReadonlyMap): void { + const shown = this.shown; + if (shown === undefined) { + this.fullyShown.clear(); + return; + } + for (const id of this.fullyShown.keys()) { + if (!shown.byId.has(id)) this.fullyShown.delete(id); + } + if (listed === undefined) return; + for (const [id, digest] of listed) { + this.fullyShown.set(id, { revision: shown.revision, digest }); + } } /** @@ -157,11 +204,11 @@ export class ScreenPresenter { const previous = this.shown; const next = indexScreen(observation); this.shown = next; - if (previous === undefined) return `${lead}${renderFull(observation)}`; + if (previous === undefined) return `${lead}${this.renderWhole(observation)}`; const closed = options.keyboardNote === false || options.lead === undefined ? '' : keyboardClosedNote(previous, next); const tabs = tabNote(previous.tabs, next.tabs); // Another tab is another document: its ids say nothing about the tab the model held. - if (tabs?.moved === true) return `${lead}${tabs.note} ${renderFull(observation)}${closed}`; + if (tabs?.moved === true) return `${lead}${tabs.note} ${this.renderWhole(observation)}${closed}`; // A tab opened is the action's effect, so an unchanged screen blames no control. const expectChange = options.expectChange === true && tabs === undefined; if (tabs !== undefined) lead = `${lead}${tabs.note} `; @@ -169,14 +216,18 @@ export class ScreenPresenter { const changes = diff.length; // A truncated screen is never called unchanged: what it left out is unknown. if (changes === 0 && !observation.truncated) { + this.noteListed(); return `${lead}${renderUnchanged(previous.revision, observation, expectChange, pixels)}${closed}`; } if (changes >= MAX_DIFF_LINES || changes > MAX_DIFF_SHARE * next.order.length) { - return `${lead}The screen changed substantially since revision ${previous.revision}. ${renderFull(observation, previous)}${closed}`; + const rendered = renderFull(observation, previous, this.fullyShown); + this.noteListed(rendered.listed); + return `${lead}The screen changed substantially since revision ${previous.revision}. ${rendered.text}${closed}`; } const assurance = observation.truncated ? 'The screen listing is truncated: nodes past the cut are not listed and none is reported removed; every listed node keeps the id you have.' : 'Every node not listed as removed is still on screen under the id you have.'; + this.noteListed(); return [ `${lead}Screen changes since revision ${previous.revision} (now revision ${observation.revision}${describeLocation(observation)}, ${String(next.nodes)} nodes): ${describeCounts(diff)}. ${assurance}`, ...diff, @@ -273,21 +324,41 @@ function describeCounts(diff: readonly string[]): string { return counts === '' ? 'no listed node changed' : counts; } +/** A rendered full screen, with the subtrees it listed in full and what each read. */ +interface RenderedFullScreen { + readonly text: string; + /** Node ids whose subtree the rendering listed in full, each with the digest of its node lines. */ + readonly listed: ReadonlyMap; +} + /** One full screen, headed so the elision can find it later. A screen that * follows one the model holds folds its large unchanged subtrees: the root - * line stays, with the count of the nodes inside and the revision they were - * last seen at, and the nodes inside are left out. The first screen of a + * line stays, with the count of the nodes inside and the revision that last + * listed them in full, and the nodes inside are left out. The first screen of a * step has nothing to fold against, and a truncated listing never folds, * since the nodes past the cut are unknown. */ -function renderFull(observation: ExecutorObservation, previous?: ShownScreen): string { +function renderFull( + observation: ExecutorObservation, + previous?: ShownScreen, + fullyShown?: ReadonlyMap, +): RenderedFullScreen { if (observation.treeUnavailable === true) { - return `Current screen (revision ${observation.revision}${describeLocation(observation)}, semantic capture unavailable):\n${observation.text}`; + return { + text: `Current screen (revision ${observation.revision}${describeLocation(observation)}, semantic capture unavailable):\n${observation.text}`, + listed: new Map(), + }; } const nodes = indexScreen(observation).nodes; - const text = previous !== undefined && !observation.truncated - ? foldUnchangedSubtrees(observation, previous) - : observation.text; - return `Current screen (revision ${observation.revision}${describeLocation(observation)}, ${String(nodes)} nodes):\n${text}`; + const head = `Current screen (revision ${observation.revision}${describeLocation(observation)}, ${String(nodes)} nodes):\n`; + if (previous === undefined || fullyShown === undefined || observation.truncated) { + const entries = treeEntries(observation); + return { + text: `${head}${observation.text}`, + listed: observation.truncated ? new Map() : subtreeDigests(entries, spanEnds(entries)), + }; + } + const folded = foldUnchangedSubtrees(observation, previous, fullyShown); + return { text: `${head}${folded.text}`, listed: folded.listed }; } /** A node line of a screen listing, with its depth and id. */ @@ -297,6 +368,58 @@ interface TreeEntry { readonly id: string | undefined; } +/** A screen listing split into lines, each with its depth and node id. */ +function treeEntries(observation: ExecutorObservation): TreeEntry[] { + return observation.text.split('\n').map((raw) => ({ + raw, + indent: indentOf(raw), + id: lineId(raw), + })); +} + +/** + * The end of every node's span: the first line after it that is not one of + * its descendants, a marker line with no node id or a line at its depth or + * above. A marker line closes every open span: no subtree reaches past one. + */ +function spanEnds(entries: readonly TreeEntry[]): number[] { + const ends = Array.from({ length: entries.length }, () => entries.length); + const open: number[] = []; + for (let index = 0; index < entries.length; index += 1) { + const entry = entries[index]!; + if (entry.id === undefined) { + while (open.length > 0) ends[open.pop()!] = index; + continue; + } + while (open.length > 0 && entries[open[open.length - 1]!]!.indent >= entry.indent) { + ends[open.pop()!] = index; + } + open.push(index); + } + return ends; +} + +/** + * The digest of every node's subtree: its node lines, focus stripped and + * unindented, in document order. Focus moves with every action, so it is + * left out the way the fold's own comparison leaves it out. Lines are joined + * with a separator no node line carries, so two different subtrees cannot + * digest alike by regrouping lines. + */ +function subtreeDigests(entries: readonly TreeEntry[], ends: readonly number[]): Map { + const digests = new Map(); + for (let index = 0; index < entries.length; index += 1) { + const entry = entries[index]!; + if (entry.id === undefined) continue; + const lines: string[] = []; + for (let inner = index; inner < ends[index]!; inner += 1) { + lines.push(withoutFocus(entries[inner]!.raw).trimStart()); + } + digests.set(entry.id, sha256Hex(lines.join('\n'))); + } + return digests; +} + /** Leading spaces of one listing line: its depth in the tree. */ function indentOf(line: string): number { return line.length - line.trimStart().length; @@ -310,18 +433,25 @@ function hasFocus(line: string): boolean { /** * The screen listing with every large unchanged subtree replaced by its root - * line plus a summary. A subtree folds only when it is not the document - * root, holds enough nodes to be worth the hiding, carries no focused node, - * and reads exactly as the model last saw it, with the same node ids: a node - * added, changed, or removed inside it keeps the whole subtree listed, and - * its children are then considered on their own. + * line plus a summary, and the subtrees the listing left in full. A subtree + * folds only when it is not the document root, holds enough nodes to be + * worth the hiding, carries no focused node, reads exactly as the model last + * saw it, with the same node ids, and still reads as it did at the revision + * the fold line names: a change update in between may have listed it only in + * part, or an earlier screen may itself have folded it, so the last + * observation's revision is no proof the model holds it whole. A subtree + * that fails the digest check is listed in full instead, which puts it on + * record at this revision. */ -function foldUnchangedSubtrees(observation: ExecutorObservation, previous: ShownScreen): string { - const entries: TreeEntry[] = observation.text.split('\n').map((raw) => ({ - raw, - indent: indentOf(raw), - id: lineId(raw), - })); +function foldUnchangedSubtrees( + observation: ExecutorObservation, + previous: ShownScreen, + fullyShown: ReadonlyMap, +): { readonly text: string; readonly listed: ReadonlyMap } { + const entries = treeEntries(observation); + const ends = spanEnds(entries); + const digests = subtreeDigests(entries, ends); + const foldedSpans: Array<{ readonly start: number; readonly end: number }> = []; const out: string[] = []; let index = 0; while (index < entries.length) { @@ -331,22 +461,51 @@ function foldUnchangedSubtrees(observation: ExecutorObservation, previous: Shown index += 1; continue; } - let end = index + 1; - while (end < entries.length) { - const next = entries[end]!; - if (next.id === undefined || next.indent <= entry.indent) break; - end += 1; - } + const end = ends[index]!; const inside = end - index - 1; - if (index > 0 && entry.indent > 0 && inside >= MIN_FOLD_NODES && subtreeUnchanged(entries, index, end, previous)) { - out.push(`${entry.raw} (${String(inside)} nodes inside, unchanged since revision ${previous.revision})`); + const shown = fullyShown.get(entry.id); + if ( + index > 0 && entry.indent > 0 && inside >= MIN_FOLD_NODES && + shown !== undefined && digests.get(entry.id) === shown.digest && + subtreeUnchanged(entries, index, end, previous) + ) { + out.push(`${entry.raw} (${String(inside)} nodes inside, unchanged since revision ${shown.revision})`); + foldedSpans.push({ start: index, end }); index = end; } else { out.push(entry.raw); index += 1; } } - return out.join('\n'); + return { text: out.join('\n'), listed: fullyListedSubtrees(entries, ends, digests, foldedSpans) }; +} + +/** + * The subtrees a folded listing left in full: every node whose span holds no + * line a fold hid. A folded root's own span is not fully listed, its hidden + * nodes are not, and neither is a subtree holding a folded one: the fold + * line it would name later has to be a screen that showed all of it. + */ +function fullyListedSubtrees( + entries: readonly TreeEntry[], + ends: readonly number[], + digests: ReadonlyMap, + foldedSpans: readonly { readonly start: number; readonly end: number }[], +): ReadonlyMap { + const hidden = Array.from({ length: entries.length }, () => 0); + for (const span of foldedSpans) { + for (let index = span.start + 1; index < span.end; index += 1) hidden[index] = 1; + } + const prefix = Array.from({ length: entries.length + 1 }, () => 0); + for (let index = 0; index < entries.length; index += 1) prefix[index + 1] = prefix[index]! + hidden[index]!; + const listed = new Map(); + for (let index = 0; index < entries.length; index += 1) { + const entry = entries[index]!; + if (entry.id === undefined) continue; + if (prefix[ends[index]!]! - prefix[index]! !== 0) continue; + listed.set(entry.id, digests.get(entry.id)!); + } + return listed; } /** diff --git a/packages/core/src/agent/transcript-compaction.ts b/packages/core/src/agent/transcript-compaction.ts index 0f8ecba730..d2370d28c7 100644 --- a/packages/core/src/agent/transcript-compaction.ts +++ b/packages/core/src/agent/transcript-compaction.ts @@ -86,9 +86,25 @@ function rewriteOldestParts(messages: ModelMessage[], count: number, rewrite: Pa * How many of the newest full screens stay verbatim in the transcript. One: * a whole new screen means the page changed mostly, and the screen it * replaced is dead weight on every later turn. Change updates never elide. + * A screen a surviving screen's fold lines depend on stays too: a folded + * line names the revision that last listed its subtree in full, and that + * screen is the only place the model can still read it. */ const FULL_SCREEN_PRESERVE_COUNT = 1; +/** A fold line's dependency: `(12 nodes inside, unchanged since revision b1)`. */ +const FOLD_DEPENDENCY_PATTERN = /\(\d+ nodes inside, unchanged since revision ([^)]+)\)/g; + +/** The revision a full screen's heading carries, or undefined when it carries none. */ +function screenRevision(text: string): string | undefined { + return /Current screen \(revision ([^,)]+),/.exec(text)?.[1]; +} + +/** The revisions a full screen's fold lines depend on. */ +function foldDependencies(text: string): string[] { + return [...text.matchAll(FOLD_DEPENDENCY_PATTERN)].map((match) => match[1]!); +} + /** * Bytes of superseded full screens a transcript carries before they are * elided. Re-reading a stale screen from the cache costs a tenth of sending @@ -104,23 +120,56 @@ export interface CompactScreenHistoryOptions { /** * Elides full screens the transcript no longer needs: every full screen but - * the newest few is reduced to its lead and a notice, in the opening prompt - * and in tool results alike, once the stale screens together outgrow the - * budget. Diffs are never touched. Returns the input array unchanged when - * nothing qualifies, so the caller can skip the override. + * the newest is reduced to its lead and a notice, in the opening prompt + * and in tool results alike, once the screens that may be elided together + * outgrow the budget. Diffs are never touched. A screen a surviving screen's + * fold lines depend on is never elided, with the screens it in turn depends + * on: the fold line names the revision that last listed its subtree in full, + * and eliding it would leave the model a pointer to content it no longer + * holds. Returns the input array unchanged when nothing qualifies, so the + * caller can skip the override. */ export function compactScreenHistory( messages: ModelMessage[], options: CompactScreenHistoryOptions = {}, ): ModelMessage[] { - const screens = transcriptParts(messages).map(fullScreenText).filter((text) => text !== undefined); - const stale = screens.length - FULL_SCREEN_PRESERVE_COUNT; - if (stale <= 0) return messages; - const staleBytes = screens.slice(0, stale).reduce((bytes, text) => bytes + Buffer.byteLength(text, 'utf8'), 0); + const screens = transcriptParts(messages) + .map(fullScreenText) + .filter((text) => text !== undefined) + .map((text) => ({ text, revision: screenRevision(text) })); + const preserved = new Set(); + for (let at = Math.max(0, screens.length - FULL_SCREEN_PRESERVE_COUNT); at < screens.length; at += 1) { + preserved.add(at); + } + // Follow each preserved screen's fold lines to the screens they depend on, + // transitively: a preserved screen's fold may name a screen that itself + // folded another subtree, and only that screen lists it in full. + const queue = [...preserved]; + while (queue.length > 0) { + const index = queue.pop()!; + for (const dependency of foldDependencies(screens[index]!.text)) { + screens.forEach((screen, at) => { + if (screen.revision === dependency && !preserved.has(at)) { + preserved.add(at); + queue.push(at); + } + }); + } + } + const elidable = screens.filter((_, index) => !preserved.has(index)); + if (elidable.length === 0) return messages; + const staleBytes = elidable.reduce((bytes, screen) => bytes + Buffer.byteLength(screen.text, 'utf8'), 0); if (staleBytes <= (options.keepStaleBytes ?? KEEP_STALE_SCREEN_BYTES)) return messages; - return rewriteOldestParts(messages, stale, { - user: (part) => (part.type === 'text' && isFullScreen(part.text) ? { ...part, text: elideScreen(part.text) } : undefined), - tool: elideResultScreen, + const preservedRevisions = new Set( + [...preserved].map((index) => screens[index]!.revision).filter((revision) => revision !== undefined), + ); + const keep = (text: string): boolean => { + const revision = screenRevision(text); + return revision !== undefined && preservedRevisions.has(revision); + }; + return rewriteOldestParts(messages, elidable.length, { + user: (part) => (part.type === 'text' && isFullScreen(part.text) && !keep(part.text) ? { ...part, text: elideScreen(part.text) } : undefined), + tool: (part) => elideResultScreen(part, keep), }); } @@ -132,19 +181,21 @@ function fullScreenText(part: UserPart | ToolPart): string | undefined { /** * The tool result with its full screen elided, or undefined when it carries - * none. A result with a screenshot attached loses only the screen text and - * keeps its other items: the image, or the notice that replaced it. + * none or the screen is one a surviving screen depends on. A result with a + * screenshot attached loses only the screen text and keeps its other items: + * the image, or the notice that replaced it. */ -function elideResultScreen(part: ToolPart): ToolPart | undefined { +function elideResultScreen(part: ToolPart, keep: (text: string) => boolean): ToolPart | undefined { if (part.type !== 'tool-result') return undefined; const { output } = part; if (output.type === 'text') { - return isFullScreen(output.value) ? { ...part, output: { ...output, value: elideScreen(output.value) } } : undefined; + return isFullScreen(output.value) && !keep(output.value) ? { ...part, output: { ...output, value: elideScreen(output.value) } } : undefined; } if (output.type !== 'content' || !output.value.some(isScreenItem)) return undefined; + if (output.value.every((item) => !isScreenItem(item) || keep(item.text))) return undefined; return { ...part, - output: { ...output, value: output.value.map((item) => (isScreenItem(item) ? { ...item, text: elideScreen(item.text) } : item)) }, + output: { ...output, value: output.value.map((item) => (isScreenItem(item) && !keep(item.text) ? { ...item, text: elideScreen(item.text) } : item)) }, }; } diff --git a/packages/e2e/tests/unit/agent-screen-update.test.ts b/packages/e2e/tests/unit/agent-screen-update.test.ts index 847f704fdb..5f7b9e4151 100644 --- a/packages/e2e/tests/unit/agent-screen-update.test.ts +++ b/packages/e2e/tests/unit/agent-screen-update.test.ts @@ -264,6 +264,18 @@ describe('ScreenPresenter folds large unchanged subtrees in the full-screen fall ' #n37 link "Back to orders"', ]; + const MAIN_SEARCH = [ + ' #n40 main "Order search"', + ' #n41 heading "Find an order"', + ' #n42 textbox "Order number"', + ' #n43 button "Search"', + ' #n44 text "No order selected"', + ' #n45 button "Clear"', + ' #n46 link "All orders"', + ' #n47 text "Search by number or customer"', + ' #n48 button "Help"', + ]; + function orderScreen(main: readonly string[], sidebarCount: number): string[] { return ['#n1 document "Orders"', ...sidebar(sidebarCount), ...main]; } @@ -355,6 +367,65 @@ describe('ScreenPresenter folds large unchanged subtrees in the full-screen fall expect(text).toContain(' #n3 link "Item 1"'); expect(text).toContain(' #n14 link "Item 12"'); }); + + it('names the revision that last listed the subtree in full, not an intervening diff', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + // The sidebar is not listed at b2: the diff names only the changed heading. + const changed = orderScreen(MAIN_LIST, 12).map((line) => + (line.includes('#n21 ') ? ' #n21 heading "Orders (2)"' : line), + ); + const diff = presenter.update(screen('b2', changed)); + expect(diff).toContain('changed #n21 heading "Orders (2)"'); + expect(diff).not.toContain('Current screen'); + const text = presenter.update(screen('b3', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b2.'); + expect(text).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + expect(text).not.toContain('unchanged since revision b2'); + }); + + it('names the original revision when the previous screen itself folded the subtree', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const first = presenter.update(screen('b2', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(first).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + const second = presenter.update(screen('b3', orderScreen(MAIN_SEARCH, 12)), { lead: 'Tapped #n43.' }); + expect(second).toContain('The screen changed substantially since revision b2.'); + expect(second).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + expect(second).not.toContain('unchanged since revision b2'); + }); + + it('lists a changed subtree in full again, then folds it against that listing', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const renamed = orderScreen(MAIN_LIST, 12).map((line) => + (line.includes('#n5 ') ? ' #n5 link "Item 3 (new)"' : line), + ); + expect(presenter.update(screen('b2', renamed))).toContain('changed #n5 link "Item 3 (new)"'); + // The sidebar changed at b2, so b3 may not fold it against b1: the model + // holds the old item text there, not the new one. + const rename = (lines: readonly string[]): string[] => + lines.map((line) => (line.includes('#n5 ') ? ' #n5 link "Item 3 (new)"' : line)); + const relisted = presenter.update(screen('b3', rename(orderScreen(MAIN_DETAILS, 12))), { lead: 'Tapped #n22.' }); + expect(relisted).toContain('The screen changed substantially since revision b2.'); + expect(relisted).not.toContain('nodes inside, unchanged since revision'); + expect(relisted).toContain(' #n5 link "Item 3 (new)"'); + const folded = presenter.update(screen('b4', rename(orderScreen(MAIN_SEARCH, 12))), { lead: 'Tapped #n43.' }); + expect(folded).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b3)'); + expect(folded).not.toContain('unchanged since revision b1'); + }); + + it('still folds against the original revision when a change was undone', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const renamed = orderScreen(MAIN_LIST, 12).map((line) => + (line.includes('#n5 ') ? ' #n5 link "Item 3 (new)"' : line), + ); + presenter.update(screen('b2', renamed)); + presenter.update(screen('b3', orderScreen(MAIN_LIST, 12))); + const text = presenter.update(screen('b4', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(text).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + }); }); describe('ScreenPresenter and the on-screen keyboard', () => { diff --git a/packages/e2e/tests/unit/transcript-compaction.test.ts b/packages/e2e/tests/unit/transcript-compaction.test.ts index a66d45f4d2..3041d604fd 100644 --- a/packages/e2e/tests/unit/transcript-compaction.test.ts +++ b/packages/e2e/tests/unit/transcript-compaction.test.ts @@ -67,6 +67,21 @@ function textChars(messages: readonly ModelMessage[]): number { }, 0); } +/** A full screen whose subtree folds against an earlier revision. */ +function foldedScreenResult(id: string, revision: string, dependsOn: string, subtree = 'Sidebar'): ModelMessage { + return { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: id, + toolName: 'tap', + output: { type: 'text', value: `Tapped #n3.\n\nThe screen changed substantially since revision b0. Current screen (revision ${revision}, 14 nodes):\n#n1 document\n #n2 navigation "${subtree}" (12 nodes inside, unchanged since revision ${dependsOn})\n #n20 main "Orders"` }, + }, + ], + }; +} + function changesResult(id: string): ModelMessage { return { role: 'tool', @@ -153,6 +168,56 @@ describe('compactScreenHistory', () => { expect(compacted[2]).toEqual(messages[2]); }); + it('keeps a full screen a surviving folded screen depends on', () => { + const messages = [ + opening, + changesResult('c1'), + fullScreenResult('c2', 'b2'), + foldedScreenResult('c4', 'b4', 'b0'), + ]; + const compacted = compactScreenHistory(messages, { keepStaleBytes: 0 }); + expect(compacted).not.toBe(messages); + // The opening screen is the only place the folded sidebar is listed in + // full, so it stays verbatim; the middle screen, which nothing names, + // is elided. + expect(compacted[0]).toEqual(messages[0]); + expect(compacted[1]).toEqual(messages[1]); + expect(toolOutput(compacted[2]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); + expect(compacted[3]).toEqual(messages[3]); + }); + + it('elides a screen no surviving fold depends on, even when it is the oldest', () => { + const messages = [ + opening, + changesResult('c1'), + fullScreenResult('c2', 'b2'), + foldedScreenResult('c4', 'b4', 'b2'), + ]; + const compacted = compactScreenHistory(messages, { keepStaleBytes: 0 }); + expect(compacted).not.toBe(messages); + expect(compacted[0]!.content).toContain('[earlier screen elided'); + expect(compacted[2]).toEqual(messages[2]); + expect(compacted[3]).toEqual(messages[3]); + }); + + it('keeps a dependency screen transitively, through a screen that itself folded', () => { + const messages = [ + opening, + fullScreenResult('c1', 'bx'), + foldedScreenResult('c2', 'b1', 'b0', 'Footer'), + foldedScreenResult('c3', 'b2', 'b1'), + ]; + const compacted = compactScreenHistory(messages, { keepStaleBytes: 0 }); + expect(compacted).not.toBe(messages); + // The newest screen names b1, and b1 names b0: both stay, so every fold + // line still points at a screen the model holds. Only the screen no + // fold names is elided. + expect(compacted[0]).toEqual(messages[0]); + expect(toolOutput(compacted[1]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); + expect(compacted[2]).toEqual(messages[2]); + expect(compacted[3]).toEqual(messages[3]); + }); + it('shrinks a pixel-mode history on the budget-free pass the overflow retry runs after the cache-friendly pass kept it', () => { const messages = [opening, pixelScreenResult('c1', 'b1', 40), changesResult('c2'), fullScreenResult('c3', 'b3'), pixelScreenResult('c4', 'b4', 40)]; const kept = compactScreenHistory(messages); diff --git a/skills/e2e/references/agent.md b/skills/e2e/references/agent.md index 29e3266da4..24eed88c19 100644 --- a/skills/e2e/references/agent.md +++ b/skills/e2e/references/agent.md @@ -356,8 +356,8 @@ values, or a `Secret`'s value; password fields masked. The first screen of a step arrives whole; later action results report what changed, keyed by node ids stable while an element exists, or the whole screen when most changed, with large subtrees that have not changed since the last screen -folded to one line each naming the node count and the revision they were last -seen at. `observe` still returns the complete tree. Pixels arrive through `vision` on a judgment or the act loop's +folded to one line each naming the node count and the revision that last +listed them in full. `observe` still returns the complete tree. Pixels arrive through `vision` on a judgment or the act loop's `screenshot` and point tools, with secure fields masked. When the browser engine's tree capture times out, the model gets a screenshot and a warning, a judgment needs `vision: true`, and no control may be inferred From b9ff882f8b72ff4f7d228589e341763686a6652a Mon Sep 17 00:00:00 2001 From: Varshith Puli Date: Sat, 14 Nov 2026 16:05:52 +0000 Subject: [PATCH 04/12] fix(plugins): sync generated skill reference with fold-line wording The fold-line change updated skills/e2e/references/agent.md but the generated plugins/e2e copy was not re-synced, so pnpm plugins:check failed on the PR. Regenerated with pnpm plugins:sync. Signed-off-by: Varshith Puli --- plugins/e2e/skills/e2e/references/agent.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/e2e/skills/e2e/references/agent.md b/plugins/e2e/skills/e2e/references/agent.md index c564dbb333..24eed88c19 100644 --- a/plugins/e2e/skills/e2e/references/agent.md +++ b/plugins/e2e/skills/e2e/references/agent.md @@ -355,7 +355,9 @@ summaries, and your context; never raw HTML, cookies, headers, environment values, or a `Secret`'s value; password fields masked. The first screen of a step arrives whole; later action results report what changed, keyed by node ids stable while an element exists, or the whole screen when -most changed. Pixels arrive through `vision` on a judgment or the act loop's +most changed, with large subtrees that have not changed since the last screen +folded to one line each naming the node count and the revision that last +listed them in full. `observe` still returns the complete tree. Pixels arrive through `vision` on a judgment or the act loop's `screenshot` and point tools, with secure fields masked. When the browser engine's tree capture times out, the model gets a screenshot and a warning, a judgment needs `vision: true`, and no control may be inferred From a30b6fd7327622bec223ce0bfc6928e62080942e Mon Sep 17 00:00:00 2001 From: Varshith Puli Date: Sat, 14 Nov 2026 19:24:48 +0000 Subject: [PATCH 05/12] fix(agent): clear fold records on truncated renders and drop the redundant previous-screen check A truncated full screen recorded nothing and folded nothing, so the old fold record survived it while the screen it named could already be elided from the transcript: the next fallback then folded against a revision the model no longer holds. A full render now replaces the record with exactly what it listed in full or folded, so a truncated render clears it entirely and a later fold can never name a screen that render did not show. The comparison with the previous screen in the fold check is gone: the digest already covers the subtree node ids and lines, and the check could suppress a valid fold when an intervening change was undone. The focused node guard stays, as its own check. Digests are now computed only for subtrees large enough to fold, the fold-line pattern is exported from screen-update.ts for the transcript elision to share, and hasFocus shares the state-bracket parse with withoutFocus. The skill reference now says observe returns what changed for the built-in agent, with the whole tree only for the MCP observe, and the changeset is one user-facing sentence. Signed-off-by: Varshith Puli --- .changeset/mcp-fold-unchanged-subtrees.md | 2 +- packages/core/src/agent/screen-update.ts | 174 +++++++++--------- .../core/src/agent/transcript-compaction.ts | 5 +- .../tests/unit/agent-screen-update.test.ts | 114 ++++++++++++ .../tests/unit/transcript-compaction.test.ts | 17 ++ plugins/e2e/skills/e2e/references/agent.md | 4 +- skills/e2e/references/agent.md | 4 +- 7 files changed, 226 insertions(+), 94 deletions(-) diff --git a/.changeset/mcp-fold-unchanged-subtrees.md b/.changeset/mcp-fold-unchanged-subtrees.md index 1be3551116..c3969fb529 100644 --- a/.changeset/mcp-fold-unchanged-subtrees.md +++ b/.changeset/mcp-fold-unchanged-subtrees.md @@ -2,4 +2,4 @@ "e2e": patch --- -When an action changes most of the screen, the whole-screen fallback now folds large unchanged subtrees (10 or more nodes) to a one-line summary naming the node count and the revision that last listed them in full, instead of repeating them. A fold line never names a revision that only reported the subtree as a change, or that itself folded it. Transcript elision keeps the screens those fold lines depend on, so a folded line never names a screen the model no longer holds. `observe` still returns the complete tree. +When an action changes most of the screen, large subtrees that did not change are shown as one line instead of being repeated. diff --git a/packages/core/src/agent/screen-update.ts b/packages/core/src/agent/screen-update.ts index 1a2e776050..7b52efb0b7 100644 --- a/packages/core/src/agent/screen-update.ts +++ b/packages/core/src/agent/screen-update.ts @@ -42,14 +42,15 @@ const MIN_FOLD_NODES = 10; /** Every full screen is introduced by this phrase; the transcript elision looks for it. */ export const FULL_SCREEN_PATTERN = /Current screen \(revision /; +/** A fold line's dependency: `(12 nodes inside, unchanged since revision b1)`; the transcript elision looks for it. */ +export const FOLD_DEPENDENCY_PATTERN = /\(\d+ nodes inside, unchanged since revision ([^)]+)\)/g; + /** The screen as the model last received it, indexed for comparison. */ interface ShownScreen { readonly revision: string; /** Node lines in document order, without indentation or focus; the truncation marker is not a node. */ readonly order: readonly string[]; readonly byId: ReadonlyMap; - /** Leading spaces of each node line, so a subtree can be found in `order`. */ - readonly indent: ReadonlyMap; readonly nodes: number; /** The listing is cut short (byte limit or engine cap): nodes beyond it exist but are not listed. */ readonly truncated: boolean; @@ -161,30 +162,44 @@ export class ScreenPresenter { */ private renderWhole(observation: ExecutorObservation): string { const rendered = renderFull(observation); - this.noteListed(rendered.listed); + this.noteListed(rendered.listed, rendered.folded); return rendered.text; } /** - * Records the subtrees `listed` names as last listed in full at the newest - * revision, and drops the record for nodes no longer on screen. Called with - * no map by a render that listed nothing in full, a diff or an unchanged - * screen, which only prunes: a subtree the model last saw whole stays on - * record until a node in it leaves the screen. + * Records the subtrees a full render listed in full or folded, and drops + * every other record. Called with no map by a render that listed nothing + * in full, a diff or an unchanged screen, which only prunes: a subtree + * the model last saw whole stays on record until a node in it leaves the + * screen. A full render replaces the record with exactly what it showed: + * a subtree it neither listed nor folded is not on record afterwards, so + * a later fold can never name a screen this render did not show. A + * truncated render lists and folds nothing, so it clears the record + * entirely: the screens the old records named may already be elided from + * the transcript, and a fold naming one would point at content the model + * no longer holds. */ - private noteListed(listed?: ReadonlyMap): void { + private noteListed(listed?: ReadonlyMap, folded?: ReadonlySet): void { const shown = this.shown; if (shown === undefined) { this.fullyShown.clear(); return; } - for (const id of this.fullyShown.keys()) { - if (!shown.byId.has(id)) this.fullyShown.delete(id); + if (listed === undefined) { + for (const id of this.fullyShown.keys()) { + if (!shown.byId.has(id)) this.fullyShown.delete(id); + } + return; + } + const next = new Map(); + for (const id of folded ?? []) { + const kept = this.fullyShown.get(id); + if (kept !== undefined) next.set(id, kept); } - if (listed === undefined) return; for (const [id, digest] of listed) { - this.fullyShown.set(id, { revision: shown.revision, digest }); + next.set(id, { revision: shown.revision, digest }); } + this.fullyShown = next; } /** @@ -220,8 +235,8 @@ export class ScreenPresenter { return `${lead}${renderUnchanged(previous.revision, observation, expectChange, pixels)}${closed}`; } if (changes >= MAX_DIFF_LINES || changes > MAX_DIFF_SHARE * next.order.length) { - const rendered = renderFull(observation, previous, this.fullyShown); - this.noteListed(rendered.listed); + const rendered = renderFull(observation, this.fullyShown); + this.noteListed(rendered.listed, rendered.folded); return `${lead}The screen changed substantially since revision ${previous.revision}. ${rendered.text}${closed}`; } const assurance = observation.truncated @@ -324,11 +339,13 @@ function describeCounts(diff: readonly string[]): string { return counts === '' ? 'no listed node changed' : counts; } -/** A rendered full screen, with the subtrees it listed in full and what each read. */ +/** A rendered full screen, with the subtrees it listed in full and what each read, and the subtrees it folded. */ interface RenderedFullScreen { readonly text: string; /** Node ids whose subtree the rendering listed in full, each with the digest of its node lines. */ readonly listed: ReadonlyMap; + /** Node ids whose subtree the rendering folded, keeping the record of the revision that last listed it in full. */ + readonly folded: ReadonlySet; } /** One full screen, headed so the elision can find it later. A screen that @@ -339,26 +356,27 @@ interface RenderedFullScreen { * since the nodes past the cut are unknown. */ function renderFull( observation: ExecutorObservation, - previous?: ShownScreen, fullyShown?: ReadonlyMap, ): RenderedFullScreen { if (observation.treeUnavailable === true) { return { text: `Current screen (revision ${observation.revision}${describeLocation(observation)}, semantic capture unavailable):\n${observation.text}`, listed: new Map(), + folded: new Set(), }; } const nodes = indexScreen(observation).nodes; const head = `Current screen (revision ${observation.revision}${describeLocation(observation)}, ${String(nodes)} nodes):\n`; - if (previous === undefined || fullyShown === undefined || observation.truncated) { + if (fullyShown === undefined || observation.truncated) { const entries = treeEntries(observation); return { text: `${head}${observation.text}`, listed: observation.truncated ? new Map() : subtreeDigests(entries, spanEnds(entries)), + folded: new Set(), }; } - const folded = foldUnchangedSubtrees(observation, previous, fullyShown); - return { text: `${head}${folded.text}`, listed: folded.listed }; + const folded = foldUnchangedSubtrees(observation, fullyShown); + return { text: `${head}${folded.text}`, listed: folded.listed, folded: folded.folded }; } /** A node line of a screen listing, with its depth and id. */ @@ -400,17 +418,20 @@ function spanEnds(entries: readonly TreeEntry[]): number[] { } /** - * The digest of every node's subtree: its node lines, focus stripped and - * unindented, in document order. Focus moves with every action, so it is - * left out the way the fold's own comparison leaves it out. Lines are joined - * with a separator no node line carries, so two different subtrees cannot - * digest alike by regrouping lines. + * The digest of every subtree large enough to fold: its node lines, focus + * stripped and unindented, in document order. Focus moves with every action, + * so it is left out the way the fold's own comparison leaves it out. Lines + * are joined with a separator no node line carries, so two different subtrees + * cannot digest alike by regrouping lines. A smaller subtree never folds, so + * it needs no digest: it is listed in full whenever a screen goes out whole, + * which puts it on record at that revision if it later grows large enough. */ function subtreeDigests(entries: readonly TreeEntry[], ends: readonly number[]): Map { const digests = new Map(); for (let index = 0; index < entries.length; index += 1) { const entry = entries[index]!; if (entry.id === undefined) continue; + if (ends[index]! - index - 1 < MIN_FOLD_NODES) continue; const lines: string[] = []; for (let inner = index; inner < ends[index]!; inner += 1) { lines.push(withoutFocus(entries[inner]!.raw).trimStart()); @@ -427,31 +448,41 @@ function indentOf(line: string): number { /** True when the line's state bracket names the focused node. */ function hasFocus(line: string): boolean { - const states = / \[([^\]]*)\]$/.exec(line)?.[1]; - return states !== undefined && states.split(' ').includes('focused'); + const bracket = stateBracket(line); + return bracket !== undefined && bracket.states.split(' ').includes('focused'); +} + +/** + * The trailing state bracket of a line, with the line before it, or + * undefined when the line carries none. The states bracket is always the + * last token of a line. + */ +function stateBracket(line: string): { readonly states: string; readonly before: string } | undefined { + const match = / \[([^\]]*)\]$/.exec(line); + return match === null ? undefined : { states: match[1]!, before: line.slice(0, match.index) }; } /** * The screen listing with every large unchanged subtree replaced by its root * line plus a summary, and the subtrees the listing left in full. A subtree * folds only when it is not the document root, holds enough nodes to be - * worth the hiding, carries no focused node, reads exactly as the model last - * saw it, with the same node ids, and still reads as it did at the revision - * the fold line names: a change update in between may have listed it only in - * part, or an earlier screen may itself have folded it, so the last - * observation's revision is no proof the model holds it whole. A subtree - * that fails the digest check is listed in full instead, which puts it on - * record at this revision. + * worth the hiding, carries no focused node, and still reads as it did at + * the revision the fold line names: the digest covers its node ids and lines, + * so no comparison with the screen before it is needed. A change update in + * between may have listed it only in part, or an earlier screen may itself + * have folded it, so the last observation's revision is no proof the model + * holds it whole. A subtree that fails the digest check is listed in full + * instead, which puts it on record at this revision. */ function foldUnchangedSubtrees( observation: ExecutorObservation, - previous: ShownScreen, fullyShown: ReadonlyMap, -): { readonly text: string; readonly listed: ReadonlyMap } { +): { readonly text: string; readonly listed: ReadonlyMap; readonly folded: ReadonlySet } { const entries = treeEntries(observation); const ends = spanEnds(entries); const digests = subtreeDigests(entries, ends); const foldedSpans: Array<{ readonly start: number; readonly end: number }> = []; + const foldedIds = new Set(); const out: string[] = []; let index = 0; while (index < entries.length) { @@ -467,17 +498,26 @@ function foldUnchangedSubtrees( if ( index > 0 && entry.indent > 0 && inside >= MIN_FOLD_NODES && shown !== undefined && digests.get(entry.id) === shown.digest && - subtreeUnchanged(entries, index, end, previous) + !subtreeHoldsFocus(entries, index, end) ) { out.push(`${entry.raw} (${String(inside)} nodes inside, unchanged since revision ${shown.revision})`); foldedSpans.push({ start: index, end }); + foldedIds.add(entry.id); index = end; } else { out.push(entry.raw); index += 1; } } - return { text: out.join('\n'), listed: fullyListedSubtrees(entries, ends, digests, foldedSpans) }; + return { text: out.join('\n'), listed: fullyListedSubtrees(entries, ends, digests, foldedSpans), folded: foldedIds }; +} + +/** True when a node in the span holds focus: the fold would hide where focus is. */ +function subtreeHoldsFocus(entries: readonly TreeEntry[], start: number, end: number): boolean { + for (let index = start; index < end; index += 1) { + if (hasFocus(entries[index]!.raw)) return true; + } + return false; } /** @@ -503,49 +543,13 @@ function fullyListedSubtrees( const entry = entries[index]!; if (entry.id === undefined) continue; if (prefix[ends[index]!]! - prefix[index]! !== 0) continue; - listed.set(entry.id, digests.get(entry.id)!); + const digest = digests.get(entry.id); + if (digest === undefined) continue; + listed.set(entry.id, digest); } return listed; } -/** - * True when every node line in the span reads as the model last saw it and - * the span holds the same node ids it held then. Focus alone never counts - * as a change, but a subtree holding the focused node never folds: the fold - * would hide where focus is. - */ -function subtreeUnchanged(entries: readonly TreeEntry[], start: number, end: number, previous: ShownScreen): boolean { - const current = new Set(); - for (let index = start; index < end; index += 1) { - const entry = entries[index]!; - if (entry.id === undefined || hasFocus(entry.raw)) return false; - if (previous.byId.get(entry.id) !== withoutFocus(entry.raw).trimStart()) return false; - current.add(entry.id); - } - const before = previousSubtreeIds(previous, entries[start]!.id!); - if (before === undefined || before.size !== current.size) return false; - for (const id of current) if (!before.has(id)) return false; - return true; -} - -/** The node ids of the subtree rooted at `rootId` in the shown screen. */ -function previousSubtreeIds(previous: ShownScreen, rootId: string): Set | undefined { - const rootIndex = previous.order.findIndex((line) => lineId(line) === rootId); - if (rootIndex === -1) return undefined; - const rootIndent = previous.indent.get(rootId); - if (rootIndent === undefined) return undefined; - const ids = new Set(); - for (let index = rootIndex; index < previous.order.length; index += 1) { - const id = lineId(previous.order[index]!); - if (id === undefined) return undefined; - const indent = previous.indent.get(id); - if (indent === undefined) return undefined; - if (index > rootIndex && indent <= rootIndent) break; - ids.add(id); - } - return ids; -} - /** * What the screenshot going out with an update says against the one the * model held: `none` when there is no pair to compare (no screenshot, or the @@ -619,22 +623,20 @@ function lineId(line: string): string | undefined { } /** - * The line without its focus state. Focus moves with every action and on its + * The node lines with focus stripped. Focus moves with every action and on its * own, so a diff that reported it would drown the changes that matter and - * make a control that did nothing look like it did something. The states - * bracket is always the last token of a line. The full screen keeps focus. + * make a control that did nothing look like it did something. The full screen keeps focus. */ function withoutFocus(line: string): string { - return line.replace(/ \[([^\]]*)\]$/, (_match, states: string) => { - const kept = states.split(' ').filter((state) => state !== 'focused'); - return kept.length === 0 ? '' : ` [${kept.join(' ')}]`; - }); + const bracket = stateBracket(line); + if (bracket === undefined) return line; + const kept = bracket.states.split(' ').filter((state) => state !== 'focused'); + return kept.length === 0 ? bracket.before : `${bracket.before} [${kept.join(' ')}]`; } function indexScreen(observation: ExecutorObservation): ShownScreen { const order: string[] = []; const byId = new Map(); - const indent = new Map(); for (const raw of observation.text.split('\n')) { const id = lineId(raw); if (id === undefined) continue; @@ -643,13 +645,11 @@ function indexScreen(observation: ExecutorObservation): ShownScreen { const line = withoutFocus(raw).trimStart(); order.push(line); byId.set(id, line); - indent.set(id, indentOf(raw)); } return { revision: observation.revision, order, byId, - indent, nodes: byId.size, truncated: observation.truncated, keyboard: order.some((line) => KEYBOARD_LINE.test(line)), diff --git a/packages/core/src/agent/transcript-compaction.ts b/packages/core/src/agent/transcript-compaction.ts index d2370d28c7..418e13e61a 100644 --- a/packages/core/src/agent/transcript-compaction.ts +++ b/packages/core/src/agent/transcript-compaction.ts @@ -11,7 +11,7 @@ */ import type { FilePart, ModelMessage, TextPart, ToolResultPart, UserContent } from 'ai'; -import { FULL_SCREEN_PATTERN, toolResultTexts } from './screen-update.ts'; +import { FOLD_DEPENDENCY_PATTERN, FULL_SCREEN_PATTERN, toolResultTexts } from './screen-update.ts'; /** A part of a user message; a string message counts as one text part. */ type UserPart = Exclude[number]; @@ -92,9 +92,6 @@ function rewriteOldestParts(messages: ModelMessage[], count: number, rewrite: Pa */ const FULL_SCREEN_PRESERVE_COUNT = 1; -/** A fold line's dependency: `(12 nodes inside, unchanged since revision b1)`. */ -const FOLD_DEPENDENCY_PATTERN = /\(\d+ nodes inside, unchanged since revision ([^)]+)\)/g; - /** The revision a full screen's heading carries, or undefined when it carries none. */ function screenRevision(text: string): string | undefined { return /Current screen \(revision ([^,)]+),/.exec(text)?.[1]; diff --git a/packages/e2e/tests/unit/agent-screen-update.test.ts b/packages/e2e/tests/unit/agent-screen-update.test.ts index 5f7b9e4151..d85ed25283 100644 --- a/packages/e2e/tests/unit/agent-screen-update.test.ts +++ b/packages/e2e/tests/unit/agent-screen-update.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from 'vitest'; +import type { ModelMessage, ToolResultPart } from 'ai'; import type { ExecutorObservation } from '@e2e-dev/core/lib/agent/executor'; import { ScreenPresenter } from '@e2e-dev/core/lib/agent/screen-update'; +import { compactScreenHistory } from '@e2e-dev/core/lib/agent/transcript-compaction'; function screen(revision: string, lines: readonly string[], extra: Partial = {}): ExecutorObservation { return { @@ -21,6 +23,22 @@ const HOME = [ ' #n6 textbox "Email"', ]; +/** One tool result carrying a rendered screen as its text. */ +function toolResult(id: string, text: string): ModelMessage { + return { + role: 'tool', + content: [{ type: 'tool-result', toolCallId: id, toolName: 'tap', output: { type: 'text', value: text } }], + }; +} + +/** The text of a tool message's one text result. */ +function toolText(message: ModelMessage): string { + const [part] = message.content as ToolResultPart[]; + const output = part!.output; + if (output.type !== 'text') throw new Error('expected a text tool result'); + return output.value; +} + describe('ScreenPresenter', () => { it('retires semantic history through repeated unavailable captures and shows recovery whole', () => { const presenter = new ScreenPresenter(); @@ -426,6 +444,102 @@ describe('ScreenPresenter folds large unchanged subtrees in the full-screen fall const text = presenter.update(screen('b4', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); expect(text).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); }); + + it('folds against the original listing when an intervening change was undone, even though the previous screen differs', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const renamed = orderScreen(MAIN_LIST, 12).map((line) => + (line.includes('#n5 ') ? ' #n5 link "Item 3 (new)"' : line), + ); + expect(presenter.update(screen('b2', renamed))).toContain('changed #n5 link "Item 3 (new)"'); + // b3 changes most of the screen and puts the sidebar back as b1 listed + // it: the fold names b1, since the digest is what the model holds, not + // the screen before it. + const text = presenter.update(screen('b3', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b2.'); + expect(text).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + }); + + it('a truncated full screen clears the fold record, so the next fallback lists the subtree in full instead of naming an elided screen', () => { + const presenter = new ScreenPresenter(); + const b1 = presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const b2 = presenter.update(screen('b2', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(b2).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + // b3 is a truncated full screen: it records nothing and folds nothing. + const extra = Array.from({ length: 12 }, (_, i) => ` #n${String(i + 60)} button "Extra ${String(i + 1)}"`); + const b3 = presenter.update(screen('b3', [...orderScreen(MAIN_SEARCH, 12), ...extra], { truncated: true }), { lead: 'Tapped #n43.' }); + expect(b3).toContain('The screen changed substantially since revision b2.'); + expect(b3).not.toContain('nodes inside, unchanged since revision'); + // By the time b4 renders, compaction has already elided b1 and b2: the + // truncated screen carries no fold, so nothing preserves them. + const messages = [b1, b2, b3].map((text, index) => toolResult(`c${String(index)}`, text)); + const compacted = compactScreenHistory(messages, { keepStaleBytes: 0 }); + const b1Text = toolText(compacted[0]!); + expect(b1Text).toContain('[earlier screen elided'); + expect(b1Text).not.toContain('#n3 link "Item 1"'); + // The record b3 cleared means b4 cannot fold against b1: the sidebar is + // listed in full again instead of naming a screen the model no longer holds. + const b4 = presenter.update(screen('b4', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(b4).toContain('The screen changed substantially since revision b3.'); + expect(b4).not.toContain('nodes inside, unchanged since revision'); + expect(b4).toContain(' #n3 link "Item 1"'); + expect(b4).toContain(' #n14 link "Item 12"'); + }); + + it('drops the fold record when a subtree leaves the screen, so a re-added subtree is listed in full', () => { + const presenter = new ScreenPresenter(); + const extras = [ + ' #n50 group "Extras"', + ...Array.from({ length: 24 }, (_, i) => ` #n${String(60 + i)} button "Extra ${String(i + 1)}"`), + ]; + const wide = (main: readonly string[], sidebarCount: number): string[] => + ['#n1 document "Orders"', ...sidebar(sidebarCount), ...main, ...extras]; + presenter.initial(screen('b1', wide(MAIN_LIST, 12))); + // The sidebar leaves in a change update: 13 removed lines of 43 nodes. + const removed = presenter.update(screen('b2', ['#n1 document "Orders"', ...MAIN_LIST, ...extras])); + expect(removed).toContain('removed #n2 navigation "Sidebar"'); + expect(removed).not.toContain('Current screen'); + const readded = presenter.update(screen('b3', wide(MAIN_LIST, 12))); + expect(readded).toContain('added #n2 navigation "Sidebar"'); + expect(readded).not.toContain('Current screen'); + // The record was dropped when the sidebar left, so the fallback lists it + // in full even though it reads exactly as b1 listed it. + const renamedExtras = extras.map((line, index) => + (index > 0 ? line.replace(/Extra (\d+)/, 'Extra $1 (new)') : line), + ); + const text = presenter.update(screen('b4', ['#n1 document "Orders"', ...sidebar(12), ...MAIN_DETAILS, ...renamedExtras]), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b3.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain(' #n3 link "Item 1"'); + }); + + it('never folds a top-level root, even one holding enough unchanged nodes', () => { + const presenter = new ScreenPresenter(); + // A second top-level root with its leaves directly under it: the root + // itself is the only node that could fold, and its indent is 0. + const second = [ + '#n50 document "Second"', + ...Array.from({ length: 12 }, (_, i) => ` #n${String(60 + i)} link "Second ${String(i + 1)}"`), + ]; + const first = [ + '#n1 document "First"', + ' #n2 heading "First"', + ' #n3 button "First 1"', + ' #n4 button "First 2"', + ' #n5 button "First 3"', + ]; + presenter.initial(screen('b1', [...first, ...second])); + const changedFirst = [ + '#n1 document "First (new)"', + ' #n2 heading "First changed"', + ...Array.from({ length: 9 }, (_, i) => ` #n${String(70 + i)} button "New ${String(i + 1)}"`), + ]; + const text = presenter.update(screen('b2', [...changedFirst, ...second]), { lead: 'Tapped #n3.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain('#n50 document "Second"'); + expect(text).toContain(' #n60 link "Second 1"'); + }); }); describe('ScreenPresenter and the on-screen keyboard', () => { diff --git a/packages/e2e/tests/unit/transcript-compaction.test.ts b/packages/e2e/tests/unit/transcript-compaction.test.ts index 3041d604fd..0703656570 100644 --- a/packages/e2e/tests/unit/transcript-compaction.test.ts +++ b/packages/e2e/tests/unit/transcript-compaction.test.ts @@ -218,6 +218,23 @@ describe('compactScreenHistory', () => { expect(compacted[3]).toEqual(messages[3]); }); + it('leaves a screenshot result whose screens are all kept untouched, without spending the elision budget', () => { + const messages = [ + pixelScreenResult('c1', 'b1'), + fullScreenResult('c2', 'b2'), + fullScreenResult('c3', 'b3'), + foldedScreenResult('c4', 'b4', 'b1'), + ]; + const compacted = compactScreenHistory(messages, { keepStaleBytes: 0 }); + expect(compacted).not.toBe(messages); + // The opening screenshot result carries only the kept b1 screen, so it + // is returned untouched and the budget still covers both elidable screens. + expect(compacted[0]).toBe(messages[0]); + expect(toolOutput(compacted[1]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); + expect(toolOutput(compacted[2]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); + expect(compacted[3]).toBe(messages[3]); + }); + it('shrinks a pixel-mode history on the budget-free pass the overflow retry runs after the cache-friendly pass kept it', () => { const messages = [opening, pixelScreenResult('c1', 'b1', 40), changesResult('c2'), fullScreenResult('c3', 'b3'), pixelScreenResult('c4', 'b4', 40)]; const kept = compactScreenHistory(messages); diff --git a/plugins/e2e/skills/e2e/references/agent.md b/plugins/e2e/skills/e2e/references/agent.md index 24eed88c19..d54aa7e22a 100644 --- a/plugins/e2e/skills/e2e/references/agent.md +++ b/plugins/e2e/skills/e2e/references/agent.md @@ -357,7 +357,9 @@ screen of a step arrives whole; later action results report what changed, keyed by node ids stable while an element exists, or the whole screen when most changed, with large subtrees that have not changed since the last screen folded to one line each naming the node count and the revision that last -listed them in full. `observe` still returns the complete tree. Pixels arrive through `vision` on a judgment or the act loop's +listed them in full. `observe` returns what changed since the last screen, +not the whole tree; only the MCP `observe` returns the complete tree every +time. Pixels arrive through `vision` on a judgment or the act loop's `screenshot` and point tools, with secure fields masked. When the browser engine's tree capture times out, the model gets a screenshot and a warning, a judgment needs `vision: true`, and no control may be inferred diff --git a/skills/e2e/references/agent.md b/skills/e2e/references/agent.md index 24eed88c19..d54aa7e22a 100644 --- a/skills/e2e/references/agent.md +++ b/skills/e2e/references/agent.md @@ -357,7 +357,9 @@ screen of a step arrives whole; later action results report what changed, keyed by node ids stable while an element exists, or the whole screen when most changed, with large subtrees that have not changed since the last screen folded to one line each naming the node count and the revision that last -listed them in full. `observe` still returns the complete tree. Pixels arrive through `vision` on a judgment or the act loop's +listed them in full. `observe` returns what changed since the last screen, +not the whole tree; only the MCP `observe` returns the complete tree every +time. Pixels arrive through `vision` on a judgment or the act loop's `screenshot` and point tools, with secure fields masked. When the browser engine's tree capture times out, the model gets a screenshot and a warning, a judgment needs `vision: true`, and no control may be inferred From 8db7cbba03146d98be4246460468827b65c1acc0 Mon Sep 17 00:00:00 2001 From: Varshith Puli Date: Sat, 14 Nov 2026 22:02:13 +0000 Subject: [PATCH 06/12] test(agent): make the kept-screens budget test discriminate The screenshot-result test claimed the kept b1 and b4 screens spend none of the elision budget, but with keepStaleBytes 0 any elidable screen triggers elision, so counting the kept screens in the budget produced the same output. Pass a budget that exactly covers the elidable b2 and b3 screens instead: the transcript stays untouched only when the kept screens spend none of it. The keepStaleBytes 0 pass still pins the screenshot result returning untouched while the elidable screens go. Signed-off-by: Varshith Puli --- .../tests/unit/transcript-compaction.test.ts | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/e2e/tests/unit/transcript-compaction.test.ts b/packages/e2e/tests/unit/transcript-compaction.test.ts index 0703656570..5498d51c09 100644 --- a/packages/e2e/tests/unit/transcript-compaction.test.ts +++ b/packages/e2e/tests/unit/transcript-compaction.test.ts @@ -53,6 +53,16 @@ function toolOutput(message: ModelMessage): ToolResultPart['output'] { return part!.output; } +/** The full-screen text a tool message's result carries. */ +function screenText(message: ModelMessage): string { + const output = toolOutput(message); + if (output.type === 'text') return output.value; + if (output.type !== 'content') throw new Error('expected a tool result carrying a full screen'); + const textItem = output.value.find((item) => item.type === 'text'); + if (textItem === undefined) throw new Error('expected a tool result carrying a full screen'); + return textItem.text; +} + /** Characters of text a transcript carries, the text beside a screenshot included; the overflow retry's measure. */ function textChars(messages: readonly ModelMessage[]): number { return messages.reduce((total, message) => { @@ -225,10 +235,20 @@ describe('compactScreenHistory', () => { fullScreenResult('c3', 'b3'), foldedScreenResult('c4', 'b4', 'b1'), ]; + // Only b2 and b3 may be elided: b1 stays as the fold dependency of b4, + // and b4 stays as the newest screen. A budget that exactly covers the + // elidable screens leaves the transcript alone only when the kept + // screens spend none of it; counting b1 and b4 would push the total + // over budget and elide b2 and b3. + const elidableBytes = [messages[1]!, messages[2]!].reduce( + (bytes, message) => bytes + Buffer.byteLength(screenText(message), 'utf8'), + 0, + ); + expect(compactScreenHistory(messages, { keepStaleBytes: elidableBytes })).toBe(messages); const compacted = compactScreenHistory(messages, { keepStaleBytes: 0 }); expect(compacted).not.toBe(messages); // The opening screenshot result carries only the kept b1 screen, so it - // is returned untouched and the budget still covers both elidable screens. + // is returned untouched while the elidable screens go. expect(compacted[0]).toBe(messages[0]); expect(toolOutput(compacted[1]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); expect(toolOutput(compacted[2]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); From 8014923823bd56247b8d65d684caabec475237f0 Mon Sep 17 00:00:00 2001 From: Varshith Puli Date: Sun, 15 Nov 2026 01:45:05 +0000 Subject: [PATCH 07/12] fix(agent): clear fold records on overflow shrink and cover the text-result keep The overflow retry clips every screen to its head, including one a surviving fold line names, so the next screen kept folding against text the model no longer holds. The loop now tells the act executor when it shrinks, and the executor drops the step presenter's fold records, so the next full render lists everything in full. Clearing the records instead of leaving depended-on screens uncut keeps the retry able to shrink: a single huge screen would otherwise leave it nothing to cut. The same change adds the missing test for a depended-on screen that arrived as a plain-text tool result, drops the dead index check in the fold, shares the screen-revision pattern instead of duplicating its regex, exports the fold pattern without /g so later .test() calls stay stateless, and fixes the agent.md fold wording. Signed-off-by: Varshith Puli --- packages/core/src/agent/default-agent.ts | 6 ++ packages/core/src/agent/screen-update.ts | 18 +++++- packages/core/src/agent/tool-loop.ts | 9 +++ .../core/src/agent/transcript-compaction.ts | 8 +-- .../tests/unit/agent-screen-update.test.ts | 23 ++++++- .../e2e/tests/unit/tool-loop-overflow.test.ts | 63 ++++++++++++++++++- .../tests/unit/transcript-compaction.test.ts | 22 +++++++ plugins/e2e/skills/e2e/references/agent.md | 6 +- skills/e2e/references/agent.md | 6 +- 9 files changed, 146 insertions(+), 15 deletions(-) diff --git a/packages/core/src/agent/default-agent.ts b/packages/core/src/agent/default-agent.ts index 2cd709564d..224d8784a7 100644 --- a/packages/core/src/agent/default-agent.ts +++ b/packages/core/src/agent/default-agent.ts @@ -63,6 +63,11 @@ function presenterFor(context: StepExecutorContext): ScreenPresenter { return presenter; } +/** Drops the fold records of the step's presenter, if it has one: the overflow retry may have cut the screens they name. */ +function clearPresenterFolds(context: StepExecutorContext): void { + presenters.get(context)?.clearFoldRecords(); +} + /** What the built-in agent is built from: an agents entry's `system` and its validated `tools`. */ export interface BuiltInAgentOptions { /** Appended to the built-in execution rules; read by act and exploring assert loops. */ @@ -88,6 +93,7 @@ function createActLoop(options: BuiltInAgentOptions): StepExecutor { ...BUILT_IN_AGENT, system: rules(BASE_RULES, options.system), prepareMessages: (messages) => compactScreenshotHistory(compactScreenHistory(messages)), + onOverflowShrink: clearPresenterFolds, tools: (context, helpers) => loopTools(context, helpers, options.tools), buildPrompt: async (context) => { // The opening look is tree-only, unless no listed interactive node diff --git a/packages/core/src/agent/screen-update.ts b/packages/core/src/agent/screen-update.ts index 7b52efb0b7..c917fbd74c 100644 --- a/packages/core/src/agent/screen-update.ts +++ b/packages/core/src/agent/screen-update.ts @@ -42,8 +42,11 @@ const MIN_FOLD_NODES = 10; /** Every full screen is introduced by this phrase; the transcript elision looks for it. */ export const FULL_SCREEN_PATTERN = /Current screen \(revision /; -/** A fold line's dependency: `(12 nodes inside, unchanged since revision b1)`; the transcript elision looks for it. */ -export const FOLD_DEPENDENCY_PATTERN = /\(\d+ nodes inside, unchanged since revision ([^)]+)\)/g; +/** A full screen's heading with its revision: `Current screen (revision b1, ...`; the transcript elision reads it. */ +export const SCREEN_REVISION_PATTERN = /Current screen \(revision ([^,)]+),/; + +/** A fold line's dependency: `(12 nodes inside, unchanged since revision b1)`; the transcript elision looks for it. No `/g`, so a later `.test()` or `.exec()` call is not stateful; a caller that needs every match builds its own global copy. */ +export const FOLD_DEPENDENCY_PATTERN = /\(\d+ nodes inside, unchanged since revision ([^)]+)\)/; /** The screen as the model last received it, indexed for comparison. */ interface ShownScreen { @@ -166,6 +169,15 @@ export class ScreenPresenter { return rendered.text; } + /** + * Forgets every subtree the model last saw whole. The overflow retry cuts + * the screens those records name, so the next full render lists everything + * in full instead of folding against text the model no longer holds. + */ + clearFoldRecords(): void { + this.fullyShown.clear(); + } + /** * Records the subtrees a full render listed in full or folded, and drops * every other record. Called with no map by a render that listed nothing @@ -496,7 +508,7 @@ function foldUnchangedSubtrees( const inside = end - index - 1; const shown = fullyShown.get(entry.id); if ( - index > 0 && entry.indent > 0 && inside >= MIN_FOLD_NODES && + entry.indent > 0 && inside >= MIN_FOLD_NODES && shown !== undefined && digests.get(entry.id) === shown.digest && !subtreeHoldsFocus(entries, index, end) ) { diff --git a/packages/core/src/agent/tool-loop.ts b/packages/core/src/agent/tool-loop.ts index c82999f71f..c63ac391f0 100644 --- a/packages/core/src/agent/tool-loop.ts +++ b/packages/core/src/agent/tool-loop.ts @@ -164,6 +164,14 @@ export interface ToolLoopExecutorOptions { * effort). Defaults to the config-resolved `agent.providerOptions`. */ readonly providerOptions?: ProviderOptions; + /** + * Called when a context-overflow retry shrinks the history and resends it: + * the screens the presenter recorded as listed in full may have been cut + * to their head, so the executor forgets what it recorded and the next + * screen renders in full instead of folding against text the model no + * longer holds. Receives the step's context. + */ + readonly onOverflowShrink?: (context: StepExecutorContext) => void; /** * Between-turn history preparation: compaction, and anything the executor * wants the model to read before its next turn. Runs before the loop's own @@ -565,6 +573,7 @@ class LoopRun { } this.overflowNote = ' again after the history was shrunk once'; this.turnOffset = this.turnsUsed; + this.options.onOverflowShrink?.(this.context); this.note(`context overflow before turn ${String(this.turnsUsed + 1)}: history shrunk from ${String(before)} to ${String(after)} chars, retrying once`); return shrunk; } diff --git a/packages/core/src/agent/transcript-compaction.ts b/packages/core/src/agent/transcript-compaction.ts index 418e13e61a..6b63b34273 100644 --- a/packages/core/src/agent/transcript-compaction.ts +++ b/packages/core/src/agent/transcript-compaction.ts @@ -11,7 +11,7 @@ */ import type { FilePart, ModelMessage, TextPart, ToolResultPart, UserContent } from 'ai'; -import { FOLD_DEPENDENCY_PATTERN, FULL_SCREEN_PATTERN, toolResultTexts } from './screen-update.ts'; +import { FOLD_DEPENDENCY_PATTERN, FULL_SCREEN_PATTERN, SCREEN_REVISION_PATTERN, toolResultTexts } from './screen-update.ts'; /** A part of a user message; a string message counts as one text part. */ type UserPart = Exclude[number]; @@ -94,12 +94,12 @@ const FULL_SCREEN_PRESERVE_COUNT = 1; /** The revision a full screen's heading carries, or undefined when it carries none. */ function screenRevision(text: string): string | undefined { - return /Current screen \(revision ([^,)]+),/.exec(text)?.[1]; + return SCREEN_REVISION_PATTERN.exec(text)?.[1]; } -/** The revisions a full screen's fold lines depend on. */ +/** The revisions a full screen's fold lines depend on. The exported pattern carries no `/g`, so the global copy for `matchAll` is built here. */ function foldDependencies(text: string): string[] { - return [...text.matchAll(FOLD_DEPENDENCY_PATTERN)].map((match) => match[1]!); + return [...text.matchAll(new RegExp(FOLD_DEPENDENCY_PATTERN, 'g'))].map((match) => match[1]!); } /** diff --git a/packages/e2e/tests/unit/agent-screen-update.test.ts b/packages/e2e/tests/unit/agent-screen-update.test.ts index d85ed25283..0cfd266649 100644 --- a/packages/e2e/tests/unit/agent-screen-update.test.ts +++ b/packages/e2e/tests/unit/agent-screen-update.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import type { ModelMessage, ToolResultPart } from 'ai'; import type { ExecutorObservation } from '@e2e-dev/core/lib/agent/executor'; -import { ScreenPresenter } from '@e2e-dev/core/lib/agent/screen-update'; +import { FOLD_DEPENDENCY_PATTERN, ScreenPresenter } from '@e2e-dev/core/lib/agent/screen-update'; import { compactScreenHistory } from '@e2e-dev/core/lib/agent/transcript-compaction'; function screen(revision: string, lines: readonly string[], extra: Partial = {}): ExecutorObservation { @@ -486,6 +486,19 @@ describe('ScreenPresenter folds large unchanged subtrees in the full-screen fall expect(b4).toContain(' #n14 link "Item 12"'); }); + it('lists the subtree in full again after the fold records are cleared', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const folded = presenter.update(screen('b2', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(folded).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + presenter.clearFoldRecords(); + const text = presenter.update(screen('b3', orderScreen(MAIN_SEARCH, 12)), { lead: 'Tapped #n43.' }); + expect(text).toContain('The screen changed substantially since revision b2.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain(' #n3 link "Item 1"'); + expect(text).toContain(' #n14 link "Item 12"'); + }); + it('drops the fold record when a subtree leaves the screen, so a re-added subtree is listed in full', () => { const presenter = new ScreenPresenter(); const extras = [ @@ -513,6 +526,14 @@ describe('ScreenPresenter folds large unchanged subtrees in the full-screen fall expect(text).toContain(' #n3 link "Item 1"'); }); + it('exports a fold dependency pattern a repeated .test() can reuse', () => { + const line = ' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'; + expect(FOLD_DEPENDENCY_PATTERN.test(line)).toBe(true); + expect(FOLD_DEPENDENCY_PATTERN.test(line)).toBe(true); + expect(FOLD_DEPENDENCY_PATTERN.exec(line)?.[1]).toBe('b1'); + expect(FOLD_DEPENDENCY_PATTERN.exec(line)?.[1]).toBe('b1'); + }); + it('never folds a top-level root, even one holding enough unchanged nodes', () => { const presenter = new ScreenPresenter(); // A second top-level root with its leaves directly under it: the root diff --git a/packages/e2e/tests/unit/tool-loop-overflow.test.ts b/packages/e2e/tests/unit/tool-loop-overflow.test.ts index 9f2f0d0f31..a531a296a8 100644 --- a/packages/e2e/tests/unit/tool-loop-overflow.test.ts +++ b/packages/e2e/tests/unit/tool-loop-overflow.test.ts @@ -4,7 +4,7 @@ import { createToolLoopExecutor } from '@e2e-dev/core/lib/agent/tool-loop'; import { projectTools } from '@e2e-dev/core/lib/agent/default-agent'; import { defineTool } from '@e2e-dev/core/lib/agent/tool'; import { resolveConfig } from '../../src/config/resolve-run.ts'; -import { defineEngine, type EngineHandle } from '@e2e-dev/core/lib/engine/index'; +import { defineEngine, type EngineHandle, type SemanticNode } from '@e2e-dev/core/lib/engine/index'; import { createEngineSession } from '@e2e-dev/core/lib/engine/session'; import { Deadline } from '@e2e-dev/core/lib/internal/time'; import { AttemptBudget } from '@e2e-dev/core/lib/run/budget'; @@ -75,6 +75,32 @@ const changingBigScreen = (variant: () => number) => ), }); +/** A dense page whose sidebar sits at the end, as the review's repro: about 900 nodes, the sidebar last, so the overflow clip cuts it off the screen a later fold would name. The rows change per model turn, never per capture: the observation feed re-captures until the screen settles, and a variant that moved on every capture would never settle. */ +const foldingBigScreen = (variant: () => number) => + defineEngine({ + name: 'fake', + version: '1', + spiVersion: 1, + observe: async () => { + const rows = Array.from({ length: 880 }, (_, i) => ({ + ref: { id: `n${String(i)}`, revision: '' }, + role: 'text', + name: `v${String(variant())} row ${String(i)} ${'x'.repeat(20)}`, + })); + const sidebar: SemanticNode = { + ref: { id: 'n900', revision: '' }, + role: 'navigation', + name: 'Sidebar', + children: Array.from({ length: 12 }, (_, i) => ({ + ref: { id: `n${String(901 + i)}`, revision: '' }, + role: 'link', + name: `Item ${String(i + 1)}`, + })), + }; + return snapshot([...rows, sidebar]); + }, + }); + const OVERFLOW = 'prompt is too long: 300000 tokens > 200000 maximum'; /** A read-only project tool, so the loop has a first turn to spend before the refusal. */ @@ -132,6 +158,41 @@ describe('tool loop context overflow', () => { expect(step.metrics?.modelCalls).toBe(3); }); + it('lists the next screen in full after the overflow retry cut the screen its folds named', async () => { + let variant = 0; + const model = installFakeLoopModel(({ turn }) => { + variant = turn; + if (turn === 1) return [{ toolName: 'observe', input: {} }]; + if (turn === 2) throw new Error(OVERFLOW); + if (turn === 3) return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'read the sidebar' } }]; + }); + const { fixtures, steps } = runtime(foldingBigScreen(() => variant), { agents: { default: { model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(4); + // The second look folded the sidebar against the opening screen. The + // revision it names is whatever the settling captures minted, so the + // test reads it off the fold line instead of assuming b1. + const folded = loopCalls[1]!.lastToolResult; + const dependency = /\(\d+ nodes inside, unchanged since revision (b\d+)\)/.exec(folded)?.[1]; + expect(dependency).toMatch(/^b\d+$/); + // The retry cut the opening screen's tail, sidebar included, and the + // second screen's fold line with it. + expect(loopCalls[2]!.prompt).toContain("more characters cut: the request exceeded the model's context window"); + expect(loopCalls[2]!.prompt).not.toContain('link "Item 1"'); + // The next look lists the sidebar in full instead of folding against a + // screen the model no longer holds whole. + const next = loopCalls[3]!.lastToolResult; + expect(next).toMatch(/Current screen \(revision b\d+/); + expect(next).toContain('#n901 link "Item 1"'); + expect(next).not.toContain('unchanged since revision'); + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + expect(step.metrics?.modelCalls).toBe(3); + }); + it('reports CONTEXT_OVERFLOW when the shrunk request is refused again', async () => { const model = installFakeLoopModel(({ turn }) => { if (turn === 1) return [{ toolName: 'peek', input: {} }]; diff --git a/packages/e2e/tests/unit/transcript-compaction.test.ts b/packages/e2e/tests/unit/transcript-compaction.test.ts index 5498d51c09..1f0a3ed301 100644 --- a/packages/e2e/tests/unit/transcript-compaction.test.ts +++ b/packages/e2e/tests/unit/transcript-compaction.test.ts @@ -196,6 +196,28 @@ describe('compactScreenHistory', () => { expect(compacted[3]).toEqual(messages[3]); }); + it('keeps a depended-on screen that arrived as a plain-text tool result', () => { + const messages = [ + opening, + fullScreenResult('c1', 'b1'), + fullScreenResult('c2', 'b2'), + fullScreenResult('c3', 'b2a'), + foldedScreenResult('c4', 'b3', 'b2'), + ]; + const compacted = compactScreenHistory(messages, { keepStaleBytes: 0 }); + expect(compacted).not.toBe(messages); + // The sidebar was last listed in full by the b2 text result, so b3's + // fold names b2 and that result stays verbatim. The screen after b2 + // keeps the rewrite budget alive past it, so the keep-check on the text + // result is actually exercised: without it, b2 would be elided and b2a + // would survive instead. + expect(compacted[0]!.content).toContain('[earlier screen elided'); + expect(toolOutput(compacted[1]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); + expect(compacted[2]).toBe(messages[2]); + expect(toolOutput(compacted[3]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); + expect(compacted[4]).toBe(messages[4]); + }); + it('elides a screen no surviving fold depends on, even when it is the oldest', () => { const messages = [ opening, diff --git a/plugins/e2e/skills/e2e/references/agent.md b/plugins/e2e/skills/e2e/references/agent.md index d54aa7e22a..d5f9516476 100644 --- a/plugins/e2e/skills/e2e/references/agent.md +++ b/plugins/e2e/skills/e2e/references/agent.md @@ -355,9 +355,9 @@ summaries, and your context; never raw HTML, cookies, headers, environment values, or a `Secret`'s value; password fields masked. The first screen of a step arrives whole; later action results report what changed, keyed by node ids stable while an element exists, or the whole screen when -most changed, with large subtrees that have not changed since the last screen -folded to one line each naming the node count and the revision that last -listed them in full. `observe` returns what changed since the last screen, +most changed, with large subtrees unchanged since the screen that last +listed them in full folded to one line each, naming the node count and that +revision. `observe` returns what changed since the last screen, not the whole tree; only the MCP `observe` returns the complete tree every time. Pixels arrive through `vision` on a judgment or the act loop's `screenshot` and point tools, with secure fields masked. When diff --git a/skills/e2e/references/agent.md b/skills/e2e/references/agent.md index d54aa7e22a..d5f9516476 100644 --- a/skills/e2e/references/agent.md +++ b/skills/e2e/references/agent.md @@ -355,9 +355,9 @@ summaries, and your context; never raw HTML, cookies, headers, environment values, or a `Secret`'s value; password fields masked. The first screen of a step arrives whole; later action results report what changed, keyed by node ids stable while an element exists, or the whole screen when -most changed, with large subtrees that have not changed since the last screen -folded to one line each naming the node count and the revision that last -listed them in full. `observe` returns what changed since the last screen, +most changed, with large subtrees unchanged since the screen that last +listed them in full folded to one line each, naming the node count and that +revision. `observe` returns what changed since the last screen, not the whole tree; only the MCP `observe` returns the complete tree every time. Pixels arrive through `vision` on a judgment or the act loop's `screenshot` and point tools, with secure fields masked. When From b864b97e05f21a0132d336f8aa8eddfa85066699 Mon Sep 17 00:00:00 2001 From: Varshith Puli Date: Sun, 15 Nov 2026 07:09:06 +0000 Subject: [PATCH 08/12] fix(agent): share one screen presenter per step and forget its folds on overflow shrink A custom loop built from createGrammarTools made its own presenter, so the overflow retry that cut the screens its folds named never cleared them: the next observe still folded the sidebar against a cut revision. The presenter registry now lives beside ScreenPresenter, createGrammarTools defaults to the step's shared presenter, and the loop forgets the folds itself when it shrinks, which covers the built-in agent and a custom loop alike and drops the onOverflowShrink callback from the public options. A custom-loop case next to the built-in fold test fails before this change and passes after it. Signed-off-by: Varshith Puli --- packages/core/src/agent/default-agent.ts | 20 +-------- packages/core/src/agent/primitives.ts | 6 ++- packages/core/src/agent/screen-update.ts | 30 ++++++++++++- packages/core/src/agent/tool-loop.ts | 17 +++---- .../e2e/tests/unit/tool-loop-overflow.test.ts | 44 +++++++++++++++++++ 5 files changed, 85 insertions(+), 32 deletions(-) diff --git a/packages/core/src/agent/default-agent.ts b/packages/core/src/agent/default-agent.ts index 224d8784a7..0b4d026e5b 100644 --- a/packages/core/src/agent/default-agent.ts +++ b/packages/core/src/agent/default-agent.ts @@ -16,7 +16,7 @@ import type { ExecutorObservation, ExecutorObserveOptions, ReplayedPrefix, StepE import { interactiveNodeCount } from './observation.ts'; import { testTitleLine } from './prompts.ts'; import { grammarTools } from './primitives.ts'; -import { ScreenPresenter } from './screen-update.ts'; +import { presenterFor } from './screen-update.ts'; import { compactScreenHistory, compactScreenshotHistory } from './transcript-compaction.ts'; import { createToolLoopExecutor, type ToolLoopHelpers } from './tool-loop.ts'; import { toolAppliesTo, withToolContext } from './tool.ts'; @@ -51,23 +51,6 @@ const ASSERT_RULES = `You are judging exactly one assertion against the current - Each scroll result contains a fresh observation. When screenshots are shown, each result includes fresh pixels too. - Conclude passed when the collected evidence establishes the assertion, failed with ASSERTION_FAILED when it contradicts the assertion, and failed with ASSERTION_INCONCLUSIVE when the available evidence or remaining budget cannot establish an answer.`; -/** One presenter per dispatched step, shared by the opening prompt and the tools that follow it. */ -const presenters = new WeakMap(); - -function presenterFor(context: StepExecutorContext): ScreenPresenter { - let presenter = presenters.get(context); - if (presenter === undefined) { - presenter = new ScreenPresenter(); - presenters.set(context, presenter); - } - return presenter; -} - -/** Drops the fold records of the step's presenter, if it has one: the overflow retry may have cut the screens they name. */ -function clearPresenterFolds(context: StepExecutorContext): void { - presenters.get(context)?.clearFoldRecords(); -} - /** What the built-in agent is built from: an agents entry's `system` and its validated `tools`. */ export interface BuiltInAgentOptions { /** Appended to the built-in execution rules; read by act and exploring assert loops. */ @@ -93,7 +76,6 @@ function createActLoop(options: BuiltInAgentOptions): StepExecutor { ...BUILT_IN_AGENT, system: rules(BASE_RULES, options.system), prepareMessages: (messages) => compactScreenshotHistory(compactScreenHistory(messages)), - onOverflowShrink: clearPresenterFolds, tools: (context, helpers) => loopTools(context, helpers, options.tools), buildPrompt: async (context) => { // The opening look is tree-only, unless no listed interactive node diff --git a/packages/core/src/agent/primitives.ts b/packages/core/src/agent/primitives.ts index 003371796a..df481ff052 100644 --- a/packages/core/src/agent/primitives.ts +++ b/packages/core/src/agent/primitives.ts @@ -24,7 +24,7 @@ import { cacheTokenFields, readCost } from './model/sdk.ts'; import { OperationQueue } from './operation-queue.ts'; import { imagePointToViewport, POINT_VERBS } from './point-tap.ts'; import { SAMPLE_NAMES, SAMPLE_PREFIX } from './sample-files.ts'; -import { ScreenPresenter, type ScreenOutput, type ScreenUpdateOptions } from './screen-update.ts'; +import { ScreenPresenter, presenterFor, type ScreenOutput, type ScreenUpdateOptions } from './screen-update.ts'; /** Codes the model may pick when concluding; runtime codes are runtime-assigned. */ const MODEL_ERROR_CODES = [ @@ -912,7 +912,9 @@ function createPointTools( */ function verbKit(context: StepExecutorContext, options: GrammarToolSetup) { const guard = options.guard ?? ((body: () => Promise) => body()); - const screen: ScreenPresenter = options.screen ?? new ScreenPresenter(); + // No presenter of its own: the step's shared one, so a custom loop built + // from these tools folds against the same screens the loop itself shrinks. + const screen: ScreenPresenter = options.screen ?? presenterFor(context); const queue = new OperationQueue(); const inOrder = (body: () => Promise): Promise => queue.run(body); /** The screen after an action: the changes since the one the model holds and, once the step shows pixels, a fresh screenshot. */ diff --git a/packages/core/src/agent/screen-update.ts b/packages/core/src/agent/screen-update.ts index c917fbd74c..cedc781f32 100644 --- a/packages/core/src/agent/screen-update.ts +++ b/packages/core/src/agent/screen-update.ts @@ -25,7 +25,7 @@ */ import type { ToolResultPart } from 'ai'; -import type { ExecutorObservation, ExecutorPixels } from './executor.ts'; +import type { ExecutorObservation, ExecutorPixels, StepExecutorContext } from './executor.ts'; import { sha256Hex } from '../internal/ids.ts'; import { interactiveNodeCount, pixelDigest } from './observation.ts'; import type { VisionDegradation } from '../run/steps.ts'; @@ -323,6 +323,34 @@ export class ScreenPresenter { } } +/** + * One presenter per dispatched step, shared by the opening prompt and the + * grammar tools. The built-in agent passes its own explicitly, and a custom + * loop built from `createGrammarTools` gets the same one by default, so the + * loop that shrinks the history can forget the folds both of them recorded. + * Keyed by the step's context object, so a step's presenter dies with it. + */ +const presenters = new WeakMap(); + +/** The step's presenter, creating it on first use. */ +export function presenterFor(context: StepExecutorContext): ScreenPresenter { + let presenter = presenters.get(context); + if (presenter === undefined) { + presenter = new ScreenPresenter(); + presenters.set(context, presenter); + } + return presenter; +} + +/** + * Drops the fold records of the step's presenter, if it has one: the overflow + * retry may have cut the screens they name, so the next full render lists + * everything in full instead of folding against text the model no longer holds. + */ +export function clearPresenterFolds(context: StepExecutorContext): void { + presenters.get(context)?.clearFoldRecords(); +} + /** The line under a screenshot: its pixel size and the coordinate space the point verbs read. */ function screenshotNote(pixels: Pick): string { return `Screenshot attached: ${String(pixels.width)} by ${String(pixels.height)} pixels${ diff --git a/packages/core/src/agent/tool-loop.ts b/packages/core/src/agent/tool-loop.ts index c63ac391f0..1e89dc8ef9 100644 --- a/packages/core/src/agent/tool-loop.ts +++ b/packages/core/src/agent/tool-loop.ts @@ -28,7 +28,7 @@ import { } from './model/tool-choice.ts'; import { withStallGuard } from './model/stall.ts'; import { providerHints, type ProviderHints, type ProviderModelRef } from './model/provider-hints.ts'; -import { isScreenOutput, toolResultTexts } from './screen-update.ts'; +import { clearPresenterFolds, isScreenOutput, toolResultTexts } from './screen-update.ts'; import { shapeOf } from './message-shape.ts'; import { compactScreenHistory } from './transcript-compaction.ts'; import { AgentError, isAgentError } from './error.ts'; @@ -164,14 +164,6 @@ export interface ToolLoopExecutorOptions { * effort). Defaults to the config-resolved `agent.providerOptions`. */ readonly providerOptions?: ProviderOptions; - /** - * Called when a context-overflow retry shrinks the history and resends it: - * the screens the presenter recorded as listed in full may have been cut - * to their head, so the executor forgets what it recorded and the next - * screen renders in full instead of folding against text the model no - * longer holds. Receives the step's context. - */ - readonly onOverflowShrink?: (context: StepExecutorContext) => void; /** * Between-turn history preparation: compaction, and anything the executor * wants the model to read before its next turn. Runs before the loop's own @@ -573,7 +565,12 @@ class LoopRun { } this.overflowNote = ' again after the history was shrunk once'; this.turnOffset = this.turnsUsed; - this.options.onOverflowShrink?.(this.context); + // The screens the step's presenter recorded as listed in full may have + // been cut to their head, so it forgets what it recorded and the next + // screen renders in full instead of folding against text the model no + // longer holds. The presenter is shared by the opening prompt and the + // grammar tools, built-in or custom, so one clear covers both paths. + clearPresenterFolds(this.context); this.note(`context overflow before turn ${String(this.turnsUsed + 1)}: history shrunk from ${String(before)} to ${String(after)} chars, retrying once`); return shrunk; } diff --git a/packages/e2e/tests/unit/tool-loop-overflow.test.ts b/packages/e2e/tests/unit/tool-loop-overflow.test.ts index a531a296a8..dc9d7369e1 100644 --- a/packages/e2e/tests/unit/tool-loop-overflow.test.ts +++ b/packages/e2e/tests/unit/tool-loop-overflow.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { z } from 'zod'; +import { createGrammarTools } from '@e2e-dev/core/lib/agent/primitives'; import { createToolLoopExecutor } from '@e2e-dev/core/lib/agent/tool-loop'; import { projectTools } from '@e2e-dev/core/lib/agent/default-agent'; import { defineTool } from '@e2e-dev/core/lib/agent/tool'; @@ -193,6 +194,49 @@ describe('tool loop context overflow', () => { expect(step.metrics?.modelCalls).toBe(3); }); + it('lists the next screen in full after the overflow retry cut the screen its folds named, for a custom loop built from createGrammarTools', async () => { + let variant = 0; + const model = installFakeLoopModel(({ turn }) => { + variant = turn; + if (turn === 1) return [{ toolName: 'observe', input: {} }]; + if (turn === 2) throw new Error(OVERFLOW); + if (turn === 3) return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'read the sidebar' } }]; + }); + // The documented custom brain: the loop chassis plus the grammar tools, + // with no presenter of its own. Its first observe lists the sidebar in + // full, so the second look must not fold against the cut screen. + const executor = createToolLoopExecutor({ + name: 'custom-loop', + tools: (context) => createGrammarTools(context), + buildPrompt: () => 'read the big output', + }); + const { fixtures, steps } = runtime(foldingBigScreen(() => variant), { agents: { default: { executor, model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(4); + // The first look listed the sidebar in full, at whatever revision the + // settling captures minted. + const first = loopCalls[1]!.lastToolResult; + expect(first).toContain('#n901 link "Item 1"'); + expect(first).not.toContain('unchanged since revision'); + // The retry cut the first screen's tail, sidebar included: the opening + // prompt carries no screen here, so the cut shows in the observe result + // the retried request carries. + expect(loopCalls[2]!.lastToolResult).toContain("more characters cut: the request exceeded the model's context window"); + expect(loopCalls[2]!.lastToolResult).not.toContain('link "Item 1"'); + // The next look lists the sidebar in full instead of folding against a + // screen the model no longer holds whole. + const next = loopCalls[3]!.lastToolResult; + expect(next).toMatch(/Current screen \(revision b\d+/); + expect(next).toContain('#n901 link "Item 1"'); + expect(next).not.toContain('unchanged since revision'); + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + expect(step.metrics?.modelCalls).toBe(3); + }); + it('reports CONTEXT_OVERFLOW when the shrunk request is refused again', async () => { const model = installFakeLoopModel(({ turn }) => { if (turn === 1) return [{ toolName: 'peek', input: {} }]; From 68b12ab4af53532a033e0751ca5010a6bbf74b96 Mon Sep 17 00:00:00 2001 From: Varshith Puli Date: Sun, 15 Nov 2026 10:07:07 +0000 Subject: [PATCH 09/12] docs(agent): correct the screen option doc to name the shared presenter The GrammarToolSetup screen doc still said a tool set without screen gets a fresh presenter per step. Since the shared-presenter change, the default is presenterFor(context), the step's shared presenter, the same one the opening prompt uses. The old wording also implied the first look returns the whole screen, which no longer holds once the opening prompt has shown it. Signed-off-by: Varshith Puli --- packages/core/src/agent/primitives.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/agent/primitives.ts b/packages/core/src/agent/primitives.ts index df481ff052..ae478932e7 100644 --- a/packages/core/src/agent/primitives.ts +++ b/packages/core/src/agent/primitives.ts @@ -228,8 +228,8 @@ interface GrammarToolSetup extends GrammarToolOptions { * Renders screens for the model and remembers what it has seen, so every * result after the first reports the changes rather than the whole tree. * Shared with the opening prompt by the built-in agent and with the session - * by `e2e mcp`; a fresh one per step otherwise, so the first look - * (`observe`) returns the whole screen. + * by `e2e mcp`; without one, the step's shared presenter, the same one the + * opening prompt uses. */ readonly screen?: ScreenPresenter; /** From bcd42e9d31bd657210b55ad4423a8bfbf102353d Mon Sep 17 00:00:00 2001 From: Varshith Puli Date: Sun, 15 Nov 2026 13:14:15 +0000 Subject: [PATCH 10/12] fix(agent): bind a fresh screen presenter per loop run Two custom loops sharing one step context shared one presenter, so the second loop's first observe reported the first loop's screen unchanged, a screen its model never saw. A loop run now binds a fresh presenter for its context at the start of run, before tools and buildPrompt, and clears its folds on overflow shrink through its own reference. presenterFor stays as the fallback for tools used outside a loop. Adds the two-loop case to the overflow tests, and drops the now redundant screen option in the built-in agent. Signed-off-by: Varshith Puli --- packages/core/src/agent/default-agent.ts | 2 +- packages/core/src/agent/screen-update.ts | 21 +++++---- packages/core/src/agent/tool-loop.ts | 17 ++++--- .../e2e/tests/unit/tool-loop-overflow.test.ts | 45 +++++++++++++++++++ 4 files changed, 68 insertions(+), 17 deletions(-) diff --git a/packages/core/src/agent/default-agent.ts b/packages/core/src/agent/default-agent.ts index 0b4d026e5b..88fb94d5d8 100644 --- a/packages/core/src/agent/default-agent.ts +++ b/packages/core/src/agent/default-agent.ts @@ -143,7 +143,7 @@ function loopTools( ): ToolSet { return { ...guardedTools(helpers, projectTools(context, tools)), - ...grammarTools(context, { guard: helpers.guard, screen: presenterFor(context) }), + ...grammarTools(context, { guard: helpers.guard }), }; } diff --git a/packages/core/src/agent/screen-update.ts b/packages/core/src/agent/screen-update.ts index cedc781f32..ee89e8b3a5 100644 --- a/packages/core/src/agent/screen-update.ts +++ b/packages/core/src/agent/screen-update.ts @@ -325,10 +325,10 @@ export class ScreenPresenter { /** * One presenter per dispatched step, shared by the opening prompt and the - * grammar tools. The built-in agent passes its own explicitly, and a custom - * loop built from `createGrammarTools` gets the same one by default, so the - * loop that shrinks the history can forget the folds both of them recorded. - * Keyed by the step's context object, so a step's presenter dies with it. + * grammar tools. A loop run binds a fresh one at the start of its run, so + * the presenter state matches that run's transcript alone: two loops run + * against one step context each see their own first screen. Keyed by the + * step's context object, so a step's presenter dies with it. */ const presenters = new WeakMap(); @@ -343,12 +343,15 @@ export function presenterFor(context: StepExecutorContext): ScreenPresenter { } /** - * Drops the fold records of the step's presenter, if it has one: the overflow - * retry may have cut the screens they name, so the next full render lists - * everything in full instead of folding against text the model no longer holds. + * Binds a fresh presenter to the step's context, replacing whatever it held: + * a loop run owns the transcript, not the step context, so each run starts + * from a screen its own model has seen. `presenterFor` stays the fallback + * for tools used outside a loop. */ -export function clearPresenterFolds(context: StepExecutorContext): void { - presenters.get(context)?.clearFoldRecords(); +export function bindPresenter(context: StepExecutorContext): ScreenPresenter { + const presenter = new ScreenPresenter(); + presenters.set(context, presenter); + return presenter; } /** The line under a screenshot: its pixel size and the coordinate space the point verbs read. */ diff --git a/packages/core/src/agent/tool-loop.ts b/packages/core/src/agent/tool-loop.ts index 1e89dc8ef9..0cdb76ae8e 100644 --- a/packages/core/src/agent/tool-loop.ts +++ b/packages/core/src/agent/tool-loop.ts @@ -28,7 +28,7 @@ import { } from './model/tool-choice.ts'; import { withStallGuard } from './model/stall.ts'; import { providerHints, type ProviderHints, type ProviderModelRef } from './model/provider-hints.ts'; -import { clearPresenterFolds, isScreenOutput, toolResultTexts } from './screen-update.ts'; +import { bindPresenter, isScreenOutput, toolResultTexts, type ScreenPresenter } from './screen-update.ts'; import { shapeOf } from './message-shape.ts'; import { compactScreenHistory } from './transcript-compaction.ts'; import { AgentError, isAgentError } from './error.ts'; @@ -257,6 +257,8 @@ class LoopRun { * verdict, and the note completes its message. */ private overflowNote: string | undefined; + /** The presenter this run bound for its context, so a shrink clears this run's folds, never another run's. */ + private presenter: ScreenPresenter | undefined; /** How this step asks for tool calls; flips to `auto` when the model rejects a forced choice. */ private toolChoice: ToolChoiceMode; /** The last turn that ran, for continuing after a reply without tool calls. */ @@ -305,6 +307,11 @@ class LoopRun { } async run(): Promise { + // A fresh presenter per run: the transcript belongs to this loop, not + // the step context, so a second loop run against the same context + // starts from a screen its own model has seen. Bound before the tools + // and the opening prompt, which both read it through `presenterFor`. + this.presenter = bindPresenter(this.context); const helpers = this.helpers(); const tools: ToolSet = { ...this.options.tools(this.context, helpers), @@ -565,12 +572,8 @@ class LoopRun { } this.overflowNote = ' again after the history was shrunk once'; this.turnOffset = this.turnsUsed; - // The screens the step's presenter recorded as listed in full may have - // been cut to their head, so it forgets what it recorded and the next - // screen renders in full instead of folding against text the model no - // longer holds. The presenter is shared by the opening prompt and the - // grammar tools, built-in or custom, so one clear covers both paths. - clearPresenterFolds(this.context); + // The retry may have cut the screens this run's folds name, so the next screen renders in full. + this.presenter?.clearFoldRecords(); this.note(`context overflow before turn ${String(this.turnsUsed + 1)}: history shrunk from ${String(before)} to ${String(after)} chars, retrying once`); return shrunk; } diff --git a/packages/e2e/tests/unit/tool-loop-overflow.test.ts b/packages/e2e/tests/unit/tool-loop-overflow.test.ts index dc9d7369e1..9d1ac92ba6 100644 --- a/packages/e2e/tests/unit/tool-loop-overflow.test.ts +++ b/packages/e2e/tests/unit/tool-loop-overflow.test.ts @@ -3,6 +3,7 @@ import { z } from 'zod'; import { createGrammarTools } from '@e2e-dev/core/lib/agent/primitives'; import { createToolLoopExecutor } from '@e2e-dev/core/lib/agent/tool-loop'; import { projectTools } from '@e2e-dev/core/lib/agent/default-agent'; +import type { StepExecutor } from '@e2e-dev/core/lib/agent/executor'; import { defineTool } from '@e2e-dev/core/lib/agent/tool'; import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine, type EngineHandle, type SemanticNode } from '@e2e-dev/core/lib/engine/index'; @@ -237,6 +238,50 @@ describe('tool loop context overflow', () => { expect(step.metrics?.modelCalls).toBe(3); }); + it('gives a second custom loop sharing one step context its own first screen', async () => { + const model = installFakeLoopModel(({ turn }) => { + if (turn === 1) return [{ toolName: 'observe', input: {} }]; + if (turn === 2) return [{ toolName: 'observe', input: {} }]; + if (turn === 3) return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'loop A read the sidebar' } }]; + if (turn === 4) return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'loop B read the sidebar' } }]; + }); + // The review's repro: one custom runStep that runs two loops against the + // same step context, each built from createGrammarTools. The screen never + // changes, so a presenter shared across the loops would answer loop B's + // first look with the screen loop A already showed. + const loop = () => + createToolLoopExecutor({ + name: 'custom-loop', + tools: (context) => createGrammarTools(context), + buildPrompt: () => 'read the big output', + }); + const executor: StepExecutor = { + name: 'two-loops', + runStep: async (context) => { + await loop().runStep(context); + return loop().runStep(context); + }, + }; + const { fixtures, steps } = runtime(foldingBigScreen(() => 0), { agents: { default: { executor, model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(5); + // Loop A's second look saw the screen it already holds, so the shared + // presenter has state loop B never saw. + const second = loopCalls[2]!.lastToolResult; + expect(second).toContain('Screen unchanged since revision'); + // Loop B's first look lists the sidebar in full, as its own first screen, + // instead of reporting the first loop's screen unchanged. + const loopBFirst = loopCalls[4]!.lastToolResult; + expect(loopBFirst).toMatch(/Current screen \(revision b\d+/); + expect(loopBFirst).toContain('#n901 link "Item 1"'); + expect(loopBFirst).not.toContain('unchanged since revision'); + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + }); + it('reports CONTEXT_OVERFLOW when the shrunk request is refused again', async () => { const model = installFakeLoopModel(({ turn }) => { if (turn === 1) return [{ toolName: 'peek', input: {} }]; From 7a72306a75bb23dc8d0b4b079b289bef1bbba9ac Mon Sep 17 00:00:00 2001 From: Varshith Puli Date: Sun, 15 Nov 2026 19:30:34 +0000 Subject: [PATCH 11/12] fix(agent): restore the previous screen presenter when a loop run ends Grammar tools used outside a loop shared the presenter's state: bindPresenter was never undone, and presenterFor created and stored one shared presenter for every toolset built from the context. A second toolset's first observe reported the first toolset's screen unchanged, a screen its model never saw, and the same happened for tools built after a loop run. A loop run now saves the previous binding and restores it in finally, and verbKit reads the bound presenter without creating one, falling back to a fresh presenter of its own per toolset. presenterFor stays as the fallback for the opening prompt inside a run. Adds both repro cases to the overflow tests, plus a fold test proving a marker line inside a subtree keeps the lines past it listed in full. Also cuts the bindPresenter comment in run to one line and rewords the screen option docs, which named a shared presenter that no longer exists. Signed-off-by: Varshith Puli --- packages/core/src/agent/primitives.ts | 13 +-- packages/core/src/agent/screen-update.ts | 35 ++++++-- packages/core/src/agent/tool-loop.ts | 18 ++-- .../tests/unit/agent-screen-update.test.ts | 24 ++++++ .../e2e/tests/unit/tool-loop-overflow.test.ts | 84 ++++++++++++++++++- 5 files changed, 153 insertions(+), 21 deletions(-) diff --git a/packages/core/src/agent/primitives.ts b/packages/core/src/agent/primitives.ts index ae478932e7..4c33b8e9bb 100644 --- a/packages/core/src/agent/primitives.ts +++ b/packages/core/src/agent/primitives.ts @@ -24,7 +24,7 @@ import { cacheTokenFields, readCost } from './model/sdk.ts'; import { OperationQueue } from './operation-queue.ts'; import { imagePointToViewport, POINT_VERBS } from './point-tap.ts'; import { SAMPLE_NAMES, SAMPLE_PREFIX } from './sample-files.ts'; -import { ScreenPresenter, presenterFor, type ScreenOutput, type ScreenUpdateOptions } from './screen-update.ts'; +import { ScreenPresenter, boundPresenter, type ScreenOutput, type ScreenUpdateOptions } from './screen-update.ts'; /** Codes the model may pick when concluding; runtime codes are runtime-assigned. */ const MODEL_ERROR_CODES = [ @@ -228,8 +228,8 @@ interface GrammarToolSetup extends GrammarToolOptions { * Renders screens for the model and remembers what it has seen, so every * result after the first reports the changes rather than the whole tree. * Shared with the opening prompt by the built-in agent and with the session - * by `e2e mcp`; without one, the step's shared presenter, the same one the - * opening prompt uses. + * by `e2e mcp`; without one, the loop run's presenter when built inside a + * loop, else a fresh presenter of its own. */ readonly screen?: ScreenPresenter; /** @@ -912,9 +912,10 @@ function createPointTools( */ function verbKit(context: StepExecutorContext, options: GrammarToolSetup) { const guard = options.guard ?? ((body: () => Promise) => body()); - // No presenter of its own: the step's shared one, so a custom loop built - // from these tools folds against the same screens the loop itself shrinks. - const screen: ScreenPresenter = options.screen ?? presenterFor(context); + // No presenter of its own: the loop run's when built inside one, else a + // fresh presenter per toolset, so tools used outside a loop never share + // screen state with a run or with each other. + const screen: ScreenPresenter = options.screen ?? boundPresenter(context) ?? new ScreenPresenter(); const queue = new OperationQueue(); const inOrder = (body: () => Promise): Promise => queue.run(body); /** The screen after an action: the changes since the one the model holds and, once the step shows pixels, a fresh screenshot. */ diff --git a/packages/core/src/agent/screen-update.ts b/packages/core/src/agent/screen-update.ts index ee89e8b3a5..579adbaef4 100644 --- a/packages/core/src/agent/screen-update.ts +++ b/packages/core/src/agent/screen-update.ts @@ -324,11 +324,12 @@ export class ScreenPresenter { } /** - * One presenter per dispatched step, shared by the opening prompt and the - * grammar tools. A loop run binds a fresh one at the start of its run, so - * the presenter state matches that run's transcript alone: two loops run - * against one step context each see their own first screen. Keyed by the - * step's context object, so a step's presenter dies with it. + * One presenter per loop run, bound to the step's context for the run and + * restored after it, so the presenter state matches that run's transcript + * alone: two loops run against one step context each see their own first + * screen. Tools built outside a run read the binding while one is in + * progress, else use a presenter of their own. Keyed by the step's context + * object, so a run's presenter dies with it. */ const presenters = new WeakMap(); @@ -342,11 +343,17 @@ export function presenterFor(context: StepExecutorContext): ScreenPresenter { return presenter; } +/** The presenter bound to the step's context, if a loop run bound one: reads only, never creates. */ +export function boundPresenter(context: StepExecutorContext): ScreenPresenter | undefined { + return presenters.get(context); +} + /** - * Binds a fresh presenter to the step's context, replacing whatever it held: - * a loop run owns the transcript, not the step context, so each run starts - * from a screen its own model has seen. `presenterFor` stays the fallback - * for tools used outside a loop. + * Binds a fresh presenter to the step's context, replacing whatever it held, + * and returns it. A loop run binds one at the start of its run, so each run + * starts from a screen its own model has seen, and restores the previous + * binding when the run ends. `presenterFor` stays the fallback for the + * opening prompt inside a run. */ export function bindPresenter(context: StepExecutorContext): ScreenPresenter { const presenter = new ScreenPresenter(); @@ -354,6 +361,16 @@ export function bindPresenter(context: StepExecutorContext): ScreenPresenter { return presenter; } +/** + * Restores the binding a loop run replaced when the run ends, or drops the + * entry when there was none, so tools built outside a loop never share a + * run's presenter. + */ +export function restorePresenter(context: StepExecutorContext, previous: ScreenPresenter | undefined): void { + if (previous === undefined) presenters.delete(context); + else presenters.set(context, previous); +} + /** The line under a screenshot: its pixel size and the coordinate space the point verbs read. */ function screenshotNote(pixels: Pick): string { return `Screenshot attached: ${String(pixels.width)} by ${String(pixels.height)} pixels${ diff --git a/packages/core/src/agent/tool-loop.ts b/packages/core/src/agent/tool-loop.ts index 0cdb76ae8e..62a703fd55 100644 --- a/packages/core/src/agent/tool-loop.ts +++ b/packages/core/src/agent/tool-loop.ts @@ -28,7 +28,7 @@ import { } from './model/tool-choice.ts'; import { withStallGuard } from './model/stall.ts'; import { providerHints, type ProviderHints, type ProviderModelRef } from './model/provider-hints.ts'; -import { bindPresenter, isScreenOutput, toolResultTexts, type ScreenPresenter } from './screen-update.ts'; +import { bindPresenter, boundPresenter, isScreenOutput, restorePresenter, toolResultTexts, type ScreenPresenter } from './screen-update.ts'; import { shapeOf } from './message-shape.ts'; import { compactScreenHistory } from './transcript-compaction.ts'; import { AgentError, isAgentError } from './error.ts'; @@ -307,11 +307,19 @@ class LoopRun { } async run(): Promise { - // A fresh presenter per run: the transcript belongs to this loop, not - // the step context, so a second loop run against the same context - // starts from a screen its own model has seen. Bound before the tools - // and the opening prompt, which both read it through `presenterFor`. + // A fresh presenter per run, restored after it: the transcript belongs to this loop, not the step context. + const previous = boundPresenter(this.context); this.presenter = bindPresenter(this.context); + try { + return await this.runLoop(); + } finally { + restorePresenter(this.context, previous); + this.presenter = undefined; + } + } + + /** The loop itself: tools, opening prompt, and model turns until a verdict. */ + private async runLoop(): Promise { const helpers = this.helpers(); const tools: ToolSet = { ...this.options.tools(this.context, helpers), diff --git a/packages/e2e/tests/unit/agent-screen-update.test.ts b/packages/e2e/tests/unit/agent-screen-update.test.ts index 0cfd266649..facc4e9fd4 100644 --- a/packages/e2e/tests/unit/agent-screen-update.test.ts +++ b/packages/e2e/tests/unit/agent-screen-update.test.ts @@ -377,6 +377,30 @@ describe('ScreenPresenter folds large unchanged subtrees in the full-screen fall expect(text).toContain(' #n3 link "Item 1"'); }); + it('does not let a fold swallow lines past a marker line inside the subtree', () => { + const presenter = new ScreenPresenter(); + // A marker line with no node id sits inside the sidebar, between its + // eleventh and twelfth links: it closes the sidebar's span there. + const sidebarWithMarker = [ + ' #n2 navigation "Sidebar"', + ...Array.from({ length: 11 }, (_, i) => ` #n${String(i + 3)} link "Item ${String(i + 1)}"`), + ' [End of visible items]', + ' #n14 link "Item 12"', + ]; + presenter.initial(screen('b1', ['#n1 document "Orders"', ...sidebarWithMarker, ...MAIN_LIST])); + const text = presenter.update( + screen('b2', ['#n1 document "Orders"', ...sidebarWithMarker, ...MAIN_DETAILS]), + { lead: 'Tapped #n22.' }, + ); + expect(text).toContain('The screen changed substantially since revision b1.'); + // The fold covers only the eleven links before the marker. + expect(text).toContain(' #n2 navigation "Sidebar" (11 nodes inside, unchanged since revision b1)'); + expect(text).not.toContain('#n3 link "Item 1"'); + // The marker and the link past it are still listed in full. + expect(text).toContain(' [End of visible items]'); + expect(text).toContain(' #n14 link "Item 12"'); + }); + it('sends the first screen whole, with no folding', () => { const presenter = new ScreenPresenter(); const text = presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); diff --git a/packages/e2e/tests/unit/tool-loop-overflow.test.ts b/packages/e2e/tests/unit/tool-loop-overflow.test.ts index 9d1ac92ba6..e3c136ba5f 100644 --- a/packages/e2e/tests/unit/tool-loop-overflow.test.ts +++ b/packages/e2e/tests/unit/tool-loop-overflow.test.ts @@ -3,7 +3,7 @@ import { z } from 'zod'; import { createGrammarTools } from '@e2e-dev/core/lib/agent/primitives'; import { createToolLoopExecutor } from '@e2e-dev/core/lib/agent/tool-loop'; import { projectTools } from '@e2e-dev/core/lib/agent/default-agent'; -import type { StepExecutor } from '@e2e-dev/core/lib/agent/executor'; +import type { StepExecutor, StepExecutorContext } from '@e2e-dev/core/lib/agent/executor'; import { defineTool } from '@e2e-dev/core/lib/agent/tool'; import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine, type EngineHandle, type SemanticNode } from '@e2e-dev/core/lib/engine/index'; @@ -108,6 +108,16 @@ const OVERFLOW = 'prompt is too long: 300000 tokens > 200000 maximum'; /** A read-only project tool, so the loop has a first turn to spend before the refusal. */ const peek = defineTool({ inputSchema: z.object({}), execute: async () => 'looked' }, { mutates: false }); +/** The text of one direct `observe` call on a toolset built from the context, outside any loop. */ +async function observeDirectly(context: StepExecutorContext): Promise { + const tools = createGrammarTools(context); + const observe = tools.observe; + if (observe?.execute === undefined) throw new Error('no observe tool'); + // The ToolSet type erases each tool's input to never; observe takes no input. + const output = await observe.execute({} as never, { toolCallId: 'observe', messages: [], context: undefined }); + return typeof output === 'string' ? output : output.text; +} + describe('tool loop context overflow', () => { it.each(['act', 'assert'] as const)('shrinks overflow history for a %s loop unless inconclusiveOnOverflow is set', async kind => { const model = installFakeLoopModel(({ turn }) => { @@ -282,6 +292,78 @@ describe('tool loop context overflow', () => { expect(step.status).toBe('passed'); }); + it('gives each toolset built outside a loop its own first screen', async () => { + let first: string | undefined; + let second: string | undefined; + // The review's repro: a custom runStep builds createGrammarTools twice, + // once per model call, and calls observe on each. The screen never + // changes, so a presenter shared across the toolsets would answer the + // second look with the screen the first toolset already showed. + const executor: StepExecutor = { + name: 'two-toolsets', + runStep: async (context) => { + first = await observeDirectly(context); + second = await observeDirectly(context); + return { status: 'passed', summary: 'read the sidebar twice' }; + }, + }; + const model = installFakeLoopModel(() => [{ toolName: 'complete_step', input: { status: 'passed', summary: 'unused' } }]); + const { fixtures, steps } = runtime(foldingBigScreen(() => 0), { agents: { default: { executor, model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(0); + // Each toolset's first look lists the sidebar in full, as its own first + // screen, instead of reporting the other toolset's screen unchanged. + for (const look of [first, second]) { + expect(look).toMatch(/Current screen \(revision b\d+/); + expect(look).toContain('#n901 link "Item 1"'); + expect(look).not.toContain('unchanged since revision'); + } + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + }); + + it('gives tools built after a loop run their own first screen', async () => { + let after: string | undefined; + const model = installFakeLoopModel(({ turn }) => { + if (turn === 1) return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'loop read the sidebar' } }]; + }); + // The review's second repro: a loop run, then createGrammarTools built + // from the same step context and observed. The screen never changes, so + // a presenter shared with the run would answer with the screen the + // loop's model already holds. + const loop = () => + createToolLoopExecutor({ + name: 'custom-loop', + tools: (context) => createGrammarTools(context), + buildPrompt: () => 'read the big output', + }); + const executor: StepExecutor = { + name: 'loop-then-tools', + runStep: async (context) => { + const verdict = await loop().runStep(context); + after = await observeDirectly(context); + return verdict; + }, + }; + const { fixtures, steps } = runtime(foldingBigScreen(() => 0), { agents: { default: { executor, model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(2); + // The loop's first look listed the sidebar in full. + expect(loopCalls[1]!.lastToolResult).toContain('#n901 link "Item 1"'); + expect(loopCalls[1]!.lastToolResult).not.toContain('unchanged since revision'); + // The tools built after the run get their own first screen. + expect(after).toMatch(/Current screen \(revision b\d+/); + expect(after).toContain('#n901 link "Item 1"'); + expect(after).not.toContain('unchanged since revision'); + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + }); + it('reports CONTEXT_OVERFLOW when the shrunk request is refused again', async () => { const model = installFakeLoopModel(({ turn }) => { if (turn === 1) return [{ toolName: 'peek', input: {} }]; From eb3d233de68552ba8270ff2407854a884f295569 Mon Sep 17 00:00:00 2001 From: Varshith Puli Date: Mon, 16 Nov 2026 01:24:02 +0000 Subject: [PATCH 12/12] fix(agent): stack screen presenters per loop run so concurrent runs leave no stale binding A loop run restored the single binding it saw at start, so when two runs overlapped the last one to end put back a presenter whose run was over, and tools built afterwards read that run's screen as unchanged. The per-context binding is now a stack: bind pushes a fresh presenter, the run removes its own presenter by identity when it ends, and boundPresenter reads the top. A run that ends while another run is still bound leaves that run's presenter in place, and a run that ends after an overlapping run never puts back a presenter whose run is over. The built-in agent also reads its run's presenter in buildPrompt before the first await, so an overlapping run's opening prompt cannot pick up the other run's presenter. The screen option doc now says only e2e mcp passes a presenter, and the stale step's presenter wording is corrected to the loop run's presenter. Tests: a Promise.all of two loop runs followed by an observe on tools built afterwards, and a loop nested inside another loop's tool on the same context. The concurrent test fails on the old code (the later observe returns Screen unchanged since revision b2); the nested test fails if a run reuses the outer binding instead of binding fresh. Signed-off-by: Varshith Puli --- packages/core/src/agent/default-agent.ts | 19 ++- packages/core/src/agent/primitives.ts | 8 +- packages/core/src/agent/screen-update.ts | 63 +++++----- packages/core/src/agent/tool-loop.ts | 13 +- .../e2e/tests/unit/tool-loop-overflow.test.ts | 114 ++++++++++++++++++ 5 files changed, 175 insertions(+), 42 deletions(-) diff --git a/packages/core/src/agent/default-agent.ts b/packages/core/src/agent/default-agent.ts index 88fb94d5d8..5548dcd8fc 100644 --- a/packages/core/src/agent/default-agent.ts +++ b/packages/core/src/agent/default-agent.ts @@ -16,7 +16,7 @@ import type { ExecutorObservation, ExecutorObserveOptions, ReplayedPrefix, StepE import { interactiveNodeCount } from './observation.ts'; import { testTitleLine } from './prompts.ts'; import { grammarTools } from './primitives.ts'; -import { presenterFor } from './screen-update.ts'; +import { boundPresenter, presenterFor, type ScreenPresenter } from './screen-update.ts'; import { compactScreenHistory, compactScreenshotHistory } from './transcript-compaction.ts'; import { createToolLoopExecutor, type ToolLoopHelpers } from './tool-loop.ts'; import { toolAppliesTo, withToolContext } from './tool.ts'; @@ -78,6 +78,11 @@ function createActLoop(options: BuiltInAgentOptions): StepExecutor { prepareMessages: (messages) => compactScreenshotHistory(compactScreenHistory(messages)), tools: (context, helpers) => loopTools(context, helpers, options.tools), buildPrompt: async (context) => { + // The run's presenter is read before the first await: another run on + // this context may bind its own presenter while this one waits on + // observe, and the opening prompt must read this run's, not the other + // run's. + const presenter = boundPresenter(context) ?? presenterFor(context); // The opening look is tree-only, unless no listed interactive node // exists at all: then the opening prompt carries a screenshot, because the tree describes a canvas, a game, // or a semantics-free native screen too poorly to act on, and the model @@ -90,7 +95,7 @@ function createActLoop(options: BuiltInAgentOptions): StepExecutor { const extra: string[] = []; if (context.replayedPrefix !== undefined) extra.push(formatReplayedPrefix(context.replayedPrefix)); if (context.ledger !== '') extra.push(`Previously completed steps:\n${context.ledger}`); - return openingPrompt(context, `Execute this test step: ${context.step.instruction}`, extra, observation); + return openingPrompt(context, `Execute this test step: ${context.step.instruction}`, extra, observation, presenter); }, }); } @@ -106,8 +111,11 @@ function createAssertionLoop(options: BuiltInAgentOptions): StepExecutor { system: rules(ASSERT_RULES, options.system), inconclusiveOnOverflow: true, tools: (context, helpers) => loopTools(context, helpers, options.tools), - buildPrompt: async (context) => - openingPrompt(context, `Judge whether this assertion holds: ${context.step.instruction}`, [], await context.observe()), + buildPrompt: async (context) => { + // The run's presenter is read before the first await, as the act loop does. + const presenter = boundPresenter(context) ?? presenterFor(context); + return openingPrompt(context, `Judge whether this assertion holds: ${context.step.instruction}`, [], await context.observe(), presenter); + }, }); return { ...loop, runStep: (context) => loop.runStep(withVision(context)) }; } @@ -153,6 +161,7 @@ function openingPrompt( lead: string, extra: readonly string[], observation: ExecutorObservation, + presenter: ScreenPresenter, ): string | ModelMessage[] { const title = testTitleLine(context.attempt.titlePath); const parts = [ @@ -161,7 +170,7 @@ function openingPrompt( ...(context.step.params === undefined ? [] : [`Step parameters:\n${JSON.stringify(context.step.params)}`]), ...extra, ]; - const opening = presenterFor(context).open(observation); + const opening = presenter.open(observation); if (typeof opening === 'string') return [...parts, opening].join('\n\n'); return [ { diff --git a/packages/core/src/agent/primitives.ts b/packages/core/src/agent/primitives.ts index 4c33b8e9bb..318c5271fe 100644 --- a/packages/core/src/agent/primitives.ts +++ b/packages/core/src/agent/primitives.ts @@ -227,9 +227,9 @@ interface GrammarToolSetup extends GrammarToolOptions { /** * Renders screens for the model and remembers what it has seen, so every * result after the first reports the changes rather than the whole tree. - * Shared with the opening prompt by the built-in agent and with the session - * by `e2e mcp`; without one, the loop run's presenter when built inside a - * loop, else a fresh presenter of its own. + * Passed by `e2e mcp` to share with the session; without one, the loop + * run's presenter when built inside a loop, else a fresh presenter of its + * own. */ readonly screen?: ScreenPresenter; /** @@ -898,7 +898,7 @@ function createPointTools( /** * What every verb runs through: the loop guard, the - * step's presenter, one queue for the tool bodies (an action and its look + * loop run's presenter, one queue for the tool bodies (an action and its look * at the result run together, so a batched turn gets one coherent result * per action), and the two moves each verb makes: present the screen after * an action, and perform an action then present. A failed action still diff --git a/packages/core/src/agent/screen-update.ts b/packages/core/src/agent/screen-update.ts index 579adbaef4..a62c5a0dfc 100644 --- a/packages/core/src/agent/screen-update.ts +++ b/packages/core/src/agent/screen-update.ts @@ -324,51 +324,58 @@ export class ScreenPresenter { } /** - * One presenter per loop run, bound to the step's context for the run and - * restored after it, so the presenter state matches that run's transcript - * alone: two loops run against one step context each see their own first - * screen. Tools built outside a run read the binding while one is in - * progress, else use a presenter of their own. Keyed by the step's context - * object, so a run's presenter dies with it. + * One presenter per loop run, stacked on the step's context for the run, so + * the presenter state matches that run's transcript alone: two loops run + * against one step context each see their own first screen, and a run nested + * inside another run's tool reads its own. Tools built outside a run read + * the top binding while one is in progress, else use a presenter of their + * own. Keyed by the step's context object, so a run's presenter dies with it. */ -const presenters = new WeakMap(); +const presenterStacks = new WeakMap(); -/** The step's presenter, creating it on first use. */ +/** + * The loop run's presenter, or a fresh presenter of its own when no run + * bound one: the opening prompt inside a run reads the run's presenter, and + * a prompt built outside a run never shares screen state with one. + */ export function presenterFor(context: StepExecutorContext): ScreenPresenter { - let presenter = presenters.get(context); - if (presenter === undefined) { - presenter = new ScreenPresenter(); - presenters.set(context, presenter); - } - return presenter; + return boundPresenter(context) ?? new ScreenPresenter(); } -/** The presenter bound to the step's context, if a loop run bound one: reads only, never creates. */ +/** The presenter the innermost loop run bound to the step's context, if one bound one: reads only, never creates. */ export function boundPresenter(context: StepExecutorContext): ScreenPresenter | undefined { - return presenters.get(context); + return presenterStacks.get(context)?.at(-1); } /** - * Binds a fresh presenter to the step's context, replacing whatever it held, - * and returns it. A loop run binds one at the start of its run, so each run - * starts from a screen its own model has seen, and restores the previous - * binding when the run ends. `presenterFor` stays the fallback for the - * opening prompt inside a run. + * Binds a fresh presenter to the step's context, pushed onto the stack above + * whatever runs already bound, and returns it. A loop run binds one at the + * start of its run, so each run starts from a screen its own model has seen, + * and removes its own presenter when the run ends. `presenterFor` stays the + * fallback for the opening prompt inside a run. */ export function bindPresenter(context: StepExecutorContext): ScreenPresenter { const presenter = new ScreenPresenter(); - presenters.set(context, presenter); + const stack = presenterStacks.get(context); + if (stack === undefined) presenterStacks.set(context, [presenter]); + else stack.push(presenter); return presenter; } /** - * Restores the binding a loop run replaced when the run ends, or drops the - * entry when there was none, so tools built outside a loop never share a - * run's presenter. + * Removes the loop run's own presenter from the stack when the run ends, by + * identity: a run that ends while another run is still bound above it leaves + * that run's presenter in place, and a run that ends after an overlapping + * run never puts back a presenter whose run is over. Tools built outside a + * loop then find no binding and use a presenter of their own. */ -export function restorePresenter(context: StepExecutorContext, previous: ScreenPresenter | undefined): void { - if (previous === undefined) presenters.delete(context); - else presenters.set(context, previous); +export function restorePresenter(context: StepExecutorContext, presenter: ScreenPresenter): void { + const stack = presenterStacks.get(context); + if (stack === undefined) return; + const index = stack.lastIndexOf(presenter); + if (index === -1) return; + stack.splice(index, 1); + if (stack.length === 0) presenterStacks.delete(context); } /** The line under a screenshot: its pixel size and the coordinate space the point verbs read. */ diff --git a/packages/core/src/agent/tool-loop.ts b/packages/core/src/agent/tool-loop.ts index 62a703fd55..5d88be7933 100644 --- a/packages/core/src/agent/tool-loop.ts +++ b/packages/core/src/agent/tool-loop.ts @@ -28,7 +28,7 @@ import { } from './model/tool-choice.ts'; import { withStallGuard } from './model/stall.ts'; import { providerHints, type ProviderHints, type ProviderModelRef } from './model/provider-hints.ts'; -import { bindPresenter, boundPresenter, isScreenOutput, restorePresenter, toolResultTexts, type ScreenPresenter } from './screen-update.ts'; +import { bindPresenter, isScreenOutput, restorePresenter, toolResultTexts, type ScreenPresenter } from './screen-update.ts'; import { shapeOf } from './message-shape.ts'; import { compactScreenHistory } from './transcript-compaction.ts'; import { AgentError, isAgentError } from './error.ts'; @@ -307,13 +307,16 @@ class LoopRun { } async run(): Promise { - // A fresh presenter per run, restored after it: the transcript belongs to this loop, not the step context. - const previous = boundPresenter(this.context); - this.presenter = bindPresenter(this.context); + // A fresh presenter per run, removed after it: the transcript belongs to + // this loop, not the step context. Removed by identity, so a run that ends + // while another run on this context is still going leaves that run's + // presenter in place and never puts back one whose run is over. + const presenter = bindPresenter(this.context); + this.presenter = presenter; try { return await this.runLoop(); } finally { - restorePresenter(this.context, previous); + restorePresenter(this.context, presenter); this.presenter = undefined; } } diff --git a/packages/e2e/tests/unit/tool-loop-overflow.test.ts b/packages/e2e/tests/unit/tool-loop-overflow.test.ts index e3c136ba5f..1ee6d03639 100644 --- a/packages/e2e/tests/unit/tool-loop-overflow.test.ts +++ b/packages/e2e/tests/unit/tool-loop-overflow.test.ts @@ -292,6 +292,120 @@ describe('tool loop context overflow', () => { expect(step.status).toBe('passed'); }); + it('gives tools built after two concurrent runs on one context their own first screen', async () => { + let after: string | undefined; + // Content-scripted, not turn-scripted: the two runs interleave, so no + // absolute turn belongs to either loop. + const model = installFakeLoopModel((call) => { + if (call.lastToolResult === '') return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'loop read the sidebar' } }]; + }); + const loop = () => + createToolLoopExecutor({ + name: 'custom-loop', + tools: (context) => createGrammarTools(context), + buildPrompt: () => 'read the big output', + }); + // The review's repro: one custom runStep that runs two loops against the + // same step context at once, then builds grammar tools from it. The + // screen never changes, so a presenter left bound by either run would + // answer the later look with the screen that run already showed. + const executor: StepExecutor = { + name: 'two-concurrent-loops', + runStep: async (context) => { + const [first] = await Promise.all([loop().runStep(context), loop().runStep(context)]); + after = await observeDirectly(context); + return first!; + }, + }; + const { fixtures, steps } = runtime(foldingBigScreen(() => 0), { agents: { default: { executor, model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(4); + // Each loop's first look lists the sidebar in full, as its own first screen. + for (const call of loopCalls) { + if (call.lastToolResult === '') continue; + expect(call.lastToolResult).toMatch(/Current screen \(revision b\d+/); + expect(call.lastToolResult).toContain('#n901 link "Item 1"'); + expect(call.lastToolResult).not.toContain('unchanged since revision'); + } + // The tools built after both runs get their own first screen, instead of + // reporting the screen the run that ended last already showed. + expect(after).toMatch(/Current screen \(revision b\d+/); + expect(after).toContain('#n901 link "Item 1"'); + expect(after).not.toContain('unchanged since revision'); + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + }); + + it('gives a loop nested inside another loop\'s tool its own first screen, and the outer loop keeps its own', async () => { + let outerContext: StepExecutorContext | undefined; + const inner = () => + createToolLoopExecutor({ + name: 'inner-loop', + tools: (context) => createGrammarTools(context), + buildPrompt: () => 'read the big output', + }); + // A project tool that runs a second loop against the same step context, + // as a custom executor that delegates part of a step would. + const nest = defineTool( + { + inputSchema: z.object({}), + execute: async () => { + const verdict = await inner().runStep(outerContext!); + return `nested loop ${verdict.status}`; + }, + }, + { mutates: false }, + ); + const model = installFakeLoopModel((call) => { + // The inner loop offers no `nest` tool, so the tool names tell the + // loops apart whatever order their turns interleave in. + if (call.toolNames.includes('nest')) { + if (call.lastToolResult === '') return [{ toolName: 'nest', input: {} }]; + if (call.lastToolResult.includes('nested loop passed')) return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'outer read the sidebar after the nested run' } }]; + } + if (call.lastToolResult === '') return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'inner read the sidebar' } }]; + }); + const outer = createToolLoopExecutor({ + name: 'outer-loop', + tools: (context, helpers) => ({ + ...projectTools(context, { nest }), + ...createGrammarTools(context, { guard: helpers.guard }), + }), + buildPrompt: () => 'read the big output', + }); + const executor: StepExecutor = { + name: 'nested-loops', + runStep: async (context) => { + outerContext = context; + return outer.runStep(context); + }, + }; + const { fixtures, steps } = runtime(foldingBigScreen(() => 0), { agents: { default: { executor, model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(5); + // The inner loop's first look and the outer loop's look after the nested + // run each list the sidebar in full, as their own first screens: the + // nested run never touched the outer presenter's screen, and the outer + // look never reuses the inner run's. A run that reused the outer + // binding instead of binding fresh would answer the outer look with the + // screen the inner run already showed. + for (const call of loopCalls) { + if (call.lastToolResult === '' || call.lastToolResult.includes('nested loop passed')) continue; + expect(call.lastToolResult).toMatch(/Current screen \(revision b\d+/); + expect(call.lastToolResult).toContain('#n901 link "Item 1"'); + expect(call.lastToolResult).not.toContain('unchanged since revision'); + } + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + }); + it('gives each toolset built outside a loop its own first screen', async () => { let first: string | undefined; let second: string | undefined;