diff --git a/.changeset/caret-change-wait.md b/.changeset/caret-change-wait.md new file mode 100644 index 000000000..81ce2d897 --- /dev/null +++ b/.changeset/caret-change-wait.md @@ -0,0 +1,6 @@ +--- +"e2e": patch +"@e2e-dev/core": patch +--- + +An agent step no longer waits two seconds after a caret key that changes nothing. ArrowLeft, ArrowRight, Home, End, or select-all pressed in a text field that already holds the focus waits one second for the screen to react, not two. A tap, ArrowUp and ArrowDown, the first key that gives a field the focus, and a key batched after another action still wait the full window. A replay of a recorded rich-text editing flow takes about 14.6 s instead of 20.5 s. diff --git a/packages/core/src/agent/action-dispatcher.ts b/packages/core/src/agent/action-dispatcher.ts index f93c68a24..e707ca433 100644 --- a/packages/core/src/agent/action-dispatcher.ts +++ b/packages/core/src/agent/action-dispatcher.ts @@ -42,7 +42,7 @@ import { describePointAction, describePointDrag, describePointHit, hitTest, POIN import { nodeReading, normalizeReading, pageLanded, pageMoved, readPage, type PageReading } from './reading.ts'; import { SETTLE_POLL_MS, type AgentObservation, type RedactedNode, type SemanticAgentObservation } from './observation.ts'; import { authorizeSecretFill } from './secrets.ts'; -import { HELD_STILL_MS, SETTLE_AFTER } from './settle-policy.ts'; +import { changeWaitAfter, HELD_STILL_MS, SETTLE_AFTER, type SettleSubject } from './settle-policy.ts'; import type { StepAccounting } from './step-accounting.ts'; import type { StepTraceSession } from './step-cache.ts'; import { authorizeUploadPaths } from './upload-paths.ts'; @@ -373,7 +373,7 @@ export class ActionDispatcher { const name = `${PROJECT_TOOL_EVENT_PREFIX}${call.name}`; const value = await instrumentPhase(this.runtime, { api: this.accounting.api, kind: 'engine', phase: 'agent.action', name }, body); if (call.mutates) { - this.armAfter('tool'); + this.armAfter({ name: 'tool' }); await this.notify('tool', (fields) => ({ ...fields, action: `${PROJECT_TOOL_EVENT_PREFIX}${call.name}`, summary: `ran ${call.name}` })); } return value; @@ -751,7 +751,7 @@ export class ActionDispatcher { this.accounting.checkpoint(cause); throw cause; } - this.armAfter(action.name); + this.armAfter(action); this.recordTrace(action); await this.notify(action.name, (fields) => actionEvent(action, { ...fields, summary: describeAction(action, redaction).summary })); } @@ -835,9 +835,9 @@ export class ActionDispatcher { * would wait against the destination screen. An action whose effect the * tree cannot show arms nothing. */ - private armAfter(name: RecordedAction['name']): void { - if (name === 'scrollUntil' && !this.verbs.has('scrollTo')) return; - const { changeWaitMs } = SETTLE_AFTER[name]; + private armAfter(action: SettleSubject): void { + if (action.name === 'scrollUntil' && !this.verbs.has('scrollTo')) return; + const changeWaitMs = changeWaitAfter(action, action.name === 'pressKey' ? this.feed.focusedNode() : undefined); if (changeWaitMs !== undefined) this.feed.armChange(changeWaitMs); } diff --git a/packages/core/src/agent/observation-feed.ts b/packages/core/src/agent/observation-feed.ts index 21ca53d2a..fd7ada242 100644 --- a/packages/core/src/agent/observation-feed.ts +++ b/packages/core/src/agent/observation-feed.ts @@ -243,15 +243,30 @@ export class ObservationFeed { }; } + /** The node that had the focus on the newest screen, when the screen lists nodes and exactly one has it. */ + focusedNode(): RedactedNode | undefined { + const latest = this.newest; + if (latest === undefined || latest.kind === 'pixels') return undefined; + const focused = [...latest.nodes.values()].filter((node) => node.states?.focused === true); + return focused.length === 1 ? focused[0] : undefined; + } + /** * Gives the screen `waitMs` from this completed action to leave the newest * observation's shape. Captures and time before the next observation count - * toward the window. Nothing is armed without a comparable shape. + * toward the window. Nothing is armed without a comparable shape. A window + * still pending, because actions batched in one turn had no look between + * them, is never shortened: the later action keeps the earlier one's deadline. */ armChange(waitMs: number): void { if (this.newest === undefined) return; const shape = changeShape(this.newest); - this.pendingChange = shape === undefined ? undefined : { shape, deadlineMs: Date.now() + waitMs }; + if (shape === undefined) { + this.pendingChange = undefined; + return; + } + const deadlineMs = Math.max(Date.now() + waitMs, this.pendingChange?.deadlineMs ?? 0); + this.pendingChange = { shape, deadlineMs }; } /** diff --git a/packages/core/src/agent/settle-policy.ts b/packages/core/src/agent/settle-policy.ts index b7e57ec11..0daf6da0f 100644 --- a/packages/core/src/agent/settle-policy.ts +++ b/packages/core/src/agent/settle-policy.ts @@ -6,11 +6,13 @@ * leave the shape the action was resolved against (the dispatcher arms it, * and the next settled look waits out what remains), and how far the * replay's look before the next action settles. A new action kind does not - * compile until it declares both, and the live loop and replay can never - * disagree about the same action. + * compile until it declares both. The window is the table's, except that + * `changeWaitAfter` shortens it for a caret key by reading the focus on the + * newest screen the loop or the replay holds. */ import type { RecordedAction } from '../cache/trace.ts'; +import { parseKey } from '../engine/contract.ts'; /** * How far one look settles. @@ -72,6 +74,71 @@ interface SettleAfter { readonly look: SettleMode; } +/** + * The change wait after a caret key in a text field. A selection that changes + * shows in the first capture after the key, but a handler on `selectionchange` + * (a rich-text toolbar, a mention list) may update the tree a beat later, and + * the tree cannot say whether one is listening. The wait is the span a screen + * must hold still to count as settled, so a handler that updates within a + * second is read, at half the cost of the full window. One slower than that + * is read on the next look, as it is after any action with a shorter window. + */ +const CARET_CHANGE_WAIT_MS = HELD_STILL_MS; + +/** Roles of a field a caret lives in. A `combobox` is left out: its keys open and move a popup. */ +const CARET_ROLES: ReadonlySet = new Set(['textbox', 'searchbox']); + +/** + * Keys that only move a caret or extend a selection, alone or with modifiers. + * Narrower than the agent's focus-only keys (`primitives.ts`): Tab and the + * vertical keys leave the field or move a popup, so they keep the full wait. + */ +const CARET_KEYS: ReadonlySet = new Set(['ArrowLeft', 'ArrowRight', 'Home', 'End']); + +/** The action a settle policy reads: its kind, its key, and the node it ran against. */ +export interface SettleSubject { + readonly name: RecordedAction['name']; + readonly key?: string; + readonly node?: { readonly role?: string | undefined; readonly states?: { readonly focused?: boolean | undefined } | undefined }; +} + +/** True for a text field that held the focus when the action reached it. */ +function focusedTextField(node: SettleSubject['node']): boolean { + return node?.states?.focused === true && CARET_ROLES.has(node.role ?? ''); +} + +/** + * Whether the key moved a caret in a text field that already held the + * focus, an effect the tree mostly cannot show. A selection it changes is + * read at once, so the change wait only ever runs out on one that changed + * nothing: a key at the end of the text, a second select-all. A tap is left + * out, focused field or not: a click can open a popup or a panel that takes + * longer than the short window to render. So is the first focus of a field, + * since what a field opens when it takes the focus may arrive a beat later. + * `focused` is the field a key press went to, which a keyboard action does + * not name. + */ +function movesCaretOnly(action: SettleSubject, focused: SettleSubject['node']): boolean { + if (action.name !== 'press' && action.name !== 'pressKey') return false; + if (!focusedTextField(action.name === 'press' ? action.node : focused)) return false; + const parsed = parseKey(action.key ?? ''); + if (parsed === undefined || parsed.modifiers.includes('Alt')) return false; + if (parsed.key.kind === 'named') return CARET_KEYS.has(parsed.key.name); + const [only, ...rest] = parsed.modifiers; + return parsed.key.char === 'a' && rest.length === 0 && (only === 'Control' || only === 'Meta' || only === 'ControlOrMeta'); +} + +/** + * How long after the committed action the screen has to leave the shape it + * was resolved against (`SETTLE_AFTER`'s window, shortened for an action + * that only moves a caret): absent when the tree cannot show the effect. + */ +export function changeWaitAfter(action: SettleSubject, focused?: SettleSubject['node']): number | undefined { + const { changeWaitMs } = SETTLE_AFTER[action.name]; + if (changeWaitMs === undefined || changeWaitMs <= CARET_CHANGE_WAIT_MS) return changeWaitMs; + return movesCaretOnly(action, focused) ? CARET_CHANGE_WAIT_MS : changeWaitMs; +} + /** * The settle policy per action kind. A `tool` entry is a gap: replay stops * at it, so its `look` is never taken, while the live loop arms its change diff --git a/packages/e2e/tests/helpers/scripted-scene.ts b/packages/e2e/tests/helpers/scripted-scene.ts index 247416e8a..000ae7d22 100644 --- a/packages/e2e/tests/helpers/scripted-scene.ts +++ b/packages/e2e/tests/helpers/scripted-scene.ts @@ -65,7 +65,7 @@ export interface Scene { perform(id: string, action: LocatorAction): void; /** `keyboard.type` into the focused field. */ type(text: string, replace: boolean): void; - /** `keyboard.press` on the focused field. */ + /** `keyboard.press` on the focused field, which sees it as a `press` on itself. */ press(key: string): void; } @@ -238,7 +238,9 @@ export function createScene(build: (stage: Stage) => ScriptedNode[], initialLoca node.value = replace ? text : `${node.value ?? ''}${text}`; }, press(key) { - applyKey(focused(), key); + const node = focused(); + applyKey(node, key); + node.on?.({ kind: 'press', key }, node); }, }; } diff --git a/packages/e2e/tests/integration/agent-caret-settle.test.ts b/packages/e2e/tests/integration/agent-caret-settle.test.ts new file mode 100644 index 000000000..c4843bfda --- /dev/null +++ b/packages/e2e/tests/integration/agent-caret-settle.test.ts @@ -0,0 +1,110 @@ +/** A caret key in a focused text field waits a second for its effect; a tap, a first focus, or a batched earlier action still waits the full window. */ + +import { describe, expect, it } from 'vitest'; +import type { StepExecutorContext, StepExecutor } from '@e2e-dev/core/lib/agent/executor'; +import { createFakeEngine, FAKE_APP, FAKE_APP_URL } from '../helpers/fake-engine.ts'; +import { nodeIdFor } from '../helpers/fake-loop-model.ts'; +import { createProject, runExisting } from '../helpers/run-project.ts'; +import type { ScriptedNode } from '../helpers/scripted-scene.ts'; + +const SUITE = `import { test } from 'e2e'; +test('acts on the field', async ({ app, agent }) => { + await app.open(); + await agent.act('act on the field'); +});`; + +/** How long after the action the field's popup opens, and whether the field already held the focus. */ +interface Case { + readonly field: 'focused' | 'unfocused'; + readonly opensAfterMs: number | undefined; +} + +interface Ids { + readonly field: string; + readonly button: string; +} + +type Act = (context: StepExecutorContext, ids: Ids) => Promise; + +/** Runs one action on a text field in a scene whose popup opens `opensAfterMs` after it, and reports the look after it. */ +async function lookAfter(scenario: Case, act: Act): Promise<{ readonly text: string; readonly ms: number }> { + let seen = { text: '', ms: 0 }; + const fake = createFakeEngine({ + actions: ['tap', 'press'], + keyboard: true, + scene: (stage) => { + const popup: ScriptedNode = { id: 'popup', role: 'status', name: 'Suggestions', text: 'closed' }; + const open = (): void => { + if (scenario.opensAfterMs !== undefined) stage.after(scenario.opensAfterMs, () => { popup.text = 'open'; }); + }; + return [popup, { + id: 'button', role: 'button', name: 'Go', + rect: { x: 0, y: 80, width: 100, height: 40 }, + on: (action) => { if (action.kind === 'tap') open(); }, + }, { + id: 'field', role: 'textbox', name: 'Search', value: 'release approved', + rect: { x: 0, y: 20, width: 100, height: 40 }, + ...(scenario.field === 'focused' ? { states: { focused: true } } : {}), + on: (action) => { if (action.kind === 'tap' || action.kind === 'press') open(); }, + }]; + }, + }); + const executor: StepExecutor = { + name: 'caret-settle', version: '1', + async runStep(context) { + const first = await context.observe(); + await act(context, { field: nodeIdFor(first.text, /textbox "Search"/u), button: nodeIdFor(first.text, /button "Go"/u) }); + const started = Date.now(); + const after = await context.observe(); + seen = { text: after.text, ms: Date.now() - started }; + return { status: 'passed', summary: 'acted on the field' }; + }, + }; + const project = createProject({ 'tests/caret.e2e.ts': SUITE }); + try { + const outcome = await runExisting(project, { + appUrl: FAKE_APP_URL, + config: { tests: 'tests/**/*.e2e.ts', targets: [{ name: 'surface', platform: 'web', engine: fake.engine, app: FAKE_APP }], agents: { default: { executor } } }, + }); + expect(outcome.exitCode, JSON.stringify(outcome.report.run.errors)).toBe(0); + return seen; + } finally { + project.cleanup(); + } +} + +const tapField: Act = (context, { field }) => context.actions.tap({ id: field }); +const pressField: Act = (context, { field }) => context.actions.press({ id: field }, 'ArrowRight'); +const pressKey: Act = (context) => context.actions.pressKey('ArrowRight'); +const batched: Act = async (context, { button }) => { + await context.actions.tap({ id: button }); + await context.actions.pressKey('End'); +}; + +describe('the look after a caret key in a text field', () => { + it.each([ + ['a first tap on the field', 'unfocused', tapField], + ['a tap on the focused field', 'focused', tapField], + ['a caret key pressed on the field before it held the focus', 'unfocused', pressField], + ['a tap on a button followed by a caret key in the same turn', 'focused', batched], + ] as const)('still waits for what %s opens', async (_name, field, act) => { + const look = await lookAfter({ field, opensAfterMs: 700 }, act); + expect(look.text).toContain('open'); + }); + + it.each([ + ['a caret key pressed on the focused field', pressField], + ['a caret key sent to the focused field', pressKey], + ] as const)('still reads what a handler opens 700 ms after %s', async (_name, act) => { + const look = await lookAfter({ field: 'focused', opensAfterMs: 700 }, act); + expect(look.text).toContain('open'); + }); + + it.each([ + ['a caret key pressed on the focused field', pressField], + ['a caret key sent to the focused field', pressKey], + ] as const)('does not wait out the whole change window for %s that changes nothing', async (_name, act) => { + const look = await lookAfter({ field: 'focused', opensAfterMs: undefined }, act); + expect(look.ms).toBeLessThan(1600); + }); +}); diff --git a/packages/e2e/tests/unit/settle-policy.test.ts b/packages/e2e/tests/unit/settle-policy.test.ts new file mode 100644 index 000000000..39a64c56a --- /dev/null +++ b/packages/e2e/tests/unit/settle-policy.test.ts @@ -0,0 +1,57 @@ +/** The change wait an action arms: the table's window, shortened only for a caret key in a focused text field. */ + +import { describe, expect, it } from 'vitest'; +import { changeWaitAfter, SETTLE_AFTER } from '@e2e-dev/core/lib/agent/settle-policy'; + +const full = SETTLE_AFTER.tap.changeWaitMs; +const caretWait = 1000; +const textbox = { role: 'textbox', states: { focused: true } }; +const unfocused = { role: 'textbox' }; + +describe('changeWaitAfter', () => { + it('keeps the table window for an action that can change the screen', () => { + expect(changeWaitAfter({ name: 'tap', node: { role: 'button' } })).toBe(full); + expect(changeWaitAfter({ name: 'press', node: textbox, key: 'Enter' })).toBe(full); + expect(changeWaitAfter({ name: 'press', node: { role: 'combobox' }, key: 'ArrowRight' })).toBe(full); + expect(changeWaitAfter({ name: 'secondaryTap', node: textbox })).toBe(full); + expect(changeWaitAfter({ name: 'type', node: textbox })).toBe(full); + }); + + it('keeps the window for a tap, even on the focused field, since a click can open a panel', () => { + expect(changeWaitAfter({ name: 'tap', node: textbox })).toBe(full); + expect(changeWaitAfter({ name: 'doubleTap', node: { role: 'searchbox', states: { focused: true } } })).toBe(full); + expect(changeWaitAfter({ name: 'tap', node: unfocused })).toBe(full); + }); + + it('keeps the window for the first focus of a text field, which may open something', () => { + expect(changeWaitAfter({ name: 'press', node: unfocused, key: 'ArrowRight' })).toBe(full); + expect(changeWaitAfter({ name: 'pressKey', key: 'ArrowRight' }, unfocused)).toBe(full); + }); + + it.each(['ArrowLeft', 'ArrowRight', 'Home', 'End', 'Shift+ArrowRight', 'Control+Shift+ArrowLeft', 'Control+a', 'Meta+a'])( + 'shortens the window for the caret key %s in a text field', + (key) => { + expect(changeWaitAfter({ name: 'press', node: textbox, key })).toBe(caretWait); + expect(changeWaitAfter({ name: 'pressKey', key }, textbox)).toBe(caretWait); + }, + ); + + it.each(['ArrowUp', 'ArrowDown', 'Enter', 'Tab', 'Backspace', 'a', 'Alt+ArrowLeft', 'Shift+a', 'Control+Enter'])( + 'keeps the window for %s', + (key) => { + expect(changeWaitAfter({ name: 'press', node: textbox, key })).toBe(full); + }, + ); + + it('keeps the window for a caret key outside a text field, or on a focus nobody can name', () => { + expect(changeWaitAfter({ name: 'press', node: { role: 'tab', states: { focused: true } }, key: 'ArrowRight' })).toBe(full); + expect(changeWaitAfter({ name: 'press', node: { role: 'slider', states: { focused: true } }, key: 'Home' })).toBe(full); + expect(changeWaitAfter({ name: 'pressKey', key: 'ArrowRight' })).toBe(full); + expect(changeWaitAfter({ name: 'pressKey', key: 'ArrowRight' }, { role: 'listbox', states: { focused: true } })).toBe(full); + }); + + it('never lengthens a window and leaves an absent one absent', () => { + expect(changeWaitAfter({ name: 'scroll' })).toBe(SETTLE_AFTER.scroll.changeWaitMs); + expect(changeWaitAfter({ name: 'typeSecret', node: textbox })).toBeUndefined(); + }); +});