Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/caret-change-wait.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
"e2e": patch
"@e2e-dev/core": patch
---

An agent step no longer waits two seconds after a caret key that changes nothing. ArrowLeft, ArrowRight, Home, End, or select-all pressed in a text field that already holds the focus waits one second for the screen to react, not two. A tap, ArrowUp and ArrowDown, the first key that gives a field the focus, and a key batched after another action still wait the full window. A replay of a recorded rich-text editing flow takes about 14.6 s instead of 20.5 s.
12 changes: 6 additions & 6 deletions packages/core/src/agent/action-dispatcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import { describePointAction, describePointDrag, describePointHit, hitTest, POIN
import { nodeReading, normalizeReading, pageLanded, pageMoved, readPage, type PageReading } from './reading.ts';
import { SETTLE_POLL_MS, type AgentObservation, type RedactedNode, type SemanticAgentObservation } from './observation.ts';
import { authorizeSecretFill } from './secrets.ts';
import { HELD_STILL_MS, SETTLE_AFTER } from './settle-policy.ts';
import { changeWaitAfter, HELD_STILL_MS, SETTLE_AFTER, type SettleSubject } from './settle-policy.ts';
import type { StepAccounting } from './step-accounting.ts';
import type { StepTraceSession } from './step-cache.ts';
import { authorizeUploadPaths } from './upload-paths.ts';
Expand Down Expand Up @@ -373,7 +373,7 @@ export class ActionDispatcher {
const name = `${PROJECT_TOOL_EVENT_PREFIX}${call.name}`;
const value = await instrumentPhase(this.runtime, { api: this.accounting.api, kind: 'engine', phase: 'agent.action', name }, body);
if (call.mutates) {
this.armAfter('tool');
this.armAfter({ name: 'tool' });
await this.notify('tool', (fields) => ({ ...fields, action: `${PROJECT_TOOL_EVENT_PREFIX}${call.name}`, summary: `ran ${call.name}` }));
}
return value;
Expand Down Expand Up @@ -751,7 +751,7 @@ export class ActionDispatcher {
this.accounting.checkpoint(cause);
throw cause;
}
this.armAfter(action.name);
this.armAfter(action);
this.recordTrace(action);
await this.notify(action.name, (fields) => actionEvent(action, { ...fields, summary: describeAction(action, redaction).summary }));
}
Expand Down Expand Up @@ -835,9 +835,9 @@ export class ActionDispatcher {
* would wait against the destination screen. An action whose effect the
* tree cannot show arms nothing.
*/
private armAfter(name: RecordedAction['name']): void {
if (name === 'scrollUntil' && !this.verbs.has('scrollTo')) return;
const { changeWaitMs } = SETTLE_AFTER[name];
private armAfter(action: SettleSubject): void {
if (action.name === 'scrollUntil' && !this.verbs.has('scrollTo')) return;
const changeWaitMs = changeWaitAfter(action, action.name === 'pressKey' ? this.feed.focusedNode() : undefined);
if (changeWaitMs !== undefined) this.feed.armChange(changeWaitMs);
}

Expand Down
19 changes: 17 additions & 2 deletions packages/core/src/agent/observation-feed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -243,15 +243,30 @@ export class ObservationFeed {
};
}

/** The node that had the focus on the newest screen, when the screen lists nodes and exactly one has it. */
focusedNode(): RedactedNode | undefined {
const latest = this.newest;
if (latest === undefined || latest.kind === 'pixels') return undefined;
const focused = [...latest.nodes.values()].filter((node) => node.states?.focused === true);
return focused.length === 1 ? focused[0] : undefined;
}

/**
* Gives the screen `waitMs` from this completed action to leave the newest
* observation's shape. Captures and time before the next observation count
* toward the window. Nothing is armed without a comparable shape.
* toward the window. Nothing is armed without a comparable shape. A window
* still pending, because actions batched in one turn had no look between
* them, is never shortened: the later action keeps the earlier one's deadline.
*/
armChange(waitMs: number): void {
if (this.newest === undefined) return;
const shape = changeShape(this.newest);
this.pendingChange = shape === undefined ? undefined : { shape, deadlineMs: Date.now() + waitMs };
if (shape === undefined) {
this.pendingChange = undefined;
return;
}
const deadlineMs = Math.max(Date.now() + waitMs, this.pendingChange?.deadlineMs ?? 0);
this.pendingChange = { shape, deadlineMs };
}

/**
Expand Down
71 changes: 69 additions & 2 deletions packages/core/src/agent/settle-policy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@
* leave the shape the action was resolved against (the dispatcher arms it,
* and the next settled look waits out what remains), and how far the
* replay's look before the next action settles. A new action kind does not
* compile until it declares both, and the live loop and replay can never
* disagree about the same action.
* compile until it declares both. The window is the table's, except that
* `changeWaitAfter` shortens it for a caret key by reading the focus on the
* newest screen the loop or the replay holds.
*/

import type { RecordedAction } from '../cache/trace.ts';
import { parseKey } from '../engine/contract.ts';

/**
* How far one look settles.
Expand Down Expand Up @@ -72,6 +74,71 @@ interface SettleAfter {
readonly look: SettleMode;
}

/**
* The change wait after a caret key in a text field. A selection that changes
* shows in the first capture after the key, but a handler on `selectionchange`
* (a rich-text toolbar, a mention list) may update the tree a beat later, and
* the tree cannot say whether one is listening. The wait is the span a screen
* must hold still to count as settled, so a handler that updates within a
* second is read, at half the cost of the full window. One slower than that
* is read on the next look, as it is after any action with a shorter window.
*/
const CARET_CHANGE_WAIT_MS = HELD_STILL_MS;

/** Roles of a field a caret lives in. A `combobox` is left out: its keys open and move a popup. */
const CARET_ROLES: ReadonlySet<string> = new Set(['textbox', 'searchbox']);

/**
* Keys that only move a caret or extend a selection, alone or with modifiers.
* Narrower than the agent's focus-only keys (`primitives.ts`): Tab and the
* vertical keys leave the field or move a popup, so they keep the full wait.
*/
const CARET_KEYS: ReadonlySet<string> = new Set(['ArrowLeft', 'ArrowRight', 'Home', 'End']);

/** The action a settle policy reads: its kind, its key, and the node it ran against. */
export interface SettleSubject {
readonly name: RecordedAction['name'];
readonly key?: string;
readonly node?: { readonly role?: string | undefined; readonly states?: { readonly focused?: boolean | undefined } | undefined };
}

/** True for a text field that held the focus when the action reached it. */
function focusedTextField(node: SettleSubject['node']): boolean {
return node?.states?.focused === true && CARET_ROLES.has(node.role ?? '');
}

/**
* Whether the key moved a caret in a text field that already held the
* focus, an effect the tree mostly cannot show. A selection it changes is
* read at once, so the change wait only ever runs out on one that changed
* nothing: a key at the end of the text, a second select-all. A tap is left
* out, focused field or not: a click can open a popup or a panel that takes
* longer than the short window to render. So is the first focus of a field,
* since what a field opens when it takes the focus may arrive a beat later.
* `focused` is the field a key press went to, which a keyboard action does
* not name.
*/
function movesCaretOnly(action: SettleSubject, focused: SettleSubject['node']): boolean {
if (action.name !== 'press' && action.name !== 'pressKey') return false;
if (!focusedTextField(action.name === 'press' ? action.node : focused)) return false;
const parsed = parseKey(action.key ?? '');
if (parsed === undefined || parsed.modifiers.includes('Alt')) return false;
if (parsed.key.kind === 'named') return CARET_KEYS.has(parsed.key.name);
const [only, ...rest] = parsed.modifiers;
return parsed.key.char === 'a' && rest.length === 0 && (only === 'Control' || only === 'Meta' || only === 'ControlOrMeta');
}

/**
* How long after the committed action the screen has to leave the shape it
* was resolved against (`SETTLE_AFTER`'s window, shortened for an action
* that only moves a caret): absent when the tree cannot show the effect.
*/
export function changeWaitAfter(action: SettleSubject, focused?: SettleSubject['node']): number | undefined {
const { changeWaitMs } = SETTLE_AFTER[action.name];
if (changeWaitMs === undefined || changeWaitMs <= CARET_CHANGE_WAIT_MS) return changeWaitMs;
return movesCaretOnly(action, focused) ? CARET_CHANGE_WAIT_MS : changeWaitMs;
}

/**
* The settle policy per action kind. A `tool` entry is a gap: replay stops
* at it, so its `look` is never taken, while the live loop arms its change
Expand Down
6 changes: 4 additions & 2 deletions packages/e2e/tests/helpers/scripted-scene.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ export interface Scene {
perform(id: string, action: LocatorAction): void;
/** `keyboard.type` into the focused field. */
type(text: string, replace: boolean): void;
/** `keyboard.press` on the focused field. */
/** `keyboard.press` on the focused field, which sees it as a `press` on itself. */
press(key: string): void;
}

Expand Down Expand Up @@ -238,7 +238,9 @@ export function createScene(build: (stage: Stage) => ScriptedNode[], initialLoca
node.value = replace ? text : `${node.value ?? ''}${text}`;
},
press(key) {
applyKey(focused(), key);
const node = focused();
applyKey(node, key);
node.on?.({ kind: 'press', key }, node);
},
};
}
Expand Down
110 changes: 110 additions & 0 deletions packages/e2e/tests/integration/agent-caret-settle.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
/** A caret key in a focused text field waits a second for its effect; a tap, a first focus, or a batched earlier action still waits the full window. */

import { describe, expect, it } from 'vitest';
import type { StepExecutorContext, StepExecutor } from '@e2e-dev/core/lib/agent/executor';
import { createFakeEngine, FAKE_APP, FAKE_APP_URL } from '../helpers/fake-engine.ts';
import { nodeIdFor } from '../helpers/fake-loop-model.ts';
import { createProject, runExisting } from '../helpers/run-project.ts';
import type { ScriptedNode } from '../helpers/scripted-scene.ts';

const SUITE = `import { test } from 'e2e';
test('acts on the field', async ({ app, agent }) => {
await app.open();
await agent.act('act on the field');
});`;

/** How long after the action the field's popup opens, and whether the field already held the focus. */
interface Case {
readonly field: 'focused' | 'unfocused';
readonly opensAfterMs: number | undefined;
}

interface Ids {
readonly field: string;
readonly button: string;
}

type Act = (context: StepExecutorContext, ids: Ids) => Promise<void>;

/** Runs one action on a text field in a scene whose popup opens `opensAfterMs` after it, and reports the look after it. */
async function lookAfter(scenario: Case, act: Act): Promise<{ readonly text: string; readonly ms: number }> {
let seen = { text: '', ms: 0 };
const fake = createFakeEngine({
actions: ['tap', 'press'],
keyboard: true,
scene: (stage) => {
const popup: ScriptedNode = { id: 'popup', role: 'status', name: 'Suggestions', text: 'closed' };
const open = (): void => {
if (scenario.opensAfterMs !== undefined) stage.after(scenario.opensAfterMs, () => { popup.text = 'open'; });
};
return [popup, {
id: 'button', role: 'button', name: 'Go',
rect: { x: 0, y: 80, width: 100, height: 40 },
on: (action) => { if (action.kind === 'tap') open(); },
}, {
id: 'field', role: 'textbox', name: 'Search', value: 'release approved',
rect: { x: 0, y: 20, width: 100, height: 40 },
...(scenario.field === 'focused' ? { states: { focused: true } } : {}),
on: (action) => { if (action.kind === 'tap' || action.kind === 'press') open(); },
}];
},
});
const executor: StepExecutor = {
name: 'caret-settle', version: '1',
async runStep(context) {
const first = await context.observe();
await act(context, { field: nodeIdFor(first.text, /textbox "Search"/u), button: nodeIdFor(first.text, /button "Go"/u) });
const started = Date.now();
const after = await context.observe();
seen = { text: after.text, ms: Date.now() - started };
return { status: 'passed', summary: 'acted on the field' };
},
};
const project = createProject({ 'tests/caret.e2e.ts': SUITE });
try {
const outcome = await runExisting(project, {
appUrl: FAKE_APP_URL,
config: { tests: 'tests/**/*.e2e.ts', targets: [{ name: 'surface', platform: 'web', engine: fake.engine, app: FAKE_APP }], agents: { default: { executor } } },
});
expect(outcome.exitCode, JSON.stringify(outcome.report.run.errors)).toBe(0);
return seen;
} finally {
project.cleanup();
}
}

const tapField: Act = (context, { field }) => context.actions.tap({ id: field });
const pressField: Act = (context, { field }) => context.actions.press({ id: field }, 'ArrowRight');
const pressKey: Act = (context) => context.actions.pressKey('ArrowRight');
const batched: Act = async (context, { button }) => {
await context.actions.tap({ id: button });
await context.actions.pressKey('End');
};

describe('the look after a caret key in a text field', () => {
it.each([
['a first tap on the field', 'unfocused', tapField],
['a tap on the focused field', 'focused', tapField],
['a caret key pressed on the field before it held the focus', 'unfocused', pressField],
['a tap on a button followed by a caret key in the same turn', 'focused', batched],
] as const)('still waits for what %s opens', async (_name, field, act) => {
const look = await lookAfter({ field, opensAfterMs: 700 }, act);
expect(look.text).toContain('open');
});

it.each([
['a caret key pressed on the focused field', pressField],
['a caret key sent to the focused field', pressKey],
] as const)('still reads what a handler opens 700 ms after %s', async (_name, act) => {
const look = await lookAfter({ field: 'focused', opensAfterMs: 700 }, act);
expect(look.text).toContain('open');
});

it.each([
['a caret key pressed on the focused field', pressField],
['a caret key sent to the focused field', pressKey],
] as const)('does not wait out the whole change window for %s that changes nothing', async (_name, act) => {
const look = await lookAfter({ field: 'focused', opensAfterMs: undefined }, act);
expect(look.ms).toBeLessThan(1600);
});
});
57 changes: 57 additions & 0 deletions packages/e2e/tests/unit/settle-policy.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/** The change wait an action arms: the table's window, shortened only for a caret key in a focused text field. */

import { describe, expect, it } from 'vitest';
import { changeWaitAfter, SETTLE_AFTER } from '@e2e-dev/core/lib/agent/settle-policy';

const full = SETTLE_AFTER.tap.changeWaitMs;
const caretWait = 1000;
const textbox = { role: 'textbox', states: { focused: true } };
const unfocused = { role: 'textbox' };

describe('changeWaitAfter', () => {
it('keeps the table window for an action that can change the screen', () => {
expect(changeWaitAfter({ name: 'tap', node: { role: 'button' } })).toBe(full);
expect(changeWaitAfter({ name: 'press', node: textbox, key: 'Enter' })).toBe(full);
expect(changeWaitAfter({ name: 'press', node: { role: 'combobox' }, key: 'ArrowRight' })).toBe(full);
expect(changeWaitAfter({ name: 'secondaryTap', node: textbox })).toBe(full);
expect(changeWaitAfter({ name: 'type', node: textbox })).toBe(full);
});

it('keeps the window for a tap, even on the focused field, since a click can open a panel', () => {
expect(changeWaitAfter({ name: 'tap', node: textbox })).toBe(full);
expect(changeWaitAfter({ name: 'doubleTap', node: { role: 'searchbox', states: { focused: true } } })).toBe(full);
expect(changeWaitAfter({ name: 'tap', node: unfocused })).toBe(full);
});

it('keeps the window for the first focus of a text field, which may open something', () => {
expect(changeWaitAfter({ name: 'press', node: unfocused, key: 'ArrowRight' })).toBe(full);
expect(changeWaitAfter({ name: 'pressKey', key: 'ArrowRight' }, unfocused)).toBe(full);
});

it.each(['ArrowLeft', 'ArrowRight', 'Home', 'End', 'Shift+ArrowRight', 'Control+Shift+ArrowLeft', 'Control+a', 'Meta+a'])(
'shortens the window for the caret key %s in a text field',
(key) => {
expect(changeWaitAfter({ name: 'press', node: textbox, key })).toBe(caretWait);
expect(changeWaitAfter({ name: 'pressKey', key }, textbox)).toBe(caretWait);
},
);

it.each(['ArrowUp', 'ArrowDown', 'Enter', 'Tab', 'Backspace', 'a', 'Alt+ArrowLeft', 'Shift+a', 'Control+Enter'])(
'keeps the window for %s',
(key) => {
expect(changeWaitAfter({ name: 'press', node: textbox, key })).toBe(full);
},
);

it('keeps the window for a caret key outside a text field, or on a focus nobody can name', () => {
expect(changeWaitAfter({ name: 'press', node: { role: 'tab', states: { focused: true } }, key: 'ArrowRight' })).toBe(full);
expect(changeWaitAfter({ name: 'press', node: { role: 'slider', states: { focused: true } }, key: 'Home' })).toBe(full);
expect(changeWaitAfter({ name: 'pressKey', key: 'ArrowRight' })).toBe(full);
expect(changeWaitAfter({ name: 'pressKey', key: 'ArrowRight' }, { role: 'listbox', states: { focused: true } })).toBe(full);
});

it('never lengthens a window and leaves an absent one absent', () => {
expect(changeWaitAfter({ name: 'scroll' })).toBe(SETTLE_AFTER.scroll.changeWaitMs);
expect(changeWaitAfter({ name: 'typeSecret', node: textbox })).toBeUndefined();
});
});
Loading