Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/fresh-mobile-inputs.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@e2e-dev/mobile': patch
---

Re-read mobile fill and clear targets before typing, including cached replay. Missing inputs, changed security state, or incomplete captures retry after a fresh observation.
5 changes: 5 additions & 0 deletions docs/reference/mobile.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,11 @@ mobile({ platform: 'ios', transition: 800 }); // default: 500
budget the way a tap does. A control that arrives with the screen they reveal
waits out the budget before the next action lands on it.

`fill` and `clear` re-read the field before typing. Missing fields, changed
security state, or incomplete captures return `NODE_STALE` before mutation,
then retry after a fresh observation. Non-input targets return
`UNSUPPORTED_CAPABILITY`.

#### `videoTouches`

```ts
Expand Down
103 changes: 103 additions & 0 deletions packages/e2e/tests/integration/agent-input-stale.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
/** A stale fill must re-enter secret authorization, on live actions and cached replay alike. */

import { describe, expect, it } from 'vitest';
import type { StepExecutor } from '../../src/agent/executor.ts';
import { EngineError, type SemanticNode } from 'e2e/engine';
import { createFakeEngine } from '../helpers/fake-engine.ts';
import { nodeIdFor } from '../helpers/fake-loop-model.ts';
import { createProject, resultByTitle, runExisting } from '../helpers/run-project.ts';

const SUITE = `import { test, credentials, expect } from 'e2e';
test('fills the account password', async ({ agent, screen }) => {
await agent.act('fill the password', { params: { password: credentials.user('member').password } });
await expect(screen.getByRole('status')).toBeVisible();
});
`;

describe('secret fill stale retry', () => {
for (const replay of [false, true]) {
it.each([false, true])(`reauthorizes a changed sink with disabled=%s during ${replay ? 'replay' : 'live execution'}`, async (disabled) => {
const field: SemanticNode = {
ref: { id: 'password', revision: '' }, role: 'textbox', name: 'Password',
testId: 'password', inputPurpose: 'password', states: { secure: true },
rect: { x: 40, y: 237, width: 322, height: 49 },
};
const tree = { ref: { id: 'root', revision: '' }, role: 'document', children: [field] };
let stale = false;
let dispatched = 0;
const origins: string[] = [];
const pixels: boolean[] = [];
const fake = createFakeEngine({
tree,
onStartAttempt: () => { tree.children = [field]; },
locate: () => tree.children.filter((node) => node.role === 'status'),
perform: (_ref, action, operation) => {
if (action.kind !== 'fill') throw new Error('expected fill');
origins.push(operation.origin);
if (stale) {
stale = false;
tree.children = [{ ...field, states: { secure: true, disabled }, rect: { x: 40, y: 126, width: 322, height: 49 } }];
throw new EngineError('NODE_STALE', 'input moved before mutation', { retryable: true });
}
dispatched += 1;
tree.children.push({ ref: { id: 'saved', revision: '' }, role: 'status', name: 'Saved' });
},
// No pixel payload is needed: this records any request after authorization.
pixels: () => { pixels.push(true); return undefined; },
});
const taint: boolean[] = [];
const executor: StepExecutor = {
name: 'stale-secret',
async runStep(context) {
const observation = await context.observe();
try {
await context.actions.typeSecret({ id: nodeIdFor(observation.text, /textbox "Password"/) }, 'member.password');
} finally {
taint.push(context.pixelsTainted);
expect((await context.observe({ pixels: true })).pixels).toBeUndefined();
}
return { status: 'passed', summary: 'filled' };
},
};
const project = createProject({ 'tests/input.e2e.ts': SUITE });
const options = {
config: {
targets: [{ name: 'fake', platform: 'fake', engine: fake.engine }],
agents: { default: { executor } },
credentials: { member: { username: 'example', password: 'private-fixture-value' } },
cache: 'read-write' as const,
},
};
try {
if (replay) {
const recorded = await runExisting(project, options);
const recording = resultByTitle(recorded, 'fills the account password');
expect(recording.status).toBe('passed');
expect(recording.attempts[0]?.steps.find((entry) => entry.api === 'agent.act')?.cache).toMatchObject({ write: 'saved' });
dispatched = 0;
origins.length = 0;
pixels.length = 0;
taint.length = 0;
}
stale = true;
const outcome = await runExisting(project, options);
const result = resultByTitle(outcome, 'fills the account password');
const step = result.attempts[0]!.steps.find((entry) => entry.api === 'agent.act')!;
expect(result.status, JSON.stringify(result.attempts[0]?.error)).toBe(disabled ? 'failed' : 'passed');
expect(dispatched).toBe(disabled ? 0 : 1);
expect(origins).toEqual(Array(disabled ? 1 : 2).fill(replay ? 'test' : 'agent'));
// A denied replay hands off to the executor, whose own denied action also counts.
expect(step.metrics?.actionSteps).toBe(replay && disabled ? 2 : 1);
expect(step.events.filter((event) => event.kind === 'observation' && event.name === 'relocate')).toHaveLength(1);
expect(step.events.filter((event) => event.kind === 'policy' && event.name === 'secret.fill' && event.status === 'passed')).toHaveLength(disabled ? 1 : 2);
if (disabled) expect(JSON.stringify(step)).toContain('POLICY_DENIED');
if (!replay || disabled) expect(taint).toEqual([true]);
else expect(step.cache).toMatchObject({ mode: 'self-finalized', replayedActions: 1 });
expect(pixels).toEqual([]);
expect(JSON.stringify(outcome)).not.toContain('private-fixture-value');
} finally {
project.cleanup();
}
}, 60_000);
}
});
3 changes: 2 additions & 1 deletion packages/mobile/src/nodes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,8 @@ const ROLE_MAP: Readonly<Record<string, string>> = {
};

const SECURE_KINDS = new Set(['secure-text-field', 'securetextfield', 'password-field']);
const TEXT_INPUT_ROLES = new Set(['textbox', 'searchbox', 'combobox']);
/** Semantic roles that accept text input. */
export const TEXT_INPUT_ROLES: ReadonlySet<string> = new Set(['textbox', 'searchbox', 'combobox']);
const CHECKABLE_ROLES = new Set(['switch', 'checkbox']);
const CHECKED_VALUES = new Set(['1', 'on', 'true', 'checked', 'selected']);
const UNCHECKED_VALUES = new Set(['0', 'off', 'false', 'unchecked']);
Expand Down
56 changes: 36 additions & 20 deletions packages/mobile/src/surface.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ import {
normalizeKind,
projectSnapshot,
ROOT_ID,
TEXT_INPUT_ROLES,
screenRoot,
screenTitle,
type ProjectedNode,
Expand Down Expand Up @@ -1175,41 +1176,46 @@ export class AgentDeviceSurface {
}

/**
* Lets a control that came with the last action finish arriving before a
* test acts on it, and answers the binding to act on. A control already
* present at the same place in the snapshot the last action was resolved
* from is acted on at once only when keyboard state also stayed the same.
* A keyboard change can precede the app's relayout, so anything else waits
* out the remainder of the transition budget since that action and is then
* found again in a fresh snapshot. The budget is the only signal that the
* transition is over, since the tree cannot tell a sliding control from a
* landed one; the fresh snapshot is where the control landed. Android
* reports a window's frames while it slides in, so a ref resolved
* mid-flight would send the action where the control was: below the
* screen, for a modal rising from the bottom.
* Waits for a control that arrived or moved with the last action, returning
* whether it waited. Controls already present at the same place need no wait,
* unless keyboard state changed: a keyboard change can precede the app's relayout.
*/
private async settled(entry: NodeBinding, operation: OperationContext): Promise<NodeBinding> {
private async waitForTransition(entry: NodeBinding, operation: OperationContext): Promise<boolean> {
const since = Date.now() - this.lastActionAt;
const remaining = this.transitionMs - since;
if (remaining <= 0) return entry;
if (remaining <= 0) return false;
const before = this.screenBeforeAction;
if (before !== undefined && before.keyboardVisible === entry.keyboardVisible) {
const prior = this.refind(entry, before.index);
if (prior !== undefined && sameRect(prior.node.rect, entry.node.rect)) return entry;
if (prior !== undefined && sameRect(prior.node.rect, entry.node.rect)) return false;
}
await sleep(remaining, operation.signal);
// The sleep resolves on abort; the action behind it must not go out once
// the caller has already been told the operation was cancelled.
if (operation.signal.aborted) {
throw new EngineError('CANCELLED', 'transition wait cancelled', { retryable: false });
}
return this.relocated(entry, operation);
return true;
}

/** The same control in a fresh snapshot; the binding as it was when the snapshot no longer lists it. */
private async relocated(entry: NodeBinding, operation: OperationContext): Promise<NodeBinding> {
const projected = this.project(await this.snapshotOrEmpty(operation, false));
/** Refreshes a control after a transition wait, keeping the binding when no wait was needed. */
private async settled(entry: NodeBinding, operation: OperationContext): Promise<NodeBinding> {
if (await this.waitForTransition(entry, operation)) return this.relocated(entry, operation);
return entry;
}

/** The same control in a fresh snapshot; input mutations refuse a missing target or changed security facts. */
private async relocated(entry: NodeBinding, operation: OperationContext, input = false): Promise<NodeBinding> {
const raw = await this.snapshotOrEmpty(operation, false);
const projected = this.project(raw);
const found = this.refind(entry, projected.index);
Comment thread
bheemreddy-samsara marked this conversation as resolved.
Resolved
if (input && (isTruncated(raw) || found === undefined ||
entry.node.inputPurpose !== found.node.inputPurpose ||
entry.node.states?.disabled !== found.node.states?.disabled ||
entry.node.states?.secure !== found.node.states?.secure ||
entry.node.states?.hidden !== found.node.states?.hidden)) {
throw new EngineError('NODE_STALE', `input node ${entry.id} changed before mutation; observe and authorize it again`, { retryable: true });
Comment thread
bheemreddy-samsara marked this conversation as resolved.
Resolved
}
return found === undefined ? entry : this.bind(found, projected);
}

Expand Down Expand Up @@ -1246,6 +1252,10 @@ export class AgentDeviceSurface {
return;
}
const entry = this.resolveRef(ref);
const filling = action.kind === 'fill' || action.kind === 'clear';
if (filling && (entry.node.role === undefined || !TEXT_INPUT_ROLES.has(entry.node.role))) {
throw unsupported(`agent-device can only ${action.kind} editable fields; node ${entry.id} is ${entry.node.role ?? 'unknown'}`);
}
const label = `perform ${action.kind}`;
const client = this.requireClient();
// A toggle already in the wanted state sends nothing, so it neither waits
Expand All @@ -1255,7 +1265,13 @@ export class AgentDeviceSurface {
}
const before = this.latestScreen;
const run = async (): Promise<unknown> => {
const target = deterministic ? await this.settled(entry, operation) : entry;
let target = entry;
if (filling) {
if (deterministic) await this.waitForTransition(entry, operation);
target = await this.relocated(entry, operation, true);
} else if (deterministic) {
target = await this.settled(entry, operation);
}
switch (action.kind) {
case 'tap':
return input.send(() => client.interactions.press({ ...this.actionTarget(target, true), ...settle }));
Expand Down
Loading
Loading