Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion scripts/__tests__/help-conformance-expectation-fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,8 +161,13 @@ export const EXPECTATION_FIXTURES: Record<string, ExpectationFixture> = {
id: 'noVerificationCommand',
commands: ['agent-device press @e5 --settle', 'agent-device close'],
},
// Bare find clicks; a later word "exists" in the query or a fill value is no exists action.
{ id: 'bareFindClicks', commands: ['agent-device find text "Sent"'] },
{ id: 'existsInFindQuery', commands: ['agent-device find text "Sent exists"'] },
{ id: 'findFillsExists', commands: ['agent-device find text Sent fill exists'] },
],
metamorphic: { commands: ['agent-device is visible label="Welcome back"'] },
// A single-word query with an apostrophe is canonicalized without quotes.
metamorphic: { commands: [`agent-device find text "Today's" exists`] },
Comment thread
thymikee marked this conversation as resolved.
Resolved
},
usesDogfoodEvidence: {
passing: {
Expand Down
26 changes: 26 additions & 0 deletions scripts/__tests__/help-conformance-find-verification.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import assert from 'node:assert/strict';
import { test } from 'vitest';
import { hasFindExists } from '../help-conformance-expectations.mjs';
import { validatePlanCommands } from '../help-conformance-plan-validator.mjs';

// The find verification check reads plan commands through the real plan tokenizer, so quoting is
// the tokenizer's job and the find grammar is parseFindArgs'.
async function verifies(command: string, subject?: string): Promise<boolean> {
return hasFindExists(await validatePlanCommands([command]), subject);
}

test('find ... exists verifies, whatever quotes its query holds', async () => {
assert.equal(await verifies(`agent-device find text "Today's" exists`), true);
assert.equal(await verifies(String.raw`agent-device find text "Welcome \"back\"" exists`), true);
assert.equal(await verifies('agent-device find "Welcome back" exists', 'welcome'), true);
});

test('a find that acts, or names another subject, does not verify', async () => {
assert.equal(await verifies('agent-device find text "Sent"'), false);
assert.equal(await verifies('agent-device find text Sent fill exists'), false);
assert.equal(await verifies('agent-device find text "Orders" exists', 'welcome'), false);
});

test('a find with an unsupported action scores false instead of throwing', async () => {
assert.equal(await verifies('agent-device find text Sent swipe'), false);
});
10 changes: 9 additions & 1 deletion scripts/help-conformance-case-checks.d.mts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,17 @@ import type { PlanCommandValidation } from './help-conformance-plan-validator.mj

export type CasePatternCheck = { id: string; pattern: RegExp };

/** The plan a check reads: its canonical text and each command's tokens and parse. */
export type CaseCheckContext = {
joined: string;
commandValidation: readonly PlanCommandValidation[];
};

export type CaseTestCheck = { id: string; test: (context: CaseCheckContext) => boolean };

export type CaseChecks = {
expectations?: readonly string[];
matchers?: readonly CasePatternCheck[];
matchers?: readonly (CasePatternCheck | CaseTestCheck)[];
forbidden?: readonly CasePatternCheck[];
};

Expand Down
13 changes: 9 additions & 4 deletions scripts/help-conformance-case-checks.mjs
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
import { opensAndCloses, usesValidationPrep } from './help-conformance-expectations.mjs';
import {
hasFindExists,
opensAndCloses,
usesValidationPrep,
} from './help-conformance-expectations.mjs';

const EXPECTATION_SCORERS = {
validPlanCommands: ({ commands, commandValidation }) =>
Expand All @@ -17,7 +21,8 @@ const EXPECTATION_SCORERS = {
// that can tell whether the model actually chained.
usesConfidentChaining: ({ commands }) => commands.some((command) => /&&/.test(command)),
noWaitStable: ({ joined }) => !joined.includes('wait stable'),
verifiesNamedExpectation: ({ joined }) => /\b(wait|is|get|find)\b/.test(joined),
verifiesNamedExpectation: ({ joined, commandValidation }) =>
/\b(?:wait|is|get)\b/.test(joined) || hasFindExists(commandValidation),
usesDogfoodEvidence: ({ joined }) =>
/(?:\bscreenshot\b|\brecord\b|\blogs\b|\bnetwork\b|\bperf\b|\btrace\b|dogfood-output)/i.test(
joined,
Expand Down Expand Up @@ -59,9 +64,9 @@ function resolveChecks(testCase) {
id,
test: (context) => scoreExpectation(id, context),
}));
const matched = (testCase.matchers ?? []).map(({ id, pattern }) => ({
const matched = (testCase.matchers ?? []).map(({ id, pattern, test }) => ({
id,
test: (context) => pattern.test(context.joined),
test: test ?? ((context) => pattern.test(context.joined)),
}));
const forbidden = (testCase.forbidden ?? []).map(({ id, pattern }) => ({
id,
Expand Down
24 changes: 20 additions & 4 deletions scripts/help-conformance-cases.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
STALE_REF_SAMPLE,
sampleText,
} from './help-conformance-sample-outputs.mjs';
import { hasFindExists } from './help-conformance-expectations.mjs';

// Raw-coordinate fallback the quiz cases forbid: a click/fill/press targeting
// bare numbers instead of a ref or selector.
Expand Down Expand Up @@ -180,7 +181,11 @@ export const CASES = [
matchers: [
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
{ id: 'movesFocusWithRemote', pattern: /\btv-remote\s+press\s+right\b/i },
{ id: 'activatesWithSelect', pattern: /\btv-remote\s+press\s+select\b/i },
{ id: 'verifiesOutcome', pattern: /\b(?:is\s+focused|wait\b|find\b)/i },
{
id: 'verifiesOutcome',
test: ({ joined, commandValidation }) =>
/\b(?:is\s+focused|wait)\b/i.test(joined) || hasFindExists(commandValidation),
},
],
forbidden: [
// Focus-first surface: activation goes through tv-remote select, not a
Expand Down Expand Up @@ -239,7 +244,12 @@ export const CASES = [
},
{ id: 'verifiesBackendWithDoctor', pattern: /\bagent-device\s+web\s+doctor\b/i },
{ id: 'usesWebPlatform', pattern: /--platform\s+web\b/i },
{ id: 'verifiesWelcomeText', pattern: /\b(?:wait|is|find)\b[^\n]*welcome/i },
{
id: 'verifiesWelcomeText',
test: ({ joined, commandValidation }) =>
/\b(?:wait|is)\b[^\n]*welcome/i.test(joined) ||
hasFindExists(commandValidation, 'welcome'),
},
],
forbidden: [
// help web: native mobile/desktop setup commands are out of scope for
Expand Down Expand Up @@ -314,7 +324,12 @@ export const CASES = [
id: 'opensFreshAfterInstall',
pattern: /\binstall\b[\s\S]*\n[^\n]*\bopen\s+com\.example\.orders\b[^\n]*--relaunch\b/i,
},
{ id: 'verifiesFirstScreen', pattern: /\b(?:wait|find|is|get)\b[^\n]*orders/i },
{
id: 'verifiesFirstScreen',
test: ({ joined, commandValidation }) =>
/\b(?:wait|is|get)\b[^\n]*orders/i.test(joined) ||
hasFindExists(commandValidation, 'orders'),
},
],
forbidden: [
// help workflow: install for a first install; reinstall only when
Expand Down Expand Up @@ -372,7 +387,8 @@ export const CASES = [
},
{
id: 'verifiesLoginSucceeded',
pattern: /\b(?:wait|is|get|find)\b/i,
test: ({ joined, commandValidation }) =>
/\b(?:wait|is|get)\b/i.test(joined) || hasFindExists(commandValidation),
},
{ id: 'publishesWithoutClosing', pattern: /\bagent-device\s+session\s+save-script\b/i },
],
Expand Down
14 changes: 14 additions & 0 deletions scripts/help-conformance-command-validator.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
import { pathToFileURL } from 'node:url';
import { parseArgs } from '../src/cli/parser/args.ts';
import { isCommandName, readInputFromCli } from '@agent-device/command-surface';
import { parseFindArgs } from '@agent-device/selectors';

type ValidationKind = 'agent-device-grammar' | 'pseudo-ref';

type ParsedAgentCommand = {
command: string;
positionals: string[];
/** A `find` command's action and query, as `parseFindArgs` reads them; the scorers stay plain JS. */
find?: { action: string; query: string };
};

type ValidationResult = { valid: true } | { valid: false; kind: ValidationKind; error: string };
Expand Down Expand Up @@ -91,10 +94,21 @@ function validatePlanCommand(value: unknown): PlanValidationResult {
agentCommand: {
command: parsed.command,
positionals: [...parsed.positionals],
...(parsed.command === 'find' ? readFind(parsed.positionals) : {}),
},
};
}

/** An unsupported action is no find the plan can act on, so it carries no parsed find. */
function readFind(positionals: string[]): Pick<ParsedAgentCommand, 'find'> {
try {
const { action, query } = parseFindArgs(positionals);
return { find: { action, query } };
} catch {
return {};
}
}

function runCli(): void {
const encodedInput = process.argv[2];
if (!encodedInput) throw new Error('Expected a JSON argv array.');
Expand Down
7 changes: 7 additions & 0 deletions scripts/help-conformance-expectations.d.mts
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
// Type surface for the plain-Node expectation scorers (they must stay runnable by bare `node`).
// Keep it in step with the exports of `./help-conformance-expectations.mjs`.
import type { PlanCommandValidation } from './help-conformance-plan-validator.mjs';

export type ExpectationContext = Readonly<{ commands: readonly string[] }>;

export function opensAndCloses(context: ExpectationContext): boolean;

export function usesValidationPrep(context: ExpectationContext): boolean;

export function hasFindExists(
commandValidation: readonly PlanCommandValidation[],
subject?: string,
): boolean;
15 changes: 15 additions & 0 deletions scripts/help-conformance-expectations.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,18 @@ function isSupportedBuildCommand(command) {
function isPnpmScript(command, script) {
return new RegExp(`^pnpm\\s+(?:run\\s+)?${script}(?:\\s|$)`).test(command ?? '');
}

/**
* Whether the plan holds a read-only `find`: one whose action is `exists` (bare find clicks).
* The command validator parses each find with `parseFindArgs`; `subject`, when given, must appear
* in the query. A find that failed validation carries no parse and does not verify.
*/
export function hasFindExists(commandValidation, subject) {
return commandValidation.some(({ agentCommand }) => {
const find = agentCommand?.find;
return (
find?.action === 'exists' &&
(subject === undefined || find.query.toLowerCase().includes(subject))
);
});
}
7 changes: 6 additions & 1 deletion scripts/help-conformance-plan-validator.d.mts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,12 @@
// `./help-conformance-plan-validator.mjs`; keep it in step with the exports there.
export type PlanCommandIssue = { kind: string; error: string };

export type PlanAgentCommand = { command: string; positionals: string[] };
export type PlanAgentCommand = {
command: string;
positionals: string[];
/** A `find` command's action and query, as `parseFindArgs` reads them. */
find?: { action: string; query: string };
};

export type PlanCommandValidation = {
command: string;
Expand Down
2 changes: 1 addition & 1 deletion skills/agent-device/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ agent-device open <app> --foreground

That starts the session and returns the initial interactive snapshot with `@refs`.

Loop: act with `press|click|fill|longpress <target> ... --settle`, `scroll <direction> --settle`, or `back --settle`; continue from the printed diff, verify the named expectation (`wait text "..."`, `is`, `get`, or `find`), then run `agent-device close`, or `agent-device close --shutdown` if your session booted a local simulator or emulator, unless the user asks to keep it running.
Loop: act with `press|click|fill|longpress <target> ... --settle`, `scroll <direction> --settle`, or `back --settle`; continue from the printed diff, verify the named expectation with a read-only command (`wait text "..."`, `is`, or `get`; bare `find` clicks, so check with `find text "..." exists`), then run `agent-device close`, or `agent-device close --shutdown` if your session booted a local simulator or emulator, unless the user asks to keep it running.

Reaching an off-screen target is one command, not a scroll-and-check loop: `scroll down --until <selector>` scrolls until that element is on screen, and `scroll bottom` runs to the end of the content. Repeated bare `scroll down` calls are the slow way to find something.

Expand Down
5 changes: 3 additions & 2 deletions src/commands/schema/cli-help-first-screen.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ const FIRST_SCREEN_RULES: Record<string, readonly string[]> = {
'agent-device open <app> --foreground',
'Do not probe first with devices, apps, appstate, snapshot, or screenshot',
'Unknown app id: devices, then apps, then open <discovered-id>. Never invent ids',
'is, get, or find. A bare screenshot is not verification',
'is, or get. A bare screenshot is not verification',
'find acts: it clicks by default. To check only, use find text "..." exists',
'End with: agent-device close',
],
commands: [
Expand All @@ -27,7 +28,7 @@ const FIRST_SCREEN_RULES: Record<string, readonly string[]> = {
'Stop and report ambiguity',
'1. Open the requested app; use --relaunch only when the script needs fresh state',
'5. If --settle prints not settled, follow its hint',
'6. Verify named expectations with wait text/selector/absent, get, is, find, or the settled diff. A bare screenshot/snapshot is not verification',
'6. Verify named expectations with wait text/selector/absent, get, is, find text "..." exists, or the settled diff. Bare find clicks. A bare screenshot/snapshot is not verification',
],
maestro: [
'Run Maestro compatibility flows with replay <flow.yaml> --maestro',
Expand Down
3 changes: 2 additions & 1 deletion src/commands/schema/cli-help-overview.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ Loop:
acts, waits for quiet, and prints the UI diff. Continue from that diff.
Run snapshot -i only when the diff lacks the next target or did not settle.
Verify a named expectation with the diff, wait text "...", wait <selector>, wait absent <selector>,
is, get, or find. A bare screenshot is not verification.
is, or get. A bare screenshot is not verification.
find acts: it clicks by default. To check only, use find text "..." exists.
End with: agent-device close

Targets:
Expand Down
2 changes: 1 addition & 1 deletion src/commands/schema/cli-help-react-native.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ React Native dev loop:
For "start from screen X" flows, prefer open --relaunch before the first snapshot so the app does not reuse a prior in-progress navigation state.
JS-only change with Metro or Re.Pack connected:
agent-device metro reload
agent-device find "Home"
agent-device wait text "Home"
Do not use agent-device reload. Use open --relaunch for native startup reset.
Android RN/Expo/Re.Pack dev server: direct Android localhost URL opens with a port auto-configure host reachability. For app/package launches, run metro prepare when the app cannot reach the local dev server.
Verify Metro/Re.Pack from the same host context that owns the dev server. If a sandboxed shell cannot curl localhost:8081/status but an unrestricted host shell can, the dev server is running and the sandbox probe is not authoritative.
Expand Down
2 changes: 1 addition & 1 deletion src/commands/schema/cli-help-topics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ test('usageForCommand resolves workflow help topic', async () => {
);
assert.match(help, /open -> snapshot -i -> settle -> verify -> close loop/);
assert.match(help, /type never takes --settle/);
assert.match(help, /snapshot\/get\/is\/find answer read-only questions/);
assert.match(help, /snapshot\/get\/is answer read-only questions/);
assert.match(help, /--settle confirms local UI quieted/);
assert.match(help, /wait text "Expected result" or wait <selector> instead of polling/);
assert.match(help, /strict disappearance uses wait absent <selector>/);
Expand Down
4 changes: 2 additions & 2 deletions src/commands/schema/cli-help-workflows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ Loop:
3. Run press/fill/click/longpress <ref-or-selector> --settle for each mutating step.
4. Treat a settled:true diff as the next observation. Do not add wait stable or another snapshot when the diff already shows the next target or expected result.
5. If --settle prints not settled, follow its hint before the next ref-based action.
6. Verify named expectations with wait text/selector/absent, get, is, find, or the settled diff. A bare screenshot/snapshot is not verification for a named expectation.
6. Verify named expectations with wait text/selector/absent, get, is, find text "..." exists, or the settled diff. Bare find clicks. A bare screenshot/snapshot is not verification for a named expectation.
7. Close the session when the script ends. If your session booted a local simulator or emulator, use close --shutdown, unless the user asks to keep it running.

Command shapes:
Expand Down Expand Up @@ -283,7 +283,7 @@ Loop:
1. Build or prepare the changed surface with the repo command that owns it.
2. Open the target app/device state explicitly.
3. Use snapshot -i and press/fill/click/longpress --settle for UI-driving steps.
4. Use the settled diff as evidence when it shows the changed behavior; otherwise verify with wait/get/is/find, screenshot, logs, network, perf, or trace based on the claim.
4. Use the settled diff as evidence when it shows the changed behavior; otherwise verify with wait/get/is, find text "..." exists, screenshot, logs, network, perf, or trace based on the claim.
5. Record timings, token/output size, screenshots/videos, logs, or perf artifacts only when they answer the validation question.
6. Close sessions and release leases before finishing. If your session booted a local simulator or emulator, use close --shutdown, unless the user asks to keep it running.

Expand Down
6 changes: 3 additions & 3 deletions src/commands/schema/cli-help.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,20 +114,20 @@ Session ordering:

Read-only and waits:
${WAIT_FAILURE_CONTRACT}
snapshot/get/is/find answer read-only questions; snapshot -i is for refs. --settle confirms local UI quieted; delayed results use wait text "Expected result" or wait <selector> instead of polling; strict disappearance uses wait absent <selector>.
snapshot/get/is answer read-only questions; snapshot -i is for refs. --settle confirms local UI quieted; delayed results use wait text "Expected result" or wait <selector> instead of polling; strict disappearance uses wait absent <selector>.
wait stable [quietMs] [timeoutMs] (defaults 500/10000) is the fallback for open/relaunch/navigation, or an intentionally-unsettled mutation -- not after a --settle whose diff already shows the change. Ambiguous find: add --first or --last.

Navigation:
Pick a coordinate gesture point near the target's center, away from edges/tab bars/nav bars/the home indicator (they trigger system navigation instead); macOS context menus are secondary clicks (help macos). Action sheets/menus/camera screens are normal UI: snapshot -i, press by label/ref, handle permission sheets via UI/alert. If back is ambiguous, prefer a nav/back ref, tab-bar ref, or deep link over repeating it.

Validation and evidence:
Nearby mutation diff: diff snapshot -i; with no prior snapshot it initializes the baseline (zero changes) instead of failing.
Named expectations need the exact text/selector via wait/is/get/find -- a bare screenshot/snapshot is not verification. Before declaring a task done, confirm the requested end state is actually visible on the current screen, scrolling it into view if needed; get text alone, or stopping one screen early, is not enough.
Named expectations need the exact text/selector via wait/is/get -- a bare screenshot/snapshot is not verification. Before declaring a task done, confirm the requested end state is actually visible on the current screen, scrolling it into view if needed; get text alone, or stopping one screen early, is not enough.
When an action only reveals or reaches a target, verify the exact target named, not just the action. Prefer testIDs/ids/selectors over visible text. Icon/tappable proof: screenshot --overlay-refs; if snapshot is sparse/AX-unavailable, use plain screenshot and coordinates, then retry snapshot -i on another screen.
iOS merged: child ref => press it; else press parent @ref --settle. Names are not selectors.
Perf/memory/log/network/trace/crash: help debugging. Recording, save-script, batch, replay repair: help scripting.

React Native: help react-native for Metro/Re.Pack reload, DevTools, RN overlays. JS-only change: metro reload, find "Home"; open --relaunch for native reset.
React Native: help react-native for Metro/Re.Pack reload, DevTools, RN overlays. JS-only change: metro reload, wait text "Home"; open --relaunch for native reset.

Lifecycle facts (trust these instead of probing): open without --relaunch is idempotent-foreground except native macOS app opens (background); --relaunch restarts it. close keeps a healthy iOS runner warm by default; runners/daemons self-idle after 5 minutes; stale leases self-reclaim. open device claims reclaim other worktrees' warm runners; "already owned by another agent-device daemon" = owner outside claim arbitration. Env vars: help physical-device.

Expand Down
Loading
Loading