From 69e1724fc794fa10621bc75b840983438aa1a159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 18:30:07 +0100 Subject: [PATCH 1/7] docs(help): point verification at read-only commands, not bare find Bare find clicks by default, so listing it as a verification step made agents tap while checking. Root help, help manual-qa, the MCP guide and the skill card now name wait, is and get, and say find acts unless given exists. The help conformance scorers no longer count bare find as verification. Fixes #4485 Co-Authored-By: Claude Opus 5.5 --- scripts/help-conformance-case-checks.mjs | 4 +++- scripts/help-conformance-cases.mjs | 12 +++++++++--- skills/agent-device/SKILL.md | 2 +- src/commands/schema/cli-help-first-screen.test.ts | 5 +++-- src/commands/schema/cli-help-overview.ts | 3 ++- src/commands/schema/cli-help-workflows.ts | 2 +- src/mcp/server-guide.ts | 2 +- 7 files changed, 20 insertions(+), 10 deletions(-) diff --git a/scripts/help-conformance-case-checks.mjs b/scripts/help-conformance-case-checks.mjs index 4a8f998638..1f12e6e4e9 100644 --- a/scripts/help-conformance-case-checks.mjs +++ b/scripts/help-conformance-case-checks.mjs @@ -17,7 +17,9 @@ const EXPECTATION_SCORERS = { // that can tell whether the model actually chained. usesConfidentChaining: ({ commands }) => commands.some((command) => /&&/.test(command)), noWaitStable: ({ joined }) => !joined.includes('wait stable'), - verifiesNamedExpectation: ({ joined }) => /\b(wait|is|get|find)\b/.test(joined), + // Bare find clicks, so only find ... exists verifies. + verifiesNamedExpectation: ({ joined }) => + /\b(wait|is|get)\b|\bfind\b[^\n]*\bexists\b/.test(joined), usesDogfoodEvidence: ({ joined }) => /(?:\bscreenshot\b|\brecord\b|\blogs\b|\bnetwork\b|\bperf\b|\btrace\b|dogfood-output)/i.test( joined, diff --git a/scripts/help-conformance-cases.mjs b/scripts/help-conformance-cases.mjs index f9f7d4e7fa..c7449ae0c6 100644 --- a/scripts/help-conformance-cases.mjs +++ b/scripts/help-conformance-cases.mjs @@ -180,7 +180,7 @@ export const CASES = [ matchers: [ { id: 'movesFocusWithRemote', pattern: /\btv-remote\s+press\s+right\b/i }, { id: 'activatesWithSelect', pattern: /\btv-remote\s+press\s+select\b/i }, - { id: 'verifiesOutcome', pattern: /\b(?:is\s+focused|wait\b|find\b)/i }, + { id: 'verifiesOutcome', pattern: /\b(?:is\s+focused|wait\b|find\b[^\n]*\bexists\b)/i }, ], forbidden: [ // Focus-first surface: activation goes through tv-remote select, not a @@ -239,7 +239,10 @@ export const CASES = [ }, { id: 'verifiesBackendWithDoctor', pattern: /\bagent-device\s+web\s+doctor\b/i }, { id: 'usesWebPlatform', pattern: /--platform\s+web\b/i }, - { id: 'verifiesWelcomeText', pattern: /\b(?:wait|is|find)\b[^\n]*welcome/i }, + { + id: 'verifiesWelcomeText', + pattern: /\b(?:wait|is)\b[^\n]*welcome|\bfind\b[^\n]*welcome[^\n]*\bexists\b/i, + }, ], forbidden: [ // help web: native mobile/desktop setup commands are out of scope for @@ -314,7 +317,10 @@ export const CASES = [ id: 'opensFreshAfterInstall', pattern: /\binstall\b[\s\S]*\n[^\n]*\bopen\s+com\.example\.orders\b[^\n]*--relaunch\b/i, }, - { id: 'verifiesFirstScreen', pattern: /\b(?:wait|find|is|get)\b[^\n]*orders/i }, + { + id: 'verifiesFirstScreen', + pattern: /\b(?:wait|is|get)\b[^\n]*orders|\bfind\b[^\n]*orders[^\n]*\bexists\b/i, + }, ], forbidden: [ // help workflow: install for a first install; reinstall only when diff --git a/skills/agent-device/SKILL.md b/skills/agent-device/SKILL.md index 5da005d56f..f4fabd84d1 100644 --- a/skills/agent-device/SKILL.md +++ b/skills/agent-device/SKILL.md @@ -13,7 +13,7 @@ agent-device open --foreground That starts the session and returns the initial interactive snapshot with `@refs`. -Loop: act with `press|click|fill|longpress ... --settle`, `scroll --settle`, or `back --settle`; continue from the printed diff, verify the named expectation (`wait text "..."`, `is`, `get`, or `find`), then run `agent-device close`, or `agent-device close --shutdown` if your session booted a local simulator or emulator, unless the user asks to keep it running. +Loop: act with `press|click|fill|longpress ... --settle`, `scroll --settle`, or `back --settle`; continue from the printed diff, verify the named expectation with a read-only command (`wait text "..."`, `is`, or `get`; bare `find` clicks, so check with `find exists`), then run `agent-device close`, or `agent-device close --shutdown` if your session booted a local simulator or emulator, unless the user asks to keep it running. Reaching an off-screen target is one command, not a scroll-and-check loop: `scroll down --until ` scrolls until that element is on screen, and `scroll bottom` runs to the end of the content. Repeated bare `scroll down` calls are the slow way to find something. diff --git a/src/commands/schema/cli-help-first-screen.test.ts b/src/commands/schema/cli-help-first-screen.test.ts index 93747dc00f..9298f3d181 100644 --- a/src/commands/schema/cli-help-first-screen.test.ts +++ b/src/commands/schema/cli-help-first-screen.test.ts @@ -15,7 +15,8 @@ const FIRST_SCREEN_RULES: Record = { 'agent-device open --foreground', 'Do not probe first with devices, apps, appstate, snapshot, or screenshot', 'Unknown app id: devices, then apps, then open . Never invent ids', - 'is, get, or find. A bare screenshot is not verification', + 'is, or get. A bare screenshot is not verification', + 'find acts: it clicks by default. To check only, use find exists', 'End with: agent-device close', ], commands: [ @@ -27,7 +28,7 @@ const FIRST_SCREEN_RULES: Record = { 'Stop and report ambiguity', '1. Open the requested app; use --relaunch only when the script needs fresh state', '5. If --settle prints not settled, follow its hint', - '6. Verify named expectations with wait text/selector/absent, get, is, find, or the settled diff. A bare screenshot/snapshot is not verification', + '6. Verify named expectations with wait text/selector/absent, get, is, find exists, or the settled diff. Bare find clicks. A bare screenshot/snapshot is not verification', ], maestro: [ 'Run Maestro compatibility flows with replay --maestro', diff --git a/src/commands/schema/cli-help-overview.ts b/src/commands/schema/cli-help-overview.ts index f9bd11775a..59c43149a7 100644 --- a/src/commands/schema/cli-help-overview.ts +++ b/src/commands/schema/cli-help-overview.ts @@ -26,7 +26,8 @@ Loop: acts, waits for quiet, and prints the UI diff. Continue from that diff. Run snapshot -i only when the diff lacks the next target or did not settle. Verify a named expectation with the diff, wait text "...", wait , wait absent , - is, get, or find. A bare screenshot is not verification. + is, or get. A bare screenshot is not verification. + find acts: it clicks by default. To check only, use find exists. End with: agent-device close Targets: diff --git a/src/commands/schema/cli-help-workflows.ts b/src/commands/schema/cli-help-workflows.ts index 83bbd29b48..8206672402 100644 --- a/src/commands/schema/cli-help-workflows.ts +++ b/src/commands/schema/cli-help-workflows.ts @@ -28,7 +28,7 @@ Loop: 3. Run press/fill/click/longpress --settle for each mutating step. 4. Treat a settled:true diff as the next observation. Do not add wait stable or another snapshot when the diff already shows the next target or expected result. 5. If --settle prints not settled, follow its hint before the next ref-based action. - 6. Verify named expectations with wait text/selector/absent, get, is, find, or the settled diff. A bare screenshot/snapshot is not verification for a named expectation. + 6. Verify named expectations with wait text/selector/absent, get, is, find exists, or the settled diff. Bare find clicks. A bare screenshot/snapshot is not verification for a named expectation. 7. Close the session when the script ends. If your session booted a local simulator or emulator, use close --shutdown, unless the user asks to keep it running. Command shapes: diff --git a/src/mcp/server-guide.ts b/src/mcp/server-guide.ts index ae509802cc..cadd634d5c 100644 --- a/src/mcp/server-guide.ts +++ b/src/mcp/server-guide.ts @@ -19,7 +19,7 @@ export const MCP_SERVER_INSTRUCTIONS = `agent-device drives iOS, Android, tvOS, Start: known app -> call open {app, foreground: true} at once; do not probe with devices, apps, appstate, snapshot, or screenshot first. open returns the initial interactive snapshot with @refs. Unknown app id: devices, then apps, then open the discovered id; never invent ids. Existing session: continue from its state, do not reopen. -Loop: press/click/fill/longpress/hover/scroll/back with settle: true; the response is the settled UI diff, continue from it. To reach an off-screen target, scroll {direction, until: ""} scrolls until it is on screen in one call, and direction "bottom" runs to the end of the content; repeated bare scrolls are the slow way to find something. snapshot {interactiveOnly: true} only when the diff lacks the next target or did not settle. Verify with wait {kind: "text", text}, wait {selector}, wait {absent: selector}, is, get, or find; a bare screenshot is not verification. End with close. +Loop: press/click/fill/longpress/hover/scroll/back with settle: true; the response is the settled UI diff, continue from it. To reach an off-screen target, scroll {direction, until: ""} scrolls until it is on screen in one call, and direction "bottom" runs to the end of the content; repeated bare scrolls are the slow way to find something. snapshot {interactiveOnly: true} only when the diff lacks the next target or did not settle. Verify with wait {kind: "text", text}, wait {selector}, wait {absent: selector}, is, or get; a bare screenshot is not verification. find acts and clicks by default; to check only, pass action: "exists". End with close. Targets are top-level arguments: press {ref: "@e12"}, click {selector: "id=submit"}, press {x, y}. Copy refs byte-for-byte (@e12, @e12~s4; keep @ and any ~sN). Refs go stale after mutations. Prefer refs, then id/label/role selectors; coordinates last. On a sparse/AX-unavailable warning its refs and selectors are invalid: screenshot, read the image, press {x, y}, then snapshot the changed screen. From 4c7a756467b24957f7e1b35065315f26f9724f17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 18:48:54 +0100 Subject: [PATCH 2/7] docs(help): drop bare find from every verification line Address review: help workflow, react-native and validate still offered bare find as read-only, and the conformance matchers accepted any later word "exists". One shared matcher now requires exists as the find action, with scorer counterexamples for bare and mutating find. Co-Authored-By: Claude Opus 5.5 --- .../help-conformance-expectation-fixtures.ts | 6 +++++- scripts/help-conformance-case-checks.mjs | 12 ++++++++---- scripts/help-conformance-cases.mjs | 18 ++++++++++++++---- scripts/help-conformance-expectations.mjs | 9 +++++++++ src/commands/schema/cli-help-react-native.ts | 2 +- src/commands/schema/cli-help-topics.test.ts | 2 +- src/commands/schema/cli-help-workflows.ts | 2 +- src/commands/schema/cli-help.ts | 6 +++--- 8 files changed, 42 insertions(+), 15 deletions(-) diff --git a/scripts/__tests__/help-conformance-expectation-fixtures.ts b/scripts/__tests__/help-conformance-expectation-fixtures.ts index ea84c6b799..d1e09a8950 100644 --- a/scripts/__tests__/help-conformance-expectation-fixtures.ts +++ b/scripts/__tests__/help-conformance-expectation-fixtures.ts @@ -161,8 +161,12 @@ export const EXPECTATION_FIXTURES: Record = { id: 'noVerificationCommand', commands: ['agent-device press @e5 --settle', 'agent-device close'], }, + // Bare find clicks; a later word "exists" in the query or a fill value is no exists action. + { id: 'bareFindClicks', commands: ['agent-device find text "Sent"'] }, + { id: 'existsInFindQuery', commands: ['agent-device find text "Sent exists"'] }, + { id: 'findFillsExists', commands: ['agent-device find text Sent fill exists'] }, ], - metamorphic: { commands: ['agent-device is visible label="Welcome back"'] }, + metamorphic: { commands: ['agent-device find text "Welcome back" exists'] }, }, usesDogfoodEvidence: { passing: { diff --git a/scripts/help-conformance-case-checks.mjs b/scripts/help-conformance-case-checks.mjs index 1f12e6e4e9..f196bdd795 100644 --- a/scripts/help-conformance-case-checks.mjs +++ b/scripts/help-conformance-case-checks.mjs @@ -1,4 +1,10 @@ -import { opensAndCloses, usesValidationPrep } from './help-conformance-expectations.mjs'; +import { + findExistsSource, + opensAndCloses, + usesValidationPrep, +} from './help-conformance-expectations.mjs'; + +const VERIFICATION = new RegExp(String.raw`\b(?:wait|is|get)\b|${findExistsSource()}`); const EXPECTATION_SCORERS = { validPlanCommands: ({ commands, commandValidation }) => @@ -17,9 +23,7 @@ const EXPECTATION_SCORERS = { // that can tell whether the model actually chained. usesConfidentChaining: ({ commands }) => commands.some((command) => /&&/.test(command)), noWaitStable: ({ joined }) => !joined.includes('wait stable'), - // Bare find clicks, so only find ... exists verifies. - verifiesNamedExpectation: ({ joined }) => - /\b(wait|is|get)\b|\bfind\b[^\n]*\bexists\b/.test(joined), + verifiesNamedExpectation: ({ joined }) => VERIFICATION.test(joined), usesDogfoodEvidence: ({ joined }) => /(?:\bscreenshot\b|\brecord\b|\blogs\b|\bnetwork\b|\bperf\b|\btrace\b|dogfood-output)/i.test( joined, diff --git a/scripts/help-conformance-cases.mjs b/scripts/help-conformance-cases.mjs index c7449ae0c6..ea15e5862e 100644 --- a/scripts/help-conformance-cases.mjs +++ b/scripts/help-conformance-cases.mjs @@ -14,6 +14,7 @@ import { STALE_REF_SAMPLE, sampleText, } from './help-conformance-sample-outputs.mjs'; +import { findExistsSource } from './help-conformance-expectations.mjs'; // Raw-coordinate fallback the quiz cases forbid: a click/fill/press targeting // bare numbers instead of a ref or selector. @@ -180,7 +181,10 @@ export const CASES = [ matchers: [ { id: 'movesFocusWithRemote', pattern: /\btv-remote\s+press\s+right\b/i }, { id: 'activatesWithSelect', pattern: /\btv-remote\s+press\s+select\b/i }, - { id: 'verifiesOutcome', pattern: /\b(?:is\s+focused|wait\b|find\b[^\n]*\bexists\b)/i }, + { + id: 'verifiesOutcome', + pattern: new RegExp(String.raw`\b(?:is\s+focused|wait)\b|${findExistsSource()}`, 'i'), + }, ], forbidden: [ // Focus-first surface: activation goes through tv-remote select, not a @@ -241,7 +245,10 @@ export const CASES = [ { id: 'usesWebPlatform', pattern: /--platform\s+web\b/i }, { id: 'verifiesWelcomeText', - pattern: /\b(?:wait|is)\b[^\n]*welcome|\bfind\b[^\n]*welcome[^\n]*\bexists\b/i, + pattern: new RegExp( + String.raw`\b(?:wait|is)\b[^\n]*welcome|${findExistsSource('welcome')}`, + 'i', + ), }, ], forbidden: [ @@ -319,7 +326,10 @@ export const CASES = [ }, { id: 'verifiesFirstScreen', - pattern: /\b(?:wait|is|get)\b[^\n]*orders|\bfind\b[^\n]*orders[^\n]*\bexists\b/i, + pattern: new RegExp( + String.raw`\b(?:wait|is|get)\b[^\n]*orders|${findExistsSource('orders')}`, + 'i', + ), }, ], forbidden: [ @@ -378,7 +388,7 @@ export const CASES = [ }, { id: 'verifiesLoginSucceeded', - pattern: /\b(?:wait|is|get|find)\b/i, + pattern: new RegExp(String.raw`\b(?:wait|is|get)\b|${findExistsSource()}`, 'i'), }, { id: 'publishesWithoutClosing', pattern: /\bagent-device\s+session\s+save-script\b/i }, ], diff --git a/scripts/help-conformance-expectations.mjs b/scripts/help-conformance-expectations.mjs index a682e150ad..d873effd78 100644 --- a/scripts/help-conformance-expectations.mjs +++ b/scripts/help-conformance-expectations.mjs @@ -19,3 +19,12 @@ function isSupportedBuildCommand(command) { function isPnpmScript(command, script) { return new RegExp(`^pnpm\\s+(?:run\\s+)?${script}(?:\\s|$)`).test(command ?? ''); } + +/** + * A find whose action is `exists`, the only read-only find: bare find clicks. `subject` must + * appear in the query, which the optional locator precedes and the action follows. + */ +export function findExistsSource(subject = '') { + const unquoted = subject ? String.raw`[^\s"']*${subject}[^\s"']*` : String.raw`[^\s"']+`; + return String.raw`\bfind\s+(?:(?:text|label|value|role|id)\s+)?(?:"[^"\n]*${subject}[^"\n]*"|'[^'\n]*${subject}[^'\n]*'|${unquoted})\s+exists(?:\s|$)`; +} diff --git a/src/commands/schema/cli-help-react-native.ts b/src/commands/schema/cli-help-react-native.ts index 6db1cedead..79b123d7bc 100644 --- a/src/commands/schema/cli-help-react-native.ts +++ b/src/commands/schema/cli-help-react-native.ts @@ -20,7 +20,7 @@ React Native dev loop: For "start from screen X" flows, prefer open --relaunch before the first snapshot so the app does not reuse a prior in-progress navigation state. JS-only change with Metro or Re.Pack connected: agent-device metro reload - agent-device find "Home" + agent-device wait text "Home" Do not use agent-device reload. Use open --relaunch for native startup reset. Android RN/Expo/Re.Pack dev server: direct Android localhost URL opens with a port auto-configure host reachability. For app/package launches, run metro prepare when the app cannot reach the local dev server. Verify Metro/Re.Pack from the same host context that owns the dev server. If a sandboxed shell cannot curl localhost:8081/status but an unrestricted host shell can, the dev server is running and the sandbox probe is not authoritative. diff --git a/src/commands/schema/cli-help-topics.test.ts b/src/commands/schema/cli-help-topics.test.ts index 430b19a4de..c4575dbe96 100644 --- a/src/commands/schema/cli-help-topics.test.ts +++ b/src/commands/schema/cli-help-topics.test.ts @@ -188,7 +188,7 @@ test('usageForCommand resolves workflow help topic', async () => { ); assert.match(help, /open -> snapshot -i -> settle -> verify -> close loop/); assert.match(help, /type never takes --settle/); - assert.match(help, /snapshot\/get\/is\/find answer read-only questions/); + assert.match(help, /snapshot\/get\/is answer read-only questions/); assert.match(help, /--settle confirms local UI quieted/); assert.match(help, /wait text "Expected result" or wait instead of polling/); assert.match(help, /strict disappearance uses wait absent /); diff --git a/src/commands/schema/cli-help-workflows.ts b/src/commands/schema/cli-help-workflows.ts index 8206672402..05b4f4dcec 100644 --- a/src/commands/schema/cli-help-workflows.ts +++ b/src/commands/schema/cli-help-workflows.ts @@ -283,7 +283,7 @@ Loop: 1. Build or prepare the changed surface with the repo command that owns it. 2. Open the target app/device state explicitly. 3. Use snapshot -i and press/fill/click/longpress --settle for UI-driving steps. - 4. Use the settled diff as evidence when it shows the changed behavior; otherwise verify with wait/get/is/find, screenshot, logs, network, perf, or trace based on the claim. + 4. Use the settled diff as evidence when it shows the changed behavior; otherwise verify with wait/get/is, find exists, screenshot, logs, network, perf, or trace based on the claim. 5. Record timings, token/output size, screenshots/videos, logs, or perf artifacts only when they answer the validation question. 6. Close sessions and release leases before finishing. If your session booted a local simulator or emulator, use close --shutdown, unless the user asks to keep it running. diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 73d95483eb..58d2aa4a09 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -114,7 +114,7 @@ Session ordering: Read-only and waits: ${WAIT_FAILURE_CONTRACT} - snapshot/get/is/find answer read-only questions; snapshot -i is for refs. --settle confirms local UI quieted; delayed results use wait text "Expected result" or wait instead of polling; strict disappearance uses wait absent . + snapshot/get/is answer read-only questions; snapshot -i is for refs. --settle confirms local UI quieted; delayed results use wait text "Expected result" or wait instead of polling; strict disappearance uses wait absent . wait stable [quietMs] [timeoutMs] (defaults 500/10000) is the fallback for open/relaunch/navigation, or an intentionally-unsettled mutation -- not after a --settle whose diff already shows the change. Ambiguous find: add --first or --last. Navigation: @@ -122,12 +122,12 @@ Navigation: Validation and evidence: Nearby mutation diff: diff snapshot -i; with no prior snapshot it initializes the baseline (zero changes) instead of failing. - Named expectations need the exact text/selector via wait/is/get/find -- a bare screenshot/snapshot is not verification. Before declaring a task done, confirm the requested end state is actually visible on the current screen, scrolling it into view if needed; get text alone, or stopping one screen early, is not enough. + Named expectations need the exact text/selector via wait/is/get -- a bare screenshot/snapshot is not verification. Before declaring a task done, confirm the requested end state is actually visible on the current screen, scrolling it into view if needed; get text alone, or stopping one screen early, is not enough. When an action only reveals or reaches a target, verify the exact target named, not just the action. Prefer testIDs/ids/selectors over visible text. Icon/tappable proof: screenshot --overlay-refs; if snapshot is sparse/AX-unavailable, use plain screenshot and coordinates, then retry snapshot -i on another screen. iOS merged: child ref => press it; else press parent @ref --settle. Names are not selectors. Perf/memory/log/network/trace/crash: help debugging. Recording, save-script, batch, replay repair: help scripting. -React Native: help react-native for Metro/Re.Pack reload, DevTools, RN overlays. JS-only change: metro reload, find "Home"; open --relaunch for native reset. +React Native: help react-native for Metro/Re.Pack reload, DevTools, RN overlays. JS-only change: metro reload, wait text "Home"; open --relaunch for native reset. Lifecycle facts (trust these instead of probing): open without --relaunch is idempotent-foreground except native macOS app opens (background); --relaunch restarts it. close keeps a healthy iOS runner warm by default; runners/daemons self-idle after 5 minutes; stale leases self-reclaim. open device claims reclaim other worktrees' warm runners; "already owned by another agent-device daemon" = owner outside claim arbitration. Env vars: help physical-device. From de5c32c40175754ddda420b7aa6703fa1fefc7ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 19:01:04 +0100 Subject: [PATCH 3/7] test(help): accept escaped quotes in a find ... exists query Address review: the canonical plan keeps escaped quotes, and the double-quoted query pattern stopped at the first one. Co-Authored-By: Claude Opus 5.5 --- scripts/__tests__/help-conformance-expectation-fixtures.ts | 3 ++- scripts/help-conformance-expectations.mjs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/__tests__/help-conformance-expectation-fixtures.ts b/scripts/__tests__/help-conformance-expectation-fixtures.ts index d1e09a8950..c57129b330 100644 --- a/scripts/__tests__/help-conformance-expectation-fixtures.ts +++ b/scripts/__tests__/help-conformance-expectation-fixtures.ts @@ -166,7 +166,8 @@ export const EXPECTATION_FIXTURES: Record = { { id: 'existsInFindQuery', commands: ['agent-device find text "Sent exists"'] }, { id: 'findFillsExists', commands: ['agent-device find text Sent fill exists'] }, ], - metamorphic: { commands: ['agent-device find text "Welcome back" exists'] }, + // An escaped quote inside the query still reads as one query. + metamorphic: { commands: [String.raw`agent-device find text "Welcome \"back\"" exists`] }, }, usesDogfoodEvidence: { passing: { diff --git a/scripts/help-conformance-expectations.mjs b/scripts/help-conformance-expectations.mjs index d873effd78..6c9f0b0e0c 100644 --- a/scripts/help-conformance-expectations.mjs +++ b/scripts/help-conformance-expectations.mjs @@ -26,5 +26,7 @@ function isPnpmScript(command, script) { */ export function findExistsSource(subject = '') { const unquoted = subject ? String.raw`[^\s"']*${subject}[^\s"']*` : String.raw`[^\s"']+`; - return String.raw`\bfind\s+(?:(?:text|label|value|role|id)\s+)?(?:"[^"\n]*${subject}[^"\n]*"|'[^'\n]*${subject}[^'\n]*'|${unquoted})\s+exists(?:\s|$)`; + // A double-quoted query may hold escaped quotes, as the plan canonicalizer emits them. + const doubleQuoted = String.raw`"(?:[^"\\\n]|\\.)*${subject}(?:[^"\\\n]|\\.)*"`; + return String.raw`\bfind\s+(?:(?:text|label|value|role|id)\s+)?(?:${doubleQuoted}|'[^'\n]*${subject}[^'\n]*'|${unquoted})\s+exists(?:\s|$)`; } From fde2b547e13507b48e95b4ff8628af016a535937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 21:47:50 +0100 Subject: [PATCH 4/7] docs(help): show a concrete find ... exists example Address review: find exists hid that the locator is text, label, value, role or id followed by the query. Co-Authored-By: Claude Opus 5.5 --- skills/agent-device/SKILL.md | 2 +- src/commands/schema/cli-help-first-screen.test.ts | 4 ++-- src/commands/schema/cli-help-overview.ts | 2 +- src/commands/schema/cli-help-workflows.ts | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/skills/agent-device/SKILL.md b/skills/agent-device/SKILL.md index f4fabd84d1..96f7649726 100644 --- a/skills/agent-device/SKILL.md +++ b/skills/agent-device/SKILL.md @@ -13,7 +13,7 @@ agent-device open --foreground That starts the session and returns the initial interactive snapshot with `@refs`. -Loop: act with `press|click|fill|longpress ... --settle`, `scroll --settle`, or `back --settle`; continue from the printed diff, verify the named expectation with a read-only command (`wait text "..."`, `is`, or `get`; bare `find` clicks, so check with `find exists`), then run `agent-device close`, or `agent-device close --shutdown` if your session booted a local simulator or emulator, unless the user asks to keep it running. +Loop: act with `press|click|fill|longpress ... --settle`, `scroll --settle`, or `back --settle`; continue from the printed diff, verify the named expectation with a read-only command (`wait text "..."`, `is`, or `get`; bare `find` clicks, so check with `find text "..." exists`), then run `agent-device close`, or `agent-device close --shutdown` if your session booted a local simulator or emulator, unless the user asks to keep it running. Reaching an off-screen target is one command, not a scroll-and-check loop: `scroll down --until ` scrolls until that element is on screen, and `scroll bottom` runs to the end of the content. Repeated bare `scroll down` calls are the slow way to find something. diff --git a/src/commands/schema/cli-help-first-screen.test.ts b/src/commands/schema/cli-help-first-screen.test.ts index 9298f3d181..296091ea69 100644 --- a/src/commands/schema/cli-help-first-screen.test.ts +++ b/src/commands/schema/cli-help-first-screen.test.ts @@ -16,7 +16,7 @@ const FIRST_SCREEN_RULES: Record = { 'Do not probe first with devices, apps, appstate, snapshot, or screenshot', 'Unknown app id: devices, then apps, then open . Never invent ids', 'is, or get. A bare screenshot is not verification', - 'find acts: it clicks by default. To check only, use find exists', + 'find acts: it clicks by default. To check only, use find text "..." exists', 'End with: agent-device close', ], commands: [ @@ -28,7 +28,7 @@ const FIRST_SCREEN_RULES: Record = { 'Stop and report ambiguity', '1. Open the requested app; use --relaunch only when the script needs fresh state', '5. If --settle prints not settled, follow its hint', - '6. Verify named expectations with wait text/selector/absent, get, is, find exists, or the settled diff. Bare find clicks. A bare screenshot/snapshot is not verification', + '6. Verify named expectations with wait text/selector/absent, get, is, find text "..." exists, or the settled diff. Bare find clicks. A bare screenshot/snapshot is not verification', ], maestro: [ 'Run Maestro compatibility flows with replay --maestro', diff --git a/src/commands/schema/cli-help-overview.ts b/src/commands/schema/cli-help-overview.ts index 59c43149a7..51fa29cf6c 100644 --- a/src/commands/schema/cli-help-overview.ts +++ b/src/commands/schema/cli-help-overview.ts @@ -27,7 +27,7 @@ Loop: Run snapshot -i only when the diff lacks the next target or did not settle. Verify a named expectation with the diff, wait text "...", wait , wait absent , is, or get. A bare screenshot is not verification. - find acts: it clicks by default. To check only, use find exists. + find acts: it clicks by default. To check only, use find text "..." exists. End with: agent-device close Targets: diff --git a/src/commands/schema/cli-help-workflows.ts b/src/commands/schema/cli-help-workflows.ts index 05b4f4dcec..beac717dfe 100644 --- a/src/commands/schema/cli-help-workflows.ts +++ b/src/commands/schema/cli-help-workflows.ts @@ -28,7 +28,7 @@ Loop: 3. Run press/fill/click/longpress --settle for each mutating step. 4. Treat a settled:true diff as the next observation. Do not add wait stable or another snapshot when the diff already shows the next target or expected result. 5. If --settle prints not settled, follow its hint before the next ref-based action. - 6. Verify named expectations with wait text/selector/absent, get, is, find exists, or the settled diff. Bare find clicks. A bare screenshot/snapshot is not verification for a named expectation. + 6. Verify named expectations with wait text/selector/absent, get, is, find text "..." exists, or the settled diff. Bare find clicks. A bare screenshot/snapshot is not verification for a named expectation. 7. Close the session when the script ends. If your session booted a local simulator or emulator, use close --shutdown, unless the user asks to keep it running. Command shapes: @@ -283,7 +283,7 @@ Loop: 1. Build or prepare the changed surface with the repo command that owns it. 2. Open the target app/device state explicitly. 3. Use snapshot -i and press/fill/click/longpress --settle for UI-driving steps. - 4. Use the settled diff as evidence when it shows the changed behavior; otherwise verify with wait/get/is, find exists, screenshot, logs, network, perf, or trace based on the claim. + 4. Use the settled diff as evidence when it shows the changed behavior; otherwise verify with wait/get/is, find text "..." exists, screenshot, logs, network, perf, or trace based on the claim. 5. Record timings, token/output size, screenshots/videos, logs, or perf artifacts only when they answer the validation question. 6. Close sessions and release leases before finishing. If your session booted a local simulator or emulator, use close --shutdown, unless the user asks to keep it running. From 9651c3e7d6965dd27e51e31a20c68fa48b16948b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 09:02:58 +0100 Subject: [PATCH 5/7] test(help): classify find verification with parseFindArgs Address review: the regex find grammar missed a single-word query with an apostrophe (find text "Today's" exists), because the canonical plan leaves it unquoted. One token-based predicate, hasFindExists, now reads each find command's parsed positionals through parseFindArgs, checks for the exists action and the optional subject, and serves both the scorer and the four case matchers. Co-Authored-By: Claude Opus 5.5 --- .../help-conformance-expectation-fixtures.ts | 4 +-- scripts/help-conformance-case-checks.d.mts | 10 +++++++- scripts/help-conformance-case-checks.mjs | 11 ++++---- scripts/help-conformance-cases.mjs | 22 ++++++++-------- scripts/help-conformance-expectations.mjs | 25 +++++++++++++------ 5 files changed, 45 insertions(+), 27 deletions(-) diff --git a/scripts/__tests__/help-conformance-expectation-fixtures.ts b/scripts/__tests__/help-conformance-expectation-fixtures.ts index c57129b330..b716694688 100644 --- a/scripts/__tests__/help-conformance-expectation-fixtures.ts +++ b/scripts/__tests__/help-conformance-expectation-fixtures.ts @@ -166,8 +166,8 @@ export const EXPECTATION_FIXTURES: Record = { { id: 'existsInFindQuery', commands: ['agent-device find text "Sent exists"'] }, { id: 'findFillsExists', commands: ['agent-device find text Sent fill exists'] }, ], - // An escaped quote inside the query still reads as one query. - metamorphic: { commands: [String.raw`agent-device find text "Welcome \"back\"" exists`] }, + // A single-word query with an apostrophe is canonicalized without quotes. + metamorphic: { commands: [`agent-device find text "Today's" exists`] }, }, usesDogfoodEvidence: { passing: { diff --git a/scripts/help-conformance-case-checks.d.mts b/scripts/help-conformance-case-checks.d.mts index a28832723d..05e762f0ec 100644 --- a/scripts/help-conformance-case-checks.d.mts +++ b/scripts/help-conformance-case-checks.d.mts @@ -4,9 +4,17 @@ import type { PlanCommandValidation } from './help-conformance-plan-validator.mj export type CasePatternCheck = { id: string; pattern: RegExp }; +/** The plan a check reads: its canonical text and each command's tokens and parse. */ +export type CaseCheckContext = { + joined: string; + commandValidation: readonly PlanCommandValidation[]; +}; + +export type CaseTestCheck = { id: string; test: (context: CaseCheckContext) => boolean }; + export type CaseChecks = { expectations?: readonly string[]; - matchers?: readonly CasePatternCheck[]; + matchers?: readonly (CasePatternCheck | CaseTestCheck)[]; forbidden?: readonly CasePatternCheck[]; }; diff --git a/scripts/help-conformance-case-checks.mjs b/scripts/help-conformance-case-checks.mjs index f196bdd795..efc1702b8f 100644 --- a/scripts/help-conformance-case-checks.mjs +++ b/scripts/help-conformance-case-checks.mjs @@ -1,11 +1,9 @@ import { - findExistsSource, + hasFindExists, opensAndCloses, usesValidationPrep, } from './help-conformance-expectations.mjs'; -const VERIFICATION = new RegExp(String.raw`\b(?:wait|is|get)\b|${findExistsSource()}`); - const EXPECTATION_SCORERS = { validPlanCommands: ({ commands, commandValidation }) => commands.length > 0 && commandValidation.every(({ issues }) => issues.length === 0), @@ -23,7 +21,8 @@ const EXPECTATION_SCORERS = { // that can tell whether the model actually chained. usesConfidentChaining: ({ commands }) => commands.some((command) => /&&/.test(command)), noWaitStable: ({ joined }) => !joined.includes('wait stable'), - verifiesNamedExpectation: ({ joined }) => VERIFICATION.test(joined), + verifiesNamedExpectation: ({ joined, commandValidation }) => + /\b(?:wait|is|get)\b/.test(joined) || hasFindExists(commandValidation), usesDogfoodEvidence: ({ joined }) => /(?:\bscreenshot\b|\brecord\b|\blogs\b|\bnetwork\b|\bperf\b|\btrace\b|dogfood-output)/i.test( joined, @@ -65,9 +64,9 @@ function resolveChecks(testCase) { id, test: (context) => scoreExpectation(id, context), })); - const matched = (testCase.matchers ?? []).map(({ id, pattern }) => ({ + const matched = (testCase.matchers ?? []).map(({ id, pattern, test }) => ({ id, - test: (context) => pattern.test(context.joined), + test: test ?? ((context) => pattern.test(context.joined)), })); const forbidden = (testCase.forbidden ?? []).map(({ id, pattern }) => ({ id, diff --git a/scripts/help-conformance-cases.mjs b/scripts/help-conformance-cases.mjs index ea15e5862e..c5594a2c73 100644 --- a/scripts/help-conformance-cases.mjs +++ b/scripts/help-conformance-cases.mjs @@ -14,7 +14,7 @@ import { STALE_REF_SAMPLE, sampleText, } from './help-conformance-sample-outputs.mjs'; -import { findExistsSource } from './help-conformance-expectations.mjs'; +import { hasFindExists } from './help-conformance-expectations.mjs'; // Raw-coordinate fallback the quiz cases forbid: a click/fill/press targeting // bare numbers instead of a ref or selector. @@ -183,7 +183,8 @@ export const CASES = [ { id: 'activatesWithSelect', pattern: /\btv-remote\s+press\s+select\b/i }, { id: 'verifiesOutcome', - pattern: new RegExp(String.raw`\b(?:is\s+focused|wait)\b|${findExistsSource()}`, 'i'), + test: ({ joined, commandValidation }) => + /\b(?:is\s+focused|wait)\b/i.test(joined) || hasFindExists(commandValidation), }, ], forbidden: [ @@ -245,10 +246,9 @@ export const CASES = [ { id: 'usesWebPlatform', pattern: /--platform\s+web\b/i }, { id: 'verifiesWelcomeText', - pattern: new RegExp( - String.raw`\b(?:wait|is)\b[^\n]*welcome|${findExistsSource('welcome')}`, - 'i', - ), + test: ({ joined, commandValidation }) => + /\b(?:wait|is)\b[^\n]*welcome/i.test(joined) || + hasFindExists(commandValidation, 'welcome'), }, ], forbidden: [ @@ -326,10 +326,9 @@ export const CASES = [ }, { id: 'verifiesFirstScreen', - pattern: new RegExp( - String.raw`\b(?:wait|is|get)\b[^\n]*orders|${findExistsSource('orders')}`, - 'i', - ), + test: ({ joined, commandValidation }) => + /\b(?:wait|is|get)\b[^\n]*orders/i.test(joined) || + hasFindExists(commandValidation, 'orders'), }, ], forbidden: [ @@ -388,7 +387,8 @@ export const CASES = [ }, { id: 'verifiesLoginSucceeded', - pattern: new RegExp(String.raw`\b(?:wait|is|get)\b|${findExistsSource()}`, 'i'), + test: ({ joined, commandValidation }) => + /\b(?:wait|is|get)\b/i.test(joined) || hasFindExists(commandValidation), }, { id: 'publishesWithoutClosing', pattern: /\bagent-device\s+session\s+save-script\b/i }, ], diff --git a/scripts/help-conformance-expectations.mjs b/scripts/help-conformance-expectations.mjs index 6c9f0b0e0c..0edbd1a10d 100644 --- a/scripts/help-conformance-expectations.mjs +++ b/scripts/help-conformance-expectations.mjs @@ -1,3 +1,5 @@ +import { parseFindArgs } from '@agent-device/selectors'; + export function opensAndCloses({ commands }) { return hasAgentDeviceCommand(commands, 'open') && hasAgentDeviceCommand(commands, 'close'); } @@ -21,12 +23,21 @@ function isPnpmScript(command, script) { } /** - * A find whose action is `exists`, the only read-only find: bare find clicks. `subject` must - * appear in the query, which the optional locator precedes and the action follows. + * Whether the plan holds a read-only `find`: one whose action is `exists` (bare find clicks). + * `parseFindArgs` owns the find grammar; `subject`, when given, must appear in the query. */ -export function findExistsSource(subject = '') { - const unquoted = subject ? String.raw`[^\s"']*${subject}[^\s"']*` : String.raw`[^\s"']+`; - // A double-quoted query may hold escaped quotes, as the plan canonicalizer emits them. - const doubleQuoted = String.raw`"(?:[^"\\\n]|\\.)*${subject}(?:[^"\\\n]|\\.)*"`; - return String.raw`\bfind\s+(?:(?:text|label|value|role|id)\s+)?(?:${doubleQuoted}|'[^'\n]*${subject}[^'\n]*'|${unquoted})\s+exists(?:\s|$)`; +export function hasFindExists(commandValidation, subject) { + return commandValidation.some((command) => { + const positionals = findPositionals(command); + if (!positionals) return false; + const { action, query } = parseFindArgs(positionals); + return action === 'exists' && (subject === undefined || query.toLowerCase().includes(subject)); + }); +} + +/** A find command's positionals: parsed when it validated, else its tokens without flags. */ +function findPositionals({ tokens, agentCommand }) { + if (agentCommand) return agentCommand.command === 'find' ? agentCommand.positionals : undefined; + if (tokens[0] !== 'agent-device' || tokens[1] !== 'find') return undefined; + return tokens.slice(2).filter((token) => !token.startsWith('-')); } From 26b520f9ba3df92037d398011c6f0f59a14dee94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 09:10:22 +0100 Subject: [PATCH 6/7] test(help): keep scoring a find with an unsupported action Address review: parseFindArgs throws on an unsupported action, so an invalid find plan crashed the scorer; it now scores false. A focused test drives the check through the real plan tokenizer and restores the escaped-quote case next to the apostrophe one. Co-Authored-By: Claude Opus 5.5 --- ...help-conformance-find-verification.test.ts | 26 +++++++++++++++++++ scripts/help-conformance-expectations.d.mts | 7 +++++ scripts/help-conformance-expectations.mjs | 16 ++++++++++-- vitest.config.ts | 1 + 4 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 scripts/__tests__/help-conformance-find-verification.test.ts diff --git a/scripts/__tests__/help-conformance-find-verification.test.ts b/scripts/__tests__/help-conformance-find-verification.test.ts new file mode 100644 index 0000000000..418d7c95a1 --- /dev/null +++ b/scripts/__tests__/help-conformance-find-verification.test.ts @@ -0,0 +1,26 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { hasFindExists } from '../help-conformance-expectations.mjs'; +import { validatePlanCommands } from '../help-conformance-plan-validator.mjs'; + +// The find verification check reads plan commands through the real plan tokenizer, so quoting is +// the tokenizer's job and the find grammar is parseFindArgs'. +async function verifies(command: string, subject?: string): Promise { + return hasFindExists(await validatePlanCommands([command]), subject); +} + +test('find ... exists verifies, whatever quotes its query holds', async () => { + assert.equal(await verifies(`agent-device find text "Today's" exists`), true); + assert.equal(await verifies(String.raw`agent-device find text "Welcome \"back\"" exists`), true); + assert.equal(await verifies('agent-device find "Welcome back" exists', 'welcome'), true); +}); + +test('a find that acts, or names another subject, does not verify', async () => { + assert.equal(await verifies('agent-device find text "Sent"'), false); + assert.equal(await verifies('agent-device find text Sent fill exists'), false); + assert.equal(await verifies('agent-device find text "Orders" exists', 'welcome'), false); +}); + +test('a find with an unsupported action scores false instead of throwing', async () => { + assert.equal(await verifies('agent-device find text Sent swipe'), false); +}); diff --git a/scripts/help-conformance-expectations.d.mts b/scripts/help-conformance-expectations.d.mts index aca3b38019..addef6d319 100644 --- a/scripts/help-conformance-expectations.d.mts +++ b/scripts/help-conformance-expectations.d.mts @@ -1,7 +1,14 @@ // Type surface for the plain-Node expectation scorers (they must stay runnable by bare `node`). // Keep it in step with the exports of `./help-conformance-expectations.mjs`. +import type { PlanCommandValidation } from './help-conformance-plan-validator.mjs'; + export type ExpectationContext = Readonly<{ commands: readonly string[] }>; export function opensAndCloses(context: ExpectationContext): boolean; export function usesValidationPrep(context: ExpectationContext): boolean; + +export function hasFindExists( + commandValidation: readonly PlanCommandValidation[], + subject?: string, +): boolean; diff --git a/scripts/help-conformance-expectations.mjs b/scripts/help-conformance-expectations.mjs index 0edbd1a10d..eafee9844c 100644 --- a/scripts/help-conformance-expectations.mjs +++ b/scripts/help-conformance-expectations.mjs @@ -30,11 +30,23 @@ export function hasFindExists(commandValidation, subject) { return commandValidation.some((command) => { const positionals = findPositionals(command); if (!positionals) return false; - const { action, query } = parseFindArgs(positionals); - return action === 'exists' && (subject === undefined || query.toLowerCase().includes(subject)); + const parsed = tryParseFindArgs(positionals); + return ( + parsed?.action === 'exists' && + (subject === undefined || parsed.query.toLowerCase().includes(subject)) + ); }); } +/** An unsupported action is a plan the model got wrong, not a read-only find, so it scores false. */ +function tryParseFindArgs(positionals) { + try { + return parseFindArgs(positionals); + } catch { + return undefined; + } +} + /** A find command's positionals: parsed when it validated, else its tokens without flags. */ function findPositionals({ tokens, agentCommand }) { if (agentCommand) return agentCommand.command === 'find' ? agentCommand.positionals : undefined; diff --git a/vitest.config.ts b/vitest.config.ts index 2c3a895974..0335d9caac 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -157,6 +157,7 @@ export default defineConfig({ 'scripts/__tests__/help-conformance-bench.test.ts', 'scripts/__tests__/help-conformance-error-recovery-coverage.test.ts', 'scripts/__tests__/help-conformance-expectation-falsification.test.ts', + 'scripts/__tests__/help-conformance-find-verification.test.ts', 'scripts/__tests__/help-conformance-sample-outputs.test.ts', 'scripts/__tests__/help-conformance-topic-coverage.test.ts', 'scripts/__tests__/eager-closure-budgets.test.ts', From 955ebd06db676e23363387b91909ba1260d9dd91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 09:18:01 +0100 Subject: [PATCH 7/7] test(help): parse find commands in the TS command validator Address review: importing @agent-device/selectors into the scorer modules broke bare node on the Node 22 floor, where .ts imports need type stripping. The command validator, which already runs with --experimental-strip-types, now attaches each find command's parseFindArgs action and query, and the plain-JS scorer reads that. The scorer modules import no TypeScript again. Co-Authored-By: Claude Opus 5.5 --- scripts/help-conformance-command-validator.ts | 14 +++++++++ scripts/help-conformance-expectations.mjs | 31 ++++--------------- scripts/help-conformance-plan-validator.d.mts | 7 ++++- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/scripts/help-conformance-command-validator.ts b/scripts/help-conformance-command-validator.ts index a18f588ce1..d5b6114553 100644 --- a/scripts/help-conformance-command-validator.ts +++ b/scripts/help-conformance-command-validator.ts @@ -1,12 +1,15 @@ import { pathToFileURL } from 'node:url'; import { parseArgs } from '../src/cli/parser/args.ts'; import { isCommandName, readInputFromCli } from '@agent-device/command-surface'; +import { parseFindArgs } from '@agent-device/selectors'; type ValidationKind = 'agent-device-grammar' | 'pseudo-ref'; type ParsedAgentCommand = { command: string; positionals: string[]; + /** A `find` command's action and query, as `parseFindArgs` reads them; the scorers stay plain JS. */ + find?: { action: string; query: string }; }; type ValidationResult = { valid: true } | { valid: false; kind: ValidationKind; error: string }; @@ -91,10 +94,21 @@ function validatePlanCommand(value: unknown): PlanValidationResult { agentCommand: { command: parsed.command, positionals: [...parsed.positionals], + ...(parsed.command === 'find' ? readFind(parsed.positionals) : {}), }, }; } +/** An unsupported action is no find the plan can act on, so it carries no parsed find. */ +function readFind(positionals: string[]): Pick { + try { + const { action, query } = parseFindArgs(positionals); + return { find: { action, query } }; + } catch { + return {}; + } +} + function runCli(): void { const encodedInput = process.argv[2]; if (!encodedInput) throw new Error('Expected a JSON argv array.'); diff --git a/scripts/help-conformance-expectations.mjs b/scripts/help-conformance-expectations.mjs index eafee9844c..7c1d5863cc 100644 --- a/scripts/help-conformance-expectations.mjs +++ b/scripts/help-conformance-expectations.mjs @@ -1,5 +1,3 @@ -import { parseFindArgs } from '@agent-device/selectors'; - export function opensAndCloses({ commands }) { return hasAgentDeviceCommand(commands, 'open') && hasAgentDeviceCommand(commands, 'close'); } @@ -24,32 +22,15 @@ function isPnpmScript(command, script) { /** * Whether the plan holds a read-only `find`: one whose action is `exists` (bare find clicks). - * `parseFindArgs` owns the find grammar; `subject`, when given, must appear in the query. + * The command validator parses each find with `parseFindArgs`; `subject`, when given, must appear + * in the query. A find that failed validation carries no parse and does not verify. */ export function hasFindExists(commandValidation, subject) { - return commandValidation.some((command) => { - const positionals = findPositionals(command); - if (!positionals) return false; - const parsed = tryParseFindArgs(positionals); + return commandValidation.some(({ agentCommand }) => { + const find = agentCommand?.find; return ( - parsed?.action === 'exists' && - (subject === undefined || parsed.query.toLowerCase().includes(subject)) + find?.action === 'exists' && + (subject === undefined || find.query.toLowerCase().includes(subject)) ); }); } - -/** An unsupported action is a plan the model got wrong, not a read-only find, so it scores false. */ -function tryParseFindArgs(positionals) { - try { - return parseFindArgs(positionals); - } catch { - return undefined; - } -} - -/** A find command's positionals: parsed when it validated, else its tokens without flags. */ -function findPositionals({ tokens, agentCommand }) { - if (agentCommand) return agentCommand.command === 'find' ? agentCommand.positionals : undefined; - if (tokens[0] !== 'agent-device' || tokens[1] !== 'find') return undefined; - return tokens.slice(2).filter((token) => !token.startsWith('-')); -} diff --git a/scripts/help-conformance-plan-validator.d.mts b/scripts/help-conformance-plan-validator.d.mts index ae6f0a264a..64f8b55b19 100644 --- a/scripts/help-conformance-plan-validator.d.mts +++ b/scripts/help-conformance-plan-validator.d.mts @@ -3,7 +3,12 @@ // `./help-conformance-plan-validator.mjs`; keep it in step with the exports there. export type PlanCommandIssue = { kind: string; error: string }; -export type PlanAgentCommand = { command: string; positionals: string[] }; +export type PlanAgentCommand = { + command: string; + positionals: string[]; + /** A `find` command's action and query, as `parseFindArgs` reads them. */ + find?: { action: string; query: string }; +}; export type PlanCommandValidation = { command: string;