From 0751d1aa38ffad88db47c652a02d3388eaecf006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 15:24:52 +0100 Subject: [PATCH 01/12] feat(mcp)!: preload loop tools, trim schemas, and shorten errors Claude Code defers MCP tool schemas, so each tool a run needs costs a tool search turn. Mark the 'loop' framework tier (open, snapshot, press, fill, back, wait, scroll, screenshot) with _meta "anthropic/alwaysLoad", and give every tool an "anthropic/searchHint" from its command text. Make the always-loaded schemas lean: debug, the MCP response controls (cost, responseLevel), and the target alias leave every tool schema. The controls become operator settings (AGENT_DEVICE_COST, AGENT_DEVICE_RESPONSE_LEVEL); mcpOutputFormat is removed. Loop tool descriptions carry their usage guidance from the command help text, batch documents its step shape with tested examples, MCP errors put the typed reason beside the code and drop the log path unless the recovery step reads it, and MCP open text drops the Session state path. Co-Authored-By: Claude Opus 5.5 --- packages/command-output/src/cli-output.ts | 1 + .../src/management/output.test.ts | 16 ++++ .../command-output/src/management/output.ts | 10 ++- packages/command-output/src/output-common.ts | 5 ++ packages/command-output/src/output/error.ts | 39 ++++++-- packages/command-registry/src/command-text.ts | 16 +++- .../src/descriptors/device-operations.ts | 12 +-- .../src/descriptors/session-operations.ts | 2 +- .../src/flag-definitions-workflow.ts | 2 +- packages/command-registry/src/registry.ts | 4 +- packages/command-registry/src/types.ts | 7 +- packages/command-surface/src/batch/index.ts | 20 ++++- packages/command-surface/src/capture/alert.ts | 1 + packages/command-surface/src/capture/diff.ts | 1 + .../command-surface/src/capture/screenshot.ts | 3 +- .../command-surface/src/capture/settings.ts | 1 + .../command-surface/src/capture/snapshot.ts | 3 +- packages/command-surface/src/capture/wait.ts | 3 +- .../command-surface/src/command-contract.ts | 2 + .../src/command-output-schemas.ts | 2 +- .../src/common-input-fields.ts | 7 +- .../command-surface/src/debugging/index.ts | 1 + packages/command-surface/src/family/types.ts | 8 +- packages/command-surface/src/index.ts | 2 +- .../command-surface/src/input-audience.ts | 13 ++- .../command-surface/src/interaction/index.ts | 13 +++ .../src/interaction/metadata.ts | 4 +- .../command-surface/src/management/app.ts | 5 +- .../src/management/artifacts.ts | 1 + .../command-surface/src/management/device.ts | 4 + .../command-surface/src/management/doctor.ts | 1 + .../command-surface/src/management/install.ts | 4 + .../command-surface/src/management/prepare.ts | 1 + .../command-surface/src/management/push.ts | 2 + .../command-surface/src/management/session.ts | 1 + .../src/management/viewport.ts | 1 + packages/command-surface/src/metro/index.ts | 1 + .../src/observability/index.ts | 4 + packages/command-surface/src/perf/index.ts | 1 + .../src/post-action-observation-grammar.ts | 7 +- .../command-surface/src/react-native/index.ts | 1 + .../command-surface/src/recording/index.ts | 2 + packages/command-surface/src/replay/index.ts | 2 + .../src/schema/command-overrides.ts | 2 +- packages/command-surface/src/system/index.ts | 12 +++ packages/contracts/src/client-capture.ts | 10 +-- .../command-descriptor-parity.test.ts | 25 +++++- src/ai-sdk/__tests__/index.test.ts | 7 +- src/ai-sdk/index.ts | 2 +- .../command-tools-loading-meta.test.ts | 44 ++++++++++ .../command-tools-operator-inputs.test.ts | 41 ++++++++- .../__tests__/command-tools-parity.test.ts | 3 +- .../command-tools-structured-content.test.ts | 56 +++++++++--- src/mcp/__tests__/command-tools.test.ts | 88 ++++++------------- src/mcp/command-tools.ts | 80 +++++++++++------ src/mcp/server-guide.ts | 13 ++- src/mcp/tool-control-fields.ts | 48 +++++----- src/mcp/tool-error.ts | 2 +- website/docs/docs/agent-setup.md | 8 +- 59 files changed, 484 insertions(+), 193 deletions(-) create mode 100644 src/mcp/__tests__/command-tools-loading-meta.test.ts diff --git a/packages/command-output/src/cli-output.ts b/packages/command-output/src/cli-output.ts index 40da2d6d1a..339cf00796 100644 --- a/packages/command-output/src/cli-output.ts +++ b/packages/command-output/src/cli-output.ts @@ -76,6 +76,7 @@ export async function dispatchCliOutput( progress: params.progress, env: params.env, stderrIsTTY: params.stderrIsTTY, + reader: params.reader, }); if (output === undefined) return undefined; return routeResponseWarnings( diff --git a/packages/command-output/src/management/output.test.ts b/packages/command-output/src/management/output.test.ts index 1fc30af7e4..6ec2259713 100644 --- a/packages/command-output/src/management/output.test.ts +++ b/packages/command-output/src/management/output.test.ts @@ -22,6 +22,22 @@ describe('openCliOutput', () => { }); }); + test('leaves the session state directory out of the text a model reads', async () => { + const output = await openCliOutput( + { + session: 'default', + sessionStateDir: '/tmp/agent-device/sessions/cwd_123_default', + identifiers: { session: 'default' }, + }, + 'model', + ); + + expect(output.text).toBe('Opened: default'); + expect(output.data).toMatchObject({ + sessionStateDir: '/tmp/agent-device/sessions/cwd_123_default', + }); + }); + test('keeps internal open timing out of public output data', async () => { const result: AppOpenResult & { timing: { totalDurationMs: number } } = { session: 'default', diff --git a/packages/command-output/src/management/output.ts b/packages/command-output/src/management/output.ts index b3b085d2e6..afac71801d 100644 --- a/packages/command-output/src/management/output.ts +++ b/packages/command-output/src/management/output.ts @@ -22,6 +22,7 @@ import { readCommandMessage } from '@agent-device/kernel/success-text'; import { snapshotCliOutput } from '../capture/output.ts'; import { type CliOutputFormatter, + type CliOutputFormatterParams, messageCliOutput, messageOutput, resultOutput, @@ -87,11 +88,14 @@ async function sessionCliOutput( return { data, text: JSON.stringify(data, null, 2) }; } -export async function openCliOutput(result: AppOpenResult): Promise { +export async function openCliOutput( + result: AppOpenResult, + reader: CliOutputFormatterParams['reader'] = 'terminal', +): Promise { const { serializeOpenResult } = await import('../output/result-serialization.ts'); const data = serializeOpenResult(result); const lines = [readCommandMessage(data)].filter((line): line is string => Boolean(line)); - if (typeof data.sessionStateDir === 'string') { + if (reader === 'terminal' && typeof data.sessionStateDir === 'string') { lines.push(`Session state: ${data.sessionStateDir}`); } for (const warning of result.warnings ?? []) { @@ -243,7 +247,7 @@ export const managementCliOutputFormatters = { }), session: resultOutput(sessionCliOutput), artifacts: resultOutput(artifactsCliOutput), - open: resultOutput(openCliOutput), + open: ({ result, reader }) => openCliOutput(result as AppOpenResult, reader), close: resultOutput(closeCliOutput), install: resultOutput(deployCliOutput), reinstall: resultOutput(deployCliOutput), diff --git a/packages/command-output/src/output-common.ts b/packages/command-output/src/output-common.ts index 5acf9ed4a9..bebe2b2683 100644 --- a/packages/command-output/src/output-common.ts +++ b/packages/command-output/src/output-common.ts @@ -38,6 +38,11 @@ export type CliOutputFormatterParams = { env: EnvMap; /** Whether the caller's stderr is a terminal; a formatter that colors status markers reads it. */ stderrIsTTY: boolean; + /** + * Who reads the text. `terminal` (the default) is a person or script at the CLI. `model` is + * an agent reading a tool result, which gets no daemon-host state paths it would never open. + */ + reader?: 'terminal' | 'model'; }; export type CliOutputFormatter = ( diff --git a/packages/command-output/src/output/error.ts b/packages/command-output/src/output/error.ts index 6df70954d7..2d7786b995 100644 --- a/packages/command-output/src/output/error.ts +++ b/packages/command-output/src/output/error.ts @@ -1,5 +1,6 @@ import { AppError, + defaultHintForCode, normalizeError, readDispatchDisclosure, readErrorCandidateViews, @@ -25,6 +26,13 @@ export type HumanErrorView = { export type HumanErrorOptions = { /** Append the raw details dump (the `--debug` tier). */ showDetails?: boolean; + /** + * Who reads the text. `terminal` (the default) is a person or script at the CLI. `model` is + * an agent reading a tool result: the typed `details.reason` joins the code and the message + * stays one line. The diagnostics log path is left out unless the recovery step is the code's + * default, which tells the reader to read that log. + */ + reader?: 'terminal' | 'model'; /** The failing command's synopsis, such as `agent-device fill `. */ synopsis?: string; }; @@ -48,13 +56,20 @@ export function formatHumanError( const candidateViews = readErrorCandidateViews(normalized.details); const inlineCandidates = normalized.code === 'AMBIGUOUS_MATCH' ? readInlineCandidates(candidateViews) : undefined; - const { summary, truncated } = formatSummary(normalized, inlineCandidates, options.synopsis); + const forModel = options.reader === 'model'; + const { summary, truncated } = formatSummary( + normalized, + inlineCandidates, + options.synopsis, + forModel, + ); return { summary, body: formatBody(normalized, { - fullMessage: truncated, + fullMessage: truncated && !forModel, candidateViews: inlineCandidates ? [] : candidateViews, showDetails: options.showDetails, + logLine: !forModel || normalized.hint === defaultHintForCode(normalized.code), }), }; } @@ -63,6 +78,7 @@ function formatSummary( normalized: NormalizedError, inlineCandidates: string[] | undefined, synopsis: string | undefined, + withReason: boolean, ): { summary: string; truncated: boolean } { const dispatched = readDispatchDisclosure(normalized.details); const message = oneLine(normalized.message); @@ -71,8 +87,10 @@ function formatSummary( ? `${message.slice(0, SUMMARY_MESSAGE_LIMIT)}…` : message; const next = nextStep(normalized, dispatched, inlineCandidates, synopsis); + const reason = withReason ? readReason(normalized) : undefined; + const label = reason ? `${normalized.code}, ${reason}` : normalized.code; const summary = [ - `Error (${normalized.code}): ${withFullStop(shown)}`, + `Error (${label}): ${withFullStop(shown)}`, ...(dispatched ? [DISPATCH_SENTENCE[dispatched]] : []), ...(next ? [`Next: ${next}`] : []), ].join(' '); @@ -106,9 +124,20 @@ function nextStep( return parts.length > 0 ? parts.join(' ') : undefined; } +/** The typed `details.reason`, when the throw site classified the failure. */ +function readReason(normalized: NormalizedError): string | undefined { + const reason = normalized.details?.reason; + return typeof reason === 'string' && reason.length > 0 ? reason : undefined; +} + function formatBody( normalized: NormalizedError, - options: { fullMessage: boolean; candidateViews: ErrorCandidateView[]; showDetails?: boolean }, + options: { + fullMessage: boolean; + candidateViews: ErrorCandidateView[]; + showDetails?: boolean; + logLine: boolean; + }, ): string[] { const body: string[] = []; if (options.fullMessage) body.push(`Message: ${normalized.message}`); @@ -124,7 +153,7 @@ function formatBody( } const candidateLines = formatErrorCandidateViews(options.candidateViews); if (candidateLines.length > 0) body.push(candidateLines.join('\n')); - const logLine = formatLogLine(normalized); + const logLine = options.logLine ? formatLogLine(normalized) : undefined; if (logLine) body.push(logLine); // ADR 0012: the divergence compact report always renders; --debug's raw // details dump below remains the full-object view. diff --git a/packages/command-registry/src/command-text.ts b/packages/command-registry/src/command-text.ts index 6e6cf5b82e..746e000cca 100644 --- a/packages/command-registry/src/command-text.ts +++ b/packages/command-registry/src/command-text.ts @@ -8,6 +8,7 @@ * description what does it do, and when should I reach for it? * cliDetail what else does a terminal user need — flags, argument shapes, examples? * mcpDetail what else does an MCP caller need about this command's behavior? + * searchHint which words would a model searching for this tool type that the text lacks? * * `--help`, the command list, the MCP tool description and `explain` are all projections of * these four, computed where they are rendered rather than written back into the schema. That @@ -27,6 +28,12 @@ export type CommandText = { cliDetail?: string; /** Appended to the MCP tool description only: command behavior and input details. */ mcpDetail?: string; + /** + * Space-separated keywords a client's deferred-tool search matches besides the name and + * description: synonyms and task words a model types, such as "tap button" for press. + * Every MCP tool declares one; CLI-only commands need none. + */ + searchHint?: string; }; /** @@ -34,7 +41,9 @@ export type CommandText = { * metadata already carries it — repeating it here would leave a shadowed copy behind, which is * exactly the dead-literal problem this file's predecessor had. */ -export type FacetCommandText = Omit; +export type FacetCommandText = Omit & { + searchHint: string; +}; /** The `--help` body: canonical description plus any terminal-only detail. */ export function helpBody(text: CommandText): string { @@ -47,7 +56,10 @@ export function mcpBody(text: Pick): s } /** Resolves a facet's authored text against the description its metadata already holds. */ -export function resolveFacetText(text: FacetCommandText, description: string): CommandText { +export function resolveFacetText( + text: FacetCommandText, + description: string, +): CommandText & FacetCommandText { return { ...text, description }; } diff --git a/packages/command-registry/src/descriptors/device-operations.ts b/packages/command-registry/src/descriptors/device-operations.ts index bb1d6bac3d..8e4fd28f33 100644 --- a/packages/command-registry/src/descriptors/device-operations.ts +++ b/packages/command-registry/src/descriptors/device-operations.ts @@ -67,7 +67,7 @@ export const DEVICE_OPERATION_COMMAND_DESCRIPTORS = { ? { ownerFiles: ['packages/command-surface/src/capture/wait.ts'] as const } : {}), catalog: { group: 'public' }, - frameworkTier: 'core', + frameworkTier: 'loop', recordsSessionAction: true, recordingEffect: 'observes-app', // #1349: a wait's landmark may legitimately be absent when the step @@ -253,7 +253,7 @@ export const DEVICE_OPERATION_COMMAND_DESCRIPTORS = { : {}), ...TARGETED_TOUCH_INTERACTION_TRAITS, daemon: { ...TARGETED_TOUCH_INTERACTION_TRAITS.daemon, appLease: 'allowed' }, - frameworkTier: 'core', + frameworkTier: 'loop', timeoutPolicy: postActionObservationTimeoutPolicy('fill', PRESERVE_DAEMON_TIMEOUT_POLICY), postActionObservation: postActionObservation('fill'), responseDataTransform: FILL_INTERACTION_RESPONSE_DATA_TRANSFORM, @@ -309,7 +309,7 @@ export const DEVICE_OPERATION_COMMAND_DESCRIPTORS = { : {}), ...TARGETED_TOUCH_INTERACTION_TRAITS, daemon: { ...TARGETED_TOUCH_INTERACTION_TRAITS.daemon, appLease: 'allowed' }, - frameworkTier: 'core', + frameworkTier: 'loop', timeoutPolicy: postActionObservationTimeoutPolicy('press', PRESERVE_DAEMON_TIMEOUT_POLICY), postActionObservation: postActionObservation('press'), responseDataTransform: TOUCH_INTERACTION_RESPONSE_DATA_TRANSFORM, @@ -380,7 +380,7 @@ export const DEVICE_OPERATION_COMMAND_DESCRIPTORS = { ? { ownerFiles: ['packages/command-surface/src/system/index.ts'] as const } : {}), catalog: { group: 'public' }, - frameworkTier: 'core', + frameworkTier: 'loop', // R42 retires this command's capability bucket and its `dispatch` leaf together: admission is // the owner's `back` fact, and the only execution is the bound operation — the postActionObservation // timeout trait it kept is admission-independent, like `type` kept its dialog guard. @@ -483,7 +483,7 @@ export const DEVICE_OPERATION_COMMAND_DESCRIPTORS = { ? { ownerFiles: ['packages/daemon/src/execution/scroll-runtime.ts'] as const } : {}), catalog: { group: 'public' }, - frameworkTier: 'core', + frameworkTier: 'loop', // R53 retires this command's capability bucket and its `dispatch` leaf together: admission is // the owner's `scrollDirection` fact, and the only execution is the bound operation. `scroll` // was the last holder of the legacy `dispatch`/`capability` pair, which retires with it. @@ -540,7 +540,7 @@ export const DEVICE_OPERATION_COMMAND_DESCRIPTORS = { } : {}), catalog: { group: 'public' }, - frameworkTier: 'core', + frameworkTier: 'loop', recordsSessionAction: true, recordingEffect: 'observes-app', daemon: { route: 'generic', refFrameEffect: 'preserve', appLease: 'allowed' }, diff --git a/packages/command-registry/src/descriptors/session-operations.ts b/packages/command-registry/src/descriptors/session-operations.ts index 37da302c34..341a6e72dc 100644 --- a/packages/command-registry/src/descriptors/session-operations.ts +++ b/packages/command-registry/src/descriptors/session-operations.ts @@ -385,7 +385,7 @@ export const SESSION_OPERATION_COMMAND_DESCRIPTORS = [ ? { ownerFiles: ['packages/command-surface/src/management/app.ts'] as const } : {}), catalog: { group: 'public' }, - frameworkTier: 'core', + frameworkTier: 'loop', recordsSessionAction: true, recordingEffect: 'mutates-app', daemon: { diff --git a/packages/command-registry/src/flag-definitions-workflow.ts b/packages/command-registry/src/flag-definitions-workflow.ts index a3ce7f70cd..33b972d394 100644 --- a/packages/command-registry/src/flag-definitions-workflow.ts +++ b/packages/command-registry/src/flag-definitions-workflow.ts @@ -269,7 +269,7 @@ export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageDescription: 'Snapshot: name the affordances merged inside an element (iOS sim); not directly invokable — reach them via the detail screen, labeled children, or coordinates', inputDescription: - 'Name the affordances an element merged away (iOS UIAccessibilityCustomAction, React Native accessibilityActions) — a card whose reply/options controls are not separate elements still lists them here. The names are for PLANNING, not invocation: there is no API to trigger them, so reach the affordance through the element detail screen, through the same control exposed as a labeled element elsewhere, or by coordinates from its rect. iOS simulator only; costs one accessibility round trip per merged element.', + "List the custom actions an element merged away, such as a card's reply control. For planning only: they cannot be invoked, so reach the control another way. iOS simulator only; costs one accessibility round trip per merged element.", projectConfig: true, recorded: true, }, diff --git a/packages/command-registry/src/registry.ts b/packages/command-registry/src/registry.ts index 11a7a0edb4..3230fb2d02 100644 --- a/packages/command-registry/src/registry.ts +++ b/packages/command-registry/src/registry.ts @@ -443,7 +443,7 @@ export const RAW_COMMAND_DESCRIPTORS = [ ? { ownerFiles: ['packages/command-surface/src/capture/snapshot.ts'] as const } : {}), catalog: { group: 'public' }, - frameworkTier: 'core', + frameworkTier: 'loop', recordsSessionAction: true, recordingEffect: 'observes-app', daemon: { route: 'snapshot', refFrameEffect: 'preserve', appLease: 'allowed' }, @@ -809,7 +809,7 @@ export function resolveCommandRecordsSessionAction(command: string | undefined): * `undefined` for a command that never declares one. A tier is declared iff * `catalog.group === 'public'`; that rule is a runtime pin, not a type rule — * `src/__tests__/command-descriptor-parity.test.ts` asserts it and the exact - * `'core'` tool set, so a new public command cannot join a framework adapter's + * `'loop'` and `'core'` tool sets, so a new public command cannot join a framework adapter's * default set silently. Framework adapters (`agent-device/ai-sdk`, * `@agent-device/eve`) read this to build their default tool set instead of * hand-listing tool names. diff --git a/packages/command-registry/src/types.ts b/packages/command-registry/src/types.ts index 5110445825..6ee3984401 100644 --- a/packages/command-registry/src/types.ts +++ b/packages/command-registry/src/types.ts @@ -115,6 +115,11 @@ export type CommandCatalogGroup = 'public' | 'internal' | 'local-cli'; /** * Which default tool set a framework adapter (`agent-device/ai-sdk`, the * planned `@agent-device/eve` package) includes a public command in. + * - `'loop'` — the part of the core set nearly every run calls: launch, + * observe, act, navigate back, wait, scroll. A `'core'` set + * includes it, and the MCP server marks these tools + * always-loaded, so a client that defers tool schemas sends + * them in the first prompt instead of searching for them. * - `'core'` — the small, curated perceive/act loop a typical tool-calling * agent needs by default: launch, observe, interact, read, * verify, wait, navigate, and system-dialog handling. @@ -128,7 +133,7 @@ export type CommandCatalogGroup = 'public' | 'internal' | 'local-cli'; * that command's descriptor facets instead of a separate hand-maintained list * in the adapter code; `resolveCommandFrameworkTier` reads it back. */ -export type CommandFrameworkTier = 'core' | 'extended'; +export type CommandFrameworkTier = 'loop' | 'core' | 'extended'; type CommandCatalogFacetBase = { /** diff --git a/packages/command-surface/src/batch/index.ts b/packages/command-surface/src/batch/index.ts index d2ce3e4714..3b1c2115f0 100644 --- a/packages/command-surface/src/batch/index.ts +++ b/packages/command-surface/src/batch/index.ts @@ -33,11 +33,27 @@ const batchCliReader: CliReader = (_positionals, flags) => ({ * example nobody can run (#2062). */ const BATCH_STEP_EXAMPLES: readonly BatchCommandStep[] = [ - { command: 'snapshot', input: { interactiveOnly: true } }, { command: 'press', input: { ref: '@e12', settle: true } }, + { command: 'press', input: { selector: 'label=Settings', settle: true } }, { command: 'fill', input: { selector: 'id="field-email"', text: 'qa@example.com' } }, + { command: 'scroll', input: { direction: 'down', until: 'label=Privacy', settle: true } }, + { command: 'back', input: { settle: true } }, + { command: 'wait', input: { kind: 'text', text: 'Saved' } }, + { command: 'snapshot', input: { interactiveOnly: true } }, ]; +/** + * The MCP description: a model batches a walk it already knows from this text alone, without + * loading each step command's schema. The step examples are the same tested ones `help batch` prints. + */ +const BATCH_MCP_DETAIL = [ + 'Use it for a known multi-step walk, such as press A, press B, back, instead of one tool call per step.', + "steps is an array of {command, input} objects; input takes the same arguments as that command's own tool (ref, selector, or x and y for targets; settle: true for the settled diff).", + 'Steps run serially in one session; onError: "stop" halts at the first failing step, and the result reports the steps that ran.', + 'The whole batch has a 60-second work budget; split longer walks.', + `Example steps, in order: ${BATCH_STEP_EXAMPLES.map((step) => JSON.stringify(step)).join(', ')}.`, +].join(' '); + /** * `help batch` documented neither the step shape nor which commands batch accepts, so an agent * discovered both by trial: `["press @e12"]` and `{"command":"press","args":[...]}` were refused @@ -64,6 +80,8 @@ const batchCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Run multiple commands', + mcpDetail: BATCH_MCP_DETAIL, + searchHint: 'batch sequence steps multi-step walk chain several taps in one call', cliDetail: buildBatchCliDetail(), }, metadata: batchCommandMetadata, diff --git a/packages/command-surface/src/capture/alert.ts b/packages/command-surface/src/capture/alert.ts index 353e5d79d1..a2ad24ae76 100644 --- a/packages/command-surface/src/capture/alert.ts +++ b/packages/command-surface/src/capture/alert.ts @@ -51,6 +51,7 @@ export const alertCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Inspect, accept, or dismiss a platform alert', + searchHint: 'dialog popup permission prompt allow deny system alert', }, metadata: alertCommandMetadata, run: (client, input) => client.command.alert(input), diff --git a/packages/command-surface/src/capture/diff.ts b/packages/command-surface/src/capture/diff.ts index b950dfad18..7054d9b3b3 100644 --- a/packages/command-surface/src/capture/diff.ts +++ b/packages/command-surface/src/capture/diff.ts @@ -114,6 +114,7 @@ export const diffCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Diff snapshot or screenshot', + searchHint: 'compare changes visual regression baseline', cliDetail: 'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; the current image scales proportionally to the baseline width and aligns top-left. Scaled height rounds to the nearest pixel (minimum 1). Extra or missing bottom rows always count as differences, even at threshold 1. The originals remain unchanged. When dimensions differ, normalization metadata reports original, scaled and compared sizes; region coordinates use comparison pixels. Both screenshot inputs are decoded from their bytes, so a baseline or current image may be PNG or JPEG, and the diff image is always PNG. JPEG is lossy, so keep --threshold above 0 whenever either input is JPEG. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines. Use --screen-mask black for a live iOS simulator capture with a black screen outline, and use the same option when creating its baseline. --screen-mask is refused when current.png is supplied; saved images are compared unchanged.', }, diff --git a/packages/command-surface/src/capture/screenshot.ts b/packages/command-surface/src/capture/screenshot.ts index 807b1dc500..2631c21675 100644 --- a/packages/command-surface/src/capture/screenshot.ts +++ b/packages/command-surface/src/capture/screenshot.ts @@ -28,7 +28,7 @@ import { defineFieldCommandMetadata } from '../field-command-contract.ts'; const SCREENSHOT_COMMAND_NAME = 'screenshot'; const screenshotCommandDescription = - 'Capture a screenshot of the active app or web session. Input fields control capture scope, density, size, and annotations.'; + 'Capture a screenshot of the active app or web session. Input fields control capture scope, density, size, and annotations. Use it for visual evidence, or when a snapshot warns that the accessibility tree is sparse: read the image, then press by coordinates. A screenshot alone does not verify a result.'; const screenshotCommandMetadata = defineFieldCommandMetadata( SCREENSHOT_COMMAND_NAME, @@ -84,6 +84,7 @@ export const screenshotCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Capture a screenshot', + searchHint: 'image picture screen capture png visual evidence', cliDetail: "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. Use --screen-mask black to fill the iOS simulator screen outline, including rounded corners, with black for this capture. It requires simctl: physical devices, tvOS and visionOS simulators, other platforms, provider captures and XCTest runner captures are refused. A masked capture fails if simctl fails instead of falling back to an unmasked runner capture. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, as is the small indicator window macOS adds in front of a window that a screen-sharing stream captures, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.", }, diff --git a/packages/command-surface/src/capture/settings.ts b/packages/command-surface/src/capture/settings.ts index ba10f29a76..a4521481de 100644 --- a/packages/command-surface/src/capture/settings.ts +++ b/packages/command-surface/src/capture/settings.ts @@ -101,6 +101,7 @@ export const settingsCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Change OS settings and app permissions', + searchHint: 'wifi airplane location dark mode appearance permission grant revoke', cliDetail: [ 'A bare readable setting reads the value the target holds. A target that does not serve a setting refuses it with UNSUPPORTED_OPERATION before anything changes.', '', diff --git a/packages/command-surface/src/capture/snapshot.ts b/packages/command-surface/src/capture/snapshot.ts index 97bd65f67a..42f5a9264b 100644 --- a/packages/command-surface/src/capture/snapshot.ts +++ b/packages/command-surface/src/capture/snapshot.ts @@ -18,7 +18,7 @@ import { defineFieldCommandMetadata } from '../field-command-contract.ts'; const SNAPSHOT_COMMAND_NAME = 'snapshot'; const snapshotCommandDescription = - 'Capture the accessibility tree or compare it with the previous session baseline. The result includes refs for semantic interactions; the diff option reports UI changes.'; + 'Capture the accessibility tree or compare it with the previous session baseline. The result includes refs for semantic interactions; the diff option reports UI changes. interactiveOnly lists only elements you can act on. After an action with settle, take a snapshot only when its diff lacks the next target or did not settle.'; const snapshotBackendCapabilityHelp = Object.entries(SNAPSHOT_BACKEND_CAPABILITIES) .map(([backend, capability]) => { @@ -76,6 +76,7 @@ export const snapshotCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Capture or diff the accessibility tree', + searchHint: 'ui tree elements refs inspect screen hierarchy', cliDetail: `Scoped output (--scope) renders every returned ref with its ref generation pin (@eN~s) and one Refs generation line, so a pasted ref targets the ref frame that issued it; --json keeps plain refs plus the response-level refsGeneration, and --raw keeps byte-for-byte output. Regular Android ADB snapshots clip app rectangles outside measured status and navigation bars, including launcher-owned navigation; fully covered content is omitted and --raw retains reported bounds. Ref and selector taps honor measured visible bounds even for refs from raw snapshots. Regular iOS snapshots clip overflowing elements to their visible area and omit fully outside elements. In a regular (non-raw) snapshot, a listed element whose frame has no area is marked [unmeasured]: nothing confirms it is on screen. If it is inside scrollable content, reach it with scroll --until before acting on it. --json reports clipping counts in snapshotQualityNotes with code regular-geometry-clipped; --raw retains reported frames. --observe-only is local iOS snapshot-only: an already-ready compatible runner reads the session app without activation or foreground repair, refuses unavailable observation or a readable system alert over the app, and reports XCUIApplication state rather than verified foreground ownership (#2696). Regular targetActivation disclosure stays unchanged (#2682, #2694). Repeated equivalent unfiltered Android snapshots return a compact unchanged acknowledgement. Use --force-full to re-emit the tree; --json and --raw retain full output. Android Appium JSON snapshots publish a display viewport only when measurements before and after the tree agree, independent of tree extent or keyboard content resizing. On Android, JSON value carries field content rather than static labels; saved selectors and replay scripts using value= for static labels must use label= instead; a reported showing hint means value is empty, and text equal to the placeholder is preserved when hint state is unknown. HarmonyOS reads the on-screen keyboard's own system window and publishes the keyboard field of --json output: the field is visibility-only — visible or hidden per the keyboard window's z-order column; a failed probe, an unreadable or conflicting z-order, or a table without the panel publishes unmeasurable, never a guessed band. When an iOS snapshot reports a screen viewport, inspect rects with snapshot -i --json and press the intended control's center, then verify with diff snapshot -i or snapshot --diff. When a fallback cannot establish the screen origin, viewport is omitted and coordinate-derived element actions refuse with window-origin-unknown before input; the snapshot may warn about the unavailable origin. Use a plain screenshot to choose screen coordinates. An empty text field with no label or value prints its hint as [placeholder:"..."]; label=, text= and find do not match that hint. iOS backend capability contract: ${snapshotBackendCapabilityHelp}.`, }, metadata: snapshotCommandMetadata, diff --git a/packages/command-surface/src/capture/wait.ts b/packages/command-surface/src/capture/wait.ts index 04239e2a33..cd368cedcf 100644 --- a/packages/command-surface/src/capture/wait.ts +++ b/packages/command-surface/src/capture/wait.ts @@ -59,7 +59,7 @@ export const WAIT_COMMAND_OUTPUT_SCHEMAS = { } satisfies Pick, 'wait'>; const waitCommandDescription = - 'Wait for a duration, text, snapshot ref, selector to appear, selector to be strictly absent, or stable UI. Text matches a label, identifier, or value; an editable field never matches by its current value or by a label that only repeats it, so text you typed does not satisfy it. The strict absence target requires zero selector matches; stable waits until the UI stays quiet for the requested window.'; + 'Wait for a duration, text, snapshot ref, selector to appear, selector to be strictly absent, or stable UI. Text matches a label, identifier, or value; an editable field never matches by its current value or by a label that only repeats it, so text you typed does not satisfy it. The strict absence target requires zero selector matches; stable waits until the UI stays quiet for the requested window. Use it to verify an expected result, such as text that appears after a network call, instead of polling with snapshot.'; const waitCommandMetadata = defineFieldCommandMetadata(WAIT_COMMAND_NAME, waitCommandDescription, { kind: enumField(WAIT_KIND_VALUES), @@ -97,6 +97,7 @@ export const waitCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Wait for a duration, text, selector, strict absence, or stable UI', + searchHint: 'wait until appear disappear sleep verify assert visible text', }, metadata: waitCommandMetadata, run: (client, input) => client.command.wait(waitInputToOptions(input)), diff --git a/packages/command-surface/src/command-contract.ts b/packages/command-surface/src/command-contract.ts index 754dbd7d63..e6977404a6 100644 --- a/packages/command-surface/src/command-contract.ts +++ b/packages/command-surface/src/command-contract.ts @@ -33,6 +33,8 @@ export type CommandEffect = 'read' | 'write' | 'destructive'; export type CommandMetadata = CommandInputMetadata & { title: string; effect: CommandEffect; + /** Deferred-tool search keywords from the command text; see `CommandText.searchHint`. */ + searchHint?: string; }; export type CommandInputMetadata = { diff --git a/packages/command-surface/src/command-output-schemas.ts b/packages/command-surface/src/command-output-schemas.ts index c78c596f15..ae8de1288f 100644 --- a/packages/command-surface/src/command-output-schemas.ts +++ b/packages/command-surface/src/command-output-schemas.ts @@ -41,7 +41,7 @@ import { SYSTEM_COMMAND_OUTPUT_SCHEMAS } from './system/index.ts'; * descriptors' `ownerFiles`), that module authors its entries and projects them into this * map instead of being hand-listed here. Two invariants: * - NEVER strict: no `additionalProperties: false` anywhere, so the additive - * `cost` object (opted in via `--cost` / `includeCost`) and any other additive + * `cost` object (opted in via `--cost` / `AGENT_DEVICE_COST`) and any other additive * fields ride into `structuredContent` and still validate. * - Accurate, never invented: required-vs-optional, enums, `const` discriminants * and discriminated-union branches mirror the source contract types. One schema diff --git a/packages/command-surface/src/common-input-fields.ts b/packages/command-surface/src/common-input-fields.ts index 1939b6011f..a121476114 100644 --- a/packages/command-surface/src/common-input-fields.ts +++ b/packages/command-surface/src/common-input-fields.ts @@ -130,6 +130,7 @@ const COMMON_INPUT_FIELDS = { enum: DEVICE_TARGETS, description: 'Alias for deviceTarget.', }, + audience: { kind: 'alias', of: 'deviceTarget' }, }, device: { schema: { @@ -264,6 +265,10 @@ const COMMON_INPUT_FIELDS = { debug: { schema: { type: 'boolean', description: 'Enable debug diagnostics.' }, read: (record) => optionalBoolean(record, 'debug'), + // Diagnostics for whoever debugs agent-device, not per-call work for a model. + audience: operatorAudience({ + operatorPath: 'Run the same command from a terminal with --debug to see diagnostics.', + }), }, } as const satisfies Record< keyof CommonCommandInput | 'target' | 'readinessTimeoutMs', @@ -273,7 +278,7 @@ const COMMON_INPUT_FIELDS = { const COMMON_INPUT_ROWS: ReadonlyArray = Object.entries(COMMON_INPUT_FIELDS); -/** Common keys no model-facing tool schema advertises or admits. */ +/** Common keys no model-facing tool schema advertises. */ export const COMMON_INPUT_AUDIENCE: InputAudienceMap = Object.fromEntries( COMMON_INPUT_ROWS.flatMap(([key, field]) => (field.audience ? [[key, field.audience]] : [])), ); diff --git a/packages/command-surface/src/debugging/index.ts b/packages/command-surface/src/debugging/index.ts index a556073126..b2f9369a35 100644 --- a/packages/command-surface/src/debugging/index.ts +++ b/packages/command-surface/src/debugging/index.ts @@ -47,6 +47,7 @@ export const debugCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Symbolicate Apple crash artifacts', + searchHint: 'crash report symbolicate ips dsym', }, metadata: debugCommandMetadata, run: (client, input) => client.debug.symbols(input), diff --git a/packages/command-surface/src/family/types.ts b/packages/command-surface/src/family/types.ts index 90b42a409a..c3900fd14a 100644 --- a/packages/command-surface/src/family/types.ts +++ b/packages/command-surface/src/family/types.ts @@ -94,7 +94,13 @@ export function defineCommandFacet = { name: metadata.name, description: metadata.description, diff --git a/packages/command-surface/src/index.ts b/packages/command-surface/src/index.ts index f0a824cfc5..c63cad31d6 100644 --- a/packages/command-surface/src/index.ts +++ b/packages/command-surface/src/index.ts @@ -15,7 +15,7 @@ export type { CommandName } from './command-metadata.ts'; export type { CommandExecutionResult } from './command-surface.ts'; export { COMMAND_OUTPUT_SCHEMAS } from './command-output-schemas.ts'; export { COMMON_INPUT_AUDIENCE } from './common-input-fields.ts'; -export { operatorAudience, operatorInputRefusal } from './input-audience.ts'; +export { aliasInputRefusal, operatorAudience, operatorInputRefusal } from './input-audience.ts'; export { buildRequestFlags } from './command-flags.ts'; export { runCommand } from './command-surface.ts'; export { readInputFromCli } from './cli-grammar/registry.ts'; diff --git a/packages/command-surface/src/input-audience.ts b/packages/command-surface/src/input-audience.ts index 5aa21d91f1..3c26965d4c 100644 --- a/packages/command-surface/src/input-audience.ts +++ b/packages/command-surface/src/input-audience.ts @@ -36,10 +36,16 @@ export type OperatorInputSource = * `retired` keys are released keys that were removed. They are absent from * every schema but still recognized, so supplying one answers with migration * guidance instead of silently dropping the value or rejecting it as unknown. + * + * `alias` keys are a second spelling of another key. The CLI and Node input + * keep them; a model-facing tool schema advertises only the canonical key, so + * the alias costs no schema bytes, and a call that passes it is refused with + * the key to use. */ export type InputAudience = | { kind: 'operator'; source: OperatorInputSource } - | { kind: 'retired'; message: string }; + | { kind: 'retired'; message: string } + | { kind: 'alias'; of: string }; /** Non-model audiences by input key. Keys absent from the map are model-writable. */ export type InputAudienceMap = Readonly>; @@ -58,6 +64,11 @@ export function operatorAudience(source: OperatorInputSource): InputAudience { return { kind: 'operator', source }; } +/** The refusal a model-facing surface answers with for an alias key. */ +export function aliasInputRefusal(key: string, canonicalKey: string): string { + return `${key} is not accepted as a tool argument. Pass ${canonicalKey}.`; +} + /** The refusal a model-facing surface answers with, rendered from the declared source. */ export function operatorInputRefusal(key: string, source: OperatorInputSource): string { return `${key} is not accepted as a tool argument. ${operatorPathSentence(key, source)}`; diff --git a/packages/command-surface/src/interaction/index.ts b/packages/command-surface/src/interaction/index.ts index 31377677de..dce907b0f0 100644 --- a/packages/command-surface/src/interaction/index.ts +++ b/packages/command-surface/src/interaction/index.ts @@ -459,6 +459,7 @@ const clickCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Click or tap a UI target', + searchHint: 'tap press button link web mouse', }, metadata: metadata('click'), run: (client, input) => client.interactions.click(toClickOptions(input)), @@ -472,6 +473,7 @@ const pressCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Short-press a UI target', + searchHint: 'tap click touch button ref selector coordinates', cliDetail: 'The hold duration is positional on longpress, not press --hold-ms.', }, metadata: metadata('press'), @@ -486,6 +488,7 @@ const fillCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Replace text in a UI input', + searchHint: 'type enter text input field form textbox', cliDetail: 'Every positional after an @ref is the replacement text, so fill @e57 good morning enters "good morning"; quote the text when the shell must preserve exact whitespace. Clear a field with an empty text argument: fill @e57 "" (the argument must be present — fill @e57 alone is a missing argument, not a clear). When visible label text also matches a non-input element, constrain the target with editable=true, for example fill \'label="Email" editable=true\' "qa@example.com". To keep a secret out of argv, omit the text and pipe it in: printf %s "$PASSWORD" | fill @e57 --text-stdin (max 64 KiB, one trailing newline removed). On web, a coordinate fill selects the clicked field in the page and refuses with COMMAND_FAILED and details.reason text_entry_focus_not_observed, before it sends any text or key, when the click did not reach an editable text field it can reach: a read-only, disabled, date, checkbox, or select control, a field in a cross-origin frame, a sandboxed frame without allow-same-origin, or a closed shadow root, or a click that missed. The click has already landed, so run snapshot -i again and fill the field it reports by ref (fill @e2 text); a selector fill reaches the same coordinate path and hits the same limits.', }, @@ -501,6 +504,7 @@ const longPressCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Hold a UI target to open a context menu', + searchHint: 'long press hold context menu', cliDetail: 'Duration is positional, for example longpress @e12 800 or longpress 300 500 800.', }, metadata: metadata('longpress'), @@ -515,6 +519,7 @@ const hoverCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Hover the pointer over a UI target (web only)', + searchHint: 'mouse over pointer reveal menu', cliDetail: 'The pointer stays where hover left it: read the revealed UI (--settle or snapshot -i) and act on it before another click or hover moves the pointer away.', }, @@ -530,6 +535,7 @@ const swipeCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Fling between coordinates', + searchHint: 'swipe drag fling gesture coordinates', cliDetail: 'The first read after a swipe waits for two captures to agree. --no-post-gesture-stabilization skips that wait for a caller whose own polling verifies the screen.', }, @@ -545,6 +551,7 @@ const focusCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Focus input at screen coordinates', + searchHint: 'focus input field cursor', }, metadata: metadata('focus'), run: (client, input) => client.interactions.focus(input as FocusOptions), @@ -558,6 +565,7 @@ const typeCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Append text to the focused input', + searchHint: 'type keys keyboard input text append', }, metadata: metadata('type'), run: (client, input) => client.interactions.type(input as TypeTextOptions), @@ -571,6 +579,7 @@ const scrollCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Scroll in a direction or to an edge', + searchHint: 'scroll swipe down up list find off-screen element', cliDetail: 'A caller that verifies the screen by polling can add --no-post-gesture-stabilization. The scroll then returns right after its gesture, with no movement field and no scroll_no_progress refusal, and the next read skips the stabilization wait. With --settle the settle observation already reports the outcome, and the option removes the same wait from its first capture.', }, @@ -586,6 +595,7 @@ const getCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Read element text or attributes', + searchHint: 'read text value attribute label', }, metadata: metadata('get'), run: (client, input) => client.interactions.get(toGetOptions(input)), @@ -599,6 +609,7 @@ const isCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Check a UI predicate on a selector', + searchHint: 'assert check visible hidden exists enabled selected', }, metadata: metadata('is'), run: (client, input) => client.interactions.is(input as IsOptions), @@ -612,6 +623,7 @@ const findCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Find an element and act', + searchHint: 'search locate element text click', }, metadata: metadata('find'), run: (client, input) => client.interactions.find(input as FindOptions), @@ -625,6 +637,7 @@ const gestureCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Run pan, fling, swipe, pinch, rotate, transform, tap, or drag gestures', + searchHint: 'gesture multi-touch pinch zoom drag pan', cliDetail: 'Argument shapes: pan [durationMs], fling [distance], swipe , pinch [x] [y], rotate [x] [y], transform [durationMs], tap [... ] [--hold-ms ] [--count ] [--pause-ms ], or drag [sourceHoldMs] [moveMs] [destinationHoldMs]. For command plans, output only command lines. Android transform verification should use all app-observable effects, for example wait text "pan changed yes", wait text "pinch changed yes", and wait text "rotate changed yes", not exact transform values. gesture swipe accepts --no-post-gesture-stabilization to skip the stabilization wait before the next read, for a caller whose own polling verifies the screen.', }, diff --git a/packages/command-surface/src/interaction/metadata.ts b/packages/command-surface/src/interaction/metadata.ts index e2bb68dcca..306586d230 100644 --- a/packages/command-surface/src/interaction/metadata.ts +++ b/packages/command-surface/src/interaction/metadata.ts @@ -65,10 +65,10 @@ const FIND_ACTION_VALUES = [ const interactionCommandDescriptions = { click: - "Activate a UI target by snapshot ref, selector, or coordinates. This can change app state. Success reports dispatch only; settle returns the resulting UI diff and verify returns cheap post-action evidence. A ref or selector target is re-checked for motion before dispatch when the session's previous action just animated the screen, and is dispatched at its resting position; one that has not stopped refuses with target_still_moving rather than acting where it was.", + "Activate a UI target by snapshot ref, selector, or coordinates. This can change app state. Success reports dispatch only; settle returns the resulting UI diff and verify returns cheap post-action evidence. A ref or selector target is re-checked for motion before dispatch when the session's previous action just animated the screen, and is dispatched at its resting position; one that has not stopped refuses with target_still_moving rather than acting where it was. With settle, continue from the returned diff. Copy refs exactly, including any ~sN suffix.", press: "Short-press a UI target by snapshot ref, selector, or coordinates. Success reports dispatch only; settle returns the resulting UI diff and verify returns cheap post-action evidence. A ref or selector target is re-checked for motion before dispatch when the session's previous action just animated the screen, and is dispatched at its resting position; one that has not stopped refuses with target_still_moving rather than acting where it was.", - fill: 'Replace text in a UI input selected by snapshot ref, selector, or coordinates. An empty text clears the field. Unconfirmed evidence does not establish the resulting value or screen. The recordAs option keeps sensitive text out of a recorded replay while sending it to the live app.', + fill: 'Replace text in a UI input selected by snapshot ref, selector, or coordinates. An empty text clears the field. Unconfirmed evidence does not establish the resulting value or screen. The recordAs option keeps sensitive text out of a recorded replay while sending it to the live app. To append instead of replacing, use type.', longpress: 'Hold a UI target by snapshot ref, selector, or coordinates to open a context menu or perform another hold gesture. The durationMs input overrides the default hold duration.', hover: diff --git a/packages/command-surface/src/management/app.ts b/packages/command-surface/src/management/app.ts index 3e170df543..0dff572035 100644 --- a/packages/command-surface/src/management/app.ts +++ b/packages/command-surface/src/management/app.ts @@ -43,7 +43,7 @@ const appsCommandMetadata = defineFieldCommandMetadata( const openCommandMetadata = defineFieldCommandMetadata( 'open', - 'Boot the selected device when needed, then open an app, deep link, or URL in a session. The app or URL inputs select the foreground automation target. Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', + 'Boot the selected device when needed, then open an app, deep link, or URL in a session. With a known app, start with open and foreground: it returns the initial interactive snapshot with refs, so do not probe with devices, apps, or snapshot first. For an unknown app id, list devices and apps; never guess an id. The app or URL inputs select the foreground automation target. Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', { app: stringField('App name, bundle id, package, or URL.'), url: stringField('Optional URL passed with an app shell.'), @@ -195,6 +195,7 @@ export const appsCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'List installed apps or deferred provider app assets', + searchHint: 'installed apps bundle id package list', cliDetail: 'Before provider allocation, lists uploaded app assets when the selected provider exposes a catalog. On a live device, defaults to user-installed apps; use --all to include system/OEM apps.', }, @@ -210,6 +211,7 @@ export const openCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Open an app, deep link or URL, save replays', + searchHint: 'launch start app deep link url relaunch session', cliDetail: 'Use --platform to bind URL/deep-link opens to the target platform. For local or Limrun iOS simulator initial stdout/stderr, put --launch-console on this open command, for example agent-device open "Agent Device Tester" --platform ios --launch-console artifacts/launch-console.log. Expo Go/dev-client shells accept host + URL, for example agent-device open "Expo Go" exp://127.0.0.1:8081 --platform ios. macOS also supports --surface app|frontmost-app|desktop|menubar. --metro-host/--metro-port/--bundle-url choose the app\'s Metro server and save that address for later metro reload calls in the same session. A fresh open without Metro hints clears an older same-name binding; close also clears it. --launch-url supplies the app\'s launch URL; it does not select the reload server. On an iOS simulator without --launch-url, open connects a detected Expo development client through its expo-development-client URL. The app must declare a single exp+ scheme that no other installed app owns. If detection is inconclusive, the app opens normally. Bare React Native apps receive their server settings before launch. On an iOS simulator, when --launch-url raises an "Open in ?" prompt that keeps open from seeing the app, open accepts it if the URL scheme belongs to the session app (data.launchConfirmation: "accepted"); if the scheme belongs to another installed app, the prompt stays on screen and open fails with details.reason launch_confirmation_foreign_app; if the scheme owner cannot be resolved because no single installed app owns it or the lookup failed, the prompt stays on screen and open returns with data.launchConfirmation: "unanswered", launchConfirmationReason url-owner-unresolved, and a warning; a prompt that is seen but not recognized is reported the same way with launchConfirmationReason alert-unrecognized and its own warning; for url-owner-unresolved, answer the prompt with alert accept; for alert-unrecognized, read the alert with alert get first, because it can be the app\'s own alert (such as a permission prompt) rather than the launch confirmation, and answer with alert accept only when it is the launch confirmation; when agent-device cannot read whether a prompt is up at all (the alert read, the accept, or the runner itself failed, including after a re-hand of the URL), open stays green, reports no launchConfirmation plus data.launchConfirmationProbeFailure naming the failed step (alert-read, alert-accept, or runner), and appends a warning that says to check with alert get. The prompt is read whenever open cannot see the app afterwards, including on a slow host where the app\'s target discovery fails on its own deadline. When an accept leaves the app provably not running, open hands the same URL over once more and reads again; still not running, it fails with details.reason launch_confirmation_unanswered. When the runner reports its main-thread timeout for the accept and the prompt is still up, open presses Open again, but only when the time the open request has left covers a whole press and the read that checks it. A recognized prompt still visible after a failed accept, or once that time is gone, also fails with this reason, even if the app is running behind it; answer it with alert accept or alert dismiss. A simulator openurl that CoreSimulator never answers fails within 20s with details.reason ios-simulator-openurl-timeout. The prompt is recognized by the OS\'s own wording of its title and buttons in the Simulator\'s preferred languages; when the runner cannot read that wording, the prompt stays on screen as alert-unrecognized. A prompt that appears after open has already seen the app is not answered; use alert accept. A device another session is holding refuses at once; add --wait to block up to that budget for it and only then fail with DEVICE_IN_USE naming the owning session.', mcpDetail: @@ -227,6 +229,7 @@ export const closeCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Close an app or end the session', + searchHint: 'close quit stop end session', }, metadata: closeCommandMetadata, run: (client, input) => diff --git a/packages/command-surface/src/management/artifacts.ts b/packages/command-surface/src/management/artifacts.ts index 06dd9e819a..6b7d074e28 100644 --- a/packages/command-surface/src/management/artifacts.ts +++ b/packages/command-surface/src/management/artifacts.ts @@ -37,6 +37,7 @@ export const artifactsCommandFacet = defineCommandFacet({ effect: 'read', text: { summary: 'List daemon or cloud provider session artifacts', + searchHint: 'artifacts files video logs download', }, metadata: artifactsCommandMetadata, run: (client, input) => client.sessions.artifacts(input), diff --git a/packages/command-surface/src/management/device.ts b/packages/command-surface/src/management/device.ts index 6992f05e5b..cef0e5f291 100644 --- a/packages/command-surface/src/management/device.ts +++ b/packages/command-surface/src/management/device.ts @@ -141,6 +141,7 @@ const devicesCommandFacet = defineCommandFacet({ effect: 'read', text: { summary: 'List available devices and simulators', + searchHint: 'devices simulators emulators list udid serial', }, metadata: devicesCommandMetadata, run: (client, input) => client.devices.list(input), @@ -154,6 +155,7 @@ const capabilitiesCommandFacet = defineCommandFacet({ effect: 'read', text: { summary: 'List supported commands for the selected device', + searchHint: 'capabilities supported commands device features', cliDetail: 'Select an explicit target with --platform/--device/--udid/--serial.', }, metadata: capabilitiesCommandMetadata, @@ -168,6 +170,7 @@ const bootCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Boot target device/simulator', + searchHint: 'boot start simulator emulator device', }, metadata: bootCommandMetadata, run: (client, input) => client.devices.boot(input), @@ -181,6 +184,7 @@ const shutdownCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Shutdown target simulator/emulator', + searchHint: 'shutdown stop simulator emulator', cliDetail: 'For an active session, use close --shutdown.', }, metadata: shutdownCommandMetadata, diff --git a/packages/command-surface/src/management/doctor.ts b/packages/command-surface/src/management/doctor.ts index ee077622b8..f8a99cb583 100644 --- a/packages/command-surface/src/management/doctor.ts +++ b/packages/command-surface/src/management/doctor.ts @@ -80,6 +80,7 @@ export const doctorCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Diagnose device, app, dev-server, and RN/Expo readiness', + searchHint: 'diagnose doctor health check setup troubleshoot', cliDetail: 'Metro reachability is inferred from cwd/runtime. On iOS simulators it also warms the XCTest runner build cache in the background when missing. Pass --app to verify a target app on the one matching booted device without opening a session. Use --remote to check remote connection setup without probing local devices. Default output is compact; use --json for full checks and evidence.', mcpDetail: diff --git a/packages/command-surface/src/management/install.ts b/packages/command-surface/src/management/install.ts index ad2cb4f6f3..9501db990c 100644 --- a/packages/command-surface/src/management/install.ts +++ b/packages/command-surface/src/management/install.ts @@ -136,6 +136,7 @@ const installCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Install an app binary from a path', + searchHint: 'install app apk ipa binary', cliDetail: 'A HarmonyOS .hap installs from its path alone, for example agent-device install ./build/entry-default-signed.hap --platform harmonyos.', }, @@ -151,6 +152,7 @@ const reinstallCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Replace an installed app with a new build', + searchHint: 'reinstall update app build apk ipa', }, metadata: reinstallCommandMetadata, run: (client, input) => client.apps.reinstall(input), @@ -164,6 +166,7 @@ const uninstallCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Remove an installed app', + searchHint: 'uninstall delete remove app', }, metadata: uninstallCommandMetadata, run: (client, input) => client.apps.uninstall(input), @@ -177,6 +180,7 @@ const installFromSourceCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Install app builds from URLs or CI artifacts', + searchHint: 'install download build url ci artifact', }, metadata: installFromSourceCommandMetadata, run: (client, input) => client.apps.installFromSource(input), diff --git a/packages/command-surface/src/management/prepare.ts b/packages/command-surface/src/management/prepare.ts index fe345e482a..0245efcbe2 100644 --- a/packages/command-surface/src/management/prepare.ts +++ b/packages/command-surface/src/management/prepare.ts @@ -122,6 +122,7 @@ export const prepareCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Pre-warm platform helpers before automation', + searchHint: 'prepare warm up runner helpers', }, metadata: prepareCommandMetadata, run: (client, input) => client.command.prepare(input), diff --git a/packages/command-surface/src/management/push.ts b/packages/command-surface/src/management/push.ts index 51e1c18b48..dbd7562850 100644 --- a/packages/command-surface/src/management/push.ts +++ b/packages/command-surface/src/management/push.ts @@ -132,6 +132,7 @@ const pushCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Deliver a push notification payload', + searchHint: 'push notification apns fcm payload', }, metadata: pushCommandMetadata, run: (client, input) => client.apps.push(input), @@ -145,6 +146,7 @@ const triggerAppEventCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Invoke an app-defined automation event', + searchHint: 'app event trigger automation hook', }, metadata: triggerAppEventCommandMetadata, run: (client, input) => client.apps.triggerEvent(input), diff --git a/packages/command-surface/src/management/session.ts b/packages/command-surface/src/management/session.ts index d8b421b08e..36300af96a 100644 --- a/packages/command-surface/src/management/session.ts +++ b/packages/command-surface/src/management/session.ts @@ -39,6 +39,7 @@ export const sessionCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'List sessions, show the state dir, or publish a script', + searchHint: 'sessions list state script publish', }, metadata: sessionCommandMetadata, run: async (client, { action, path, force, ...input }) => { diff --git a/packages/command-surface/src/management/viewport.ts b/packages/command-surface/src/management/viewport.ts index 2853a05a0f..51d9d5d0dc 100644 --- a/packages/command-surface/src/management/viewport.ts +++ b/packages/command-surface/src/management/viewport.ts @@ -59,6 +59,7 @@ export const viewportCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Resize the active web viewport for the current session', + searchHint: 'viewport resize browser window size', }, metadata: viewportCommandMetadata, run: (client, input) => client.command.viewport(input), diff --git a/packages/command-surface/src/metro/index.ts b/packages/command-surface/src/metro/index.ts index bd5dcdaa07..cb41d71d7f 100644 --- a/packages/command-surface/src/metro/index.ts +++ b/packages/command-surface/src/metro/index.ts @@ -131,6 +131,7 @@ export const metroCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Prepare the dev server or reload apps', + searchHint: 'metro bundler reload react native dev server', cliDetail: "reload with no --metro-host/--metro-port/--bundle-url resolves against the dev server this session last bound via metro prepare or open's metro hint flags (falling back to localhost:8081 only when the session never bound one), so it never silently reloads a different project's server on the default port; pass an explicit flag to override the session hint for one call. After prepare, reload uses the local server address independently of the public or proxy address given to the device; a fresh open preserves it only when its client-config runtime points to the prepared device server, while explicit reload flags override it. For open or an explicit bundle URL, reload removes a .bundle/.jsbundle/.js entry filename and keeps the remaining path as the server mount, and when the server has no HTTP /reload route (Expo) the reload is broadcast over its /message websocket instead of trusting the app-page fallback. The binding is cleared when the session closes, and a fresh open without hint flags also clears any leftover binding from a previous same-name session. --kind expo (detected or forced) requests the virtual-entry bundle URL (.expo/.virtual-metro-entry.bundle) instead of index.bundle, since index.bundle 404s/500s against Expo dev servers in monorepos. Dependency install auto-detects the package manager from the nearest yarn.lock/pnpm-lock.yaml/bun.lock/bun.lockb/package-lock.json walking up from --project-root (bounded at the repo root), so Yarn/pnpm workspace monorepos with the lockfile at the repo root do not wrongly fall back to npm install (which fails on workspace: dependency specifiers); if install still fails, pass --no-install-deps when dependencies are already installed (for example via a monorepo root install).", mcpDetail: diff --git a/packages/command-surface/src/observability/index.ts b/packages/command-surface/src/observability/index.ts index e93f69dc6f..3907ec8881 100644 --- a/packages/command-surface/src/observability/index.ts +++ b/packages/command-surface/src/observability/index.ts @@ -158,6 +158,7 @@ export const logsCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Manage session app logs', + searchHint: 'logs console output device log', }, metadata: logsCommandMetadata, run: (client, input) => client.observability.logs(input), @@ -171,6 +172,7 @@ export const eventsCommandFacet = defineCommandFacet({ effect: 'read', text: { summary: 'Read session event timeline', + searchHint: 'events timeline history session log', }, metadata: eventsCommandMetadata, run: (client, input) => client.observability.events(input), @@ -184,6 +186,7 @@ export const networkCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Inspect HTTP(S) traffic from session logs', + searchHint: 'network http requests traffic api calls', }, metadata: networkCommandMetadata, run: (client, input) => client.observability.network(input), @@ -197,6 +200,7 @@ export const audioCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Probe audio levels', + searchHint: 'audio sound volume level probe', }, metadata: audioCommandMetadata, run: (client, input) => client.observability.audio(input), diff --git a/packages/command-surface/src/perf/index.ts b/packages/command-surface/src/perf/index.ts index e5e853c095..1d12e567fb 100644 --- a/packages/command-surface/src/perf/index.ts +++ b/packages/command-surface/src/perf/index.ts @@ -73,6 +73,7 @@ export const perfCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Check frames, memory, or native profiles', + searchHint: 'performance fps frames memory profile cpu', cliDetail: 'Use perf frames for bounded frame-health evidence and perf memory sample for a compact process-memory reading. On iOS simulators and macOS, process sampling and captures target the resolved app executable and exclude other copies with the same name. Apple xctrace and Android Simpleperf/Perfetto captures keep raw artifacts on disk; report produces bounded agent-readable evidence. For React render performance, run agent-device react-devtools profile start before the interaction and react-devtools profile stop after it.', mcpDetail: diff --git a/packages/command-surface/src/post-action-observation-grammar.ts b/packages/command-surface/src/post-action-observation-grammar.ts index 9c0e4e0062..d0213a6039 100644 --- a/packages/command-surface/src/post-action-observation-grammar.ts +++ b/packages/command-surface/src/post-action-observation-grammar.ts @@ -31,12 +31,9 @@ const verifyField = () => const settleFields = () => ({ settle: booleanField( - 'After the action, wait for the UI to go quiet and return the settled diff vs the pre-action tree in the same response. Best-effort; never fails the action.', - ), - settleQuietMs: integerField( - 'Settle: quiet window in milliseconds (default 500; 300 on the iOS simulator snapshot bridge). On iOS, a broad replacement of a non-empty screen holds at least 1500ms within timeoutMs.', - { min: 0 }, + 'Wait for the UI to go quiet, then return the settled UI diff in this response. Best-effort; never fails the action.', ), + settleQuietMs: integerField('Settle: quiet window in milliseconds (default 500).', { min: 0 }), timeoutMs: integerField('Settle: wait deadline in milliseconds (default 10000).', { min: 1 }), }); diff --git a/packages/command-surface/src/react-native/index.ts b/packages/command-surface/src/react-native/index.ts index f28ec43d14..f71ba5f7dd 100644 --- a/packages/command-surface/src/react-native/index.ts +++ b/packages/command-surface/src/react-native/index.ts @@ -40,6 +40,7 @@ export const reactNativeCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Run React Native automation helpers', + searchHint: 'react native rn expo helpers', }, metadata: reactNativeCommandMetadata, run: (client, input) => client.command.reactNative(input), diff --git a/packages/command-surface/src/recording/index.ts b/packages/command-surface/src/recording/index.ts index 396af1c48e..e28d4dafc5 100644 --- a/packages/command-surface/src/recording/index.ts +++ b/packages/command-surface/src/recording/index.ts @@ -127,6 +127,7 @@ export const recordCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Start or stop screen recording', + searchHint: 'record video screen recording mp4', cliDetail: 'MP4 verification setup starts in the background before recording output preparation, without delaying capture. If setup exceeds the stop budget, details.reason is video_validator_compile_timeout (retriable); run record stop again while setup continues. Hosts without xcrun keep container-only MP4 completion. Other compilation failures, or an unavailable validator executable or AVFoundation during verification, return video_validator_unavailable. Repair the reported failure and retry record stop; a failed compile is retried on the next start or stop. The default --scope app requires an active app session from open ; use --scope device/system to explicitly request whole-screen recording where the selected backend supports it. Android record start publishes a durable device manifest, recordings longer than the 180s adb screenrecord limit are returned as multiple MP4 chunks while the daemon stays alive, and daemon-restart recovery uses only manifest-owned chunks. Android screenrecord encodes a frame only when the screen changes, so a clip can end at the last frame the recorder encoded instead of at record stop; durationMs is host wall clock from record start until the export finished, and capturedDurationMs reports the video timeline when it can be measured, with a warning naming how much of the window that video covers. An Android manifest left by an unreachable recording is retired on the next start once its recorders are proven gone; one still owned refuses with non-retriable DEVICE_IN_USE and reason native_recovery_evidence_open naming the session to run record stop for, and a recorder that is still writing an artifact refuses with DEVICE_IN_USE and reason native_recording_artifact_claimed, which clears itself once that recorder ends at the 180s limit. HarmonyOS supports whole-screen recording on physical devices only: use --scope device/system; --fps, --quality, and --hide-touches are unsupported. Use --quality to choose medium or high export quality on supported backends. The touch overlay renders at most 30 frames a second, or at --fps when lower; a lower --fps makes record stop faster on a long recording, and an overlay export that runs past its budget returns the raw video with an overlayWarning. An iOS simulator host recording lock returns non-retriable DEVICE_IN_USE with reason apple_simulator_recording_busy. Stop the recording in its owning session; if a dead recorder left the host locked, ask the host operator to restart the CoreSimulator stream service. On any backend, record start first settles the open recording records on its device that it would refuse on and whose session this daemon no longer holds: it finishes each one as its own record stop would, so the export lands at the output path of that recording and a later record stop in that session returns it, and runs the exact-owner recovery a daemon restart runs, which discards what that recording left, only for a record that cannot finish. A record it cannot confirm cleaned, or one whose session is still held, refuses the start with reason cleanup-unconfirmed and names that session: stop or close it, then retry. A record that another device left under the same session name is never settled from this device: the start refuses and names that device, so stop it in that session or record under another session name. A record stop that cannot produce its export keeps what a retry reads back — on Android the device-side artifact and the native manifest — leaves the recording manifest of that session open, and returns its own error, so retry record stop in that session; while this daemon holds that session, only closing it disposes them, and once it does not, the next record start on that device settles them as described above. Every stopped recording also reports what became of its recorder and of the file that recorder writes to: recorder is confirmed, or lost when the session holding it died, and nativePathDisposition is retirable while that file still owes a removal or retired once its removal was verified. Both are optional disclosures, not failures; ADR 0024 declares further states (unconfirmed, pending) for the steps that gain those probes, and no stop reports them yet. record contact-sheet [--out ] is a local report over a file you already exported: it decodes frames out of that MP4 and writes one PNG holding the cells where the screen visibly changed, each labeled with its elapsed time on the clip timeline. It needs no session and no device, rebuilds any past recording, and works for every backend that exports MP4; a WebM recording is refused with reason contact_sheet_container_unsupported. The sample grid is bounded and spread across the whole clip, so it reports coverage rather than a review: a flash that opens and closes entirely between two sample times is not in the sheet, which is why the result also reports how many times it sampled and how many frames came back. Frame decoding is Apple AVFoundation tooling, so the command runs only on macOS hosts and refuses elsewhere with reason contact_sheet_unsupported_host.', }, @@ -142,6 +143,7 @@ export const traceCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Start or stop trace capture', + searchHint: 'trace tracing profile capture', }, metadata: traceCommandMetadata, run: (client, input) => client.recording.trace(input), diff --git a/packages/command-surface/src/replay/index.ts b/packages/command-surface/src/replay/index.ts index d8204a9565..a24f673b78 100644 --- a/packages/command-surface/src/replay/index.ts +++ b/packages/command-surface/src/replay/index.ts @@ -312,6 +312,7 @@ export const replayCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Replay a recorded session or Maestro flow', + searchHint: 'replay script rerun recorded flow maestro', cliDetail: 'Replay has a 60-second work budget inside a 90-second client timeout; --timeout sets a work budget of X and client timeout of X + 30000 ms. The running step receives ordinary request cancellation at the budget. When it settles, failure returns details.reason daemon_request_timeout and details.timeoutMs, preserving the failing script path and line. The daemon and other sessions stay available if it responds; an authored close may already have ended its session. A completed replay returns success even if it settles just after the deadline. Cancellation does not prove native device work stopped. A step that ignores cancellation beyond the 30-second margin may cause a transport timeout; an unresponsive daemon retains the client liveness-based reset. Direct test keeps its own timeout behavior. For Maestro YAML compatibility flows, use replay --maestro and keep the target binding such as --platform ios on the replay command. A script with no terminal close leaves its session (and daemon) running until you close it or it idle-reaps — no different from a session opened interactively. For native .ad scripts, --keep-session suppresses exactly an authored terminal close so you can continue interactively. replay export converts compatible actions to Maestro YAML locally, including app switches with explicit launchApp.appId targets, deep links (including tel: and mailto:) as openLink, and home as pressKey: Home.', }, @@ -327,6 +328,7 @@ export const testCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Run replay test suites', + searchHint: 'test suite run tests replay ad', cliDetail: "Relative globs are expanded on the caller from its working directory, whose name is treated literally. Quote glob inputs to defer expansion to test. Copied diagnostic artifacts receive numbered filenames when needed to preserve other artifacts, replay sources, timing traces, and attempt manifests. JUnit reports (--reporter junit:) replace characters forbidden by XML 1.0 with U+FFFD and preserve legal Unicode and whitespace. JSON and other reporters retain the original suite values. Custom reporter getExitCode hooks must return an integer from 0 to 255 or undefined; the highest valid code wins and cannot lower a failing suite's exit code. A nested test stops after a bounded resource refusal even with --retries; other failures keep their configured retries.", }, diff --git a/packages/command-surface/src/schema/command-overrides.ts b/packages/command-surface/src/schema/command-overrides.ts index 3c639abd6b..ccbf4e7453 100644 --- a/packages/command-surface/src/schema/command-overrides.ts +++ b/packages/command-surface/src/schema/command-overrides.ts @@ -152,7 +152,7 @@ const SCHEMA_ONLY_CLI_COMMAND_SCHEMAS = { text: { summary: 'Start MCP server', description: - 'Start the official stdio MCP server. It exposes command tools backed by the agent-device client. Tool results are compact text by default; mcpOutputFormat: "json", a non-default responseLevel, or includeCost returns JSON text. With --structured, tools also advertise output schemas and return structuredContent, including errors, for clients that read results as data. It accepts no other flags or arguments besides --structured, --help, and --version; configure it with AGENT_DEVICE_* environment variables or AGENT_DEVICE_CONFIG.', + 'Start the official stdio MCP server. It exposes command tools backed by the agent-device client. Tool results are compact text by default; a non-default AGENT_DEVICE_RESPONSE_LEVEL or AGENT_DEVICE_COST returns JSON text. With --structured, tools also advertise output schemas and return structuredContent, including errors, for clients that read results as data. It accepts no other flags or arguments besides --structured, --help, and --version; configure it with AGENT_DEVICE_* environment variables or AGENT_DEVICE_CONFIG.', }, allowedFlags: ['mcpStructured'], }, diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 39594afc8e..f59787c94b 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -630,6 +630,7 @@ const appStateCommandFacet = defineParameterlessCommandFacet({ description: appStateCommandDescription, text: { summary: 'Show the foreground app and activity', + searchHint: 'foreground app current activity state', }, run: (client, input) => client.command.appState(input), }); @@ -639,6 +640,7 @@ const backCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Navigate back in the app or system', + searchHint: 'back navigate previous screen return dismiss', }, metadata: backCommandMetadata, run: (client, input) => client.command.back(input), @@ -653,6 +655,7 @@ const homeCommandFacet = defineParameterlessCommandFacet({ description: homeCommandDescription, text: { summary: 'Go to the device home screen', + searchHint: 'home screen springboard launcher', }, run: (client, input) => client.command.home(input), }); @@ -662,6 +665,7 @@ const orientationCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Set device orientation', + searchHint: 'orientation rotate landscape portrait', }, metadata: orientationCommandMetadata, run: (client, input) => client.command.orientation(input), @@ -675,6 +679,7 @@ const foldCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Fold or unfold a foldable iPhone simulator or Android emulator', + searchHint: 'fold unfold foldable posture', cliDetail: `iPhone Duo simulators and foldable Android emulators. iOS sends a ${foldGuidance.hingeEvent} and confirms the hinge angle inside the simulator; Android sets the emulator console posture and confirms the device state. Refs and coordinates do not survive a pose change.`, }, metadata: foldCommandMetadata, @@ -690,6 +695,7 @@ const appSwitcherCommandFacet = defineParameterlessCommandFacet({ description: appSwitcherCommandDescription, text: { summary: 'Open the device app switcher', + searchHint: 'app switcher recents multitasking', }, run: (client, input) => client.command.appSwitcher(input), }); @@ -699,6 +705,7 @@ const keyboardCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Inspect, press, or dismiss the device keyboard', + searchHint: 'keyboard hide dismiss return key', }, metadata: keyboardCommandMetadata, run: (client, input) => client.command.keyboard(input), @@ -712,6 +719,7 @@ const clipboardCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Read or write device clipboard text', + searchHint: 'clipboard copy paste pasteboard', }, metadata: clipboardCommandMetadata, run: (client, input) => client.command.clipboard(input as ClipboardCommandOptions), @@ -726,6 +734,7 @@ const actionButtonCommandFacet = defineParameterlessCommandFacet({ description: actionButtonCommandDescription, text: { summary: 'Press the iPhone or iPad Action Button', + searchHint: 'action button hardware', cliDetail: 'iPhone and iPad only. The runner asks the device for the button and reports unsupported when that model has none.', }, @@ -737,6 +746,7 @@ const tvRemoteCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Press a TV remote navigation, media, or hardware button', + searchHint: 'tv remote dpad select menu play', cliDetail: 'longpress selects the 500ms preset; --duration-ms overrides it. Android TV uses the system long-press duration.', }, @@ -752,6 +762,7 @@ const hardwareButtonCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Press a phone hardware button once', + searchHint: 'hardware button volume power lock', cliDetail: 'Single press; no hold. Android presses all eleven named keys on phones and tablets: volume-up, volume-down, power, mute, sleep, wake, and the five media keys. A physical iPhone or iPad presses volume only; the other keys are never supported on Apple targets. iOS Simulators, Android TV, and tvOS refuse — use tv-remote on TV targets.', }, @@ -767,6 +778,7 @@ const telephonyCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Simulate an incoming call or SMS on an Android emulator over adb', + searchHint: 'phone call sms telephony emulator', cliDetail: 'Android emulators through the local adb executor; WebDriver (Appium) and Limrun sessions refuse it. Quote the SMS text, or pass it as the remaining words. Re-snapshot after the event: the call screen or notification can cover the app.', }, diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index 3a1952b7fd..b1ef88d5f4 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -32,13 +32,9 @@ export type CaptureSnapshotOptions = AgentDeviceRequestOverrides & AgentDeviceSelectionOptions & Omit & { /** - * Name the affordances an element merged away (iOS UIAccessibilityCustomAction, - * React Native accessibilityActions) — a card whose reply/options controls are not - * separate elements still lists them here. The names are for PLANNING, not - * invocation: there is no API to trigger them, so reach the affordance through the - * element detail screen, through the same control exposed as a labeled element - * elsewhere, or by coordinates from its rect. iOS simulator only; costs one - * accessibility round trip per merged element. + * List the custom actions an element merged away, such as a card's reply control. + * For planning only: they cannot be invoked, so reach the control another way. iOS + * simulator only; costs one accessibility round trip per merged element. */ customActions?: SnapshotCommandOptionFields['customActions']; timeoutMs?: number; diff --git a/src/__tests__/command-descriptor-parity.test.ts b/src/__tests__/command-descriptor-parity.test.ts index c7716e1a5d..19b38b116c 100644 --- a/src/__tests__/command-descriptor-parity.test.ts +++ b/src/__tests__/command-descriptor-parity.test.ts @@ -270,7 +270,10 @@ test('frameworkTier is declared iff a command is public, and never elsewhere', ( // adding a command. test("frameworkTier pins exactly the 'core' command set", () => { const core = commandDescriptors - .filter((descriptor) => resolveCommandFrameworkTier(descriptor.name) === 'core') + .filter((descriptor) => { + const tier = resolveCommandFrameworkTier(descriptor.name); + return tier === 'loop' || tier === 'core'; + }) .map((descriptor) => descriptor.name) .sort(); @@ -294,6 +297,26 @@ test("frameworkTier pins exactly the 'core' command set", () => { ]); }); +// The 'loop' tier is the part of the core set the MCP server marks always-loaded: its schemas +// land in every run's first prompt, so widening it is a reviewed cost decision. +test("frameworkTier pins exactly the 'loop' command set", () => { + const loop = commandDescriptors + .filter((descriptor) => resolveCommandFrameworkTier(descriptor.name) === 'loop') + .map((descriptor) => descriptor.name) + .sort(); + + assert.deepEqual(loop, [ + 'back', + 'fill', + 'open', + 'press', + 'screenshot', + 'scroll', + 'snapshot', + 'wait', + ]); +}); + test('targetIdentityVerification pins exactly the evidence-carrying command set (ADR 0012 / #1349)', () => { const declared = RAW_COMMAND_DESCRIPTORS.flatMap((descriptor) => { const phase = resolveTargetIdentityVerification(descriptor.name); diff --git a/src/ai-sdk/__tests__/index.test.ts b/src/ai-sdk/__tests__/index.test.ts index d148ebf7aa..679a22a91e 100644 --- a/src/ai-sdk/__tests__/index.test.ts +++ b/src/ai-sdk/__tests__/index.test.ts @@ -100,7 +100,7 @@ test('defaults to the core tool set and pins the session for every call', async const schema = rawSchemaOf(tools, 'snapshot'); assert.equal('session' in schema.properties, false, 'session is always hidden from the model'); - assert.equal('mcpOutputFormat' in schema.properties, false, 'mcpOutputFormat is always hidden'); + assert.equal('responseLevel' in schema.properties, false, 'responseLevel is always hidden'); assert.ok('platform' in schema.properties, 'platform stays visible when not pinned'); await executeOf(tools, 'snapshot')({ interactiveOnly: true }); @@ -113,7 +113,7 @@ test('MCP transport/config controls are hidden from the schema and cannot reach const { tools } = await createAgentDeviceTools({ session: 'pinned-session' }); const schema = rawSchemaOf(tools, 'snapshot'); - for (const field of ['stateDir', 'includeCost', 'responseLevel', 'mcpOutputFormat']) { + for (const field of ['stateDir', 'cost', 'responseLevel']) { assert.equal(field in schema.properties, false, `${field} must not be model-visible`); } @@ -126,9 +126,8 @@ test('MCP transport/config controls are hidden from the schema and cannot reach )({ interactiveOnly: true, stateDir: '/tmp/attacker-controlled-state-dir', - includeCost: true, + cost: true, responseLevel: 'full', - mcpOutputFormat: 'json', session: 'attacker-supplied-session', }); diff --git a/src/ai-sdk/index.ts b/src/ai-sdk/index.ts index 8c69200246..a790a237ba 100644 --- a/src/ai-sdk/index.ts +++ b/src/ai-sdk/index.ts @@ -101,7 +101,7 @@ export async function createAgentDeviceTools( const tools: ToolSet = {}; for (const definition of listCommandTools({ outputSchemas: true })) { - if (set === 'core' && resolveCommandFrameworkTier(definition.name) !== 'core') continue; + if (set === 'core' && resolveCommandFrameworkTier(definition.name) === 'extended') continue; tools[definition.name] = tool({ description: definition.description, diff --git a/src/mcp/__tests__/command-tools-loading-meta.test.ts b/src/mcp/__tests__/command-tools-loading-meta.test.ts new file mode 100644 index 0000000000..c0ce1d7361 --- /dev/null +++ b/src/mcp/__tests__/command-tools-loading-meta.test.ts @@ -0,0 +1,44 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { resolveCommandFrameworkTier } from '@agent-device/command-registry/registry'; +import { listCommandTools } from '../command-tools.ts'; + +// Claude Code defers MCP tool schemas: each tool the model needs costs a tool-search turn before +// its first call. `anthropic/alwaysLoad` sends a tool's schema in the first prompt instead, and +// `anthropic/searchHint` adds keywords to the search. +const tools = listCommandTools({ outputSchemas: false }); + +test("exactly the 'loop' framework tier is always loaded", () => { + const alwaysLoaded = tools + .filter((tool) => tool._meta['anthropic/alwaysLoad'] === true) + .map((tool) => tool.name); + const loopTier = tools + .filter((tool) => resolveCommandFrameworkTier(tool.name) === 'loop') + .map((tool) => tool.name); + assert.deepEqual(alwaysLoaded, loopTier); + assert.ok(alwaysLoaded.includes('press')); +}); + +test('every tool declares search keywords', () => { + for (const tool of tools) { + const hint = tool._meta['anthropic/searchHint']; + assert.ok(hint && hint.trim().length > 0, `${tool.name} declares no searchHint`); + } +}); + +// Always-loaded schemas land in every run's first prompt, so their total size is a per-run cost. +const ALWAYS_LOADED_BUDGET_BYTES = 26 * 1024; + +test('the always-loaded tools stay within their byte budget', () => { + const bytes = Buffer.byteLength( + JSON.stringify(tools.filter((tool) => tool._meta['anthropic/alwaysLoad'] === true)), + ); + assert.ok(bytes <= ALWAYS_LOADED_BUDGET_BYTES, `always-loaded tools are ${bytes} bytes`); +}); + +test('the batch description states the step shape and example steps', () => { + const batch = tools.find((tool) => tool.name === 'batch'); + assert.ok(batch); + assert.match(batch.description, /array of \{command, input\} objects/); + assert.match(batch.description, /\{"command":"press","input":\{"ref":"@e12","settle":true\}\}/); +}); diff --git a/src/mcp/__tests__/command-tools-operator-inputs.test.ts b/src/mcp/__tests__/command-tools-operator-inputs.test.ts index 61c8eecc1a..83374cab82 100644 --- a/src/mcp/__tests__/command-tools-operator-inputs.test.ts +++ b/src/mcp/__tests__/command-tools-operator-inputs.test.ts @@ -30,6 +30,10 @@ const OPERATOR_OWNED_KEYS = [ 'tenant', 'runId', 'leaseId', + // Diagnostics and response shaping: set once for the server, not per call. + 'debug', + 'cost', + 'responseLevel', ] as const; // The rest of the shared common input: keys a model may write, because they @@ -38,11 +42,9 @@ const MODEL_WRITABLE_COMMON_KEYS = [ 'session', 'platform', 'deviceTarget', - 'target', 'device', 'udid', 'serial', - 'debug', ] as const; test('MCP tool schemas advertise no operator-owned inputs', () => { @@ -88,6 +90,34 @@ test('MCP refuses every explicit operator-owned argument with guidance', async ( assert.deepEqual(calls, [], 'a refused operator input must never reach the command route'); }); +// The CLI and Node input keep the `target` spelling; a tool schema carries only deviceTarget. +test('MCP neither advertises nor admits the target alias of deviceTarget', async () => { + for (const tool of listCommandTools({ outputSchemas: false })) { + assert.equal( + 'target' in (tool.inputSchema.properties ?? {}), + false, + `${tool.name} advertises it`, + ); + } + const calls: unknown[] = []; + const executor = createCommandToolExecutor({ + createClient: () => ({}) as AgentDeviceClient, + runCommand: async (_client, name, input) => { + calls.push({ name, input }); + return {}; + }, + }); + + const result = await executor.execute('wait', { target: 'tv' }); + + assert.equal(result.isError, true); + assert.match( + result.content[0]?.text ?? '', + /target is not accepted as a tool argument\. Pass deviceTarget\./, + ); + assert.deepEqual(calls, []); +}); + test('MCP neither advertises nor admits the retired readiness budget', async () => { for (const tool of listCommandTools({ outputSchemas: false })) { const properties = tool.inputSchema.properties ?? {}; @@ -220,6 +250,8 @@ test('MCP still resolves operator env values outside the model-writable surface' vi.stubEnv('AGENT_DEVICE_RUN_ID', 'operator-run'); vi.stubEnv('AGENT_DEVICE_LEASE_ID', 'operator-lease'); vi.stubEnv('AGENT_DEVICE_ANDROID_DEVICE_ALLOWLIST', 'emulator-5554'); + vi.stubEnv('AGENT_DEVICE_COST', '1'); + vi.stubEnv('AGENT_DEVICE_RESPONSE_LEVEL', 'digest'); const createdConfigs: Array> = []; const calls: Array<{ name: string; input: Record }> = []; const executor = createCommandToolExecutor({ @@ -242,6 +274,11 @@ test('MCP still resolves operator env values outside the model-writable surface' assert.equal(calls[0]?.input.leaseId, 'operator-lease'); assert.equal(calls[0]?.input.androidDeviceAllowlist, 'emulator-5554'); assert.equal(createdConfigs[0]?.stateDir, '/operator/state-dir'); + assert.equal(createdConfigs[0]?.cost, true); + assert.equal(createdConfigs[0]?.responseLevel, 'digest'); + // MCP controls configure the client; they never reach the command route. + assert.equal('cost' in (calls[0]?.input ?? {}), false); + assert.equal('responseLevel' in (calls[0]?.input ?? {}), false); }); test('MCP neither advertises nor admits the CLI-only fill textStdin marker', async () => { diff --git a/src/mcp/__tests__/command-tools-parity.test.ts b/src/mcp/__tests__/command-tools-parity.test.ts index c8088c4e6d..9bf9ec89df 100644 --- a/src/mcp/__tests__/command-tools-parity.test.ts +++ b/src/mcp/__tests__/command-tools-parity.test.ts @@ -82,7 +82,8 @@ test('MCP open keeps device-selection evidence in structured content and JSON te runCommand: async () => openResult, }); - const result = await executor.execute('open', { app: 'Settings', mcpOutputFormat: 'json' }); + vi.stubEnv('AGENT_DEVICE_RESPONSE_LEVEL', 'full'); + const result = await executor.execute('open', { app: 'Settings' }); assert.deepEqual(result.structuredContent, openResult); assert.deepEqual(JSON.parse(result.content[0]?.text ?? '{}').selection, openResult.selection); diff --git a/src/mcp/__tests__/command-tools-structured-content.test.ts b/src/mcp/__tests__/command-tools-structured-content.test.ts index f65ad7cfd7..d70baa94f9 100644 --- a/src/mcp/__tests__/command-tools-structured-content.test.ts +++ b/src/mcp/__tests__/command-tools-structured-content.test.ts @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { test } from 'vitest'; +import { test, vi } from 'vitest'; import type { AgentDeviceClient } from '@agent-device/command-surface'; import { AppError } from '@agent-device/kernel/errors'; import { createCommandToolExecutor } from '../command-tools.ts'; @@ -28,7 +28,8 @@ test('the default executor returns text only, from typed and untyped tools', asy const press = await executor.execute('press', { ref: '@e2' }); const snapshot = await executor.execute('snapshot', {}); - const json = await executor.execute('snapshot', { mcpOutputFormat: 'json' }); + vi.stubEnv('AGENT_DEVICE_RESPONSE_LEVEL', 'full'); + const json = await executor.execute('snapshot', {}); assert.equal('structuredContent' in press, false); assert.equal(press.content[0]?.text, 'Tapped @e2 (10, 20)'); @@ -50,11 +51,12 @@ test('the default executor returns a failed call as error text only', async () = assert.match(result.content[0]?.text ?? '', /^Error \(COMMAND_FAILED\): snapshot failed/); }); -test('the default executor renders includeCost as JSON text, because the cost rides only on the data', async () => { +test('the default executor renders the cost setting as JSON text, because the cost rides only on the data', async () => { const waitResult = { waitedMs: 42, cost: { wallClockMs: 42, runnerRoundTrips: 0 } }; const executor = executorReturning(undefined, () => waitResult); - const result = await executor.execute('wait', { includeCost: true }); + vi.stubEnv('AGENT_DEVICE_COST', '1'); + const result = await executor.execute('wait', {}); assert.equal('structuredContent' in result, false); assert.deepEqual(JSON.parse(result.content[0]?.text ?? ''), waitResult); @@ -67,8 +69,10 @@ test('declared returns structuredContent for typed tools, and for untyped tools assert.equal(compact.structuredContent, undefined); assert.match(compact.content[0]?.text ?? '', /@e2 \[button\] "Save"/); - const json = await executor.execute('snapshot', { mcpOutputFormat: 'json' }); + vi.stubEnv('AGENT_DEVICE_RESPONSE_LEVEL', 'full'); + const json = await executor.execute('snapshot', {}); assert.deepEqual(json.structuredContent, snapshotResult); + vi.unstubAllEnvs(); const press = await executor.execute('press', { ref: '@e2' }); assert.deepEqual(press.structuredContent, { message: 'Tapped @e2 (10, 20)' }); @@ -83,15 +87,19 @@ test('a data host gets structuredContent from untyped tools by default', async ( }); test.each([ - ['a non-default responseLevel', { responseLevel: 'digest' }], - ['includeCost', { includeCost: true }], -])('declared returns structuredContent from an untyped tool with %s', async (_case, opt) => { - const executor = executorReturning('declared', () => ({ path: '/tmp/shot.png' })); + ['a non-default response level', ['AGENT_DEVICE_RESPONSE_LEVEL', 'digest']], + ['the cost setting', ['AGENT_DEVICE_COST', '1']], +] as const)( + 'declared returns structuredContent from an untyped tool with %s', + async (_case, [name, value]) => { + const executor = executorReturning('declared', () => ({ path: '/tmp/shot.png' })); - const result = await executor.execute('screenshot', opt); + vi.stubEnv(name, value); + const result = await executor.execute('screenshot', {}); - assert.deepEqual(result.structuredContent, { path: '/tmp/shot.png' }); -}); + assert.deepEqual(result.structuredContent, { path: '/tmp/shot.png' }); + }, +); test('declared returns a failed untyped tool error in structuredContent', async () => { const executor = executorReturning('declared', () => { @@ -125,7 +133,8 @@ test('MCP command tool executor renders JSON text when requested', async () => { }, }); - const result = await executor.execute('snapshot', { mcpOutputFormat: 'json' }); + vi.stubEnv('AGENT_DEVICE_RESPONSE_LEVEL', 'full'); + const result = await executor.execute('snapshot', {}); // Compact: JSON text is what a text-only client shows the model. assert.equal( @@ -134,7 +143,7 @@ test('MCP command tool executor renders JSON text when requested', async () => { ); }); -test('default MCP errors retain possible dispatch, observe-first recovery and the local log path', async () => { +test('default MCP errors retain possible dispatch and observe-first recovery, and leave out the log path', async () => { const executor = executorReturning(undefined, () => { const error = new AppError('COMMAND_FAILED', 'press reply was lost', { dispatched: 'unknown', @@ -151,5 +160,24 @@ test('default MCP errors retain possible dispatch, observe-first recovery and th const text = result.content[0]?.text ?? ''; assert.match(text, /May have reached the device\./); assert.match(text, /Run snapshot to check the screen before you retry\./); + assert.match(text, /^Error \(COMMAND_FAILED, runner_reply_lost\): press reply was lost\./); + // The default recovery step reads the log, so the path stays for that step. assert.match(text, /Log: \/tmp\/agent-device\/press\.log/); }); + +test('MCP errors with a specific recovery step leave out the log path', async () => { + const executor = executorReturning(undefined, () => { + throw new AppError('COMMAND_FAILED', 'press reply was lost', { + reason: 'runner_reply_lost', + hint: 'Run snapshot, then press the target again.', + logPath: '/tmp/agent-device/press.log', + }); + }); + + const result = await executor.execute('press', { ref: '@e2' }); + + assert.equal( + result.content[0]?.text, + 'Error (COMMAND_FAILED, runner_reply_lost): press reply was lost. Next: Run snapshot, then press the target again.', + ); +}); diff --git a/src/mcp/__tests__/command-tools.test.ts b/src/mcp/__tests__/command-tools.test.ts index 91fd106fce..2834ba7495 100644 --- a/src/mcp/__tests__/command-tools.test.ts +++ b/src/mcp/__tests__/command-tools.test.ts @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { test } from 'vitest'; +import { test, vi } from 'vitest'; import type { AgentDeviceClient } from '@agent-device/command-surface'; import { createCommandToolExecutor, listCommandTools } from '../command-tools.ts'; import { resolveCommandRecordsSessionAction } from '@agent-device/command-registry/registry'; @@ -23,10 +23,8 @@ test('MCP command tool executor hides client creation behind an execution adapte }, }); - const result = await executor.execute('wait', { - includeCost: true, - mcpOutputFormat: 'optimized', - }); + vi.stubEnv('AGENT_DEVICE_COST', '1'); + const result = await executor.execute('wait', {}); assert.deepEqual(createdConfigs, [{ cost: true }]); assert.deepEqual(calls, [ @@ -37,7 +35,7 @@ test('MCP command tool executor hides client creation behind an execution adapte }, ]); assert.deepEqual(result.structuredContent, { message: 'Ran wait', ok: true }); - // The cost rides only on the data, so includeCost renders the result as JSON text. + // The cost rides only on the data, so the cost setting renders the result as JSON text. assert.deepEqual(JSON.parse(result.content[0]?.text ?? ''), { message: 'Ran wait', ok: true }); }); @@ -69,7 +67,7 @@ test('MCP renders a non-default response level as JSON text, not misleading opti // With responseLevel:digest the daemon returns the digest shape (no `nodes`). // The optimized snapshot formatter expects `nodes` and would print // "Snapshot: 0 nodes" — contradicting structuredContent. The text must instead - // be the digest payload verbatim (JSON), even though mcpOutputFormat is optimized. + // be the digest payload verbatim (JSON). const digest = { nodeCount: 3, refs: [{ ref: 'e1', label: 'Continue' }], truncated: false }; const executor = createCommandToolExecutor({ structuredContent: 'declared', @@ -77,10 +75,8 @@ test('MCP renders a non-default response level as JSON text, not misleading opti runCommand: async () => digest, }); - const result = await executor.execute('snapshot', { - mcpOutputFormat: 'optimized', - responseLevel: 'digest', - }); + vi.stubEnv('AGENT_DEVICE_RESPONSE_LEVEL', 'digest'); + const result = await executor.execute('snapshot', {}); assert.deepEqual(result.structuredContent, digest); assert.match(result.content[0]?.text ?? '', /^\{/); @@ -88,25 +84,14 @@ test('MCP renders a non-default response level as JSON text, not misleading opti assert.doesNotMatch(result.content[0]?.text ?? '', /Snapshot: 0 nodes/); }); -test('MCP tool schemas add MCP client config fields at the MCP boundary', () => { - const devicesTool = listCommandTools({ outputSchemas: true }).find( - (tool) => tool.name === 'devices', - ); - - assert.ok(devicesTool); - assert.equal('stateDir' in (devicesTool.inputSchema.properties ?? {}), false); - assert.deepEqual( - (devicesTool.inputSchema.properties?.mcpOutputFormat as { enum?: unknown[] } | undefined)?.enum, - ['optimized', 'json'], - ); - assert.equal( - (devicesTool.inputSchema.properties?.includeCost as { type?: string } | undefined)?.type, - 'boolean', - ); - assert.deepEqual( - (devicesTool.inputSchema.properties?.responseLevel as { enum?: unknown[] } | undefined)?.enum, - ['digest', 'default', 'full'], - ); +// The MCP controls are operator settings, so no tool schema spends bytes on them. +test('MCP tool schemas advertise none of the MCP control fields', () => { + for (const tool of listCommandTools({ outputSchemas: true })) { + const properties = tool.inputSchema.properties ?? {}; + for (const key of ['stateDir', 'cost', 'responseLevel', 'mcpOutputFormat', 'includeCost']) { + assert.equal(key in properties, false, `${tool.name} advertises ${key}`); + } + } }); // #1625: `list` must be accepted by the MCP input surface, not only the CLI @@ -268,7 +253,7 @@ test('MCP swipe tool exposes bounded repetition inputs', () => { }); }); -test('MCP includeCost:true opts into agent-cost: sets client.cost, strips the arg, surfaces cost', async () => { +test('AGENT_DEVICE_COST opts into agent-cost: sets client.cost and surfaces cost', async () => { const createdConfigs: unknown[] = []; const calls: unknown[] = []; const executor = createCommandToolExecutor({ @@ -283,11 +268,12 @@ test('MCP includeCost:true opts into agent-cost: sets client.cost, strips the ar }, }); - const result = await executor.execute('wait', { includeCost: true }); + vi.stubEnv('AGENT_DEVICE_COST', '1'); + const result = await executor.execute('wait', {}); - // includeCost maps to the client `cost` config (→ meta.includeCost on the daemon). + // The cost setting maps to the client `cost` config (→ meta.includeCost on the daemon). assert.deepEqual(createdConfigs, [{ cost: true }]); - // includeCost is an MCP-boundary field and must not leak into the command input. + // It is an MCP-boundary field and must not leak into the command input. assert.deepEqual(calls, [{ name: 'wait', input: {} }]); // The daemon-provided cost rides through structuredContent unchanged. assert.deepEqual(result.structuredContent, { @@ -296,7 +282,7 @@ test('MCP includeCost:true opts into agent-cost: sets client.cost, strips the ar }); }); -test('MCP includeCost absent/false leaves the request shape untouched (no cost config)', async () => { +test('an absent or false cost setting leaves the request shape untouched (no cost config)', async () => { const createdConfigs: unknown[] = []; const executor = createCommandToolExecutor({ structuredContent: 'declared', @@ -308,7 +294,8 @@ test('MCP includeCost absent/false leaves the request shape untouched (no cost c }); const absent = await executor.execute('wait', {}); - const explicitFalse = await executor.execute('wait', { includeCost: false }); + vi.stubEnv('AGENT_DEVICE_COST', '0'); + const explicitFalse = await executor.execute('wait', {}); // Neither path sets `cost` on the client config; both are byte-identical configs. assert.deepEqual(createdConfigs, [{}, {}]); @@ -316,19 +303,7 @@ test('MCP includeCost absent/false leaves the request shape untouched (no cost c assert.equal(JSON.stringify(absent), JSON.stringify(explicitFalse)); }); -test('MCP includeCost rejects non-boolean values at the boundary', async () => { - const executor = createCommandToolExecutor({ - createClient: () => ({}) as AgentDeviceClient, - runCommand: async () => ({}), - }); - - await assert.rejects( - executor.execute('wait', { includeCost: 'yes' }), - /Expected includeCost to be a boolean\./, - ); -}); - -test('MCP responseLevel:digest opts into a verbosity level: sets client.responseLevel, strips the arg', async () => { +test('AGENT_DEVICE_RESPONSE_LEVEL opts into a verbosity level: sets client.responseLevel', async () => { const createdConfigs: unknown[] = []; const calls: unknown[] = []; const executor = createCommandToolExecutor({ @@ -343,7 +318,8 @@ test('MCP responseLevel:digest opts into a verbosity level: sets client.response }, }); - const result = await executor.execute('wait', { responseLevel: 'digest' }); + vi.stubEnv('AGENT_DEVICE_RESPONSE_LEVEL', 'digest'); + const result = await executor.execute('wait', {}); // responseLevel maps to the client `responseLevel` config (→ meta.responseLevel on the daemon). assert.deepEqual(createdConfigs, [{ responseLevel: 'digest' }]); @@ -370,18 +346,6 @@ test('MCP responseLevel absent leaves the request shape untouched (no responseLe assert.deepEqual(absent.structuredContent, { message: 'ok' }); }); -test('MCP responseLevel rejects unknown values at the boundary', async () => { - const executor = createCommandToolExecutor({ - createClient: () => ({}) as AgentDeviceClient, - runCommand: async () => ({}), - }); - - await assert.rejects( - executor.execute('wait', { responseLevel: 'verbose' }), - /Expected responseLevel to be one of 'digest', 'default', or 'full'\./, - ); -}); - test('MCP keyboard outputSchema advertises flat contract discriminants', () => { const tools = listCommandTools({ outputSchemas: true }); diff --git a/src/mcp/command-tools.ts b/src/mcp/command-tools.ts index f5849c0ced..0ad1bbb08d 100644 --- a/src/mcp/command-tools.ts +++ b/src/mcp/command-tools.ts @@ -1,5 +1,6 @@ import type { AgentDeviceClientConfig } from '@agent-device/contracts/client'; import { + aliasInputRefusal, COMMON_INPUT_AUDIENCE, commandInputExample, findCommandMetadata, @@ -26,6 +27,7 @@ import type { EnvMap, HostEnvironment } from '@agent-device/kernel/source-value' import { mcpBody } from '@agent-device/command-registry/command-text'; import { resolveStructuredBatchCommandName } from '@agent-device/command-registry/batch-policy'; import { + resolveCommandFrameworkTier, resolveCommandRecordsSessionAction, resolveCommandTimeoutPolicy, } from '@agent-device/command-registry/registry'; @@ -83,13 +85,21 @@ export type CommandToolExecutor = { errorResult: (error: unknown) => Promise; }; -type McpOutputFormat = 'optimized' | 'json'; - type McpToolConfig = { client: AgentDeviceClientConfig; - outputFormat: McpOutputFormat; }; +/** + * A common alias (`target` for `deviceTarget`) yields to a command that declares the same key + * itself: `press` retired its own nested `target`, and that migration guidance must still answer. + */ +const COMMON_ALIAS_AUDIENCE: InputAudienceMap = Object.fromEntries( + Object.entries(COMMON_INPUT_AUDIENCE).filter(([, audience]) => audience.kind === 'alias'), +); +const COMMON_INPUT_AUDIENCE_EXCEPT_ALIASES: InputAudienceMap = Object.fromEntries( + Object.entries(COMMON_INPUT_AUDIENCE).filter(([, audience]) => audience.kind !== 'alias'), +); + /** * Every `operator` key any command declares. The deny side is deliberately * global rather than per-command: a screen that steers the model into writing a @@ -119,14 +129,25 @@ const DECLARED_OPERATOR_AUDIENCE: InputAudienceMap = Object.fromEntries( */ function toolInputAudience(metadata: AdmissionMetadata): InputAudienceMap { return { + ...COMMON_ALIAS_AUDIENCE, ...metadata.inputAudience, - ...COMMON_INPUT_AUDIENCE, + ...COMMON_INPUT_AUDIENCE_EXCEPT_ALIASES, ...MCP_TOOL_CONFIG_AUDIENCE, ...CONFIG_LOADER_AUDIENCE, ...DECLARED_OPERATOR_AUDIENCE, }; } +/** + * Tool-loading hints Claude Code reads from a tool's `_meta`. A client that defers tool schemas + * loads an `alwaysLoad` tool up front instead of making the model search for it, and matches + * `searchHint` keywords when the model does search. + */ +type ToolLoadingMeta = { + 'anthropic/searchHint'?: string; + 'anthropic/alwaysLoad'?: true; +}; + /** `outputSchemas` advertises each typed tool's outputSchema; only a structured host returns data that matches it. */ export function listCommandTools(options: { outputSchemas: boolean }): Array<{ name: string; @@ -135,6 +156,7 @@ export function listCommandTools(options: { outputSchemas: boolean }): Array<{ description: string; inputSchema: JsonSchema; outputSchema?: JsonSchema; + _meta: ToolLoadingMeta; }> { return listMcpCommandMetadata().map((definition) => { // The registry is keyed by the typed-result commands only (CommandResultMap), @@ -156,10 +178,23 @@ export function listCommandTools(options: { outputSchemas: boolean }): Array<{ // Only typed commands carry an outputSchema; untyped tools stay // byte-identical to today (no key at all), additive-only. ...(outputSchema ? { outputSchema } : {}), + _meta: toolLoadingMeta(definition), }; }); } +/** The `'loop'` framework tier is the always-loaded set; see `CommandFrameworkTier`. */ +function toolLoadingMeta(definition: CommandMetadata): ToolLoadingMeta { + return { + ...(definition.searchHint === undefined + ? {} + : { 'anthropic/searchHint': definition.searchHint }), + ...(resolveCommandFrameworkTier(definition.name) === 'loop' + ? { 'anthropic/alwaysLoad': true as const } + : {}), + }; +} + export function createCommandToolExecutor(deps: CommandToolExecutorDeps = {}): CommandToolExecutor { const refPins = createToolRefPinStore(); return { @@ -321,10 +356,7 @@ function readMcpToolConfig( record: Record, binding: BindingSettings, ): McpToolConfig { - return { - client: { ...readClientConfig(record), ...readLockConfig(binding) }, - outputFormat: readMcpOutputFormat(record.mcpOutputFormat), - }; + return { client: { ...readClientConfig(record), ...readLockConfig(binding) } }; } /** Only an active lock adds keys, so the default request shape stays untouched. */ @@ -338,7 +370,7 @@ function readLockConfig(binding: BindingSettings): AgentDeviceClientConfig { function readClientConfig(record: Record): AgentDeviceClientConfig { const stateDir = record.stateDir; - const includeCost = record.includeCost; + const cost = record.cost; const responseLevel = record.responseLevel; const client: AgentDeviceClientConfig = {}; if (stateDir !== undefined) { @@ -347,13 +379,13 @@ function readClientConfig(record: Record): AgentDeviceClientCon } client.stateDir = stateDir; } - if (includeCost !== undefined) { - if (typeof includeCost !== 'boolean') { - throw new AppError('INVALID_ARGS', 'Expected includeCost to be a boolean.'); + if (cost !== undefined) { + if (typeof cost !== 'boolean') { + throw new AppError('INVALID_ARGS', 'Expected cost to be a boolean.'); } // Only set when explicitly true so the default request shape is untouched // (cost rides on response.data → structuredContent only when opted in). - if (includeCost) client.cost = true; + if (cost) client.cost = true; } // Only set when it names a known level so the default request shape is // untouched (responseLevel rides on meta.responseLevel only when opted in). @@ -374,14 +406,6 @@ function readResponseLevel(value: unknown): ResponseLevel | undefined { return level; } -function readMcpOutputFormat(outputFormat: unknown): McpOutputFormat { - if (outputFormat === undefined) return 'optimized'; - if (outputFormat !== 'optimized' && outputFormat !== 'json') { - throw new AppError('INVALID_ARGS', 'Expected mcpOutputFormat to be "optimized" or "json".'); - } - return outputFormat; -} - function stripMcpConfigFields(input: Record): Record { return Object.fromEntries( Object.entries(input).filter(([key]) => !MCP_TOOL_CONFIG_KEYS.has(key)), @@ -414,7 +438,7 @@ function advertisedInputSchema( return { ...schema, properties: Object.fromEntries( - Object.entries(schema.properties).filter(([key]) => audience[key]?.kind !== 'operator'), + Object.entries(schema.properties).filter(([key]) => audience[key] === undefined), ), }; } @@ -440,6 +464,9 @@ function findInadmissibleInput( if (keyAudience?.kind === 'operator') { return { message: operatorInputRefusal(key, keyAudience.source) }; } + if (keyAudience?.kind === 'alias') { + return { message: aliasInputRefusal(key, keyAudience.of) }; + } return { message: `${key} is not an accepted argument for the ${name} tool.`, hint: `Example call: ${name} ${JSON.stringify(commandInputExample(metadata))}`, @@ -598,6 +625,7 @@ async function renderToolText(params: { result: params.result, env: params.env, stderrIsTTY: Boolean(process.stderr.isTTY), + reader: 'model', }); if (typeof cliOutput?.text !== 'string') return renderJsonText(cliOutput?.data ?? params.result); return toolTextWithWarnings(params.name, cliOutput); @@ -615,14 +643,10 @@ function toolTextWithWarnings(name: CommandName, cliOutput: CliOutput): string { return stderr === '' ? text : `${text}${text.endsWith('\n') ? '' : '\n'}${stderr}`; } -/** Machine output requested: the JSON form, a leveled payload, or the cost that rides only on the data. */ +/** Machine output requested: a leveled payload, or the cost that rides only on the data. */ function rendersJsonText(config: McpToolConfig): boolean { const { responseLevel, cost } = config.client; - return ( - config.outputFormat === 'json' || - (responseLevel !== undefined && responseLevel !== 'default') || - cost === true - ); + return (responseLevel !== undefined && responseLevel !== 'default') || cost === true; } function renderJsonText(value: unknown): string { diff --git a/src/mcp/server-guide.ts b/src/mcp/server-guide.ts index 1eebef2f8c..7e17a788f9 100644 --- a/src/mcp/server-guide.ts +++ b/src/mcp/server-guide.ts @@ -6,6 +6,7 @@ import { } from '../commands/schema/cli-help.ts'; import { listCliCommandNames } from '@agent-device/command-registry/catalog'; import { listMcpExposedCommandNames } from '@agent-device/command-registry/registry'; +import { resolvePostActionObservationSupport } from '@agent-device/command-registry/post-action-observation'; import type { ToolResult } from './command-tools.ts'; /** @@ -13,18 +14,26 @@ import type { ToolResult } from './command-tools.ts'; * at session start (Claude Code, Codex CLI). It is paid every session and Claude Code * truncates it at 2 KB, so it holds only what changes the first few calls: start with * `open`, act → settled diff → verify → `close`, ref fidelity, sparse-AX recovery, follow - * error hints. Everything situational lives behind the `help` tool. + * error hints. Everything situational lives behind the `help` tool. Per-tool usage lives in each + * tool's description, which is the command's own help text. */ export const MCP_SERVER_INSTRUCTIONS = `agent-device drives iOS, Android, tvOS, Android TV, macOS, Linux, and web apps. Tools mirror CLI commands of the same name; CLI flags are camelCase properties (--settle -> settle: true). If your client defers tool schemas, load a tool's schema before its first call. Start: known app -> call open {app, foreground: true} at once; do not probe with devices, apps, appstate, snapshot, or screenshot first. open returns the initial interactive snapshot with @refs. Unknown app id: devices, then apps, then open the discovered id; never invent ids. Existing session: continue from its state, do not reopen. -Loop: press/click/fill/longpress/hover/scroll/back with settle: true; the response is the settled UI diff, continue from it. To reach an off-screen target, scroll {direction, until: ""} scrolls until it is on screen in one call, and direction "bottom" runs to the end of the content; repeated bare scrolls are the slow way to find something. snapshot {interactiveOnly: true} only when the diff lacks the next target or did not settle. Verify with wait {kind: "text", text}, wait {selector}, wait {absent: selector}, is, or get; a bare screenshot is not verification. find acts and clicks by default; to check only, pass action: "exists". End with close. +Loop: ${settleToolNames().join('/')} with settle: true; the response is the settled UI diff, continue from it. A known multi-step walk is one batch call: steps [{command, input}]. To reach an off-screen target, scroll {direction, until: ""} scrolls until it is on screen in one call, and direction "bottom" runs to the end of the content; repeated bare scrolls are the slow way to find something. snapshot {interactiveOnly: true} only when the diff lacks the next target or did not settle. Verify with wait {kind: "text", text}, wait {selector}, wait {absent: selector}, is, or get; a bare screenshot is not verification. find acts and clicks by default; to check only, pass action: "exists". End with close. Targets are top-level arguments: press {ref: "@e12"}, click {selector: "id=submit"}, press {x, y}. Copy refs byte-for-byte (@e12, @e12~s4; keep @ and any ~sN). Refs go stale after mutations. Prefer refs, then id/label/role selectors; coordinates last. On a sparse/AX-unavailable warning its refs and selectors are invalid: screenshot, read the image, press {x, y}, then snapshot the changed screen. Errors carry hints; follow them instead of re-planning. Call help only for specialized work (gestures, scripting, TV, macOS, web, remote, debugging) or an unclear command shape, never at startup.`; +/** The tools that take settle, from the command registry's post-action observation trait. */ +function settleToolNames(): string[] { + return listMcpExposedCommandNames().filter( + (name) => resolvePostActionObservationSupport(name) !== undefined, + ); +} + export const HELP_TOOL_NAME = 'help'; /** diff --git a/src/mcp/tool-control-fields.ts b/src/mcp/tool-control-fields.ts index fdb466927c..90f8ba44f3 100644 --- a/src/mcp/tool-control-fields.ts +++ b/src/mcp/tool-control-fields.ts @@ -6,48 +6,40 @@ import { } from '@agent-device/command-surface'; /** - * The MCP surface's own tool arguments: transport and response-shaping controls - * it adds on top of a command's input, consumes itself, and never forwards to - * the command route. One row per key, so the advertised schema, the keys - * stripped before dispatch, and the audience boundary read one declaration. + * The MCP surface's own controls: the state directory and response shaping it + * adds on top of a command's input, consumes itself, and never forwards to the + * command route. One row per key, so the keys stripped before dispatch and the + * audience boundary read one declaration. * - * Not `tool-input-config.ts`, which is the other half of the same call: that one - * merges the operator's env and config-file defaults into a call's input. + * Every row is operator-owned: the operator sets it for the whole server with + * the CLI flag's `AGENT_DEVICE_*` environment variable or config key, which + * `tool-input-config.ts` merges into each call. No tool schema advertises + * these keys, so they cost no schema bytes on any tool, and a call that passes + * one is refused with the operator path. */ type McpToolConfigFieldSpec = { + /** The schema env/config defaults resolve against; never advertised. */ schema: JsonSchema; - /** Who may write the key. Absent means the model — see `commands/input-audience.ts`. */ - audience?: InputAudience; + audience: InputAudience; }; const MCP_TOOL_CONFIG_FIELDS = { stateDir: { - schema: { type: 'string', description: 'Agent-device state directory.' }, + schema: { type: 'string' }, // Selects which daemon state directory (and therefore which daemon and // session namespace) a call resolves against: operator infrastructure, not // per-call work. audience: operatorAudience({ operatorConfig: true }), }, - mcpOutputFormat: { - schema: { - type: 'string', - enum: ['optimized', 'json'], - description: 'Result text: compact optimized (default) or the full result as JSON (larger).', - }, - }, - includeCost: { - schema: { - type: 'boolean', - description: 'Add agent cost (cost.wallClockMs, …) to the result, returned as JSON text.', - }, + // Keyed by the CLI flag (`--cost`, `--level`), so the flag's env/config default reaches the + // call through the same resolver as every other flag. + cost: { + schema: { type: 'boolean' }, + audience: operatorAudience({ operatorConfig: true }), }, responseLevel: { - schema: { - type: 'string', - enum: ['digest', 'default', 'full'], - description: - 'Result detail: digest (smallest) or full (largest), as JSON; default is compact text.', - }, + schema: { type: 'string', enum: ['digest', 'default', 'full'] }, + audience: operatorAudience({ operatorConfig: true }), }, } as const satisfies Record; @@ -78,7 +70,7 @@ export const CONFIG_LOADER_AUDIENCE: InputAudienceMap = { }; export const MCP_TOOL_CONFIG_AUDIENCE: InputAudienceMap = Object.fromEntries( - MCP_TOOL_CONFIG_ROWS.flatMap(([key, field]) => (field.audience ? [[key, field.audience]] : [])), + MCP_TOOL_CONFIG_ROWS.map(([key, field]) => [key, field.audience]), ); export function mcpToolConfigProperties(): Record { diff --git a/src/mcp/tool-error.ts b/src/mcp/tool-error.ts index e046fda68e..060a84aba7 100644 --- a/src/mcp/tool-error.ts +++ b/src/mcp/tool-error.ts @@ -6,7 +6,7 @@ export function normalizeToolError(error: unknown): NormalizedError { } export function formatToolErrorText(normalized: NormalizedError): string { - const { summary, body } = formatHumanError(normalized); + const { summary, body } = formatHumanError(normalized, { reader: 'model' }); const lines = [summary, ...body]; if (normalized.supportedOn) lines.push(`Supported on: ${normalized.supportedOn}`); return lines.join('\n'); diff --git a/website/docs/docs/agent-setup.md b/website/docs/docs/agent-setup.md index 6e71048f06..180e109e1f 100644 --- a/website/docs/docs/agent-setup.md +++ b/website/docs/docs/agent-setup.md @@ -61,13 +61,15 @@ For web automation, MCP tools can target `platform: "web"` once the managed brow MCP-only clients don't need a separately installed skill. When a client connects (`initialize` and `server/discover`), the server returns short `instructions` with the core workflow: start with `open`, act with `settle: true`, verify, `close`, how to keep refs accurate, and how to recover when the accessibility tree is sparse. An MCP-only `help` tool returns the same guides as `agent-device help `. Agents don't call `help` at startup; they call it for specialized work (gestures, scripting, TV, macOS, web, remote, debugging) or when a command is unclear. -Tool results are compact text, such as `@e4 [button] "Products"` lines and settled UI diffs. Tools declare no output schema, and results carry no `structuredContent`, because some clients show the model `structuredContent` instead of the text. Pass `mcpOutputFormat: "json"`, a non-default `responseLevel`, or `includeCost` to get the result as JSON text. +Tool results are compact text, such as `@e4 [button] "Products"` lines and settled UI diffs. Tools declare no output schema, and results carry no `structuredContent`, because some clients show the model `structuredContent` instead of the text. To get results as JSON text, set `AGENT_DEVICE_RESPONSE_LEVEL` to `digest` or `full`, or set `AGENT_DEVICE_COST=1`, in the server entry. Tool calls cannot pass these values, `debug`, or the `target` alias of `deviceTarget`. -If your client reads results as data, start the server with `agent-device mcp --structured`. Tools that have a typed result, such as `press` or `wait`, then declare an output schema and return `structuredContent` that matches it. Other tools return `structuredContent` when you ask for JSON as above. A failed call returns the error in `structuredContent`. +Clients that defer tool schemas, such as Claude Code, load the core loop tools up front: `open`, `snapshot`, `press`, `fill`, `back`, `wait`, `scroll`, and `screenshot`. Each tool also carries search keywords for the client's tool search. To run a known multi-step walk in one call, use `batch`; its description shows the step shape. + +If your client reads results as data, start the server with `agent-device mcp --structured`. Tools that have a typed result, such as `press` or `wait`, then declare an output schema and return `structuredContent` that matches it. Other tools return `structuredContent` when the server is set to return JSON as above. A failed call returns the error in `structuredContent`. Tools that act on a UI element take its target as top-level arguments, in the same words as the CLI: `press {"ref":"@e12"}`, `click {"selector":"id=\"submit\""}`, or `press {"x":120,"y":340}`. `get` takes `ref` or `selector`. -When a tool fails, the server returns an MCP tool result with `isError: true`. Check the tool result, not only whether the JSON-RPC call succeeded. A call with an argument the tool doesn't take fails with `INVALID_ARGS`, and its hint shows a call that the tool accepts. +When a tool fails, the server returns an MCP tool result with `isError: true`. Check the tool result, not only whether the JSON-RPC call succeeded. The error text is short: the error code and typed reason, a one-line message, the cause when known, and the next step. It leaves out diagnostics log paths. A call with an argument the tool doesn't take fails with `INVALID_ARGS`, and its hint shows a call that the tool accepts. The server runs one `tools/call` at a time, in the order it receives them. It answers `ping`, `tools/list`, and the handshake at once, even while a long tool call such as `test` or `wait` runs. A `notifications/cancelled` for a running or queued `tools/call` stops it, and the server sends no response for that request. Cancellation stops the daemon request, not work that already reached the device: on iOS, a command the device received keeps running there, and the session waits for it. See [Command methods](/docs/client-api#command-methods). A JSON-RPC batch cannot carry a `tools/call`; the server refuses that element with `-32600`. From 4c8ba04dbfd9cb96a43c08e7d10e02369170e67b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 15:42:25 +0100 Subject: [PATCH 02/12] fix(mcp): restore caller-facing contract facts and type the log-path rule - Settle fields state the diff is against the pre-action tree and keep the iOS 300 ms default and broad-replacement floor, tersely. - open documents initialSnapshotError on a fresh open; customActions keeps its recovery routes, with the full wording in help snapshot. - Model-facing errors keep the log path only when the code's default step reads the log (defaultHintReadsLog), and bound details.reason to one line. - batch grounds its @e12 example; searchHints carry only words the tool text lacks, enforced by a test. Co-Authored-By: Claude Opus 5.5 --- packages/command-output/src/output/error.ts | 22 +++++++++++---- packages/command-registry/src/command-text.ts | 5 ++-- .../src/flag-definitions-workflow.ts | 2 +- packages/command-surface/src/batch/index.ts | 4 +-- packages/command-surface/src/capture/alert.ts | 2 +- packages/command-surface/src/capture/diff.ts | 2 +- .../command-surface/src/capture/screenshot.ts | 2 +- .../command-surface/src/capture/settings.ts | 2 +- .../command-surface/src/capture/snapshot.ts | 4 +-- packages/command-surface/src/capture/wait.ts | 2 +- .../command-surface/src/debugging/index.ts | 2 +- .../command-surface/src/interaction/index.ts | 26 ++++++++--------- .../command-surface/src/management/app.ts | 8 +++--- .../src/management/artifacts.ts | 2 +- .../command-surface/src/management/device.ts | 8 +++--- .../command-surface/src/management/doctor.ts | 2 +- .../command-surface/src/management/install.ts | 8 +++--- .../command-surface/src/management/push.ts | 4 +-- .../command-surface/src/management/session.ts | 2 +- .../src/management/viewport.ts | 2 +- packages/command-surface/src/metro/index.ts | 2 +- .../src/observability/index.ts | 8 +++--- packages/command-surface/src/perf/index.ts | 2 +- .../src/post-action-observation-grammar.ts | 7 +++-- .../command-surface/src/react-native/index.ts | 2 +- .../command-surface/src/recording/index.ts | 4 +-- packages/command-surface/src/replay/index.ts | 4 +-- packages/command-surface/src/system/index.ts | 24 ++++++++-------- packages/contracts/src/client-capture.ts | 4 +-- packages/kernel/src/errors.ts | 5 ++++ .../command-tools-loading-meta.test.ts | 10 +++++++ .../command-tools-structured-content.test.ts | 28 ++++++++++++++++++- website/docs/docs/agent-setup.md | 2 +- 33 files changed, 135 insertions(+), 78 deletions(-) diff --git a/packages/command-output/src/output/error.ts b/packages/command-output/src/output/error.ts index 2d7786b995..6d959d5bf6 100644 --- a/packages/command-output/src/output/error.ts +++ b/packages/command-output/src/output/error.ts @@ -1,6 +1,7 @@ import { AppError, defaultHintForCode, + defaultHintReadsLog, normalizeError, readDispatchDisclosure, readErrorCandidateViews, @@ -30,7 +31,7 @@ export type HumanErrorOptions = { * Who reads the text. `terminal` (the default) is a person or script at the CLI. `model` is * an agent reading a tool result: the typed `details.reason` joins the code and the message * stays one line. The diagnostics log path is left out unless the recovery step is the code's - * default, which tells the reader to read that log. + * default and that default tells the reader to read the log. */ reader?: 'terminal' | 'model'; /** The failing command's synopsis, such as `agent-device fill `. */ @@ -69,7 +70,7 @@ export function formatHumanError( fullMessage: truncated && !forModel, candidateViews: inlineCandidates ? [] : candidateViews, showDetails: options.showDetails, - logLine: !forModel || normalized.hint === defaultHintForCode(normalized.code), + logLine: !forModel || hintReadsLog(normalized), }), }; } @@ -124,10 +125,21 @@ function nextStep( return parts.length > 0 ? parts.join(' ') : undefined; } -/** The typed `details.reason`, when the throw site classified the failure. */ +function hintReadsLog(normalized: NormalizedError): boolean { + return ( + normalized.hint === defaultHintForCode(normalized.code) && defaultHintReadsLog(normalized.code) + ); +} + +const REASON_LIMIT = 80; + +/** The typed `details.reason`, when the throw site classified the failure, as one bounded line. */ function readReason(normalized: NormalizedError): string | undefined { - const reason = normalized.details?.reason; - return typeof reason === 'string' && reason.length > 0 ? reason : undefined; + const raw = normalized.details?.reason; + if (typeof raw !== 'string') return undefined; + const reason = oneLine(raw); + if (reason.length === 0) return undefined; + return reason.length > REASON_LIMIT ? `${reason.slice(0, REASON_LIMIT)}…` : reason; } function formatBody( diff --git a/packages/command-registry/src/command-text.ts b/packages/command-registry/src/command-text.ts index 746e000cca..55c6bf3918 100644 --- a/packages/command-registry/src/command-text.ts +++ b/packages/command-registry/src/command-text.ts @@ -11,9 +11,10 @@ * searchHint which words would a model searching for this tool type that the text lacks? * * `--help`, the command list, the MCP tool description and `explain` are all projections of - * these four, computed where they are rendered rather than written back into the schema. That + * the first four, computed where they are rendered rather than written back into the schema. That * is deliberate: the field this replaced was authored on some commands and generated on others, - * which is how surfaces silently drifted apart. + * which is how surfaces silently drifted apart. `searchHint` renders nowhere in that text; it + * goes only to the MCP tool's `_meta`. * * Because `cliDetail` is the only field carrying terminal vocabulary, and the MCP surface never * reads it, MCP descriptions stay free of flags and positional syntax structurally rather than diff --git a/packages/command-registry/src/flag-definitions-workflow.ts b/packages/command-registry/src/flag-definitions-workflow.ts index 33b972d394..53807458ff 100644 --- a/packages/command-registry/src/flag-definitions-workflow.ts +++ b/packages/command-registry/src/flag-definitions-workflow.ts @@ -269,7 +269,7 @@ export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageDescription: 'Snapshot: name the affordances merged inside an element (iOS sim); not directly invokable — reach them via the detail screen, labeled children, or coordinates', inputDescription: - "List the custom actions an element merged away, such as a card's reply control. For planning only: they cannot be invoked, so reach the control another way. iOS simulator only; costs one accessibility round trip per merged element.", + "List the custom actions an element merged away, such as a card's reply control. Not invocable: reach the control via its detail screen, a labeled element elsewhere, or its rect. iOS simulator only; costs one accessibility round trip per merged element.", projectConfig: true, recorded: true, }, diff --git a/packages/command-surface/src/batch/index.ts b/packages/command-surface/src/batch/index.ts index 3b1c2115f0..f7cc14546a 100644 --- a/packages/command-surface/src/batch/index.ts +++ b/packages/command-surface/src/batch/index.ts @@ -51,7 +51,7 @@ const BATCH_MCP_DETAIL = [ "steps is an array of {command, input} objects; input takes the same arguments as that command's own tool (ref, selector, or x and y for targets; settle: true for the settled diff).", 'Steps run serially in one session; onError: "stop" halts at the first failing step, and the result reports the steps that ran.', 'The whole batch has a 60-second work budget; split longer walks.', - `Example steps, in order: ${BATCH_STEP_EXAMPLES.map((step) => JSON.stringify(step)).join(', ')}.`, + `Example steps, in order (refs such as @e12 come from the last snapshot or settled diff): ${BATCH_STEP_EXAMPLES.map((step) => JSON.stringify(step)).join(', ')}.`, ].join(' '); /** @@ -81,7 +81,7 @@ const batchCommandFacet = defineCommandFacet({ text: { summary: 'Run multiple commands', mcpDetail: BATCH_MCP_DETAIL, - searchHint: 'batch sequence steps multi-step walk chain several taps in one call', + searchHint: 'sequence chain several taps', cliDetail: buildBatchCliDetail(), }, metadata: batchCommandMetadata, diff --git a/packages/command-surface/src/capture/alert.ts b/packages/command-surface/src/capture/alert.ts index a2ad24ae76..6a04cefaa3 100644 --- a/packages/command-surface/src/capture/alert.ts +++ b/packages/command-surface/src/capture/alert.ts @@ -51,7 +51,7 @@ export const alertCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Inspect, accept, or dismiss a platform alert', - searchHint: 'dialog popup permission prompt allow deny system alert', + searchHint: 'dialog system', }, metadata: alertCommandMetadata, run: (client, input) => client.command.alert(input), diff --git a/packages/command-surface/src/capture/diff.ts b/packages/command-surface/src/capture/diff.ts index 7054d9b3b3..dd2c417061 100644 --- a/packages/command-surface/src/capture/diff.ts +++ b/packages/command-surface/src/capture/diff.ts @@ -114,7 +114,7 @@ export const diffCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Diff snapshot or screenshot', - searchHint: 'compare changes visual regression baseline', + searchHint: 'visual regression baseline', cliDetail: 'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; the current image scales proportionally to the baseline width and aligns top-left. Scaled height rounds to the nearest pixel (minimum 1). Extra or missing bottom rows always count as differences, even at threshold 1. The originals remain unchanged. When dimensions differ, normalization metadata reports original, scaled and compared sizes; region coordinates use comparison pixels. Both screenshot inputs are decoded from their bytes, so a baseline or current image may be PNG or JPEG, and the diff image is always PNG. JPEG is lossy, so keep --threshold above 0 whenever either input is JPEG. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines. Use --screen-mask black for a live iOS simulator capture with a black screen outline, and use the same option when creating its baseline. --screen-mask is refused when current.png is supplied; saved images are compared unchanged.', }, diff --git a/packages/command-surface/src/capture/screenshot.ts b/packages/command-surface/src/capture/screenshot.ts index 2631c21675..634305d3da 100644 --- a/packages/command-surface/src/capture/screenshot.ts +++ b/packages/command-surface/src/capture/screenshot.ts @@ -84,7 +84,7 @@ export const screenshotCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Capture a screenshot', - searchHint: 'image picture screen capture png visual evidence', + searchHint: 'picture screen png', cliDetail: "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. Use --screen-mask black to fill the iOS simulator screen outline, including rounded corners, with black for this capture. It requires simctl: physical devices, tvOS and visionOS simulators, other platforms, provider captures and XCTest runner captures are refused. A masked capture fails if simctl fails instead of falling back to an unmasked runner capture. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, as is the small indicator window macOS adds in front of a window that a screen-sharing stream captures, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.", }, diff --git a/packages/command-surface/src/capture/settings.ts b/packages/command-surface/src/capture/settings.ts index a4521481de..464b02e14c 100644 --- a/packages/command-surface/src/capture/settings.ts +++ b/packages/command-surface/src/capture/settings.ts @@ -101,7 +101,7 @@ export const settingsCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Change OS settings and app permissions', - searchHint: 'wifi airplane location dark mode appearance permission grant revoke', + searchHint: 'wifi airplane location dark mode permission grant revoke', cliDetail: [ 'A bare readable setting reads the value the target holds. A target that does not serve a setting refuses it with UNSUPPORTED_OPERATION before anything changes.', '', diff --git a/packages/command-surface/src/capture/snapshot.ts b/packages/command-surface/src/capture/snapshot.ts index 42f5a9264b..2f5eb5f561 100644 --- a/packages/command-surface/src/capture/snapshot.ts +++ b/packages/command-surface/src/capture/snapshot.ts @@ -76,8 +76,8 @@ export const snapshotCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Capture or diff the accessibility tree', - searchHint: 'ui tree elements refs inspect screen hierarchy', - cliDetail: `Scoped output (--scope) renders every returned ref with its ref generation pin (@eN~s) and one Refs generation line, so a pasted ref targets the ref frame that issued it; --json keeps plain refs plus the response-level refsGeneration, and --raw keeps byte-for-byte output. Regular Android ADB snapshots clip app rectangles outside measured status and navigation bars, including launcher-owned navigation; fully covered content is omitted and --raw retains reported bounds. Ref and selector taps honor measured visible bounds even for refs from raw snapshots. Regular iOS snapshots clip overflowing elements to their visible area and omit fully outside elements. In a regular (non-raw) snapshot, a listed element whose frame has no area is marked [unmeasured]: nothing confirms it is on screen. If it is inside scrollable content, reach it with scroll --until before acting on it. --json reports clipping counts in snapshotQualityNotes with code regular-geometry-clipped; --raw retains reported frames. --observe-only is local iOS snapshot-only: an already-ready compatible runner reads the session app without activation or foreground repair, refuses unavailable observation or a readable system alert over the app, and reports XCUIApplication state rather than verified foreground ownership (#2696). Regular targetActivation disclosure stays unchanged (#2682, #2694). Repeated equivalent unfiltered Android snapshots return a compact unchanged acknowledgement. Use --force-full to re-emit the tree; --json and --raw retain full output. Android Appium JSON snapshots publish a display viewport only when measurements before and after the tree agree, independent of tree extent or keyboard content resizing. On Android, JSON value carries field content rather than static labels; saved selectors and replay scripts using value= for static labels must use label= instead; a reported showing hint means value is empty, and text equal to the placeholder is preserved when hint state is unknown. HarmonyOS reads the on-screen keyboard's own system window and publishes the keyboard field of --json output: the field is visibility-only — visible or hidden per the keyboard window's z-order column; a failed probe, an unreadable or conflicting z-order, or a table without the panel publishes unmeasurable, never a guessed band. When an iOS snapshot reports a screen viewport, inspect rects with snapshot -i --json and press the intended control's center, then verify with diff snapshot -i or snapshot --diff. When a fallback cannot establish the screen origin, viewport is omitted and coordinate-derived element actions refuse with window-origin-unknown before input; the snapshot may warn about the unavailable origin. Use a plain screenshot to choose screen coordinates. An empty text field with no label or value prints its hint as [placeholder:"..."]; label=, text= and find do not match that hint. iOS backend capability contract: ${snapshotBackendCapabilityHelp}.`, + searchHint: 'inspect screen hierarchy', + cliDetail: `Scoped output (--scope) renders every returned ref with its ref generation pin (@eN~s) and one Refs generation line, so a pasted ref targets the ref frame that issued it; --json keeps plain refs plus the response-level refsGeneration, and --raw keeps byte-for-byte output. Regular Android ADB snapshots clip app rectangles outside measured status and navigation bars, including launcher-owned navigation; fully covered content is omitted and --raw retains reported bounds. Ref and selector taps honor measured visible bounds even for refs from raw snapshots. Regular iOS snapshots clip overflowing elements to their visible area and omit fully outside elements. In a regular (non-raw) snapshot, a listed element whose frame has no area is marked [unmeasured]: nothing confirms it is on screen. If it is inside scrollable content, reach it with scroll --until before acting on it. --json reports clipping counts in snapshotQualityNotes with code regular-geometry-clipped; --raw retains reported frames. --observe-only is local iOS snapshot-only: an already-ready compatible runner reads the session app without activation or foreground repair, refuses unavailable observation or a readable system alert over the app, and reports XCUIApplication state rather than verified foreground ownership (#2696). Regular targetActivation disclosure stays unchanged (#2682, #2694). Repeated equivalent unfiltered Android snapshots return a compact unchanged acknowledgement. Use --force-full to re-emit the tree; --json and --raw retain full output. Android Appium JSON snapshots publish a display viewport only when measurements before and after the tree agree, independent of tree extent or keyboard content resizing. On Android, JSON value carries field content rather than static labels; saved selectors and replay scripts using value= for static labels must use label= instead; a reported showing hint means value is empty, and text equal to the placeholder is preserved when hint state is unknown. HarmonyOS reads the on-screen keyboard's own system window and publishes the keyboard field of --json output: the field is visibility-only — visible or hidden per the keyboard window's z-order column; a failed probe, an unreadable or conflicting z-order, or a table without the panel publishes unmeasurable, never a guessed band. When an iOS snapshot reports a screen viewport, inspect rects with snapshot -i --json and press the intended control's center, then verify with diff snapshot -i or snapshot --diff. When a fallback cannot establish the screen origin, viewport is omitted and coordinate-derived element actions refuse with window-origin-unknown before input; the snapshot may warn about the unavailable origin. Use a plain screenshot to choose screen coordinates. --actions names the affordances an element merged away (iOS UIAccessibilityCustomAction, React Native accessibilityActions), so a card whose reply or options controls are not separate elements still lists them. The names are for planning, not invocation: no API triggers them, so reach the affordance through the element detail screen, through the same control exposed as a labeled element elsewhere, or by coordinates from its rect. An empty text field with no label or value prints its hint as [placeholder:"..."]; label=, text= and find do not match that hint. iOS backend capability contract: ${snapshotBackendCapabilityHelp}.`, }, metadata: snapshotCommandMetadata, run: (client, input) => client.capture.snapshot(input), diff --git a/packages/command-surface/src/capture/wait.ts b/packages/command-surface/src/capture/wait.ts index cd368cedcf..cca9441df1 100644 --- a/packages/command-surface/src/capture/wait.ts +++ b/packages/command-surface/src/capture/wait.ts @@ -97,7 +97,7 @@ export const waitCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Wait for a duration, text, selector, strict absence, or stable UI', - searchHint: 'wait until appear disappear sleep verify assert visible text', + searchHint: 'disappear sleep assert visible', }, metadata: waitCommandMetadata, run: (client, input) => client.command.wait(waitInputToOptions(input)), diff --git a/packages/command-surface/src/debugging/index.ts b/packages/command-surface/src/debugging/index.ts index b2f9369a35..f9a28a0eca 100644 --- a/packages/command-surface/src/debugging/index.ts +++ b/packages/command-surface/src/debugging/index.ts @@ -47,7 +47,7 @@ export const debugCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Symbolicate Apple crash artifacts', - searchHint: 'crash report symbolicate ips dsym', + searchHint: 'report ips', }, metadata: debugCommandMetadata, run: (client, input) => client.debug.symbols(input), diff --git a/packages/command-surface/src/interaction/index.ts b/packages/command-surface/src/interaction/index.ts index dce907b0f0..93986f641a 100644 --- a/packages/command-surface/src/interaction/index.ts +++ b/packages/command-surface/src/interaction/index.ts @@ -459,7 +459,7 @@ const clickCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Click or tap a UI target', - searchHint: 'tap press button link web mouse', + searchHint: 'press button link web mouse', }, metadata: metadata('click'), run: (client, input) => client.interactions.click(toClickOptions(input)), @@ -473,7 +473,7 @@ const pressCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Short-press a UI target', - searchHint: 'tap click touch button ref selector coordinates', + searchHint: 'tap click touch button', cliDetail: 'The hold duration is positional on longpress, not press --hold-ms.', }, metadata: metadata('press'), @@ -488,7 +488,7 @@ const fillCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Replace text in a UI input', - searchHint: 'type enter text input field form textbox', + searchHint: 'enter form textbox', cliDetail: 'Every positional after an @ref is the replacement text, so fill @e57 good morning enters "good morning"; quote the text when the shell must preserve exact whitespace. Clear a field with an empty text argument: fill @e57 "" (the argument must be present — fill @e57 alone is a missing argument, not a clear). When visible label text also matches a non-input element, constrain the target with editable=true, for example fill \'label="Email" editable=true\' "qa@example.com". To keep a secret out of argv, omit the text and pipe it in: printf %s "$PASSWORD" | fill @e57 --text-stdin (max 64 KiB, one trailing newline removed). On web, a coordinate fill selects the clicked field in the page and refuses with COMMAND_FAILED and details.reason text_entry_focus_not_observed, before it sends any text or key, when the click did not reach an editable text field it can reach: a read-only, disabled, date, checkbox, or select control, a field in a cross-origin frame, a sandboxed frame without allow-same-origin, or a closed shadow root, or a click that missed. The click has already landed, so run snapshot -i again and fill the field it reports by ref (fill @e2 text); a selector fill reaches the same coordinate path and hits the same limits.', }, @@ -504,7 +504,7 @@ const longPressCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Hold a UI target to open a context menu', - searchHint: 'long press hold context menu', + searchHint: 'long press', cliDetail: 'Duration is positional, for example longpress @e12 800 or longpress 300 500 800.', }, metadata: metadata('longpress'), @@ -519,7 +519,7 @@ const hoverCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Hover the pointer over a UI target (web only)', - searchHint: 'mouse over pointer reveal menu', + searchHint: 'mouse menu', cliDetail: 'The pointer stays where hover left it: read the revealed UI (--settle or snapshot -i) and act on it before another click or hover moves the pointer away.', }, @@ -535,7 +535,7 @@ const swipeCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Fling between coordinates', - searchHint: 'swipe drag fling gesture coordinates', + searchHint: 'drag gesture', cliDetail: 'The first read after a swipe waits for two captures to agree. --no-post-gesture-stabilization skips that wait for a caller whose own polling verifies the screen.', }, @@ -551,7 +551,7 @@ const focusCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Focus input at screen coordinates', - searchHint: 'focus input field cursor', + searchHint: 'field cursor', }, metadata: metadata('focus'), run: (client, input) => client.interactions.focus(input as FocusOptions), @@ -565,7 +565,7 @@ const typeCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Append text to the focused input', - searchHint: 'type keys keyboard input text append', + searchHint: 'keys keyboard', }, metadata: metadata('type'), run: (client, input) => client.interactions.type(input as TypeTextOptions), @@ -579,7 +579,7 @@ const scrollCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Scroll in a direction or to an edge', - searchHint: 'scroll swipe down up list find off-screen element', + searchHint: 'swipe down list', cliDetail: 'A caller that verifies the screen by polling can add --no-post-gesture-stabilization. The scroll then returns right after its gesture, with no movement field and no scroll_no_progress refusal, and the next read skips the stabilization wait. With --settle the settle observation already reports the outcome, and the option removes the same wait from its first capture.', }, @@ -595,7 +595,7 @@ const getCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Read element text or attributes', - searchHint: 'read text value attribute label', + searchHint: 'value label', }, metadata: metadata('get'), run: (client, input) => client.interactions.get(toGetOptions(input)), @@ -609,7 +609,7 @@ const isCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Check a UI predicate on a selector', - searchHint: 'assert check visible hidden exists enabled selected', + searchHint: 'assert enabled', }, metadata: metadata('is'), run: (client, input) => client.interactions.is(input as IsOptions), @@ -623,7 +623,7 @@ const findCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Find an element and act', - searchHint: 'search locate element text click', + searchHint: 'search locate', }, metadata: metadata('find'), run: (client, input) => client.interactions.find(input as FindOptions), @@ -637,7 +637,7 @@ const gestureCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Run pan, fling, swipe, pinch, rotate, transform, tap, or drag gestures', - searchHint: 'gesture multi-touch pinch zoom drag pan', + searchHint: 'multi-touch zoom', cliDetail: 'Argument shapes: pan [durationMs], fling [distance], swipe , pinch [x] [y], rotate [x] [y], transform [durationMs], tap [... ] [--hold-ms ] [--count ] [--pause-ms ], or drag [sourceHoldMs] [moveMs] [destinationHoldMs]. For command plans, output only command lines. Android transform verification should use all app-observable effects, for example wait text "pan changed yes", wait text "pinch changed yes", and wait text "rotate changed yes", not exact transform values. gesture swipe accepts --no-post-gesture-stabilization to skip the stabilization wait before the next read, for a caller whose own polling verifies the screen.', }, diff --git a/packages/command-surface/src/management/app.ts b/packages/command-surface/src/management/app.ts index 0dff572035..659b12319c 100644 --- a/packages/command-surface/src/management/app.ts +++ b/packages/command-surface/src/management/app.ts @@ -43,7 +43,7 @@ const appsCommandMetadata = defineFieldCommandMetadata( const openCommandMetadata = defineFieldCommandMetadata( 'open', - 'Boot the selected device when needed, then open an app, deep link, or URL in a session. With a known app, start with open and foreground: it returns the initial interactive snapshot with refs, so do not probe with devices, apps, or snapshot first. For an unknown app id, list devices and apps; never guess an id. The app or URL inputs select the foreground automation target. Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', + 'Boot the selected device when needed, then open an app, deep link, or URL in a session. With a known app, start with open and foreground: a fresh open returns the initial interactive snapshot with refs (or initialSnapshotError: then run snapshot). Do not probe with devices or apps first; list them only for an unknown app id, and never guess one. The app or URL inputs select the foreground automation target. Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', { app: stringField('App name, bundle id, package, or URL.'), url: stringField('Optional URL passed with an app shell.'), @@ -195,7 +195,7 @@ export const appsCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'List installed apps or deferred provider app assets', - searchHint: 'installed apps bundle id package list', + searchHint: 'bundle id package', cliDetail: 'Before provider allocation, lists uploaded app assets when the selected provider exposes a catalog. On a live device, defaults to user-installed apps; use --all to include system/OEM apps.', }, @@ -211,7 +211,7 @@ export const openCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Open an app, deep link or URL, save replays', - searchHint: 'launch start app deep link url relaunch session', + searchHint: 'relaunch', cliDetail: 'Use --platform to bind URL/deep-link opens to the target platform. For local or Limrun iOS simulator initial stdout/stderr, put --launch-console on this open command, for example agent-device open "Agent Device Tester" --platform ios --launch-console artifacts/launch-console.log. Expo Go/dev-client shells accept host + URL, for example agent-device open "Expo Go" exp://127.0.0.1:8081 --platform ios. macOS also supports --surface app|frontmost-app|desktop|menubar. --metro-host/--metro-port/--bundle-url choose the app\'s Metro server and save that address for later metro reload calls in the same session. A fresh open without Metro hints clears an older same-name binding; close also clears it. --launch-url supplies the app\'s launch URL; it does not select the reload server. On an iOS simulator without --launch-url, open connects a detected Expo development client through its expo-development-client URL. The app must declare a single exp+ scheme that no other installed app owns. If detection is inconclusive, the app opens normally. Bare React Native apps receive their server settings before launch. On an iOS simulator, when --launch-url raises an "Open in ?" prompt that keeps open from seeing the app, open accepts it if the URL scheme belongs to the session app (data.launchConfirmation: "accepted"); if the scheme belongs to another installed app, the prompt stays on screen and open fails with details.reason launch_confirmation_foreign_app; if the scheme owner cannot be resolved because no single installed app owns it or the lookup failed, the prompt stays on screen and open returns with data.launchConfirmation: "unanswered", launchConfirmationReason url-owner-unresolved, and a warning; a prompt that is seen but not recognized is reported the same way with launchConfirmationReason alert-unrecognized and its own warning; for url-owner-unresolved, answer the prompt with alert accept; for alert-unrecognized, read the alert with alert get first, because it can be the app\'s own alert (such as a permission prompt) rather than the launch confirmation, and answer with alert accept only when it is the launch confirmation; when agent-device cannot read whether a prompt is up at all (the alert read, the accept, or the runner itself failed, including after a re-hand of the URL), open stays green, reports no launchConfirmation plus data.launchConfirmationProbeFailure naming the failed step (alert-read, alert-accept, or runner), and appends a warning that says to check with alert get. The prompt is read whenever open cannot see the app afterwards, including on a slow host where the app\'s target discovery fails on its own deadline. When an accept leaves the app provably not running, open hands the same URL over once more and reads again; still not running, it fails with details.reason launch_confirmation_unanswered. When the runner reports its main-thread timeout for the accept and the prompt is still up, open presses Open again, but only when the time the open request has left covers a whole press and the read that checks it. A recognized prompt still visible after a failed accept, or once that time is gone, also fails with this reason, even if the app is running behind it; answer it with alert accept or alert dismiss. A simulator openurl that CoreSimulator never answers fails within 20s with details.reason ios-simulator-openurl-timeout. The prompt is recognized by the OS\'s own wording of its title and buttons in the Simulator\'s preferred languages; when the runner cannot read that wording, the prompt stays on screen as alert-unrecognized. A prompt that appears after open has already seen the app is not answered; use alert accept. A device another session is holding refuses at once; add --wait to block up to that budget for it and only then fail with DEVICE_IN_USE naming the owning session.', mcpDetail: @@ -229,7 +229,7 @@ export const closeCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Close an app or end the session', - searchHint: 'close quit stop end session', + searchHint: 'quit', }, metadata: closeCommandMetadata, run: (client, input) => diff --git a/packages/command-surface/src/management/artifacts.ts b/packages/command-surface/src/management/artifacts.ts index 6b7d074e28..52844a60b0 100644 --- a/packages/command-surface/src/management/artifacts.ts +++ b/packages/command-surface/src/management/artifacts.ts @@ -37,7 +37,7 @@ export const artifactsCommandFacet = defineCommandFacet({ effect: 'read', text: { summary: 'List daemon or cloud provider session artifacts', - searchHint: 'artifacts files video logs download', + searchHint: 'files video logs download', }, metadata: artifactsCommandMetadata, run: (client, input) => client.sessions.artifacts(input), diff --git a/packages/command-surface/src/management/device.ts b/packages/command-surface/src/management/device.ts index cef0e5f291..7eb3bb9b53 100644 --- a/packages/command-surface/src/management/device.ts +++ b/packages/command-surface/src/management/device.ts @@ -141,7 +141,7 @@ const devicesCommandFacet = defineCommandFacet({ effect: 'read', text: { summary: 'List available devices and simulators', - searchHint: 'devices simulators emulators list udid serial', + searchHint: 'emulators udid serial', }, metadata: devicesCommandMetadata, run: (client, input) => client.devices.list(input), @@ -155,7 +155,7 @@ const capabilitiesCommandFacet = defineCommandFacet({ effect: 'read', text: { summary: 'List supported commands for the selected device', - searchHint: 'capabilities supported commands device features', + searchHint: 'features', cliDetail: 'Select an explicit target with --platform/--device/--udid/--serial.', }, metadata: capabilitiesCommandMetadata, @@ -170,7 +170,7 @@ const bootCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Boot target device/simulator', - searchHint: 'boot start simulator emulator device', + searchHint: 'start emulator', }, metadata: bootCommandMetadata, run: (client, input) => client.devices.boot(input), @@ -184,7 +184,7 @@ const shutdownCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Shutdown target simulator/emulator', - searchHint: 'shutdown stop simulator emulator', + searchHint: 'stop', cliDetail: 'For an active session, use close --shutdown.', }, metadata: shutdownCommandMetadata, diff --git a/packages/command-surface/src/management/doctor.ts b/packages/command-surface/src/management/doctor.ts index f8a99cb583..d3d5ee4a41 100644 --- a/packages/command-surface/src/management/doctor.ts +++ b/packages/command-surface/src/management/doctor.ts @@ -80,7 +80,7 @@ export const doctorCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Diagnose device, app, dev-server, and RN/Expo readiness', - searchHint: 'diagnose doctor health check setup troubleshoot', + searchHint: 'health check setup troubleshoot', cliDetail: 'Metro reachability is inferred from cwd/runtime. On iOS simulators it also warms the XCTest runner build cache in the background when missing. Pass --app to verify a target app on the one matching booted device without opening a session. Use --remote to check remote connection setup without probing local devices. Default output is compact; use --json for full checks and evidence.', mcpDetail: diff --git a/packages/command-surface/src/management/install.ts b/packages/command-surface/src/management/install.ts index 9501db990c..eb57070af4 100644 --- a/packages/command-surface/src/management/install.ts +++ b/packages/command-surface/src/management/install.ts @@ -136,7 +136,7 @@ const installCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Install an app binary from a path', - searchHint: 'install app apk ipa binary', + searchHint: 'apk ipa', cliDetail: 'A HarmonyOS .hap installs from its path alone, for example agent-device install ./build/entry-default-signed.hap --platform harmonyos.', }, @@ -152,7 +152,7 @@ const reinstallCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Replace an installed app with a new build', - searchHint: 'reinstall update app build apk ipa', + searchHint: 'update apk ipa', }, metadata: reinstallCommandMetadata, run: (client, input) => client.apps.reinstall(input), @@ -166,7 +166,7 @@ const uninstallCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Remove an installed app', - searchHint: 'uninstall delete remove app', + searchHint: 'delete', }, metadata: uninstallCommandMetadata, run: (client, input) => client.apps.uninstall(input), @@ -180,7 +180,7 @@ const installFromSourceCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Install app builds from URLs or CI artifacts', - searchHint: 'install download build url ci artifact', + searchHint: 'fetch link hosted', }, metadata: installFromSourceCommandMetadata, run: (client, input) => client.apps.installFromSource(input), diff --git a/packages/command-surface/src/management/push.ts b/packages/command-surface/src/management/push.ts index dbd7562850..beb01f6fe8 100644 --- a/packages/command-surface/src/management/push.ts +++ b/packages/command-surface/src/management/push.ts @@ -132,7 +132,7 @@ const pushCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Deliver a push notification payload', - searchHint: 'push notification apns fcm payload', + searchHint: 'apns fcm', }, metadata: pushCommandMetadata, run: (client, input) => client.apps.push(input), @@ -146,7 +146,7 @@ const triggerAppEventCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Invoke an app-defined automation event', - searchHint: 'app event trigger automation hook', + searchHint: 'hook', }, metadata: triggerAppEventCommandMetadata, run: (client, input) => client.apps.triggerEvent(input), diff --git a/packages/command-surface/src/management/session.ts b/packages/command-surface/src/management/session.ts index 36300af96a..93fb99830c 100644 --- a/packages/command-surface/src/management/session.ts +++ b/packages/command-surface/src/management/session.ts @@ -39,7 +39,7 @@ export const sessionCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'List sessions, show the state dir, or publish a script', - searchHint: 'sessions list state script publish', + searchHint: 'current running inspect', }, metadata: sessionCommandMetadata, run: async (client, { action, path, force, ...input }) => { diff --git a/packages/command-surface/src/management/viewport.ts b/packages/command-surface/src/management/viewport.ts index 51d9d5d0dc..7d3ed761d9 100644 --- a/packages/command-surface/src/management/viewport.ts +++ b/packages/command-surface/src/management/viewport.ts @@ -59,7 +59,7 @@ export const viewportCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Resize the active web viewport for the current session', - searchHint: 'viewport resize browser window size', + searchHint: 'browser window size', }, metadata: viewportCommandMetadata, run: (client, input) => client.command.viewport(input), diff --git a/packages/command-surface/src/metro/index.ts b/packages/command-surface/src/metro/index.ts index cb41d71d7f..e2c2ce4d91 100644 --- a/packages/command-surface/src/metro/index.ts +++ b/packages/command-surface/src/metro/index.ts @@ -131,7 +131,7 @@ export const metroCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Prepare the dev server or reload apps', - searchHint: 'metro bundler reload react native dev server', + searchHint: 'bundler', cliDetail: "reload with no --metro-host/--metro-port/--bundle-url resolves against the dev server this session last bound via metro prepare or open's metro hint flags (falling back to localhost:8081 only when the session never bound one), so it never silently reloads a different project's server on the default port; pass an explicit flag to override the session hint for one call. After prepare, reload uses the local server address independently of the public or proxy address given to the device; a fresh open preserves it only when its client-config runtime points to the prepared device server, while explicit reload flags override it. For open or an explicit bundle URL, reload removes a .bundle/.jsbundle/.js entry filename and keeps the remaining path as the server mount, and when the server has no HTTP /reload route (Expo) the reload is broadcast over its /message websocket instead of trusting the app-page fallback. The binding is cleared when the session closes, and a fresh open without hint flags also clears any leftover binding from a previous same-name session. --kind expo (detected or forced) requests the virtual-entry bundle URL (.expo/.virtual-metro-entry.bundle) instead of index.bundle, since index.bundle 404s/500s against Expo dev servers in monorepos. Dependency install auto-detects the package manager from the nearest yarn.lock/pnpm-lock.yaml/bun.lock/bun.lockb/package-lock.json walking up from --project-root (bounded at the repo root), so Yarn/pnpm workspace monorepos with the lockfile at the repo root do not wrongly fall back to npm install (which fails on workspace: dependency specifiers); if install still fails, pass --no-install-deps when dependencies are already installed (for example via a monorepo root install).", mcpDetail: diff --git a/packages/command-surface/src/observability/index.ts b/packages/command-surface/src/observability/index.ts index 3907ec8881..fa6f6e6742 100644 --- a/packages/command-surface/src/observability/index.ts +++ b/packages/command-surface/src/observability/index.ts @@ -158,7 +158,7 @@ export const logsCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Manage session app logs', - searchHint: 'logs console output device log', + searchHint: 'console output device', }, metadata: logsCommandMetadata, run: (client, input) => client.observability.logs(input), @@ -172,7 +172,7 @@ export const eventsCommandFacet = defineCommandFacet({ effect: 'read', text: { summary: 'Read session event timeline', - searchHint: 'events timeline history session log', + searchHint: 'history log', }, metadata: eventsCommandMetadata, run: (client, input) => client.observability.events(input), @@ -186,7 +186,7 @@ export const networkCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Inspect HTTP(S) traffic from session logs', - searchHint: 'network http requests traffic api calls', + searchHint: 'requests api calls', }, metadata: networkCommandMetadata, run: (client, input) => client.observability.network(input), @@ -200,7 +200,7 @@ export const audioCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Probe audio levels', - searchHint: 'audio sound volume level probe', + searchHint: 'sound volume level', }, metadata: audioCommandMetadata, run: (client, input) => client.observability.audio(input), diff --git a/packages/command-surface/src/perf/index.ts b/packages/command-surface/src/perf/index.ts index 1d12e567fb..94fb7b706f 100644 --- a/packages/command-surface/src/perf/index.ts +++ b/packages/command-surface/src/perf/index.ts @@ -73,7 +73,7 @@ export const perfCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Check frames, memory, or native profiles', - searchHint: 'performance fps frames memory profile cpu', + searchHint: 'performance fps profile', cliDetail: 'Use perf frames for bounded frame-health evidence and perf memory sample for a compact process-memory reading. On iOS simulators and macOS, process sampling and captures target the resolved app executable and exclude other copies with the same name. Apple xctrace and Android Simpleperf/Perfetto captures keep raw artifacts on disk; report produces bounded agent-readable evidence. For React render performance, run agent-device react-devtools profile start before the interaction and react-devtools profile stop after it.', mcpDetail: diff --git a/packages/command-surface/src/post-action-observation-grammar.ts b/packages/command-surface/src/post-action-observation-grammar.ts index d0213a6039..fb20d95493 100644 --- a/packages/command-surface/src/post-action-observation-grammar.ts +++ b/packages/command-surface/src/post-action-observation-grammar.ts @@ -31,9 +31,12 @@ const verifyField = () => const settleFields = () => ({ settle: booleanField( - 'Wait for the UI to go quiet, then return the settled UI diff in this response. Best-effort; never fails the action.', + 'Once the UI goes quiet, return the diff vs the pre-action tree. Best-effort; never fails the action.', + ), + settleQuietMs: integerField( + 'Settle: quiet window in ms (default 500, 300 on iOS simulator; broad iOS screen swaps hold at least 1500).', + { min: 0 }, ), - settleQuietMs: integerField('Settle: quiet window in milliseconds (default 500).', { min: 0 }), timeoutMs: integerField('Settle: wait deadline in milliseconds (default 10000).', { min: 1 }), }); diff --git a/packages/command-surface/src/react-native/index.ts b/packages/command-surface/src/react-native/index.ts index f71ba5f7dd..b23bc4fc5c 100644 --- a/packages/command-surface/src/react-native/index.ts +++ b/packages/command-surface/src/react-native/index.ts @@ -40,7 +40,7 @@ export const reactNativeCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Run React Native automation helpers', - searchHint: 'react native rn expo helpers', + searchHint: 'rn expo', }, metadata: reactNativeCommandMetadata, run: (client, input) => client.command.reactNative(input), diff --git a/packages/command-surface/src/recording/index.ts b/packages/command-surface/src/recording/index.ts index e28d4dafc5..04443bb253 100644 --- a/packages/command-surface/src/recording/index.ts +++ b/packages/command-surface/src/recording/index.ts @@ -127,7 +127,7 @@ export const recordCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Start or stop screen recording', - searchHint: 'record video screen recording mp4', + searchHint: 'mp4', cliDetail: 'MP4 verification setup starts in the background before recording output preparation, without delaying capture. If setup exceeds the stop budget, details.reason is video_validator_compile_timeout (retriable); run record stop again while setup continues. Hosts without xcrun keep container-only MP4 completion. Other compilation failures, or an unavailable validator executable or AVFoundation during verification, return video_validator_unavailable. Repair the reported failure and retry record stop; a failed compile is retried on the next start or stop. The default --scope app requires an active app session from open ; use --scope device/system to explicitly request whole-screen recording where the selected backend supports it. Android record start publishes a durable device manifest, recordings longer than the 180s adb screenrecord limit are returned as multiple MP4 chunks while the daemon stays alive, and daemon-restart recovery uses only manifest-owned chunks. Android screenrecord encodes a frame only when the screen changes, so a clip can end at the last frame the recorder encoded instead of at record stop; durationMs is host wall clock from record start until the export finished, and capturedDurationMs reports the video timeline when it can be measured, with a warning naming how much of the window that video covers. An Android manifest left by an unreachable recording is retired on the next start once its recorders are proven gone; one still owned refuses with non-retriable DEVICE_IN_USE and reason native_recovery_evidence_open naming the session to run record stop for, and a recorder that is still writing an artifact refuses with DEVICE_IN_USE and reason native_recording_artifact_claimed, which clears itself once that recorder ends at the 180s limit. HarmonyOS supports whole-screen recording on physical devices only: use --scope device/system; --fps, --quality, and --hide-touches are unsupported. Use --quality to choose medium or high export quality on supported backends. The touch overlay renders at most 30 frames a second, or at --fps when lower; a lower --fps makes record stop faster on a long recording, and an overlay export that runs past its budget returns the raw video with an overlayWarning. An iOS simulator host recording lock returns non-retriable DEVICE_IN_USE with reason apple_simulator_recording_busy. Stop the recording in its owning session; if a dead recorder left the host locked, ask the host operator to restart the CoreSimulator stream service. On any backend, record start first settles the open recording records on its device that it would refuse on and whose session this daemon no longer holds: it finishes each one as its own record stop would, so the export lands at the output path of that recording and a later record stop in that session returns it, and runs the exact-owner recovery a daemon restart runs, which discards what that recording left, only for a record that cannot finish. A record it cannot confirm cleaned, or one whose session is still held, refuses the start with reason cleanup-unconfirmed and names that session: stop or close it, then retry. A record that another device left under the same session name is never settled from this device: the start refuses and names that device, so stop it in that session or record under another session name. A record stop that cannot produce its export keeps what a retry reads back — on Android the device-side artifact and the native manifest — leaves the recording manifest of that session open, and returns its own error, so retry record stop in that session; while this daemon holds that session, only closing it disposes them, and once it does not, the next record start on that device settles them as described above. Every stopped recording also reports what became of its recorder and of the file that recorder writes to: recorder is confirmed, or lost when the session holding it died, and nativePathDisposition is retirable while that file still owes a removal or retired once its removal was verified. Both are optional disclosures, not failures; ADR 0024 declares further states (unconfirmed, pending) for the steps that gain those probes, and no stop reports them yet. record contact-sheet [--out ] is a local report over a file you already exported: it decodes frames out of that MP4 and writes one PNG holding the cells where the screen visibly changed, each labeled with its elapsed time on the clip timeline. It needs no session and no device, rebuilds any past recording, and works for every backend that exports MP4; a WebM recording is refused with reason contact_sheet_container_unsupported. The sample grid is bounded and spread across the whole clip, so it reports coverage rather than a review: a flash that opens and closes entirely between two sample times is not in the sheet, which is why the result also reports how many times it sampled and how many frames came back. Frame decoding is Apple AVFoundation tooling, so the command runs only on macOS hosts and refuses elsewhere with reason contact_sheet_unsupported_host.', }, @@ -143,7 +143,7 @@ export const traceCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Start or stop trace capture', - searchHint: 'trace tracing profile capture', + searchHint: 'tracing profile', }, metadata: traceCommandMetadata, run: (client, input) => client.recording.trace(input), diff --git a/packages/command-surface/src/replay/index.ts b/packages/command-surface/src/replay/index.ts index a24f673b78..8fe72d95a8 100644 --- a/packages/command-surface/src/replay/index.ts +++ b/packages/command-surface/src/replay/index.ts @@ -312,7 +312,7 @@ export const replayCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Replay a recorded session or Maestro flow', - searchHint: 'replay script rerun recorded flow maestro', + searchHint: 'rerun', cliDetail: 'Replay has a 60-second work budget inside a 90-second client timeout; --timeout sets a work budget of X and client timeout of X + 30000 ms. The running step receives ordinary request cancellation at the budget. When it settles, failure returns details.reason daemon_request_timeout and details.timeoutMs, preserving the failing script path and line. The daemon and other sessions stay available if it responds; an authored close may already have ended its session. A completed replay returns success even if it settles just after the deadline. Cancellation does not prove native device work stopped. A step that ignores cancellation beyond the 30-second margin may cause a transport timeout; an unresponsive daemon retains the client liveness-based reset. Direct test keeps its own timeout behavior. For Maestro YAML compatibility flows, use replay --maestro and keep the target binding such as --platform ios on the replay command. A script with no terminal close leaves its session (and daemon) running until you close it or it idle-reaps — no different from a session opened interactively. For native .ad scripts, --keep-session suppresses exactly an authored terminal close so you can continue interactively. replay export converts compatible actions to Maestro YAML locally, including app switches with explicit launchApp.appId targets, deep links (including tel: and mailto:) as openLink, and home as pressKey: Home.', }, @@ -328,7 +328,7 @@ export const testCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Run replay test suites', - searchHint: 'test suite run tests replay ad', + searchHint: 'tests ad', cliDetail: "Relative globs are expanded on the caller from its working directory, whose name is treated literally. Quote glob inputs to defer expansion to test. Copied diagnostic artifacts receive numbered filenames when needed to preserve other artifacts, replay sources, timing traces, and attempt manifests. JUnit reports (--reporter junit:) replace characters forbidden by XML 1.0 with U+FFFD and preserve legal Unicode and whitespace. JSON and other reporters retain the original suite values. Custom reporter getExitCode hooks must return an integer from 0 to 255 or undefined; the highest valid code wins and cannot lower a failing suite's exit code. A nested test stops after a bounded resource refusal even with --retries; other failures keep their configured retries.", }, diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index f59787c94b..1509df4cd2 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -630,7 +630,7 @@ const appStateCommandFacet = defineParameterlessCommandFacet({ description: appStateCommandDescription, text: { summary: 'Show the foreground app and activity', - searchHint: 'foreground app current activity state', + searchHint: 'current state', }, run: (client, input) => client.command.appState(input), }); @@ -640,7 +640,7 @@ const backCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Navigate back in the app or system', - searchHint: 'back navigate previous screen return dismiss', + searchHint: 'previous return dismiss', }, metadata: backCommandMetadata, run: (client, input) => client.command.back(input), @@ -655,7 +655,7 @@ const homeCommandFacet = defineParameterlessCommandFacet({ description: homeCommandDescription, text: { summary: 'Go to the device home screen', - searchHint: 'home screen springboard launcher', + searchHint: 'springboard launcher', }, run: (client, input) => client.command.home(input), }); @@ -665,7 +665,7 @@ const orientationCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Set device orientation', - searchHint: 'orientation rotate landscape portrait', + searchHint: 'rotate landscape', }, metadata: orientationCommandMetadata, run: (client, input) => client.command.orientation(input), @@ -679,7 +679,7 @@ const foldCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Fold or unfold a foldable iPhone simulator or Android emulator', - searchHint: 'fold unfold foldable posture', + searchHint: 'flip book clamshell', cliDetail: `iPhone Duo simulators and foldable Android emulators. iOS sends a ${foldGuidance.hingeEvent} and confirms the hinge angle inside the simulator; Android sets the emulator console posture and confirms the device state. Refs and coordinates do not survive a pose change.`, }, metadata: foldCommandMetadata, @@ -695,7 +695,7 @@ const appSwitcherCommandFacet = defineParameterlessCommandFacet({ description: appSwitcherCommandDescription, text: { summary: 'Open the device app switcher', - searchHint: 'app switcher recents multitasking', + searchHint: 'recents multitasking', }, run: (client, input) => client.command.appSwitcher(input), }); @@ -705,7 +705,7 @@ const keyboardCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Inspect, press, or dismiss the device keyboard', - searchHint: 'keyboard hide dismiss return key', + searchHint: 'return', }, metadata: keyboardCommandMetadata, run: (client, input) => client.command.keyboard(input), @@ -719,7 +719,7 @@ const clipboardCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Read or write device clipboard text', - searchHint: 'clipboard copy paste pasteboard', + searchHint: 'copy paste pasteboard', }, metadata: clipboardCommandMetadata, run: (client, input) => client.command.clipboard(input as ClipboardCommandOptions), @@ -734,7 +734,7 @@ const actionButtonCommandFacet = defineParameterlessCommandFacet({ description: actionButtonCommandDescription, text: { summary: 'Press the iPhone or iPad Action Button', - searchHint: 'action button hardware', + searchHint: 'hardware', cliDetail: 'iPhone and iPad only. The runner asks the device for the button and reports unsupported when that model has none.', }, @@ -746,7 +746,7 @@ const tvRemoteCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Press a TV remote navigation, media, or hardware button', - searchHint: 'tv remote dpad select menu play', + searchHint: 'dpad play', cliDetail: 'longpress selects the 500ms preset; --duration-ms overrides it. Android TV uses the system long-press duration.', }, @@ -762,7 +762,7 @@ const hardwareButtonCommandFacet = defineCommandFacet({ effect: 'write', text: { summary: 'Press a phone hardware button once', - searchHint: 'hardware button volume power lock', + searchHint: 'lock', cliDetail: 'Single press; no hold. Android presses all eleven named keys on phones and tablets: volume-up, volume-down, power, mute, sleep, wake, and the five media keys. A physical iPhone or iPad presses volume only; the other keys are never supported on Apple targets. iOS Simulators, Android TV, and tvOS refuse — use tv-remote on TV targets.', }, @@ -778,7 +778,7 @@ const telephonyCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Simulate an incoming call or SMS on an Android emulator over adb', - searchHint: 'phone call sms telephony emulator', + searchHint: 'voice gsm cellular', cliDetail: 'Android emulators through the local adb executor; WebDriver (Appium) and Limrun sessions refuse it. Quote the SMS text, or pass it as the remaining words. Re-snapshot after the event: the call screen or notification can cover the app.', }, diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index b1ef88d5f4..7ee8da96b7 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -33,8 +33,8 @@ export type CaptureSnapshotOptions = AgentDeviceRequestOverrides & Omit & { /** * List the custom actions an element merged away, such as a card's reply control. - * For planning only: they cannot be invoked, so reach the control another way. iOS - * simulator only; costs one accessibility round trip per merged element. + * Not invocable: reach the control via its detail screen, a labeled element elsewhere, + * or its rect. iOS simulator only; costs one accessibility round trip per merged element. */ customActions?: SnapshotCommandOptionFields['customActions']; timeoutMs?: number; diff --git a/packages/kernel/src/errors.ts b/packages/kernel/src/errors.ts index 8d89d92426..ad660039ac 100644 --- a/packages/kernel/src/errors.ts +++ b/packages/kernel/src/errors.ts @@ -679,6 +679,11 @@ export function defaultHintForCode(code: string): string { return codeRecoveryHint(code) ?? 'Read the diagnostics log for the failing step.'; } +/** Whether the code's default recovery step tells the reader to read the diagnostics log. */ +export function defaultHintReadsLog(code: string): boolean { + return code === 'UNKNOWN' || codeRecoveryHint(code) === undefined; +} + function codeRecoveryHint(code: string): string | undefined { switch (code) { case 'INVALID_ARGS': diff --git a/src/mcp/__tests__/command-tools-loading-meta.test.ts b/src/mcp/__tests__/command-tools-loading-meta.test.ts index c0ce1d7361..56035ad85f 100644 --- a/src/mcp/__tests__/command-tools-loading-meta.test.ts +++ b/src/mcp/__tests__/command-tools-loading-meta.test.ts @@ -26,6 +26,16 @@ test('every tool declares search keywords', () => { } }); +test('search keywords add only words the tool name and description lack', () => { + for (const tool of tools) { + const text = new Set(`${tool.name} ${tool.description}`.toLowerCase().split(/[^a-z0-9]+/)); + const repeated = (tool._meta['anthropic/searchHint'] ?? '') + .split(/\s+/) + .filter((word) => text.has(word.toLowerCase())); + assert.deepEqual(repeated, [], `${tool.name} searchHint repeats its text`); + } +}); + // Always-loaded schemas land in every run's first prompt, so their total size is a per-run cost. const ALWAYS_LOADED_BUDGET_BYTES = 26 * 1024; diff --git a/src/mcp/__tests__/command-tools-structured-content.test.ts b/src/mcp/__tests__/command-tools-structured-content.test.ts index d70baa94f9..8f6a063399 100644 --- a/src/mcp/__tests__/command-tools-structured-content.test.ts +++ b/src/mcp/__tests__/command-tools-structured-content.test.ts @@ -143,7 +143,7 @@ test('MCP command tool executor renders JSON text when requested', async () => { ); }); -test('default MCP errors retain possible dispatch and observe-first recovery, and leave out the log path', async () => { +test('default MCP errors retain possible dispatch and observe-first recovery, and keep the log path the default step reads', async () => { const executor = executorReturning(undefined, () => { const error = new AppError('COMMAND_FAILED', 'press reply was lost', { dispatched: 'unknown', @@ -181,3 +181,29 @@ test('MCP errors with a specific recovery step leave out the log path', async () 'Error (COMMAND_FAILED, runner_reply_lost): press reply was lost. Next: Run snapshot, then press the target again.', ); }); + +test('MCP errors whose default recovery step does not read the log leave out the log path', async () => { + const executor = executorReturning(undefined, () => { + throw new AppError('INVALID_ARGS', 'missing ref', { logPath: '/tmp/agent-device/press.log' }); + }); + + const result = await executor.execute('press', {}); + + assert.doesNotMatch(result.content[0]?.text ?? '', /Log:/); +}); + +test('MCP errors keep a multi-line typed reason on the summary line', async () => { + const executor = executorReturning(undefined, () => { + throw new AppError('COMMAND_FAILED', 'press failed', { + reason: 'runner\n reply lost', + hint: 'Run snapshot, then press the target again.', + }); + }); + + const result = await executor.execute('press', { ref: '@e2' }); + + assert.match( + result.content[0]?.text ?? '', + /^Error \(COMMAND_FAILED, runner reply lost\): press failed\./, + ); +}); diff --git a/website/docs/docs/agent-setup.md b/website/docs/docs/agent-setup.md index 180e109e1f..2bf494aaca 100644 --- a/website/docs/docs/agent-setup.md +++ b/website/docs/docs/agent-setup.md @@ -69,7 +69,7 @@ If your client reads results as data, start the server with `agent-device mcp -- Tools that act on a UI element take its target as top-level arguments, in the same words as the CLI: `press {"ref":"@e12"}`, `click {"selector":"id=\"submit\""}`, or `press {"x":120,"y":340}`. `get` takes `ref` or `selector`. -When a tool fails, the server returns an MCP tool result with `isError: true`. Check the tool result, not only whether the JSON-RPC call succeeded. The error text is short: the error code and typed reason, a one-line message, the cause when known, and the next step. It leaves out diagnostics log paths. A call with an argument the tool doesn't take fails with `INVALID_ARGS`, and its hint shows a call that the tool accepts. +When a tool fails, the server returns an MCP tool result with `isError: true`. Check the tool result, not only whether the JSON-RPC call succeeded. The error text is short: the error code and typed reason, a one-line message, the cause when known, and the next step. It includes a diagnostics log path only when the next step is to read that log. A call with an argument the tool doesn't take fails with `INVALID_ARGS`, and its hint shows a call that the tool accepts. The server runs one `tools/call` at a time, in the order it receives them. It answers `ping`, `tools/list`, and the handshake at once, even while a long tool call such as `test` or `wait` runs. A `notifications/cancelled` for a running or queued `tools/call` stops it, and the server sends no response for that request. Cancellation stops the daemon request, not work that already reached the device: on iOS, a command the device received keeps running there, and the session waits for it. See [Command methods](/docs/client-api#command-methods). A JSON-RPC batch cannot carry a `tools/call`; the server refuses that element with `-32600`. From 88080f6051ccafe04aed87f236b78d6ae83e20b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 15:49:46 +0100 Subject: [PATCH 03/12] test(mcp): assert the INVALID_ARGS error before checking the log path is absent Co-Authored-By: Claude Opus 5.5 --- src/mcp/__tests__/command-tools-structured-content.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mcp/__tests__/command-tools-structured-content.test.ts b/src/mcp/__tests__/command-tools-structured-content.test.ts index 8f6a063399..4606c6734e 100644 --- a/src/mcp/__tests__/command-tools-structured-content.test.ts +++ b/src/mcp/__tests__/command-tools-structured-content.test.ts @@ -189,7 +189,10 @@ test('MCP errors whose default recovery step does not read the log leave out the const result = await executor.execute('press', {}); - assert.doesNotMatch(result.content[0]?.text ?? '', /Log:/); + assert.equal(result.isError, true); + const text = result.content[0]?.text ?? ''; + assert.match(text, /^Error \(INVALID_ARGS\): missing ref\./); + assert.doesNotMatch(text, /Log:/); }); test('MCP errors keep a multi-line typed reason on the summary line', async () => { From 8e0434741bbd902dd8b76bcc55198dda75fae9d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 16:01:21 +0100 Subject: [PATCH 04/12] fix(batch): derive the MCP batch deadline from its timeout policy Co-Authored-By: Claude Opus 5.5 --- packages/command-surface/src/batch/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/command-surface/src/batch/index.ts b/packages/command-surface/src/batch/index.ts index f7cc14546a..08cb53cfd6 100644 --- a/packages/command-surface/src/batch/index.ts +++ b/packages/command-surface/src/batch/index.ts @@ -6,6 +6,9 @@ import { defineCommandFacet, defineCommandFamilyFromFacets } from '../family/typ import { commonToClientOptions } from '../common-input-fields.ts'; import { createBatchCommandMetadata, type BatchCommandStep, type BatchInput } from './metadata.ts'; import { STRUCTURED_BATCH_COMMAND_NAMES } from '@agent-device/command-registry/batch-policy'; +import { COMPOSITE_TIMEOUT_POLICIES } from '@agent-device/command-registry/timeout-policy'; + +const BATCH_DEADLINE_SECONDS = (COMPOSITE_TIMEOUT_POLICIES.batch.daemonDeadlineMs ?? 0) / 1000; const batchCommandMetadata = createBatchCommandMetadata(); @@ -50,7 +53,7 @@ const BATCH_MCP_DETAIL = [ 'Use it for a known multi-step walk, such as press A, press B, back, instead of one tool call per step.', "steps is an array of {command, input} objects; input takes the same arguments as that command's own tool (ref, selector, or x and y for targets; settle: true for the settled diff).", 'Steps run serially in one session; onError: "stop" halts at the first failing step, and the result reports the steps that ran.', - 'The whole batch has a 60-second work budget; split longer walks.', + `The daemon stops the whole batch after ${BATCH_DEADLINE_SECONDS} seconds; split longer walks.`, `Example steps, in order (refs such as @e12 come from the last snapshot or settled diff): ${BATCH_STEP_EXAMPLES.map((step) => JSON.stringify(step)).join(', ')}.`, ].join(' '); From 067626d2b7c2f96e4a8dd66804cc9105021be3de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 16:10:21 +0100 Subject: [PATCH 05/12] fix(ai-sdk): keep tier-less commands out of the core tool set Co-Authored-By: Claude Opus 5.5 --- src/ai-sdk/__tests__/index.test.ts | 7 +++++++ src/ai-sdk/index.ts | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ai-sdk/__tests__/index.test.ts b/src/ai-sdk/__tests__/index.test.ts index 679a22a91e..288674894e 100644 --- a/src/ai-sdk/__tests__/index.test.ts +++ b/src/ai-sdk/__tests__/index.test.ts @@ -97,6 +97,13 @@ test('defaults to the core tool set and pins the session for every call', async false, 'devices is extended-tier, excluded from the core default', ); + for (const tierless of ['debug', 'metro', 'session']) { + assert.equal( + tierless in tools, + false, + `${tierless} declares no tier, so the core default leaves it out`, + ); + } const schema = rawSchemaOf(tools, 'snapshot'); assert.equal('session' in schema.properties, false, 'session is always hidden from the model'); diff --git a/src/ai-sdk/index.ts b/src/ai-sdk/index.ts index a790a237ba..3357533a7d 100644 --- a/src/ai-sdk/index.ts +++ b/src/ai-sdk/index.ts @@ -101,7 +101,8 @@ export async function createAgentDeviceTools( const tools: ToolSet = {}; for (const definition of listCommandTools({ outputSchemas: true })) { - if (set === 'core' && resolveCommandFrameworkTier(definition.name) === 'extended') continue; + const tier = resolveCommandFrameworkTier(definition.name); + if (set === 'core' && tier !== 'core' && tier !== 'loop') continue; tools[definition.name] = tool({ description: definition.description, From 44e7c76b6dbf569ab2e667b061554116d1b77c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 17:04:17 +0100 Subject: [PATCH 06/12] fix(mcp): keep the diagnostics log path and full message in agent error text A producer hint that sends the reader to the runner or diagnostics log lost the path under the default-hint rule. Agent text now keeps the Log: line whenever a log exists and the full Message: line when the summary cuts it. Co-Authored-By: Claude Opus 5.5 --- packages/command-output/src/output-common.ts | 5 ++- packages/command-output/src/output/error.ts | 22 ++++--------- packages/kernel/src/errors.ts | 5 --- .../command-tools-structured-content.test.ts | 32 ++++++++++++++----- src/mcp/command-tools.ts | 5 +-- website/docs/docs/agent-setup.md | 2 +- website/docs/docs/ai-sdk.md | 1 + 7 files changed, 39 insertions(+), 33 deletions(-) diff --git a/packages/command-output/src/output-common.ts b/packages/command-output/src/output-common.ts index bebe2b2683..0b0f6df32c 100644 --- a/packages/command-output/src/output-common.ts +++ b/packages/command-output/src/output-common.ts @@ -42,9 +42,12 @@ export type CliOutputFormatterParams = { * Who reads the text. `terminal` (the default) is a person or script at the CLI. `model` is * an agent reading a tool result, which gets no daemon-host state paths it would never open. */ - reader?: 'terminal' | 'model'; + reader?: OutputReader; }; +/** Who reads command output: a person or script at the CLI, or an agent reading a tool result. */ +export type OutputReader = 'terminal' | 'model'; + export type CliOutputFormatter = ( params: CliOutputFormatterParams, ) => CliOutput | Promise; diff --git a/packages/command-output/src/output/error.ts b/packages/command-output/src/output/error.ts index 6d959d5bf6..4bd16bf785 100644 --- a/packages/command-output/src/output/error.ts +++ b/packages/command-output/src/output/error.ts @@ -1,7 +1,5 @@ import { AppError, - defaultHintForCode, - defaultHintReadsLog, normalizeError, readDispatchDisclosure, readErrorCandidateViews, @@ -10,6 +8,7 @@ import { type NormalizedError, } from '@agent-device/kernel/errors'; import { readResponseWarnings } from '@agent-device/kernel/success-text'; +import type { OutputReader } from '../output-common.ts'; import { formatReplayDivergenceReport } from '@agent-device/ad-replay/divergence'; import { collapseWarningText } from '../output-common.ts'; @@ -29,11 +28,10 @@ export type HumanErrorOptions = { showDetails?: boolean; /** * Who reads the text. `terminal` (the default) is a person or script at the CLI. `model` is - * an agent reading a tool result: the typed `details.reason` joins the code and the message - * stays one line. The diagnostics log path is left out unless the recovery step is the code's - * default and that default tells the reader to read the log. + * an agent reading a tool result: the typed `details.reason` joins the code and the summary + * stays one line. */ - reader?: 'terminal' | 'model'; + reader?: OutputReader; /** The failing command's synopsis, such as `agent-device fill `. */ synopsis?: string; }; @@ -67,10 +65,9 @@ export function formatHumanError( return { summary, body: formatBody(normalized, { - fullMessage: truncated && !forModel, + fullMessage: truncated, candidateViews: inlineCandidates ? [] : candidateViews, showDetails: options.showDetails, - logLine: !forModel || hintReadsLog(normalized), }), }; } @@ -125,12 +122,6 @@ function nextStep( return parts.length > 0 ? parts.join(' ') : undefined; } -function hintReadsLog(normalized: NormalizedError): boolean { - return ( - normalized.hint === defaultHintForCode(normalized.code) && defaultHintReadsLog(normalized.code) - ); -} - const REASON_LIMIT = 80; /** The typed `details.reason`, when the throw site classified the failure, as one bounded line. */ @@ -148,7 +139,6 @@ function formatBody( fullMessage: boolean; candidateViews: ErrorCandidateView[]; showDetails?: boolean; - logLine: boolean; }, ): string[] { const body: string[] = []; @@ -165,7 +155,7 @@ function formatBody( } const candidateLines = formatErrorCandidateViews(options.candidateViews); if (candidateLines.length > 0) body.push(candidateLines.join('\n')); - const logLine = options.logLine ? formatLogLine(normalized) : undefined; + const logLine = formatLogLine(normalized); if (logLine) body.push(logLine); // ADR 0012: the divergence compact report always renders; --debug's raw // details dump below remains the full-object view. diff --git a/packages/kernel/src/errors.ts b/packages/kernel/src/errors.ts index ad660039ac..8d89d92426 100644 --- a/packages/kernel/src/errors.ts +++ b/packages/kernel/src/errors.ts @@ -679,11 +679,6 @@ export function defaultHintForCode(code: string): string { return codeRecoveryHint(code) ?? 'Read the diagnostics log for the failing step.'; } -/** Whether the code's default recovery step tells the reader to read the diagnostics log. */ -export function defaultHintReadsLog(code: string): boolean { - return code === 'UNKNOWN' || codeRecoveryHint(code) === undefined; -} - function codeRecoveryHint(code: string): string | undefined { switch (code) { case 'INVALID_ARGS': diff --git a/src/mcp/__tests__/command-tools-structured-content.test.ts b/src/mcp/__tests__/command-tools-structured-content.test.ts index 4606c6734e..bfaf310aa6 100644 --- a/src/mcp/__tests__/command-tools-structured-content.test.ts +++ b/src/mcp/__tests__/command-tools-structured-content.test.ts @@ -165,24 +165,25 @@ test('default MCP errors retain possible dispatch and observe-first recovery, an assert.match(text, /Log: \/tmp\/agent-device\/press\.log/); }); -test('MCP errors with a specific recovery step leave out the log path', async () => { +test('MCP errors keep the log path when a producer hint sends the reader to the log', async () => { const executor = executorReturning(undefined, () => { - throw new AppError('COMMAND_FAILED', 'press reply was lost', { - reason: 'runner_reply_lost', - hint: 'Run snapshot, then press the target again.', - logPath: '/tmp/agent-device/press.log', + throw new AppError('COMMAND_FAILED', 'press still failed after a runner restart', { + reason: 'runner_recycle_budget_exhausted', + hint: 'Check the runner log for the underlying failure before retrying.', + logPath: '/tmp/agent-device/runner.log', }); }); const result = await executor.execute('press', { ref: '@e2' }); + assert.equal(result.isError, true); assert.equal( result.content[0]?.text, - 'Error (COMMAND_FAILED, runner_reply_lost): press reply was lost. Next: Run snapshot, then press the target again.', + 'Error (COMMAND_FAILED, runner_recycle_budget_exhausted): press still failed after a runner restart. Next: Check the runner log for the underlying failure before retrying.\nLog: /tmp/agent-device/runner.log', ); }); -test('MCP errors whose default recovery step does not read the log leave out the log path', async () => { +test('MCP errors keep the log path whatever the recovery step', async () => { const executor = executorReturning(undefined, () => { throw new AppError('INVALID_ARGS', 'missing ref', { logPath: '/tmp/agent-device/press.log' }); }); @@ -192,7 +193,22 @@ test('MCP errors whose default recovery step does not read the log leave out the assert.equal(result.isError, true); const text = result.content[0]?.text ?? ''; assert.match(text, /^Error \(INVALID_ARGS\): missing ref\./); - assert.doesNotMatch(text, /Log:/); + assert.match(text, /\nLog: \/tmp\/agent-device\/press\.log$/); +}); + +test('MCP errors keep the full message when the summary cuts it', async () => { + const long = `runner reply was lost ${'x'.repeat(320)} end`; + const executor = executorReturning(undefined, () => { + throw new AppError('COMMAND_FAILED', long, { + hint: 'Run snapshot, then press the target again.', + }); + }); + + const result = await executor.execute('press', { ref: '@e2' }); + + const text = result.content[0]?.text ?? ''; + assert.match(text.split('\n')[0] ?? '', /… Next: Run snapshot/); + assert.ok(text.includes(`\nMessage: ${long}`), 'the cut summary is followed by the full message'); }); test('MCP errors keep a multi-line typed reason on the summary line', async () => { diff --git a/src/mcp/command-tools.ts b/src/mcp/command-tools.ts index 0ad1bbb08d..10559a05af 100644 --- a/src/mcp/command-tools.ts +++ b/src/mcp/command-tools.ts @@ -120,8 +120,9 @@ const DECLARED_OPERATOR_AUDIENCE: InputAudienceMap = Object.fromEntries( * operator key. `listCommandTools` and admission both read this, so a key can * never be hidden from the model yet admitted from the wire. * - * The command's own map is spread FIRST so an `operator` classification always - * outranks it. `retired` keys are admitted (the command's reader answers with + * The shared alias map is spread first, so a command that declares a field under an + * alias's name (its own `target`, for example) keeps it. The command's own map comes next, + * so an `operator` classification spread after it always outranks it. `retired` keys are admitted (the command's reader answers with * migration guidance), so a command that retired a key whose name another * declaration owns as `operator` must fail closed on the refusal, not open on * the guidance. Every operator key a command declares is already in diff --git a/website/docs/docs/agent-setup.md b/website/docs/docs/agent-setup.md index 2bf494aaca..efa3c2c9fd 100644 --- a/website/docs/docs/agent-setup.md +++ b/website/docs/docs/agent-setup.md @@ -69,7 +69,7 @@ If your client reads results as data, start the server with `agent-device mcp -- Tools that act on a UI element take its target as top-level arguments, in the same words as the CLI: `press {"ref":"@e12"}`, `click {"selector":"id=\"submit\""}`, or `press {"x":120,"y":340}`. `get` takes `ref` or `selector`. -When a tool fails, the server returns an MCP tool result with `isError: true`. Check the tool result, not only whether the JSON-RPC call succeeded. The error text is short: the error code and typed reason, a one-line message, the cause when known, and the next step. It includes a diagnostics log path only when the next step is to read that log. A call with an argument the tool doesn't take fails with `INVALID_ARGS`, and its hint shows a call that the tool accepts. +When a tool fails, the server returns an MCP tool result with `isError: true`. Check the tool result, not only whether the JSON-RPC call succeeded. The error text is short: the error code and typed reason, a one-line message, the cause when known, and the next step. When the failure flushed a diagnostics log, the text ends with its path; a message cut short in the summary follows in full. A call with an argument the tool doesn't take fails with `INVALID_ARGS`, and its hint shows a call that the tool accepts. The server runs one `tools/call` at a time, in the order it receives them. It answers `ping`, `tools/list`, and the handshake at once, even while a long tool call such as `test` or `wait` runs. A `notifications/cancelled` for a running or queued `tools/call` stops it, and the server sends no response for that request. Cancellation stops the daemon request, not work that already reached the device: on iOS, a command the device received keeps running there, and the session waits for it. See [Command methods](/docs/client-api#command-methods). A JSON-RPC batch cannot carry a `tools/call`; the server refuses that element with `-32600`. diff --git a/website/docs/docs/ai-sdk.md b/website/docs/docs/ai-sdk.md index 7c5d8061d5..f650f2368a 100644 --- a/website/docs/docs/ai-sdk.md +++ b/website/docs/docs/ai-sdk.md @@ -55,6 +55,7 @@ An `abortSignal` passed to `generateText`, `streamText`, or the agent cancels a `set: 'core'` is the default. It exposes the perceive-and-act loop: open, close, snapshot, click, press, fill, type, get, is, find, wait, back, scroll, swipe, alert, and screenshot. - Pass `set: 'all'` when the agent also needs device-management or observability commands. +- Tool calls take the same arguments as the MCP tools. Operator settings (`debug`, `cost`, `responseLevel`) and the `target` alias are refused; set `AGENT_DEVICE_COST` or `AGENT_DEVICE_RESPONSE_LEVEL` for the process, and use `deviceTarget` for a form factor. - Pass `approval: { close: 'user-approval' }` to require user approval before a command runs. `createAgentDeviceTools()` returns the map as `toolApproval`, ready to pass to [`ToolLoopAgent`](https://ai-sdk.dev/docs/agents/tool-approvals). Use the [Node.js API](/docs/client-api) when your application needs deterministic setup or other direct device control outside the agent loop. See the AI SDK reference for [`ToolLoopAgent`](https://ai-sdk.dev/docs/reference/ai-sdk-core/tool-loop-agent). From 4d365c31fdd857b239347aea1f776cbf20268950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 17:08:51 +0100 Subject: [PATCH 07/12] docs(ai-sdk): note the pinned session and platform in tool arguments Co-Authored-By: Claude Opus 5.5 --- website/docs/docs/ai-sdk.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/ai-sdk.md b/website/docs/docs/ai-sdk.md index f650f2368a..6936ec14f8 100644 --- a/website/docs/docs/ai-sdk.md +++ b/website/docs/docs/ai-sdk.md @@ -55,7 +55,7 @@ An `abortSignal` passed to `generateText`, `streamText`, or the agent cancels a `set: 'core'` is the default. It exposes the perceive-and-act loop: open, close, snapshot, click, press, fill, type, get, is, find, wait, back, scroll, swipe, alert, and screenshot. - Pass `set: 'all'` when the agent also needs device-management or observability commands. -- Tool calls take the same arguments as the MCP tools. Operator settings (`debug`, `cost`, `responseLevel`) and the `target` alias are refused; set `AGENT_DEVICE_COST` or `AGENT_DEVICE_RESPONSE_LEVEL` for the process, and use `deviceTarget` for a form factor. +- Tool calls take the MCP tools' command arguments, except that `session` is always pinned and `platform` is hidden when you pass it. Operator settings (`debug`, `cost`, `responseLevel`) and the `target` alias are refused; set `AGENT_DEVICE_COST` or `AGENT_DEVICE_RESPONSE_LEVEL` for the process, and use `deviceTarget` for a form factor. - Pass `approval: { close: 'user-approval' }` to require user approval before a command runs. `createAgentDeviceTools()` returns the map as `toolApproval`, ready to pass to [`ToolLoopAgent`](https://ai-sdk.dev/docs/agents/tool-approvals). Use the [Node.js API](/docs/client-api) when your application needs deterministic setup or other direct device control outside the agent loop. See the AI SDK reference for [`ToolLoopAgent`](https://ai-sdk.dev/docs/reference/ai-sdk-core/tool-loop-agent). From 3ede31599bf4651c67be556fffad8f0c3d2362f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 20:43:38 +0100 Subject: [PATCH 08/12] docs: say which AI SDK fields are refused and that errors include the log path Co-Authored-By: Claude Opus 5.5 --- website/docs/docs/agent-setup.md | 2 +- website/docs/docs/ai-sdk.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/docs/agent-setup.md b/website/docs/docs/agent-setup.md index efa3c2c9fd..acd5f0ca8a 100644 --- a/website/docs/docs/agent-setup.md +++ b/website/docs/docs/agent-setup.md @@ -69,7 +69,7 @@ If your client reads results as data, start the server with `agent-device mcp -- Tools that act on a UI element take its target as top-level arguments, in the same words as the CLI: `press {"ref":"@e12"}`, `click {"selector":"id=\"submit\""}`, or `press {"x":120,"y":340}`. `get` takes `ref` or `selector`. -When a tool fails, the server returns an MCP tool result with `isError: true`. Check the tool result, not only whether the JSON-RPC call succeeded. The error text is short: the error code and typed reason, a one-line message, the cause when known, and the next step. When the failure flushed a diagnostics log, the text ends with its path; a message cut short in the summary follows in full. A call with an argument the tool doesn't take fails with `INVALID_ARGS`, and its hint shows a call that the tool accepts. +When a tool fails, the server returns an MCP tool result with `isError: true`. Check the tool result, not only whether the JSON-RPC call succeeded. The error text is short: the error code and typed reason, a one-line message, the cause when known, and the next step. When the failure flushed a diagnostics log, the text includes its path; a message cut short in the summary follows in full. A call with an argument the tool doesn't take fails with `INVALID_ARGS`, and its hint shows a call that the tool accepts. The server runs one `tools/call` at a time, in the order it receives them. It answers `ping`, `tools/list`, and the handshake at once, even while a long tool call such as `test` or `wait` runs. A `notifications/cancelled` for a running or queued `tools/call` stops it, and the server sends no response for that request. Cancellation stops the daemon request, not work that already reached the device: on iOS, a command the device received keeps running there, and the session waits for it. See [Command methods](/docs/client-api#command-methods). A JSON-RPC batch cannot carry a `tools/call`; the server refuses that element with `-32600`. diff --git a/website/docs/docs/ai-sdk.md b/website/docs/docs/ai-sdk.md index 6936ec14f8..b508497b0e 100644 --- a/website/docs/docs/ai-sdk.md +++ b/website/docs/docs/ai-sdk.md @@ -55,7 +55,7 @@ An `abortSignal` passed to `generateText`, `streamText`, or the agent cancels a `set: 'core'` is the default. It exposes the perceive-and-act loop: open, close, snapshot, click, press, fill, type, get, is, find, wait, back, scroll, swipe, alert, and screenshot. - Pass `set: 'all'` when the agent also needs device-management or observability commands. -- Tool calls take the MCP tools' command arguments, except that `session` is always pinned and `platform` is hidden when you pass it. Operator settings (`debug`, `cost`, `responseLevel`) and the `target` alias are refused; set `AGENT_DEVICE_COST` or `AGENT_DEVICE_RESPONSE_LEVEL` for the process, and use `deviceTarget` for a form factor. +- Tool calls take the MCP tools' command arguments, except that `session` is always pinned and `platform` is hidden when you pass it. `debug` and the `target` alias are refused with `INVALID_ARGS`, and `cost`, `responseLevel`, and `stateDir` are dropped from the call; set `AGENT_DEVICE_COST` or `AGENT_DEVICE_RESPONSE_LEVEL` for the process, and use `deviceTarget` for a form factor. - Pass `approval: { close: 'user-approval' }` to require user approval before a command runs. `createAgentDeviceTools()` returns the map as `toolApproval`, ready to pass to [`ToolLoopAgent`](https://ai-sdk.dev/docs/agents/tool-approvals). Use the [Node.js API](/docs/client-api) when your application needs deterministic setup or other direct device control outside the agent loop. See the AI SDK reference for [`ToolLoopAgent`](https://ai-sdk.dev/docs/reference/ai-sdk-core/tool-loop-agent). From 9c4a12c4f5deabe1e5c469bd6621be9f0688366d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 08:32:58 +0100 Subject: [PATCH 09/12] fix(mcp): stop preloading screenshot Benchmark runs showed that a preloaded screenshot pulls agents into a screenshot, image read and coordinate-tap detour whenever open returns before the app is ready, so it moves back to the core tier: still in the default AI SDK set, loaded on demand over MCP. Co-Authored-By: Claude Opus 5.5 --- packages/command-registry/src/descriptors/device-operations.ts | 2 +- src/mcp/__tests__/command-tools-loading-meta.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/command-registry/src/descriptors/device-operations.ts b/packages/command-registry/src/descriptors/device-operations.ts index 8e4fd28f33..9ef94461a1 100644 --- a/packages/command-registry/src/descriptors/device-operations.ts +++ b/packages/command-registry/src/descriptors/device-operations.ts @@ -540,7 +540,7 @@ export const DEVICE_OPERATION_COMMAND_DESCRIPTORS = { } : {}), catalog: { group: 'public' }, - frameworkTier: 'loop', + frameworkTier: 'core', recordsSessionAction: true, recordingEffect: 'observes-app', daemon: { route: 'generic', refFrameEffect: 'preserve', appLease: 'allowed' }, diff --git a/src/mcp/__tests__/command-tools-loading-meta.test.ts b/src/mcp/__tests__/command-tools-loading-meta.test.ts index 56035ad85f..3201725bd2 100644 --- a/src/mcp/__tests__/command-tools-loading-meta.test.ts +++ b/src/mcp/__tests__/command-tools-loading-meta.test.ts @@ -37,7 +37,7 @@ test('search keywords add only words the tool name and description lack', () => }); // Always-loaded schemas land in every run's first prompt, so their total size is a per-run cost. -const ALWAYS_LOADED_BUDGET_BYTES = 26 * 1024; +const ALWAYS_LOADED_BUDGET_BYTES = 22 * 1024; test('the always-loaded tools stay within their byte budget', () => { const bytes = Buffer.byteLength( From 2dc0ae15ce17560b2bc81001729ad58449b626c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 08:37:30 +0100 Subject: [PATCH 10/12] test: drop screenshot from the pinned loop tier set Co-Authored-By: Claude Opus 5.5 --- src/__tests__/command-descriptor-parity.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/__tests__/command-descriptor-parity.test.ts b/src/__tests__/command-descriptor-parity.test.ts index 19b38b116c..e164e8756b 100644 --- a/src/__tests__/command-descriptor-parity.test.ts +++ b/src/__tests__/command-descriptor-parity.test.ts @@ -310,7 +310,6 @@ test("frameworkTier pins exactly the 'loop' command set", () => { 'fill', 'open', 'press', - 'screenshot', 'scroll', 'snapshot', 'wait', From 06c9d4c86a2c977087bc0b0bf40b3dcb1196a5be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 08:41:28 +0100 Subject: [PATCH 11/12] fix(mcp): keep reinstall's search keywords out of its description; format the loop pin Co-Authored-By: Claude Opus 5.5 --- packages/command-surface/src/management/install.ts | 2 +- src/__tests__/command-descriptor-parity.test.ts | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/packages/command-surface/src/management/install.ts b/packages/command-surface/src/management/install.ts index 0a15f4507f..d0f2cf3b67 100644 --- a/packages/command-surface/src/management/install.ts +++ b/packages/command-surface/src/management/install.ts @@ -152,7 +152,7 @@ const reinstallCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Replace an installed app with a new build', - searchHint: 'update apk ipa', + searchHint: 'update', }, metadata: reinstallCommandMetadata, run: (client, input) => client.apps.reinstall(input), diff --git a/src/__tests__/command-descriptor-parity.test.ts b/src/__tests__/command-descriptor-parity.test.ts index e164e8756b..dc506682f0 100644 --- a/src/__tests__/command-descriptor-parity.test.ts +++ b/src/__tests__/command-descriptor-parity.test.ts @@ -305,15 +305,7 @@ test("frameworkTier pins exactly the 'loop' command set", () => { .map((descriptor) => descriptor.name) .sort(); - assert.deepEqual(loop, [ - 'back', - 'fill', - 'open', - 'press', - 'scroll', - 'snapshot', - 'wait', - ]); + assert.deepEqual(loop, ['back', 'fill', 'open', 'press', 'scroll', 'snapshot', 'wait']); }); test('targetIdentityVerification pins exactly the evidence-carrying command set (ADR 0012 / #1349)', () => { From 83c034c8fdc13f7b830866b726c955921073bb00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 08:49:21 +0100 Subject: [PATCH 12/12] docs(mcp): screenshot is no longer always loaded Co-Authored-By: Claude Opus 5.5 --- website/docs/docs/agent-setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/agent-setup.md b/website/docs/docs/agent-setup.md index acd5f0ca8a..ea3f71566d 100644 --- a/website/docs/docs/agent-setup.md +++ b/website/docs/docs/agent-setup.md @@ -63,7 +63,7 @@ MCP-only clients don't need a separately installed skill. When a client connects Tool results are compact text, such as `@e4 [button] "Products"` lines and settled UI diffs. Tools declare no output schema, and results carry no `structuredContent`, because some clients show the model `structuredContent` instead of the text. To get results as JSON text, set `AGENT_DEVICE_RESPONSE_LEVEL` to `digest` or `full`, or set `AGENT_DEVICE_COST=1`, in the server entry. Tool calls cannot pass these values, `debug`, or the `target` alias of `deviceTarget`. -Clients that defer tool schemas, such as Claude Code, load the core loop tools up front: `open`, `snapshot`, `press`, `fill`, `back`, `wait`, `scroll`, and `screenshot`. Each tool also carries search keywords for the client's tool search. To run a known multi-step walk in one call, use `batch`; its description shows the step shape. +Clients that defer tool schemas, such as Claude Code, load the core loop tools up front: `open`, `snapshot`, `press`, `fill`, `back`, `wait`, and `scroll`; other tools, `screenshot` included, load on first use. Each tool also carries search keywords for the client's tool search. To run a known multi-step walk in one call, use `batch`; its description shows the step shape. If your client reads results as data, start the server with `agent-device mcp --structured`. Tools that have a typed result, such as `press` or `wait`, then declare an output schema and return `structuredContent` that matches it. Other tools return `structuredContent` when the server is set to return JSON as above. A failed call returns the error in `structuredContent`.