diff --git a/packages/client/src/__tests__/index.test.ts b/packages/client/src/__tests__/index.test.ts index 5ce1e774ad..ded85772d5 100644 --- a/packages/client/src/__tests__/index.test.ts +++ b/packages/client/src/__tests__/index.test.ts @@ -400,12 +400,12 @@ test('structured interaction input keeps UI target separate from deviceTarget', await runCommand(client, 'get', { deviceTarget: 'mobile', format: 'text', - target: { kind: 'ref', ref: '@e1' }, + ref: '@e1', }); await runCommand(client, 'longpress', { deviceTarget: 'mobile', durationMs: 800, - target: { kind: 'ref', ref: '@e2' }, + ref: '@e2', }); assert.equal(setup.calls.length, 2); diff --git a/packages/command-surface/src/batch/index.ts b/packages/command-surface/src/batch/index.ts index 820a35b6c3..581197a24a 100644 --- a/packages/command-surface/src/batch/index.ts +++ b/packages/command-surface/src/batch/index.ts @@ -27,18 +27,15 @@ const batchCliReader: CliReader = (_positionals, flags) => ({ /** * The step examples `help batch` prints. A step's `input` is keyed by the command's structured * field names, which no CLI help text spells out — `help press` documents the positional and the - * flags, not `target: {kind, ref}` — so these lines are the only place the terminal states them. + * flags, not `ref` or `selector` — so these lines are the only place the terminal states them. * `cli-help-examples.test.ts` reads them back out of the rendered help and runs each `input` * through its own command's `readInput`, so a renamed field fails there instead of shipping an * example nobody can run (#2062). */ const BATCH_STEP_EXAMPLES: readonly BatchCommandStep[] = [ { command: 'snapshot', input: { interactiveOnly: true } }, - { command: 'press', input: { target: { kind: 'ref', ref: '@e12' }, settle: true } }, - { - command: 'fill', - input: { target: { kind: 'selector', selector: 'id="field-email"' }, text: 'qa@example.com' }, - }, + { command: 'press', input: { ref: '@e12', settle: true } }, + { command: 'fill', input: { selector: 'id="field-email"', text: 'qa@example.com' } }, ]; /** @@ -49,7 +46,7 @@ const BATCH_STEP_EXAMPLES: readonly BatchCommandStep[] = [ */ function buildBatchCliDetail(): string { const prose = [ - 'Each step is {"command":"","input":{...}}, and input is that command\'s structured input object rather than its terminal spelling: a CLI positional becomes a named field (target, text, direction) and a CLI flag becomes a camelCase key (--settle -> "settle":true, -i -> "interactiveOnly":true). There is no positional step form; args, argv, positionals, and flags are not step fields. agent-device help lists arguments in CLI spelling only. The examples below carry the structured spelling; over MCP, the command tool schema states it in full.', + 'Each step is {"command":"","input":{...}}, and input is that command\'s structured input object rather than its terminal spelling: a CLI positional becomes a named field (for example ref, label, selector, x, y, text, direction) and a CLI flag becomes a camelCase key (--settle -> "settle":true, -i -> "interactiveOnly":true). There is no positional step form; args, argv, positionals, and flags are not step fields. agent-device help lists arguments in CLI spelling only. The examples below carry the structured spelling; over MCP, the command tool schema states it in full.', 'Steps run serially in one daemon request against the same session, in order. Mutating UI verbs are included (press, click, fill, longpress, scroll, back), which is where the round-trip saving is; --on-error stop halts at the first failing step.', 'The whole batch has a 60-second work budget inside a 90-second client timeout. The running step receives ordinary request cancellation at the budget. When it settles, failure returns COMMAND_FAILED with details.reason daemon_request_timeout and preserves executed, partialResults, and dispatch status. The daemon and other sessions stay available if it responds; a close step may already have ended its session. A completed batch returns success even if it settles just after the deadline. Cancellation does not prove native device work stopped. A step that ignores cancellation beyond the 30-second margin may cause a transport timeout; an unresponsive daemon retains the client liveness-based reset. Split longer sequences into separate requests.', `Available through batch: ${[...STRUCTURED_BATCH_COMMAND_NAMES].sort().join(', ')}.`, diff --git a/packages/command-surface/src/batch/metadata.ts b/packages/command-surface/src/batch/metadata.ts index 63ba96aba9..dc8b29b7b6 100644 --- a/packages/command-surface/src/batch/metadata.ts +++ b/packages/command-surface/src/batch/metadata.ts @@ -48,7 +48,12 @@ export function createBatchCommandMetadata( 'batch', 'Execute multiple commands in one daemon request.', fields, - { readInput: (input) => readBatchInput(input, fields) }, + { + readInput: (input) => readBatchInput(input, fields), + ...(nestedCommands.includes('snapshot') + ? { inputExample: { steps: [{ command: 'snapshot', input: { interactiveOnly: true } }] } } + : {}), + }, ); } diff --git a/packages/command-surface/src/cli-grammar/common.ts b/packages/command-surface/src/cli-grammar/common.ts index e16abff988..8aa27fc971 100644 --- a/packages/command-surface/src/cli-grammar/common.ts +++ b/packages/command-surface/src/cli-grammar/common.ts @@ -151,24 +151,11 @@ export function leaseDeviceSpecInputFromFlags( return requireLeaseDeviceSpec(flags).spec; } -export function targetInputFromClientTarget( - target: InteractionTarget | ElementTarget, -): Record { - if ('ref' in target && target.ref !== undefined) { - return compactRecord({ kind: 'ref', ref: target.ref, label: target.label }); - } - if ('selector' in target && target.selector !== undefined) { - return { kind: 'selector', selector: target.selector }; - } - const point = target as { x: number; y: number }; - return { kind: 'point', x: point.x, y: point.y }; -} - export function interactionTargetPositionals(input: InteractionTarget | CommandInput): string[] { - const target = readTargetRecord(input); + const target = input as Record; if (typeof target.ref === 'string') return [target.ref, ...optionalTargetLabel(target.label)]; if (typeof target.selector === 'string') return [target.selector]; - if (target.kind === 'point' || target.x !== undefined || target.y !== undefined) { + if (target.x !== undefined || target.y !== undefined) { return [ String(requiredTargetNumber(target.x, 'x')), String(requiredTargetNumber(target.y, 'y')), @@ -178,24 +165,12 @@ export function interactionTargetPositionals(input: InteractionTarget | CommandI } export function elementTargetPositionals(input: ElementTarget | CommandInput): string[] { - const target = readTargetRecord(input); + const target = input as Record; if (typeof target.ref === 'string') return [target.ref, ...optionalTargetLabel(target.label)]; if (typeof target.selector === 'string') return [target.selector]; throw new AppError('INVALID_ARGS', 'element command requires @ref or selector target'); } -function readTargetRecord(input: unknown): Record { - if (!input || typeof input !== 'object' || Array.isArray(input)) { - throw new AppError('INVALID_ARGS', 'Expected target object.'); - } - const record = input as Record; - const nestedTarget = record.target; - if (nestedTarget && typeof nestedTarget === 'object' && !Array.isArray(nestedTarget)) { - return nestedTarget as Record; - } - return record; -} - function requiredTargetNumber(value: unknown, field: string): number { if (typeof value !== 'number' || !Number.isFinite(value)) { throw new AppError('INVALID_ARGS', `point target requires numeric ${field}.`); diff --git a/packages/command-surface/src/cli-grammar/types.ts b/packages/command-surface/src/cli-grammar/types.ts index 1c6bce599c..a83175f7f7 100644 --- a/packages/command-surface/src/cli-grammar/types.ts +++ b/packages/command-surface/src/cli-grammar/types.ts @@ -19,9 +19,8 @@ type PointInput = { y?: number; }; -export type CommandInput = Omit & - Omit, 'batchSteps' | 'target'> & { - target?: InternalRequestOptions['target'] | Record; +export type CommandInput = Omit & + Omit, 'batchSteps'> & { action?: string; amount?: number; app?: string; diff --git a/packages/command-surface/src/command-contract.ts b/packages/command-surface/src/command-contract.ts index 480a75e265..2122c3305b 100644 --- a/packages/command-surface/src/command-contract.ts +++ b/packages/command-surface/src/command-contract.ts @@ -58,6 +58,11 @@ export type CommandMetadata = { * command prints them after its own text, and no formatter decides this per-command. */ parseableOutput?: true; + /** + * The keys of a minimal valid call that a placeholder per required property cannot state, such + * as the one UI target an interaction needs. `commandInputExample` merges them over those. + */ + inputExample?: Readonly>; }; export function defineCommandMetadata( @@ -65,3 +70,41 @@ export function defineCommandMetadata( ): CommandMetadata { return definition; } + +/** + * A minimal call for a command: a placeholder for each required property, then its declared + * example keys, which win. A model-facing surface shows it when a call names an argument the tool does not take, + * so a caller that never loaded the schema can retry once. + */ +export function commandInputExample( + metadata: Pick, 'inputSchema' | 'inputExample'>, +): Record { + return { ...requiredPropertiesExample(metadata.inputSchema), ...metadata.inputExample }; +} + +function requiredPropertiesExample(schema: JsonSchema): Record { + return Object.fromEntries( + (schema.required ?? []).map((key) => [key, placeholder(key, schema.properties?.[key])]), + ); +} + +function placeholder(key: string, schema: JsonSchema | undefined): unknown { + if (schema === undefined) return `<${key}>`; + if (schema.const !== undefined) return schema.const; + if (schema.enum !== undefined && schema.enum.length > 0) return schema.enum[0]; + if (schema.oneOf !== undefined && schema.oneOf.length > 0) + return placeholder(key, schema.oneOf[0]); + switch (Array.isArray(schema.type) ? schema.type[0] : schema.type) { + case 'object': + return requiredPropertiesExample(schema); + case 'array': + return schema.items === undefined ? [] : [placeholder(key, schema.items)]; + case 'number': + case 'integer': + return schema.minimum ?? 1; + case 'boolean': + return true; + default: + return `<${key}>`; + } +} diff --git a/packages/command-surface/src/command-input-target.test.ts b/packages/command-surface/src/command-input-target.test.ts new file mode 100644 index 0000000000..ed2c0ac937 --- /dev/null +++ b/packages/command-surface/src/command-input-target.test.ts @@ -0,0 +1,73 @@ +import { expect, test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { findCommandMetadata } from './command-metadata.ts'; + +// The UI target is top-level keys, the vocabulary an agent guesses without loading the schema. +test.for([ + [{ ref: '@e6' }, { ref: '@e6' }], + [ + { ref: '@e6', label: 'Products' }, + { ref: '@e6', label: 'Products' }, + ], + [{ selector: 'id=home/products' }, { selector: 'id=home/products' }], + [ + { x: 120, y: 340 }, + { x: 120, y: 340 }, + ], +] as const)('press reads %j as its UI target', ([input, target]) => { + expect(findCommandMetadata('press').readInput(input)).toStrictEqual(target); +}); + +test.for([ + [{}, /Expected a UI target/], + [{ ref: '@e6', selector: 'id=a' }, /got ref and selector/], + [{ selector: 'id=a', x: 1, y: 2 }, /got selector and x\/y/], + [{ x: 120 }, /Expected x and y together/], + [{ selector: 'id=a', label: 'A' }, /label applies only with ref/], +] as const)('press refuses %j', ([input, message]) => { + expect(() => findCommandMetadata('press').readInput(input)).toThrow(message); +}); + +// The schema cannot state "exactly one target": a top-level oneOf/anyOf/allOf in a tool input +// schema is rejected by some model clients. The reader enforces it, and its hint names the forms. +test.for([{}, { ref: '@e6', selector: 'id=a' }, { x: 120 }] as const)( + 'press answers %j with INVALID_ARGS and the target forms', + (input) => { + try { + findCommandMetadata('press').readInput(input); + expect.unreachable('press should refuse the input'); + } catch (error) { + expect(error).toBeInstanceOf(AppError); + expect((error as AppError).code).toBe('INVALID_ARGS'); + expect((error as AppError).details?.hint).toMatch(/\{"ref":"@e12"\}|\{"x":120,"y":340\}/); + } + }, +); + +test.for([ + ['target', { target: { kind: 'ref', ref: '@e6' } }], + ['kind', { kind: 'ref', ref: '@e6' }], +] as const)('the removed %s key answers with the top-level spelling', ([key, input]) => { + const read = () => findCommandMetadata('press').readInput(input); + expect(read).toThrow(AppError); + expect(read).toThrow(new RegExp(`${key} was removed.*\\{"ref":"@e12"\\}`)); +}); + +test('get takes a ref or selector but no point', () => { + const get = findCommandMetadata('get'); + expect(get.readInput({ format: 'text', selector: 'id=a' })).toMatchObject({ selector: 'id=a' }); + expect(get.inputSchema.properties).not.toHaveProperty('x'); + expect(get.inputSchema.properties).not.toHaveProperty('target'); +}); + +// `target` without a UI target field is the deviceTarget alias. An interaction command retires +// it, so the alias must not reappear in its schema as a string a model could write a ref into. +test('interaction schemas advertise top-level target keys and no target', () => { + for (const name of ['press', 'click', 'fill', 'longpress', 'hover'] as const) { + const properties = findCommandMetadata(name).inputSchema.properties ?? {}; + expect(Object.keys(properties)).toEqual( + expect.arrayContaining(['ref', 'label', 'selector', 'x', 'y']), + ); + expect(properties).not.toHaveProperty('target'); + } +}); diff --git a/packages/command-surface/src/command-input.ts b/packages/command-surface/src/command-input.ts index 4c15c16f3e..b2e96478ea 100644 --- a/packages/command-surface/src/command-input.ts +++ b/packages/command-surface/src/command-input.ts @@ -1,7 +1,6 @@ import type { AgentDeviceRequestOverrides, AgentDeviceSelectionOptions, - ElementTarget, InteractionTarget, } from '@agent-device/contracts/client'; import { @@ -40,22 +39,9 @@ import { optionalStringArray, readInputRecord, readRecordField, - requiredEnum, requiredNumber, - requiredString, } from './input-readers.ts'; -const INTERACTION_TARGET_KINDS = ['ref', 'selector', 'point'] as const; - -export type InteractionTargetInput = - | { kind: 'ref'; ref: string; label?: string } - | { kind: 'selector'; selector: string } - | { kind: 'point'; x: number; y: number }; - -export type ElementTargetInput = - | { kind: 'ref'; ref: string; label?: string } - | { kind: 'selector'; selector: string }; - export type SelectorSnapshotInput = { depth?: number; scope?: string; @@ -67,11 +53,13 @@ export type PointInput = { x: number; y: number }; function commandInputSchema( properties: Record, required: readonly string[] = [], + retiredKeys: ReadonlySet = new Set(), ): JsonSchema { + const common = Object.entries(commonProperties()).filter(([key]) => !retiredKeys.has(key)); return { type: 'object', properties: { - ...commonProperties(), + ...Object.fromEntries(common), ...properties, }, ...(required.length > 0 ? { required } : {}), @@ -360,18 +348,6 @@ export function customField( return optionalField(schema, read); } -export function interactionTargetField(): CommandField { - return optionalField(interactionTargetSchema(), (record, key) => - record[key] === undefined ? undefined : readInteractionTarget(record, key), - ); -} - -export function elementTargetField(): CommandField { - return optionalField(elementTargetSchema(), (record, key) => - record[key] === undefined ? undefined : readElementTarget(record, key), - ); -} - export function pointField(description: string): CommandField { return optionalField(pointSchema(description), (record, key) => record[key] === undefined ? undefined : readPoint(record, key), @@ -462,7 +438,11 @@ function leaseDeviceSpecField( @@ -489,68 +469,11 @@ export function readFieldInput( }) as InferCommandInput; } -function readInteractionTarget( - record: Record, - key: string, -): InteractionTargetInput { - const target = readRecordField(record, key); - const kind = requiredEnum(target, 'kind', INTERACTION_TARGET_KINDS); - switch (kind) { - case 'ref': - return { - kind, - ref: requiredString(target, 'ref'), - label: optionalString(target, 'label'), - }; - case 'selector': - return { kind, selector: requiredString(target, 'selector') }; - case 'point': - return { - kind, - x: requiredNumber(target, 'x'), - y: requiredNumber(target, 'y'), - }; - } -} - -function readElementTarget(record: Record, key: string): ElementTargetInput { - const target = readRecordField(record, key); - const kind = requiredEnum(target, 'kind', ['ref', 'selector'] as const); - if (kind === 'ref') { - return { - kind, - ref: requiredString(target, 'ref'), - label: optionalString(target, 'label'), - }; - } - return { kind, selector: requiredString(target, 'selector') }; -} - function readPoint(record: Record, key: string): PointInput { const point = readRecordField(record, key); return { x: requiredNumber(point, 'x'), y: requiredNumber(point, 'y') }; } -export function toClientInteractionTarget(target: InteractionTargetInput): InteractionTarget { - switch (target.kind) { - case 'ref': - return { ref: target.ref, label: target.label }; - case 'selector': - return { selector: target.selector }; - case 'point': - return { x: target.x, y: target.y }; - } -} - -export function toClientElementTarget(target: ElementTargetInput): ElementTarget { - switch (target.kind) { - case 'ref': - return { ref: target.ref, label: target.label }; - case 'selector': - return { selector: target.selector }; - } -} - export function toRepeatedOptions(input: RepeatedInput): RepeatedInput { return { count: input.count, @@ -592,6 +515,15 @@ function fieldProperties(fields: CommandFieldMap): Record { ); } +/** A retired field also retires the common key it shadows, such as the `target` device alias. */ +function retiredFieldNames(fields: CommandFieldMap): ReadonlySet { + return new Set( + Object.entries(fields).flatMap(([key, field]) => + field.audience?.kind === 'retired' ? [key] : [], + ), + ); +} + function requiredFieldNames(fields: CommandFieldMap): string[] { return Object.entries(fields).flatMap(([key, field]) => (field.required ? [key] : [])); } @@ -605,52 +537,95 @@ export function fieldAudiences(fields: CommandFieldMap): InputAudienceMap { ); } -function interactionTargetSchema(): JsonSchema { +// The UI target of an interaction is named by top-level keys, the CLI's vocabulary: +// {"ref":"@e12"}, {"selector":"id=\"submit\""}, or {"x":120,"y":340}. + +export type TargetedInput = Omit< + InferCommandInput, + 'ref' | 'label' | 'selector' | 'x' | 'y' | 'target' +> & + TTarget; + +/** The keys a minimal call example sets for a command that takes a UI target. */ +export const TARGET_INPUT_EXAMPLE = { ref: '@e12' } as const; + +const REF_FORM = '{"ref":"@e12"}'; +const SELECTOR_FORM = String.raw`{"selector":"id=\"submit\""}`; +const ELEMENT_TARGET_FORMS = `${REF_FORM} or ${SELECTOR_FORM}`; +const INTERACTION_TARGET_FORMS = `${REF_FORM}, ${SELECTOR_FORM}, or {"x":120,"y":340}`; + +/** Ref or selector, for a command that reads an element (`get`). */ +export function elementTargetFields() { + return targetFields(ELEMENT_TARGET_FORMS, 'ref or selector'); +} + +/** Ref, selector, or screen point, for a command that acts on the UI. */ +export function interactionTargetFields() { return { - oneOf: [ - ...elementTargetSchemaVariants(), - { - type: 'object', - properties: { - kind: { type: 'string', const: 'point' }, - x: { type: 'number' }, - y: { type: 'number' }, - }, - required: ['kind', 'x', 'y'], - additionalProperties: false, - }, - ], - description: 'UI target. This is separate from deviceTarget, which selects the device form.', + ...targetFields(INTERACTION_TARGET_FORMS, 'ref, selector, or x+y'), + x: numberField( + 'Screen x coordinate, with y. Pass exactly one of ref, selector, or x+y; use x+y only when no ref or selector is available.', + ), + y: numberField('Screen y coordinate, with x. Pass exactly one of ref, selector, or x+y.'), }; } -function elementTargetSchema(): JsonSchema { +function targetFields(forms: string, choice: string) { return { - oneOf: elementTargetSchemaVariants(), - description: 'UI element target by snapshot ref or selector expression.', + ref: stringField(`Snapshot element ref such as @e12. Pass exactly one of ${choice}.`), + label: stringField('Optional human label for ref.'), + selector: stringField( + `Selector expression such as id="submit" or label="Sign in". Pass exactly one of ${choice}.`, + ), + target: retiredField(`target was removed; pass the UI target at the top level: ${forms}.`), + kind: retiredField(`kind was removed; the key you pass picks the UI target: ${forms}.`), }; } -function elementTargetSchemaVariants(): JsonSchema[] { - return [ - { - type: 'object', - properties: { - kind: { type: 'string', const: 'ref' }, - ref: { type: 'string', description: 'Snapshot element ref such as @e12.' }, - label: { type: 'string', description: 'Optional human label for the ref.' }, - }, - required: ['kind', 'ref'], - additionalProperties: false, - }, - { - type: 'object', - properties: { - kind: { type: 'string', const: 'selector' }, - selector: { type: 'string', description: 'Agent-device selector expression.' }, - }, - required: ['kind', 'selector'], - additionalProperties: false, - }, - ]; +/** Reads a command's fields, then requires exactly one UI target among the ones they declare. */ +export function readTargetedInput< + TFields extends CommandFieldMap, + TTarget extends InteractionTarget = InteractionTarget, +>(input: unknown, fields: TFields): TargetedInput { + const { ref, label, selector, x, y, ...rest } = readFieldInput(input, fields) as Record< + string, + unknown + >; + const named = [ + ref === undefined ? [] : ['ref'], + selector === undefined ? [] : ['selector'], + x === undefined && y === undefined ? [] : ['x/y'], + ].flat(); + if (named.length !== 1) { + const forms = 'x' in fields ? INTERACTION_TARGET_FORMS : ELEMENT_TARGET_FORMS; + throw new AppError( + 'INVALID_ARGS', + named.length === 0 + ? 'Expected a UI target.' + : `Expected one UI target, got ${named.join(' and ')}.`, + { hint: `Pass exactly one of: ${forms}.` }, + ); + } + if (label !== undefined && ref === undefined) { + throw new AppError('INVALID_ARGS', 'label applies only with ref.', { + hint: 'Pass label together with ref, for example {"ref":"@e12","label":"Sign in"}.', + }); + } + if (named[0] === 'x/y' && (x === undefined || y === undefined)) { + throw new AppError('INVALID_ARGS', 'Expected x and y together.', { + hint: 'Pass both screen coordinates, for example {"x":120,"y":340}.', + }); + } + const target = toClientInteractionTarget({ ref, label, selector, x, y } as InteractionTarget); + return { ...rest, ...target } as TargetedInput; +} + +/** Only the target keys, so a projection never carries another key along with them. */ +export function toClientInteractionTarget( + input: TTarget, +): TTarget { + if (input.ref !== undefined) + return compactRecord({ ref: input.ref, label: input.label }) as TTarget; + if (input.selector !== undefined) return { selector: input.selector } as TTarget; + return { x: input.x, y: input.y } as TTarget; } diff --git a/packages/command-surface/src/common-input-fields.ts b/packages/command-surface/src/common-input-fields.ts index 55169762e5..14df32456c 100644 --- a/packages/command-surface/src/common-input-fields.ts +++ b/packages/command-surface/src/common-input-fields.ts @@ -128,7 +128,7 @@ const COMMON_INPUT_FIELDS = { type: 'string', enum: DEVICE_TARGETS, description: - 'Alias for deviceTarget on commands without a UI target field. Interaction commands reserve target for the UI element.', + 'Alias for deviceTarget. Commands that take a UI target (ref, selector, or x/y) accept only deviceTarget.', }, }, device: { diff --git a/packages/command-surface/src/field-command-contract.ts b/packages/command-surface/src/field-command-contract.ts index 1c82a28e5c..c3e9515c75 100644 --- a/packages/command-surface/src/field-command-contract.ts +++ b/packages/command-surface/src/field-command-contract.ts @@ -15,6 +15,8 @@ type FieldCommandPresentationOptions = { type FieldCommandOptions = FieldCommandPresentationOptions & { /** For a command that reads its own fields (`batch` validates steps, `gesture` reads a union). */ readInput: (input: unknown) => TInput; + /** See `CommandMetadata.inputExample`. */ + inputExample?: Readonly>; }; /** @@ -51,5 +53,6 @@ export function defineFieldCommandMetadata( readInput: options?.readInput ?? ((input) => readFieldInput(input, fields)), inputAudience: fieldAudiences(fields), ...(options?.parseableOutput ? { parseableOutput: true as const } : {}), + ...(options?.inputExample ? { inputExample: options.inputExample } : {}), }); } diff --git a/packages/command-surface/src/index.ts b/packages/command-surface/src/index.ts index 717c6ec5e2..c44c584f28 100644 --- a/packages/command-surface/src/index.ts +++ b/packages/command-surface/src/index.ts @@ -9,6 +9,7 @@ import { AppError } from '@agent-device/kernel/errors'; export type { AgentDeviceClient, AgentDeviceCommandClient } from './client-port.ts'; export type { CommandMetadata, JsonSchema } from './command-contract.ts'; +export { commandInputExample } from './command-contract.ts'; export type { InputAudience, InputAudienceMap } from './input-audience.ts'; export type { CommandName } from './command-metadata.ts'; export type { CommandExecutionResult } from './command-surface.ts'; diff --git a/packages/command-surface/src/input-readers.ts b/packages/command-surface/src/input-readers.ts index f52e7c5e30..fb950cfd1f 100644 --- a/packages/command-surface/src/input-readers.ts +++ b/packages/command-surface/src/input-readers.ts @@ -21,14 +21,6 @@ export function readRecordField( return value as Record; } -export function requiredString(record: Record, key: string): string { - const value = record[key]; - if (typeof value !== 'string' || value.length === 0) { - throw new AppError('INVALID_ARGS', `Expected ${key} to be a non-empty string.`); - } - return value; -} - /** * Opt-in reader for the one field where the empty string is a VALUE, not a missing input: * `fill ""` is the clear-field primitive (#2063). `requiredField` still refuses a @@ -73,18 +65,6 @@ export function optionalBoolean(record: Record, key: string): b return value; } -export function requiredEnum( - record: Record, - key: string, - values: T, -): T[number] { - const value = record[key]; - if (typeof value !== 'string' || !values.includes(value)) { - throw new AppError('INVALID_ARGS', `Expected ${key} to be one of: ${values.join(', ')}.`); - } - return value; -} - export function optionalEnum( record: Record, key: string, diff --git a/packages/command-surface/src/interaction/index.ts b/packages/command-surface/src/interaction/index.ts index af496a805b..25bebab3b1 100644 --- a/packages/command-surface/src/interaction/index.ts +++ b/packages/command-surface/src/interaction/index.ts @@ -1,5 +1,6 @@ import type { ClickOptions, + ElementTarget, FillOptions, FindOptions, FocusOptions, @@ -33,7 +34,6 @@ import { objectSchema, stringArraySchema, stringSchema, - toClientElementTarget, toClientInteractionTarget, toRepeatedOptions, toSelectorSnapshotOptions, @@ -662,7 +662,7 @@ function toClickOptions(input: ClickInput): ClickOptions { function toPressOptions(input: PressInput): PressOptions { return { ...commonToClientOptions(input), - ...toClientInteractionTarget(input.target), + ...toClientInteractionTarget(input), ...toSelectorSnapshotOptions(input), ...toRepeatedOptions(input), verify: input.verify, @@ -673,7 +673,7 @@ function toPressOptions(input: PressInput): PressOptions { function toFillOptions(input: FillInput): FillOptions { return { ...commonToClientOptions(input), - ...toClientInteractionTarget(input.target), + ...toClientInteractionTarget(input), ...toSelectorSnapshotOptions(input), text: input.text, delayMs: input.delayMs, @@ -687,7 +687,7 @@ function toFillOptions(input: FillInput): FillOptions { function toLongPressOptions(input: LongPressInput): LongPressOptions { return { ...commonToClientOptions(input), - ...toClientInteractionTarget(input.target), + ...toClientInteractionTarget(input), ...toSelectorSnapshotOptions(input), durationMs: input.durationMs, ...toSettleOptions(input), @@ -697,7 +697,7 @@ function toLongPressOptions(input: LongPressInput): LongPressOptions { function toHoverOptions(input: HoverInput): HoverOptions { return { ...commonToClientOptions(input), - ...toClientInteractionTarget(input.target), + ...toClientInteractionTarget(input), ...toSelectorSnapshotOptions(input), ...toSettleOptions(input), }; @@ -718,7 +718,7 @@ function toSettleOptions(input: { function toGetOptions(input: GetInput): GetOptions { return { ...commonToClientOptions(input), - ...toClientElementTarget(input.target), + ...toClientInteractionTarget(input), ...toSelectorSnapshotOptions(input), format: input.format, // `--record` is scoped (ADR 0012 decision 6 amendment), so it does NOT ride diff --git a/packages/command-surface/src/interaction/interactions.ts b/packages/command-surface/src/interaction/interactions.ts index f8cf737483..298ee246aa 100644 --- a/packages/command-surface/src/interaction/interactions.ts +++ b/packages/command-surface/src/interaction/interactions.ts @@ -30,7 +30,6 @@ import { request, requiredDaemonString, selectorSnapshotInputFromFlags, - targetInputFromClientTarget, } from '../cli-grammar/common.ts'; import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; import type { ScrollInputDirection } from '@agent-device/contracts/scroll-gesture'; @@ -40,7 +39,7 @@ export const interactionCliReaders = { ...commonInputFromFlags(flags), ...selectorSnapshotInputFromFlags(flags), ...repeatedInputFromFlags(flags), - target: targetInputFromClientTarget(readInteractionTargetFromPositionals(positionals)), + ...readInteractionTargetFromPositionals(positionals), button: flags.clickButton, verify: flags.verify, }), @@ -48,7 +47,7 @@ export const interactionCliReaders = { ...commonInputFromFlags(flags), ...selectorSnapshotInputFromFlags(flags), ...repeatedInputFromFlags(flags), - target: targetInputFromClientTarget(readInteractionTargetFromPositionals(positionals)), + ...readInteractionTargetFromPositionals(positionals), verify: flags.verify, }), longpress: (positionals, flags) => { @@ -56,14 +55,13 @@ export const interactionCliReaders = { return { ...commonInputFromFlags(flags), ...selectorSnapshotInputFromFlags(flags), - target: targetInputFromClientTarget(decoded), - durationMs: decoded.durationMs, + ...decoded, }; }, hover: (positionals, flags) => ({ ...commonInputFromFlags(flags), ...selectorSnapshotInputFromFlags(flags), - target: targetInputFromClientTarget(readInteractionTargetFromPositionals(positionals)), + ...readInteractionTargetFromPositionals(positionals), }), swipe: (positionals, flags) => ({ ...commonInputFromFlags(flags), @@ -91,7 +89,7 @@ export const interactionCliReaders = { return { ...commonInputFromFlags(flags), ...selectorSnapshotInputFromFlags(flags), - target: targetInputFromClientTarget(decoded.target), + ...decoded.target, text: decoded.text, delayMs: flags.delayMs, recordAs: flags.recordAs, @@ -118,7 +116,7 @@ export const interactionCliReaders = { ...observationRecordInputFromFlags(flags), ...selectorSnapshotInputFromFlags(flags), format: readGetFormat(positionals[0]), - target: targetInputFromClientTarget(readElementTargetFromPositionals(positionals.slice(1))), + ...readElementTargetFromPositionals(positionals.slice(1)), }), } satisfies Record; diff --git a/packages/command-surface/src/interaction/metadata.test.ts b/packages/command-surface/src/interaction/metadata.test.ts index 06e9bc28ff..fe6517f159 100644 --- a/packages/command-surface/src/interaction/metadata.test.ts +++ b/packages/command-surface/src/interaction/metadata.test.ts @@ -16,7 +16,7 @@ const fillMetadata = interactionCommandMetadata.find((metadata) => metadata.name test('fill accepts an empty text as the clear request', () => { if (!fillMetadata) throw new Error('expected fill metadata'); const input = fillMetadata.readInput({ - target: { kind: 'ref', ref: '@e57' }, + ref: '@e57', text: '', }) as { text: unknown }; @@ -26,7 +26,7 @@ test('fill accepts an empty text as the clear request', () => { test('fill still refuses a missing text', () => { if (!fillMetadata) throw new Error('expected fill metadata'); try { - fillMetadata.readInput({ target: { kind: 'ref', ref: '@e57' } }); + fillMetadata.readInput({ ref: '@e57' }); expect.unreachable('fill should refuse input without text'); } catch (error) { expect(error).toBeInstanceOf(AppError); @@ -38,7 +38,7 @@ test('fill still refuses a missing text', () => { test('fill refuses a non-string text', () => { if (!fillMetadata) throw new Error('expected fill metadata'); try { - fillMetadata.readInput({ target: { kind: 'ref', ref: '@e57' }, text: 42 }); + fillMetadata.readInput({ ref: '@e57', text: 42 }); expect.unreachable('fill should refuse a non-string text'); } catch (error) { expect((error as AppError).code).toBe('INVALID_ARGS'); diff --git a/packages/command-surface/src/interaction/metadata.ts b/packages/command-surface/src/interaction/metadata.ts index 3602ed8bb2..b386cc2d1e 100644 --- a/packages/command-surface/src/interaction/metadata.ts +++ b/packages/command-surface/src/interaction/metadata.ts @@ -1,4 +1,5 @@ import { CLICK_BUTTONS } from '@agent-device/contracts/click-button'; +import type { ElementTarget } from '@agent-device/contracts/client'; import { GESTURE_KINDS, type GesturePayload, @@ -26,22 +27,24 @@ import { } from '@agent-device/contracts/scroll-gesture'; import { FIND_LOCATORS } from '@agent-device/selectors'; import { + TARGET_INPUT_EXAMPLE, booleanField, - elementTargetField, + elementTargetFields, enumField, integerField, - interactionTargetField, + interactionTargetFields, numberField, operatorField, pointArrayField, pointField, postGestureStabilizationFields, + readTargetedInput, repeatedFields, requiredField, selectorSnapshotFields, stringField, type CommandFieldMap, - type InferCommandInput, + type TargetedInput, } from '../command-input.ts'; import { readCommonInput, type CommonCommandInput } from '../common-input-fields.ts'; import { optionalBoolean, readInputRecord } from '../input-readers.ts'; @@ -86,7 +89,7 @@ const interactionCommandDescriptions = { type InteractionCommandName = keyof typeof interactionCommandDescriptions; const clickFields = { - target: requiredField(interactionTargetField()), + ...interactionTargetFields(), button: enumField(CLICK_BUTTONS, 'Pointer button for platforms that support mouse buttons.'), ...selectorSnapshotFields(), ...repeatedFields(), @@ -94,14 +97,14 @@ const clickFields = { }; const pressFields = { - target: requiredField(interactionTargetField()), + ...interactionTargetFields(), ...selectorSnapshotFields(), ...repeatedFields(), ...postActionObservationFields('press'), }; const fillFields = { - target: requiredField(interactionTargetField()), + ...interactionTargetFields(), // "" is accepted and means "replace with nothing", i.e. clear the field (#2063). The key // must still be present: omitting it is a missing argument, not a clear. text: requiredField( @@ -125,14 +128,14 @@ const fillFields = { }; const longPressFields = { - target: requiredField(interactionTargetField()), + ...interactionTargetFields(), durationMs: integerField('Long press duration in milliseconds.', { min: 0 }), ...selectorSnapshotFields(), ...postActionObservationFields('longpress'), }; const hoverFields = { - target: requiredField(interactionTargetField()), + ...interactionTargetFields(), ...selectorSnapshotFields(), ...postActionObservationFields('hover'), }; @@ -194,7 +197,7 @@ const recordControlFields = () => ({ const getFields = { format: requiredField(enumField(['text', 'attrs'] as const)), - target: requiredField(elementTargetField()), + ...elementTargetFields(), ...selectorSnapshotFields(), ...recordControlFields(), }; @@ -260,12 +263,12 @@ const gestureFields = { }), }; -export type ClickInput = InferCommandInput; -export type PressInput = InferCommandInput; -export type FillInput = InferCommandInput; -export type LongPressInput = InferCommandInput; -export type HoverInput = InferCommandInput; -export type GetInput = InferCommandInput; +export type ClickInput = TargetedInput; +export type PressInput = TargetedInput; +export type FillInput = TargetedInput; +export type LongPressInput = TargetedInput; +export type HoverInput = TargetedInput; +export type GetInput = TargetedInput; export type GestureInput = CommonCommandInput & GesturePayload; @@ -273,20 +276,26 @@ export type GestureInput = CommonCommandInput & GesturePayload; export type GestureCommandInput = GestureInput & { postGestureStabilization?: boolean }; export const interactionCommandMetadata = [ - defineInteractionCommandMetadata('click', clickFields), - defineInteractionCommandMetadata('press', pressFields), - defineInteractionCommandMetadata('fill', fillFields), - defineInteractionCommandMetadata('longpress', longPressFields), - defineInteractionCommandMetadata('hover', hoverFields), + defineTargetedCommandMetadata('click', clickFields), + defineTargetedCommandMetadata('press', pressFields), + defineTargetedCommandMetadata('fill', fillFields), + defineTargetedCommandMetadata('longpress', longPressFields), + defineTargetedCommandMetadata('hover', hoverFields), defineInteractionCommandMetadata('swipe', swipeFields), defineInteractionCommandMetadata('focus', focusFields), defineInteractionCommandMetadata('type', typeFields), defineInteractionCommandMetadata('scroll', scrollFields), - defineInteractionCommandMetadata('get', getFields, { parseableOutput: true }), + defineFieldCommandMetadata('get', interactionCommandDescriptions.get, getFields, { + readInput: (input: unknown): GetInput => + readTargetedInput(input, getFields), + inputExample: TARGET_INPUT_EXAMPLE, + parseableOutput: true, + }), defineInteractionCommandMetadata('is', isFields), defineInteractionCommandMetadata('find', findFields, { parseableOutput: true }), defineFieldCommandMetadata('gesture', interactionCommandDescriptions.gesture, gestureFields, { readInput: readGestureCommandInput, + inputExample: { kind: 'pan', origin: { x: 200, y: 400 }, delta: { x: 0, y: -200 } }, }), ] as const; @@ -305,6 +314,16 @@ function readGestureCommandInput(input: unknown): GestureCommandInput { : gesture; } +function defineTargetedCommandMetadata< + const TName extends InteractionCommandName, + const TFields extends CommandFieldMap, +>(name: TName, fields: TFields) { + return defineFieldCommandMetadata(name, interactionCommandDescriptions[name], fields, { + readInput: (input: unknown): TargetedInput => readTargetedInput(input, fields), + inputExample: TARGET_INPUT_EXAMPLE, + }); +} + function defineInteractionCommandMetadata< const TName extends InteractionCommandName, const TFields extends CommandFieldMap, diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 716ebd7f53..d1c6df0492 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -357,6 +357,7 @@ const foldFieldMetadata = defineFieldCommandMetadata( const { pose, keyframes, ...common } = fields; return { ...common, ...parseFoldInput({ pose, keyframes }) }; }, + inputExample: { pose: 'half-open' }, }, ); @@ -478,6 +479,7 @@ const telephonyFieldMetadata = defineFieldCommandMetadata( const { action, number, text, ...common } = readFieldInput(input, telephonyFields); return { ...common, ...parseTelephonyEventInput({ action, number, text }) }; }, + inputExample: { action: 'call', number: '+15555550100' }, }, ); diff --git a/scripts/ios-snapshot-benchmark/proxy-client-support.ts b/scripts/ios-snapshot-benchmark/proxy-client-support.ts index daafbbb297..d3a27bbcba 100644 --- a/scripts/ios-snapshot-benchmark/proxy-client-support.ts +++ b/scripts/ios-snapshot-benchmark/proxy-client-support.ts @@ -139,7 +139,7 @@ export async function openClientFixture( openAlert: async () => fixtureOperationFromClient( await client.interactions.click({ - target: { kind: 'selector', selector: 'id="automation-open-alert"' }, + selector: 'id="automation-open-alert"', platform: 'ios', udid, }), diff --git a/scripts/perf/suite.ts b/scripts/perf/suite.ts index d487b8132b..3d110c29ce 100644 --- a/scripts/perf/suite.ts +++ b/scripts/perf/suite.ts @@ -220,7 +220,7 @@ export const SUITE: readonly SuiteScenario[] = [ command: 'get', input: { format: 'text', - target: { kind: 'selector', selector: 'id="gesture-fling-status"' }, + selector: 'id="gesture-fling-status"', }, }, { command: 'is', input: { predicate: 'visible', selector: 'id="gesture-canary-ready"' } }, diff --git a/src/__tests__/cli-grammar.test.ts b/src/__tests__/cli-grammar.test.ts index 87638b0720..0b769de7d1 100644 --- a/src/__tests__/cli-grammar.test.ts +++ b/src/__tests__/cli-grammar.test.ts @@ -26,30 +26,38 @@ test('snapshot grammar keeps interactive snapshot options focused', () => { assert.equal(options.depth, 3); }); +/** The keys a reader set, without the ones it left undefined. */ +function definedKeys(input: Record): Record { + return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== undefined)); +} + test('interaction and fill grammar share ref, selector, and point parsing', () => { - assert.deepEqual(readInputFromCli('press', ['@e3', 'Email'], BASE_FLAGS).target, { - kind: 'ref', - ref: '@e3', - label: 'Email', + const { ref, label } = readInputFromCli('press', ['@e3', 'Email'], BASE_FLAGS); + assert.deepEqual({ ref, label }, { ref: '@e3', label: 'Email' }); + assert.deepEqual( + definedKeys(readInputFromCli('fill', ['id=email', 'qa@example.com'], BASE_FLAGS)), + { + selector: 'id=email', + text: 'qa@example.com', + }, + ); + assert.deepEqual(definedKeys(readInputFromCli('fill', ['@e4', 'good', 'morning'], BASE_FLAGS)), { + ref: '@e4', + text: 'good morning', + }); + assert.deepEqual(definedKeys(readInputFromCli('fill', ['10', '20', 'hello'], BASE_FLAGS)), { + x: 10, + y: 20, + text: 'hello', }); - const selectorFill = readInputFromCli('fill', ['id=email', 'qa@example.com'], BASE_FLAGS); - assert.deepEqual(selectorFill.target, { kind: 'selector', selector: 'id=email' }); - assert.equal(selectorFill.text, 'qa@example.com'); - - const refFill = readInputFromCli('fill', ['@e4', 'good', 'morning'], BASE_FLAGS); - assert.deepEqual(refFill.target, { kind: 'ref', ref: '@e4' }); - assert.equal(refFill.text, 'good morning'); - - const pointFill = readInputFromCli('fill', ['10', '20', 'hello'], BASE_FLAGS); - assert.deepEqual(pointFill.target, { kind: 'point', x: 10, y: 20 }); - assert.equal(pointFill.text, 'hello'); assert.deepEqual(readInputFromCli('longpress', ['@e4', '800'], BASE_FLAGS), { - target: { kind: 'ref', ref: '@e4' }, + ref: '@e4', durationMs: 800, }); assert.deepEqual(readInputFromCli('longpress', ['10', '20', '800'], BASE_FLAGS), { - target: { kind: 'point', x: 10, y: 20 }, + x: 10, + y: 20, durationMs: 800, }); }); @@ -83,13 +91,17 @@ test('interaction selectors reject unquoted trailing text instead of dropping it ); // Quoted selector values and fill's trailing text payload keep working. - assert.deepEqual(readInputFromCli('press', ['text="Gesture lab"'], BASE_FLAGS).target, { - kind: 'selector', - selector: 'text="Gesture lab"', - }); - const fillAfterSelector = readInputFromCli('fill', ['id=email', 'qa@example.com'], BASE_FLAGS); - assert.deepEqual(fillAfterSelector.target, { kind: 'selector', selector: 'id=email' }); - assert.equal(fillAfterSelector.text, 'qa@example.com'); + assert.equal( + readInputFromCli('press', ['text="Gesture lab"'], BASE_FLAGS).selector, + 'text="Gesture lab"', + ); + assert.deepEqual( + definedKeys(readInputFromCli('fill', ['id=email', 'qa@example.com'], BASE_FLAGS)), + { + selector: 'id=email', + text: 'qa@example.com', + }, + ); }); test('bare snapshot refs without @ prefix throw helpful error', () => { diff --git a/src/commands/batch/cli.test.ts b/src/commands/batch/cli.test.ts index c406e78f27..17920cbc5d 100644 --- a/src/commands/batch/cli.test.ts +++ b/src/commands/batch/cli.test.ts @@ -68,7 +68,7 @@ test('batch structured interaction target is projected to positionals, not devic const result = await runCliCapture([ 'batch', '--steps', - '[{"command":"press","input":{"target":{"kind":"point","x":10,"y":20},"count":2,"platform":"ios","udid":"sim-1"}}]', + '[{"command":"press","input":{"x":10,"y":20,"count":2,"platform":"ios","udid":"sim-1"}}]', '--json', ]); @@ -207,7 +207,7 @@ test('mutating UI verbs reach daemon dispatch through batch', async () => { const result = await runCliCapture([ 'batch', '--steps', - '[{"command":"press","input":{"target":{"kind":"ref","ref":"@e12"}}},{"command":"fill","input":{"target":{"kind":"ref","ref":"@e13"},"text":"x"}}]', + '[{"command":"press","input":{"ref":"@e12"}},{"command":"fill","input":{"ref":"@e13","text":"x"}}]', '--json', ]); @@ -464,7 +464,7 @@ test('batch rejects a fill step that asks for stdin text before daemon dispatch' const result = await runCliCapture([ 'batch', '--steps', - '[{"command":"fill","input":{"target":{"kind":"ref","ref":"@e3"},"text":"x","textStdin":true}}]', + '[{"command":"fill","input":{"ref":"@e3","text":"x","textStdin":true}}]', ]); assert.equal(result.code, 1); @@ -476,7 +476,7 @@ test('batch passes an explicit fill textStdin false through to dispatch', async const result = await runCliCapture([ 'batch', '--steps', - '[{"command":"fill","input":{"target":{"kind":"ref","ref":"@e3"},"text":"x","textStdin":false}}]', + '[{"command":"fill","input":{"ref":"@e3","text":"x","textStdin":false}}]', ]); assert.equal(result.calls.length, 1); diff --git a/src/commands/schema/cli-help-examples.test.ts b/src/commands/schema/cli-help-examples.test.ts index 52a6214a75..66c12f8f80 100644 --- a/src/commands/schema/cli-help-examples.test.ts +++ b/src/commands/schema/cli-help-examples.test.ts @@ -123,7 +123,8 @@ function collectBatchStepExamples(): Array<{ line: string; steps: BatchStep[] }> } // A step's `input` is keyed by the command's structured field names, and no CLI help text spells -// those out — `help press` documents the positional and the flags, not `target: {kind, ref}`. So +// those out — `help press` shows the positional (``) and the flags, not the +// keys a step's input uses for them (`ref`, `label`, `selector`, `x`, `y`). So // the printed steps ARE the terminal's only statement of them, and the check above cannot see it: // the CLI parser validates the step envelope and stops, never reading `input`. Each printed input // therefore goes through its own command's `readInput` — the same reader the daemon, the client, diff --git a/src/commands/schema/cli-help-topics.test.ts b/src/commands/schema/cli-help-topics.test.ts index a73a2b4b90..430b19a4de 100644 --- a/src/commands/schema/cli-help-topics.test.ts +++ b/src/commands/schema/cli-help-topics.test.ts @@ -74,7 +74,7 @@ test('batch help documents the step shape and the commands batch accepts', async // A step's input is keyed by structured field names no CLI help spells out, so the examples are // this text's only statement of them. `cli-help-examples.test.ts` runs them through the readers. assert.match(help, /\n\nExamples:\n {2}agent-device batch --steps '\[/); - assert.match(help, /"command":"press","input":\{"target":\{"kind":"ref"/); + assert.match(help, /"command":"press","input":\{"ref":"@e12"/); assert.match(help, /"command":"fill",.*"text":"qa@example\.com"/); assert.match(help, /"command":"snapshot","input":\{"interactiveOnly":true\}/); }); diff --git a/src/mcp/__tests__/command-tools-nested-admission.test.ts b/src/mcp/__tests__/command-tools-nested-admission.test.ts index 6619fdc7f4..f7e628399b 100644 --- a/src/mcp/__tests__/command-tools-nested-admission.test.ts +++ b/src/mcp/__tests__/command-tools-nested-admission.test.ts @@ -77,7 +77,7 @@ test('a refused step is refused wherever it sits in the batch', async () => { const result = await executor.execute('batch', { steps: [ { command: 'snapshot', input: {} }, - { command: 'tap', input: { target: { kind: 'selector', selector: 'text=OK' } } }, + { command: 'press', input: { selector: 'text=OK' } }, { command: 'snapshot', input: { iosXctestrunFile: '/attacker/run.xctestrun' } }, ], }); @@ -94,7 +94,7 @@ test('nested admission leaves legitimate batch input untouched', async () => { const steps = [ { command: 'open', input: { app: 'settings' } }, { command: 'snapshot', input: { interactiveOnly: true, noRecord: true } }, - { command: 'tap', input: { target: { kind: 'selector', selector: 'text=OK' } } }, + { command: 'press', input: { selector: 'text=OK' } }, { command: 'type', input: { text: 'hello' }, runtime: { platform: 'ios' } }, { command: 'wait', input: { kind: 'duration', durationMs: 100, session: 'sim' } }, ]; diff --git a/src/mcp/__tests__/command-tools-operator-inputs.test.ts b/src/mcp/__tests__/command-tools-operator-inputs.test.ts index 8d6ff8f2fa..3dae1b2263 100644 --- a/src/mcp/__tests__/command-tools-operator-inputs.test.ts +++ b/src/mcp/__tests__/command-tools-operator-inputs.test.ts @@ -99,8 +99,8 @@ test('MCP neither advertises nor admits the retired readiness budget', async () }); for (const [command, input] of [ - ['press', { target: { kind: 'selector', selector: 'label=Continue' } }], - ['fill', { target: { kind: 'selector', selector: 'label=Name' }, text: 'x' }], + ['press', { selector: 'label=Continue' }], + ['fill', { selector: 'label=Name', text: 'x' }], ] as const) { const result = await executor.execute(command, { ...input, readinessTimeoutMs: 2_000 }); assert.equal(result.isError, true, command); @@ -250,7 +250,7 @@ test('MCP neither advertises nor admits the CLI-only fill textStdin marker', asy }); const result = await executor.execute('fill', { - target: { kind: 'ref', ref: '@e3' }, + ref: '@e3', text: 'secret', textStdin: true, }); diff --git a/src/mcp/__tests__/command-tools-parity.test.ts b/src/mcp/__tests__/command-tools-parity.test.ts index 54c9215789..ebed1e4b6e 100644 --- a/src/mcp/__tests__/command-tools-parity.test.ts +++ b/src/mcp/__tests__/command-tools-parity.test.ts @@ -6,7 +6,7 @@ import { createAgentDeviceClient } from '@agent-device/client'; import type { AgentDeviceDaemonTransport } from '@agent-device/contracts/client'; import type { AgentDeviceClient, CommandExecutionResult } from '@agent-device/command-surface'; import { createCommandToolExecutor, listCommandTools } from '../command-tools.ts'; -import { COMMAND_OUTPUT_SCHEMAS } from '@agent-device/command-surface'; +import { COMMAND_OUTPUT_SCHEMAS, readInputFromCli } from '@agent-device/command-surface'; import { validateAgainstSchema } from './output-schema-validator.ts'; import { mkdtempForTestSync } from '@agent-device/test-kit'; @@ -155,7 +155,8 @@ test('MCP fill projects target-bound unconfirmed verification through its advert ]); const result = await executor.execute('fill', { - target: { kind: 'point', x: 100, y: 50 }, + x: 100, + y: 50, text: fillResult.requested, }); assert.deepEqual(result.structuredContent, fillResult); @@ -237,3 +238,63 @@ test('MCP applies config-backed command defaults; explicit operator input is ref assert.ok(calls.every((request) => request.command === 'snapshot')); assert.ok(calls.every((request) => request.flags?.appsFilter === undefined)); }); + +// The CLI and MCP share one command input. A UI target spelled on the command line and the same +// target passed as tool arguments must reach runCommand as the identical object, so a shape that +// drifts on either side fails here. +const UI_TARGETS = [ + { name: 'ref', argv: ['@e6'], input: { ref: '@e6' } }, + { name: 'ref with label', argv: ['@e6', 'Products'], input: { ref: '@e6', label: 'Products' } }, + { name: 'selector', argv: ['id="home/products"'], input: { selector: 'id="home/products"' } }, + { name: 'point', argv: ['201', '160'], input: { x: 201, y: 160 } }, + { name: 'pinned ref', argv: ['@e6~s4'], input: { ref: '@e6~s4' } }, +] as const; + +const TARGETED_COMMANDS = [ + { command: 'press', before: [], after: [], input: {}, without: [] }, + { command: 'click', before: [], after: [], input: {}, without: [] }, + { command: 'longpress', before: [], after: [], input: {}, without: [] }, + { command: 'hover', before: [], after: [], input: {}, without: [] }, + // The CLI reads the words after a fill ref as its text, so fill has no label spelling. + { + command: 'fill', + before: [], + after: ['hello'], + input: { text: 'hello' }, + without: ['ref with label'], + }, + { command: 'get', before: ['text'], after: [], input: { format: 'text' }, without: ['point'] }, +] as const; + +const CLI_TARGET_SCENARIOS = TARGETED_COMMANDS.flatMap((command) => + UI_TARGETS.filter((target) => !(command.without as readonly string[]).includes(target.name)).map( + (target) => ({ + label: `${command.command} ${target.name}`, + command: command.command, + argv: [...command.before, ...target.argv, ...command.after], + input: { ...command.input, ...target.input }, + }), + ), +); + +test.for(CLI_TARGET_SCENARIOS)( + 'CLI argv and MCP input reach runCommand identically: $label', + async ({ command, argv, input }) => { + const reached: Array> = []; + const executor = createCommandToolExecutor({ + createClient: () => ({}) as AgentDeviceClient, + runCommand: async (_client, _name, commandInput) => { + reached.push(commandInput); + return {}; + }, + }); + await executor.execute(command, input); + const cliInput = readInputFromCli(command, argv, { json: false, help: false, version: false }); + + assert.deepEqual(reached, [definedKeys(cliInput)]); + }, +); + +function definedKeys(input: Record): Record { + return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== undefined)); +} diff --git a/src/mcp/__tests__/command-tools-structured-content.test.ts b/src/mcp/__tests__/command-tools-structured-content.test.ts index 5809fedb2d..5ec2f45386 100644 --- a/src/mcp/__tests__/command-tools-structured-content.test.ts +++ b/src/mcp/__tests__/command-tools-structured-content.test.ts @@ -24,7 +24,7 @@ test('MCP returns structuredContent for typed tools, and for untyped tools only const json = await executor.execute('snapshot', { mcpOutputFormat: 'json' }); assert.deepEqual(json.structuredContent, snapshotResult); - const press = await executor.execute('press', { target: { kind: 'ref', ref: '@e2' } }); + const press = await executor.execute('press', { ref: '@e2' }); assert.deepEqual(press.structuredContent, { message: 'Tapped @e2 (10, 20)' }); }); diff --git a/src/mcp/__tests__/command-tools-target-args.test.ts b/src/mcp/__tests__/command-tools-target-args.test.ts new file mode 100644 index 0000000000..af617e9b67 --- /dev/null +++ b/src/mcp/__tests__/command-tools-target-args.test.ts @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { findCommandMetadata, type AgentDeviceClient } from '@agent-device/command-surface'; +import { createCommandToolExecutor, listCommandTools } from '../command-tools.ts'; + +const ACCEPTED = 'input accepted'; + +// Reads the input as the command route does, then stops before any client call. +function createReadingExecutor() { + return createCommandToolExecutor({ + createClient: () => ({}) as AgentDeviceClient, + runCommand: async (_client, name, input) => { + findCommandMetadata(name).readInput(input); + throw new Error(ACCEPTED); + }, + }); +} + +function toolText(result: { content: Array<{ text: string }> }): string { + return result.content[0]?.text ?? ''; +} + +function exampleFromHint(text: string, prefix: string): Record { + const line = text.split('\n').find((entry) => entry.startsWith(`Hint: ${prefix}`)); + assert.ok(line, `expected a "${prefix}" hint in:\n${text}`); + return JSON.parse(line.slice(`Hint: ${prefix}`.length)) as Record; +} + +// An agent without the schema loaded guesses the CLI's vocabulary as flat arguments. +test.for([ + ['press', { ref: '@e6' }], + ['click', { selector: 'id=home/products' }], + ['click', { ref: '@e6' }], + ['press', { x: 120, y: 340 }], + ['fill', { ref: '@e3', text: 'qa@example.com' }], + ['get', { format: 'text', selector: 'id=title' }], +] as const)('MCP admits the guessed %s call %j', async ([name, input]) => { + const result = await createReadingExecutor().execute(name, input); + assert.match(toolText(result), new RegExp(ACCEPTED)); +}); + +test('an unknown argument answers with a call example the tool accepts', async () => { + const executor = createReadingExecutor(); + const refused = await executor.execute('press', { element: '@e6' }); + + assert.equal(refused.isError, true); + const text = toolText(refused); + assert.match(text, /element is not an accepted argument for the press tool/); + const example = exampleFromHint(text, 'Example call: press '); + assert.deepEqual(example, { ref: '@e12' }); + assert.match(toolText(await executor.execute('press', example)), new RegExp(ACCEPTED)); +}); + +test('every tool hint example is accepted by that tool', async () => { + const executor = createReadingExecutor(); + for (const { name } of listCommandTools()) { + const refused = await executor.execute(name, { notAnArgument: true }); + const example = exampleFromHint(toolText(refused), `Example call: ${name} `); + const retried = toolText(await executor.execute(name, example)); + assert.match(retried, new RegExp(ACCEPTED), `${name} refused ${JSON.stringify(example)}`); + } +}); + +test('an unknown argument inside a batch step answers with a step example', async () => { + const executor = createReadingExecutor(); + const refused = await executor.execute('batch', { + steps: [{ command: 'click', input: { element: '@e6' } }], + }); + + const text = toolText(refused); + assert.match(text, /batch\.steps\[0\]\.input: element is not an accepted argument/); + const step = exampleFromHint(text, 'Example step: '); + assert.deepEqual(step, { command: 'click', input: { ref: '@e12' } }); + assert.match(toolText(await executor.execute('batch', { steps: [step] })), new RegExp(ACCEPTED)); +}); + +test('the removed nested target reaches its migration guidance over MCP', async () => { + const result = await createReadingExecutor().execute('press', { + target: { kind: 'ref', ref: '@e6' }, + }); + + assert.equal(result.isError, true); + assert.match(toolText(result), /target was removed; pass the UI target at the top level/); + const press = listCommandTools().find((tool) => tool.name === 'press'); + assert.ok(press && !('target' in (press.inputSchema.properties ?? {}))); +}); diff --git a/src/mcp/__tests__/command-tools.test.ts b/src/mcp/__tests__/command-tools.test.ts index 5004191714..3b05d227ed 100644 --- a/src/mcp/__tests__/command-tools.test.ts +++ b/src/mcp/__tests__/command-tools.test.ts @@ -177,16 +177,13 @@ test('MCP pins every find-list ref so a plain follow-up press forwards pinned', ['2 matches:', '= @e5~s41 [button] "Add"', '= @e9~s41 [cell] "Add another account"'].join('\n'), ); - await executor.execute('press', { session: 'demo', target: { kind: 'ref', ref: '@e9' } }); - await executor.execute('press', { session: 'demo', target: { kind: 'ref', ref: '@e5' } }); + await executor.execute('press', { session: 'demo', ref: '@e9' }); + await executor.execute('press', { session: 'demo', ref: '@e5' }); const presses = calls.filter((call) => call.name === 'press'); assert.deepEqual( - presses.map((call) => call.input.target), - [ - { kind: 'ref', ref: '@e9~s41' }, - { kind: 'ref', ref: '@e5~s41' }, - ], + presses.map((call) => call.input.ref), + ['@e9~s41', '@e5~s41'], ); }); @@ -703,7 +700,7 @@ test('MCP renders tool text from the unpinned input so the model never sees suff }); await executor.execute('snapshot', {}); - const result = await executor.execute('press', { target: { kind: 'ref', ref: '@e2' } }); + const result = await executor.execute('press', { ref: '@e2' }); assert.doesNotMatch(result.content[0]?.text ?? '', /~s9/); }); @@ -724,12 +721,12 @@ test('MCP command tool executor pins refs for runCommand while keeping rendered await executor.execute('snapshot', { session: 'demo' }); const result = await executor.execute('press', { session: 'demo', - target: { kind: 'ref', ref: '@e2' }, + ref: '@e2', }); assert.deepEqual(runCalls[1], { name: 'press', - input: { session: 'demo', target: { kind: 'ref', ref: '@e2~s500012' } }, + input: { session: 'demo', ref: '@e2~s500012' }, }); assert.doesNotMatch(result.content[0]?.text ?? '', /~s500012/); }); @@ -802,8 +799,8 @@ test('MCP tool error is a ref-issuing result: isError, structuredContent, and pi // The error-path screen ref is pinned exactly like a successful ref-issuing // response — the caller's next command against @e5 forwards the generation. - await executor.execute('press', { target: { kind: 'ref', ref: '@e5' } }); - assert.deepEqual(runCalls[1]?.input, { target: { kind: 'ref', ref: '@e5~s12' } }); + await executor.execute('press', { ref: '@e5' }); + assert.deepEqual(runCalls[1]?.input, { ref: '@e5~s12' }); }); test('ambiguous interaction errors pin their candidate refs for the next mutation', async () => { @@ -825,18 +822,18 @@ test('ambiguous interaction errors pin their candidate refs for the next mutatio const result = await executor.execute('press', { session: 'demo', - target: { kind: 'selector', selector: 'label="Team Standup"' }, + selector: 'label="Team Standup"', }); assert.equal(result.isError, true); assert.match(result.content[0]?.text ?? '', /@e5~s42 \[cell\] "Team Standup"/); await executor.execute('press', { session: 'demo', - target: { kind: 'ref', ref: '@e5' }, + ref: '@e5', }); assert.deepEqual(runCalls[1]?.input, { session: 'demo', - target: { kind: 'ref', ref: '@e5~s42' }, + ref: '@e5~s42', }); }); @@ -920,13 +917,13 @@ test('MCP forwards record/noRecord from a get tool call through to the executed }); await executor.execute('get', { format: 'text', - target: { kind: 'ref', ref: '@e5' }, + ref: '@e5', record: true, }); assert.deepEqual(calls, [ { name: 'get', - input: { format: 'text', target: { kind: 'ref', ref: '@e5' }, record: true }, + input: { format: 'text', ref: '@e5', record: true }, }, ]); }); @@ -970,13 +967,13 @@ test('MCP forwards noRecord from a press tool call through to the executed comma }, }); await executor.execute('press', { - target: { kind: 'ref', ref: '@e5' }, + ref: '@e5', noRecord: true, }); assert.deepEqual(calls, [ { name: 'press', - input: { target: { kind: 'ref', ref: '@e5' }, noRecord: true }, + input: { ref: '@e5', noRecord: true }, }, ]); }); diff --git a/src/mcp/__tests__/router.test.ts b/src/mcp/__tests__/router.test.ts index e7cc6c72d3..16dc831c20 100644 --- a/src/mcp/__tests__/router.test.ts +++ b/src/mcp/__tests__/router.test.ts @@ -37,7 +37,8 @@ test('MCP exposes every automatable CLI command as a structured direct tool, plu const fillProperties = (fillTool.inputSchema as { properties: Record }) .properties; assert.ok(!('positionals' in fillProperties)); - assert.ok('target' in fillProperties); + assert.ok('ref' in fillProperties); + assert.ok(!('target' in fillProperties)); assert.ok('recordAs' in fillProperties); const batchTool = (response.result as { tools: Array> }).tools.find( @@ -51,11 +52,11 @@ test('MCP exposes every automatable CLI command as a structured direct tool, plu jsonrpc: '2.0', id: 2, method: 'tools/call', - params: { name: 'fill', arguments: {} }, + params: { name: 'fill', arguments: { text: 'qa' } }, }); assert.ok(invalidFillResponse && 'result' in invalidFillResponse); assert.equal((invalidFillResponse.result as { isError: boolean }).isError, true); - assert.match(JSON.stringify(invalidFillResponse.result), /Expected target to be set/); + assert.match(JSON.stringify(invalidFillResponse.result), /Expected a UI target/); const malformedArgumentsResponse = await handleMcpMessage({ jsonrpc: '2.0', diff --git a/src/mcp/__tests__/tool-ref-pins.test.ts b/src/mcp/__tests__/tool-ref-pins.test.ts index c6c61fabb5..4315010ae1 100644 --- a/src/mcp/__tests__/tool-ref-pins.test.ts +++ b/src/mcp/__tests__/tool-ref-pins.test.ts @@ -17,14 +17,10 @@ test('ref-pin store keeps per-ref provenance across snapshot and find captures', ); pins.mergeCommandResult('find', { ref: '@e5', refsGeneration: 500013 }, undefined, 'demo'); - const pinned = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e37' } }, - undefined, - ); + const pinned = pins.pinInput('press', { session: 'demo', ref: '@e37' }, undefined); assert.deepEqual(pinned, { session: 'demo', - target: { kind: 'ref', ref: '@e37~s500012' }, + ref: '@e37~s500012', }); }); @@ -39,14 +35,10 @@ test('ref-pin store pins the find-issued ref to the find generation', () => { ); pins.mergeCommandResult('find', { ref: '@e5', refsGeneration: 500013 }, undefined, 'demo'); - const pinned = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e5' } }, - undefined, - ); + const pinned = pins.pinInput('press', { session: 'demo', ref: '@e5' }, undefined); assert.deepEqual(pinned, { session: 'demo', - target: { kind: 'ref', ref: '@e5~s500013' }, + ref: '@e5~s500013', }); }); @@ -66,19 +58,11 @@ test('ref-pin store pins every listed ref from a find list response', () => { 'demo', ); - const first = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e5' } }, - undefined, - ); - const second = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e9' } }, - undefined, - ); + const first = pins.pinInput('press', { session: 'demo', ref: '@e5' }, undefined); + const second = pins.pinInput('press', { session: 'demo', ref: '@e9' }, undefined); - assert.deepEqual(first, { session: 'demo', target: { kind: 'ref', ref: '@e5~s500014' } }); - assert.deepEqual(second, { session: 'demo', target: { kind: 'ref', ref: '@e9~s500014' } }); + assert.deepEqual(first, { session: 'demo', ref: '@e5~s500014' }); + assert.deepEqual(second, { session: 'demo', ref: '@e9~s500014' }); }); test('ref-pin store pins ambiguous mutation candidates issued by an error', () => { @@ -94,14 +78,10 @@ test('ref-pin store pins ambiguous mutation candidates issued by an error', () = 'demo', ); - const pinned = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e5' } }, - undefined, - ); + const pinned = pins.pinInput('press', { session: 'demo', ref: '@e5' }, undefined); assert.deepEqual(pinned, { session: 'demo', - target: { kind: 'ref', ref: '@e5~s500014' }, + ref: '@e5~s500014', }); }); @@ -116,17 +96,13 @@ test('ref-pin store pins wait refs and get targets from the per-ref map', () => ); const wait = pins.pinInput('wait', { session: 'demo', ref: '@e2' }, undefined); - const get = pins.pinInput( - 'get', - { session: 'demo', format: 'text', target: { kind: 'ref', ref: '@e37' } }, - undefined, - ); + const get = pins.pinInput('get', { session: 'demo', format: 'text', ref: '@e37' }, undefined); assert.deepEqual(wait, { session: 'demo', ref: '@e2~s500012' }); assert.deepEqual(get, { session: 'demo', format: 'text', - target: { kind: 'ref', ref: '@e37~s500012' }, + ref: '@e37~s500012', }); }); @@ -140,12 +116,8 @@ test('ref-pin store merges digest-level snapshot refs too', () => { 'demo', ); - const pinned = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e9' } }, - undefined, - ); - assert.deepEqual(pinned, { session: 'demo', target: { kind: 'ref', ref: '@e9~s41' } }); + const pinned = pins.pinInput('press', { session: 'demo', ref: '@e9' }, undefined); + assert.deepEqual(pinned, { session: 'demo', ref: '@e9~s41' }); }); // --- #1101 --settle: interaction responses re-pin from the settled diff --- @@ -183,28 +155,20 @@ test('ref-pin store merges per-ref pins from a settle response diff', () => { 'demo', ); - const first = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e2' } }, - undefined, - ); - const second = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e4' } }, - undefined, - ); + const first = pins.pinInput('press', { session: 'demo', ref: '@e2' }, undefined); + const second = pins.pinInput('press', { session: 'demo', ref: '@e4' }, undefined); const untouched = pins.pinInput( 'get', - { session: 'demo', format: 'text', target: { kind: 'ref', ref: '@e37' } }, + { session: 'demo', format: 'text', ref: '@e37' }, undefined, ); - assert.deepEqual(first, { session: 'demo', target: { kind: 'ref', ref: '@e2~s7' } }); - assert.deepEqual(second, { session: 'demo', target: { kind: 'ref', ref: '@e4~s8' } }); + assert.deepEqual(first, { session: 'demo', ref: '@e2~s7' }); + assert.deepEqual(second, { session: 'demo', ref: '@e4~s8' }); assert.deepEqual(untouched, { session: 'demo', format: 'text', - target: { kind: 'ref', ref: '@e37~s7' }, + ref: '@e37~s7', }); }); @@ -232,12 +196,8 @@ test('ref-pin store merges digest-level settled refs too', () => { 'demo', ); - const pinned = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e4' } }, - undefined, - ); - assert.deepEqual(pinned, { session: 'demo', target: { kind: 'ref', ref: '@e4~s9' } }); + const pinned = pins.pinInput('press', { session: 'demo', ref: '@e4' }, undefined); + assert.deepEqual(pinned, { session: 'demo', ref: '@e4~s9' }); }); test('ref-pin store merges per-ref pins from a settle response unchanged-interactive tail', () => { @@ -265,12 +225,8 @@ test('ref-pin store merges per-ref pins from a settle response unchanged-interac 'demo', ); - const pinned = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e1' } }, - undefined, - ); - assert.deepEqual(pinned, { session: 'demo', target: { kind: 'ref', ref: '@e1~s9' } }); + const pinned = pins.pinInput('press', { session: 'demo', ref: '@e1' }, undefined); + assert.deepEqual(pinned, { session: 'demo', ref: '@e1~s9' }); }); test('ref-pin store leaves pins untouched for plain interaction responses', () => { @@ -284,19 +240,11 @@ test('ref-pin store leaves pins untouched for plain interaction responses', () = ); pins.mergeCommandResult('press', { ref: 'e2', x: 10, y: 20 }, undefined, 'demo'); - const first = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e2' } }, - undefined, - ); - const second = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e2' } }, - undefined, - ); + const first = pins.pinInput('press', { session: 'demo', ref: '@e2' }, undefined); + const second = pins.pinInput('press', { session: 'demo', ref: '@e2' }, undefined); - assert.deepEqual(first, { session: 'demo', target: { kind: 'ref', ref: '@e2~s7' } }); - assert.deepEqual(second, { session: 'demo', target: { kind: 'ref', ref: '@e2~s7' } }); + assert.deepEqual(first, { session: 'demo', ref: '@e2~s7' }); + assert.deepEqual(second, { session: 'demo', ref: '@e2~s7' }); }); // --- ADR 0012 decision 2: resolution diagnostics are never ref-issued/pinned --- @@ -328,25 +276,13 @@ test('ref-pin store never pins resolution.winnerDiagnostic/alternatives', () => 'demo', ); - const e2 = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e2' } }, - undefined, - ); - const e3 = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e3' } }, - undefined, - ); - const diag = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@diag-e3' } }, - undefined, - ); + const e2 = pins.pinInput('press', { session: 'demo', ref: '@e2' }, undefined); + const e3 = pins.pinInput('press', { session: 'demo', ref: '@e3' }, undefined); + const diag = pins.pinInput('press', { session: 'demo', ref: '@diag-e3' }, undefined); - assert.deepEqual(e2, { session: 'demo', target: { kind: 'ref', ref: '@e2~s7' } }); - assert.deepEqual(e3, { session: 'demo', target: { kind: 'ref', ref: '@e3~s7' } }); - assert.deepEqual(diag, { session: 'demo', target: { kind: 'ref', ref: '@diag-e3' } }); + assert.deepEqual(e2, { session: 'demo', ref: '@e2~s7' }); + assert.deepEqual(e3, { session: 'demo', ref: '@e3~s7' }); + assert.deepEqual(diag, { session: 'demo', ref: '@diag-e3' }); }); test('ref-pin store passes never-issued refs through unpinned', () => { @@ -359,12 +295,8 @@ test('ref-pin store passes never-issued refs through unpinned', () => { 'demo', ); - const pinned = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e99' } }, - undefined, - ); - assert.deepEqual(pinned, { session: 'demo', target: { kind: 'ref', ref: '@e99' } }); + const pinned = pins.pinInput('press', { session: 'demo', ref: '@e99' }, undefined); + assert.deepEqual(pinned, { session: 'demo', ref: '@e99' }); }); test('ref-pin store passes refs through unpinned when the scope has no history', () => { @@ -377,12 +309,8 @@ test('ref-pin store passes refs through unpinned when the scope has no history', 'other', ); - const pinned = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e2' } }, - undefined, - ); - assert.deepEqual(pinned, { session: 'demo', target: { kind: 'ref', ref: '@e2' } }); + const pinned = pins.pinInput('press', { session: 'demo', ref: '@e2' }, undefined); + assert.deepEqual(pinned, { session: 'demo', ref: '@e2' }); }); test('ref-pin store scopes by state dir so same-named sessions never cross-pollinate', () => { @@ -397,24 +325,24 @@ test('ref-pin store scopes by state dir so same-named sessions never cross-polli const otherState = pins.pinInput( 'press', - { session: 'demo', stateDir: '/state/b', target: { kind: 'ref', ref: '@e2' } }, + { session: 'demo', stateDir: '/state/b', ref: '@e2' }, '/state/b', ); const originalState = pins.pinInput( 'press', - { session: 'demo', stateDir: '/state/a', target: { kind: 'ref', ref: '@e2' } }, + { session: 'demo', stateDir: '/state/a', ref: '@e2' }, '/state/a', ); assert.deepEqual(otherState, { session: 'demo', stateDir: '/state/b', - target: { kind: 'ref', ref: '@e2' }, + ref: '@e2', }); assert.deepEqual(originalState, { session: 'demo', stateDir: '/state/a', - target: { kind: 'ref', ref: '@e2~s500012' }, + ref: '@e2~s500012', }); }); @@ -434,12 +362,8 @@ test('ref-pin store clears the whole scope when a ref-issuing response stops car 'demo', ); - const pinned = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e2' } }, - undefined, - ); - assert.deepEqual(pinned, { session: 'demo', target: { kind: 'ref', ref: '@e2' } }); + const pinned = pins.pinInput('press', { session: 'demo', ref: '@e2' }, undefined); + assert.deepEqual(pinned, { session: 'demo', ref: '@e2' }); }); test('ref-pin store never rewrites refs that already carry a suffix and never pins non-@ refs', () => { @@ -452,19 +376,11 @@ test('ref-pin store never rewrites refs that already carry a suffix and never pi 'demo', ); - const alreadyPinned = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e2~s3' } }, - undefined, - ); - const missingAt = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: 'e2' } }, - undefined, - ); + const alreadyPinned = pins.pinInput('press', { session: 'demo', ref: '@e2~s3' }, undefined); + const missingAt = pins.pinInput('press', { session: 'demo', ref: 'e2' }, undefined); - assert.deepEqual(alreadyPinned, { session: 'demo', target: { kind: 'ref', ref: '@e2~s3' } }); - assert.deepEqual(missingAt, { session: 'demo', target: { kind: 'ref', ref: 'e2' } }); + assert.deepEqual(alreadyPinned, { session: 'demo', ref: '@e2~s3' }); + assert.deepEqual(missingAt, { session: 'demo', ref: 'e2' }); }); test('ref-pin store bounds retained pins per scope', () => { @@ -476,19 +392,11 @@ test('ref-pin store bounds retained pins per scope', () => { } pins.mergeCommandResult('snapshot', { nodes, refsGeneration: 1 }, undefined, 'demo'); - const first = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e0' } }, - undefined, - ); - const last = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e1001' } }, - undefined, - ); + const first = pins.pinInput('press', { session: 'demo', ref: '@e0' }, undefined); + const last = pins.pinInput('press', { session: 'demo', ref: '@e1001' }, undefined); - assert.deepEqual(first, { session: 'demo', target: { kind: 'ref', ref: '@e0' } }); - assert.deepEqual(last, { session: 'demo', target: { kind: 'ref', ref: '@e1001~s1' } }); + assert.deepEqual(first, { session: 'demo', ref: '@e0' }); + assert.deepEqual(last, { session: 'demo', ref: '@e1001~s1' }); }); // --- ADR 0012 migration step 2: replay divergence is a ref-issuing error --- @@ -523,12 +431,8 @@ test('ref-pin store merges divergence screen refs and pins them for later inputs pins.mergeErrorDetails(replayDivergenceDetails(), undefined, 'demo'); - const pinned = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e5' } }, - undefined, - ); - assert.deepEqual(pinned, { session: 'demo', target: { kind: 'ref', ref: '@e5~s12' } }); + const pinned = pins.pinInput('press', { session: 'demo', ref: '@e5' }, undefined); + assert.deepEqual(pinned, { session: 'demo', ref: '@e5~s12' }); }); test('ref-pin store leaves existing pins untouched for an error without a divergence', () => { @@ -542,12 +446,8 @@ test('ref-pin store leaves existing pins untouched for an error without a diverg ); pins.mergeErrorDetails({ code: 'INVALID_ARGS', message: 'bad selector' }, undefined, 'demo'); - const pinned = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e2' } }, - undefined, - ); - assert.deepEqual(pinned, { session: 'demo', target: { kind: 'ref', ref: '@e2~s7' } }); + const pinned = pins.pinInput('press', { session: 'demo', ref: '@e2' }, undefined); + assert.deepEqual(pinned, { session: 'demo', ref: '@e2~s7' }); }); test('ref-pin store leaves existing pins untouched for a mutating find without refsGeneration', () => { @@ -561,12 +461,8 @@ test('ref-pin store leaves existing pins untouched for a mutating find without r ); pins.mergeCommandResult('find', { ref: '@e2' }, undefined, 'demo'); - const pinned = pins.pinInput( - 'press', - { session: 'demo', target: { kind: 'ref', ref: '@e2' } }, - undefined, - ); - assert.deepEqual(pinned, { session: 'demo', target: { kind: 'ref', ref: '@e2~s7' } }); + const pinned = pins.pinInput('press', { session: 'demo', ref: '@e2' }, undefined); + assert.deepEqual(pinned, { session: 'demo', ref: '@e2~s7' }); }); // --- #1638: the generic-route settle commands pin on identical terms --- @@ -607,14 +503,14 @@ test('ref-pin store merges per-ref pins from a scroll settle diff', () => { // The scrolled-in ref pins at the settle generation; a ref the settle never // republished keeps its older snapshot pin, so the daemon still warns on it. - assert.deepEqual( - pins.pinInput('press', { session: 'demo', target: { kind: 'ref', ref: '@e4' } }, undefined), - { session: 'demo', target: { kind: 'ref', ref: '@e4~s8' } }, - ); - assert.deepEqual( - pins.pinInput('press', { session: 'demo', target: { kind: 'ref', ref: '@e37' } }, undefined), - { session: 'demo', target: { kind: 'ref', ref: '@e37~s7' } }, - ); + assert.deepEqual(pins.pinInput('press', { session: 'demo', ref: '@e4' }, undefined), { + session: 'demo', + ref: '@e4~s8', + }); + assert.deepEqual(pins.pinInput('press', { session: 'demo', ref: '@e37' }, undefined), { + session: 'demo', + ref: '@e37~s7', + }); }); test('ref-pin store merges per-ref pins from a back settle tail', () => { @@ -641,10 +537,10 @@ test('ref-pin store merges per-ref pins from a back settle tail', () => { 'demo', ); - assert.deepEqual( - pins.pinInput('press', { session: 'demo', target: { kind: 'ref', ref: '@e6' } }, undefined), - { session: 'demo', target: { kind: 'ref', ref: '@e6~s12' } }, - ); + assert.deepEqual(pins.pinInput('press', { session: 'demo', ref: '@e6' }, undefined), { + session: 'demo', + ref: '@e6~s12', + }); }); test('ref-pin store leaves pins untouched for a scroll with no settle payload', () => { @@ -663,8 +559,8 @@ test('ref-pin store leaves pins untouched for a scroll with no settle payload', 'demo', ); - assert.deepEqual( - pins.pinInput('press', { session: 'demo', target: { kind: 'ref', ref: '@e2' } }, undefined), - { session: 'demo', target: { kind: 'ref', ref: '@e2~s7' } }, - ); + assert.deepEqual(pins.pinInput('press', { session: 'demo', ref: '@e2' }, undefined), { + session: 'demo', + ref: '@e2~s7', + }); }); diff --git a/src/mcp/command-tools.ts b/src/mcp/command-tools.ts index a2eda74c70..1e3ee479c3 100644 --- a/src/mcp/command-tools.ts +++ b/src/mcp/command-tools.ts @@ -1,6 +1,7 @@ import type { AgentDeviceClientConfig } from '@agent-device/contracts/client'; import { COMMON_INPUT_AUDIENCE, + commandInputExample, findCommandMetadata, isCommandName, listCommandMetadata, @@ -189,7 +190,7 @@ export function createCommandToolExecutor(deps: CommandToolExecutorDeps = {}): C findInadmissibleNestedCommandInput(metadata.inputSchema, input, name); if (rejection) { return await buildErrorToolResult( - new AppError('INVALID_ARGS', rejection), + new AppError('INVALID_ARGS', rejection.message, compactHint(rejection.hint)), refPins, undefined, input.session, @@ -386,7 +387,17 @@ function stripMcpConfigFields(input: Record): Record, 'inputSchema' | 'inputAudience'>; +type AdmissionMetadata = Pick< + CommandMetadata, + 'inputSchema' | 'inputAudience' | 'inputExample' +>; + +/** Why admission refused a call, and for an unknown argument, a call the tool accepts. */ +type AdmissionRejection = { message: string; hint?: string }; + +function compactHint(hint: string | undefined): { hint: string } | undefined { + return hint === undefined ? undefined : { hint }; +} /** * The advertised schema for a tool — exactly what `listCommandTools` exposes. @@ -412,7 +423,7 @@ function findInadmissibleInput( name: CommandName, metadata: AdmissionMetadata, input: Record, -): string | undefined { +): AdmissionRejection | undefined { const advertised = advertisedInputSchema(name, metadata).properties; const audience = toolInputAudience(metadata); for (const key of Object.keys(input)) { @@ -425,8 +436,13 @@ function findInadmissibleInput( // admit them so the command's own reader answers with migration guidance // (e.g. maxSize -> "use --scale") instead of a bare unknown-key rejection. if (keyAudience?.kind === 'retired') continue; - if (keyAudience?.kind === 'operator') return operatorInputRefusal(key, keyAudience.source); - return `${key} is not an accepted argument for the ${name} tool.`; + if (keyAudience?.kind === 'operator') { + return { message: operatorInputRefusal(key, keyAudience.source) }; + } + return { + message: `${key} is not an accepted argument for the ${name} tool.`, + hint: `Example call: ${name} ${JSON.stringify(commandInputExample(metadata))}`, + }; } return undefined; } @@ -459,7 +475,7 @@ function findInadmissibleNestedCommandInput( schema: JsonSchema | undefined, value: unknown, path: string, -): string | undefined { +): AdmissionRejection | undefined { if (!schema || value === undefined) return undefined; if (Array.isArray(value)) { return firstRejection(value, (item, index) => @@ -484,7 +500,7 @@ function findInadmissibleNestedProperty( parent: Record, key: string, path: string, -): string | undefined { +): AdmissionRejection | undefined { const commandKey = property.commandInputFor; if (commandKey === undefined) { return findInadmissibleNestedCommandInput(property, parent[key], path); @@ -501,14 +517,20 @@ function findInadmissibleNestedProperty( // daemon request flag. Reporting it is the reader's job — answering here would // bury the real error under a key complaint. if (command === undefined || !isRecord(nested)) return undefined; - const rejection = findInadmissibleInput(command, findCommandMetadata(command), nested); - return rejection === undefined ? undefined : `${path}: ${rejection}`; + const metadata = findCommandMetadata(command); + const rejection = findInadmissibleInput(command, metadata, nested); + if (rejection === undefined) return undefined; + const step = { [commandKey]: command, [key]: commandInputExample(metadata) }; + return { + message: `${path}: ${rejection.message}`, + ...(rejection.hint === undefined ? {} : { hint: `Example step: ${JSON.stringify(step)}` }), + }; } function firstRejection( items: readonly TItem[], - check: (item: TItem, index: number) => string | undefined, -): string | undefined { + check: (item: TItem, index: number) => AdmissionRejection | undefined, +): AdmissionRejection | undefined { for (const [index, item] of items.entries()) { const rejection = check(item, index); if (rejection) return rejection; diff --git a/src/mcp/server-guide.ts b/src/mcp/server-guide.ts index c1eca5a718..4f52e3ce13 100644 --- a/src/mcp/server-guide.ts +++ b/src/mcp/server-guide.ts @@ -21,7 +21,7 @@ Start: known app -> call open {app, foreground: true} at once; do not probe with Loop: press/click/fill/longpress/hover/scroll/back with settle: true; the response is the settled UI diff, continue from it. To reach an off-screen target, scroll {direction, until: ""} scrolls until it is on screen in one call, and direction "bottom" runs to the end of the content; repeated bare scrolls are the slow way to find something. snapshot {interactiveOnly: true} only when the diff lacks the next target or did not settle. Verify with wait {kind: "text", text}, wait {selector}, wait {absent: selector}, is, get, or find; a bare screenshot is not verification. End with close. -Targets: copy refs byte-for-byte (@e12, @e12~s4; keep @ and any ~sN). Refs go stale after mutations. Prefer refs, then id/label/role selectors; coordinates last. On a sparse/AX-unavailable warning its refs and selectors are invalid: screenshot, read the image, press {x, y}, then snapshot the changed screen. +Targets are top-level arguments: press {ref: "@e12"}, click {selector: "id=submit"}, press {x, y}. Copy refs byte-for-byte (@e12, @e12~s4; keep @ and any ~sN). Refs go stale after mutations. Prefer refs, then id/label/role selectors; coordinates last. On a sparse/AX-unavailable warning its refs and selectors are invalid: screenshot, read the image, press {x, y}, then snapshot the changed screen. Errors carry hints; follow them instead of re-planning. Call help only for specialized work (gestures, scripting, TV, macOS, web, remote, debugging) or an unclear command shape, never at startup.`; @@ -58,7 +58,7 @@ export const HELP_TOOL: { name: string; description: string; inputSchema: JsonSc // The guides are the CLI's own text: rewriting their prose per surface would fork one // source of truth into two. One line up front states the mapping and its exceptions. -const GUIDE_PREAMBLE = `CLI syntax over MCP: \`agent-device --flag\` = tool ; positionals are named properties (target, text, direction), flags camelCase (--settle -> settle: true, -i -> interactiveOnly: true). Terminal-only (no MCP tool; run in a shell): ${terminalOnlyCommandNames().join(', ')}.\n\n`; +const GUIDE_PREAMBLE = `CLI syntax over MCP: the guide below is written for the terminal. Call the tool named after the command and pass every argument as a named property, for example press {ref: "@e12", settle: true} or snapshot {interactiveOnly: true}. Positionals become properties such as ref, label, selector, x, y, text, and direction; options become camelCase properties. Terminal-only (no MCP tool; run in a shell): ${terminalOnlyCommandNames().join(', ')}.\n\n`; export function callHelpTool(input: Record): ToolResult { const topic = input.topic; diff --git a/src/mcp/tool-ref-pins.ts b/src/mcp/tool-ref-pins.ts index 72dbce8471..0810b511bf 100644 --- a/src/mcp/tool-ref-pins.ts +++ b/src/mcp/tool-ref-pins.ts @@ -98,7 +98,8 @@ const SETTLE_REF_ISSUING_TOOLS: ReadonlySet = new Set( .filter((name) => commandSupportsSettleObservation(name)), ); -const TARGET_REF_TOOLS: ReadonlySet = new Set([ +const REF_ARGUMENT_TOOLS: ReadonlySet = new Set([ + 'wait', 'press', 'click', 'fill', @@ -303,12 +304,10 @@ function pinPlainRefArguments( ): Record { // No remembered pins for this scope → pass refs through unpinned. if (pins === undefined || pins.size === 0) return input; - if (name === 'wait') return pinWaitRef(input, pins) ?? input; - if (TARGET_REF_TOOLS.has(name)) return pinTargetRef(input, pins) ?? input; - return input; + return REF_ARGUMENT_TOOLS.has(name) ? (pinRefArgument(input, pins) ?? input) : input; } -function pinWaitRef( +function pinRefArgument( record: Record, pins: Map, ): Record | undefined { @@ -317,16 +316,6 @@ function pinWaitRef( return pinned === record.ref ? undefined : { ...record, ref: pinned }; } -function pinTargetRef( - record: Record, - pins: Map, -): Record | undefined { - const target = asOptionalRecord(record.target); - if (target?.kind !== 'ref' || typeof target.ref !== 'string') return undefined; - const pinned = pinRef(target.ref, pins); - return pinned === target.ref ? undefined : { ...record, target: { ...target, ref: pinned } }; -} - function pinRef(ref: string, pins: Map): string { // Only pin the canonical plain form `@e12`: an existing `~` means the ref is // already pinned (or malformed — the daemon owns rejecting that), and a diff --git a/test/integration/android-emulator-e2e/live-observability-scenario.ts b/test/integration/android-emulator-e2e/live-observability-scenario.ts index 8fc85c1cc0..0e15f24dc6 100644 --- a/test/integration/android-emulator-e2e/live-observability-scenario.ts +++ b/test/integration/android-emulator-e2e/live-observability-scenario.ts @@ -110,7 +110,7 @@ async function assertBatchAndEvents(context: LiveContext): Promise { JSON.stringify([ { command: 'get', - input: { format: 'text', target: { kind: 'selector', selector: 'id="dismiss-notice"' } }, + input: { format: 'text', selector: 'id="dismiss-notice"' }, }, { command: 'is', diff --git a/test/integration/ios-simulator-e2e/live-full-scenarios.ts b/test/integration/ios-simulator-e2e/live-full-scenarios.ts index 4f25f1f521..e75dffd63e 100644 --- a/test/integration/ios-simulator-e2e/live-full-scenarios.ts +++ b/test/integration/ios-simulator-e2e/live-full-scenarios.ts @@ -286,7 +286,7 @@ export async function assertObservabilityAndArtifacts(context: LiveContext): Pro command: 'get', input: { format: 'text', - target: { kind: 'selector', selector: 'id="automation-press"' }, + selector: 'id="automation-press"', }, }, { diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index d544125aaf..cb63852342 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -870,7 +870,7 @@ async function runAndroidCaptureInteractionAndReplayWorkflow( steps: [ { command: 'press', - input: { target: { kind: 'point', x: 10, y: 20 }, count: 2, intervalMs: 1 }, + input: { x: 10, y: 20, count: 2, intervalMs: 1 }, }, ], onError: 'stop', diff --git a/website/docs/docs/agent-setup.md b/website/docs/docs/agent-setup.md index 175820d6c4..89562d8218 100644 --- a/website/docs/docs/agent-setup.md +++ b/website/docs/docs/agent-setup.md @@ -61,7 +61,9 @@ MCP-only clients don't need a separately installed skill. When a client connects Tool results are compact text. A tool that declares an output schema, such as `press` or `wait`, also returns `structuredContent` that matches it. On success, other tools, such as `snapshot` and `open`, return text only, because some clients show the model `structuredContent` instead of the text. They return `structuredContent` when you pass `mcpOutputFormat: "json"`, a non-default `responseLevel`, or `includeCost`. A failed call always returns the error in `structuredContent`. -When a tool fails, the server returns an MCP tool result with `isError: true`. Check the tool result, not only whether the JSON-RPC call succeeded. +Tools that act on a UI element take its target as top-level arguments, in the same words as the CLI: `press {"ref":"@e12"}`, `click {"selector":"id=\"submit\""}`, or `press {"x":120,"y":340}`. `get` takes `ref` or `selector`. + +When a tool fails, the server returns an MCP tool result with `isError: true`. Check the tool result, not only whether the JSON-RPC call succeeded. A call with an argument the tool doesn't take fails with `INVALID_ARGS`, and its hint shows a call that the tool accepts. The server runs one `tools/call` at a time, in the order it receives them. It answers `ping`, `tools/list`, and the handshake at once, even while a long tool call such as `test` or `wait` runs. A `notifications/cancelled` for a running or queued `tools/call` stops it, and the server sends no response for that request. Cancellation stops the daemon request, not work that already reached the device: on iOS, a command the device received keeps running there, and the session waits for it. See [Command methods](/docs/client-api#command-methods). A JSON-RPC batch cannot carry a `tools/call`; the server refuses that element with `-32600`. diff --git a/website/docs/docs/batching.md b/website/docs/docs/batching.md index 1e36b86772..12fd0b2b23 100644 --- a/website/docs/docs/batching.md +++ b/website/docs/docs/batching.md @@ -60,15 +60,15 @@ Pass `batch` a JSON array of steps: { "command": "open", "input": { "app": "settings" } }, { "command": "wait", - "input": { "kind": "selector", "selector": "label=\"Privacy & Security\"", "timeoutMs": 3000 } + "input": { "selector": "label=\"Privacy & Security\"", "timeoutMs": 3000 } }, { "command": "click", - "input": { "target": { "kind": "selector", "selector": "label=\"Privacy & Security\"" } } + "input": { "selector": "label=\"Privacy & Security\"" } }, { "command": "get", - "input": { "format": "text", "target": { "kind": "selector", "selector": "label=\"Tracking\"" } } + "input": { "format": "text", "selector": "label=\"Tracking\"" } } ] ``` @@ -146,16 +146,16 @@ Open app -> open thread -> type -> send [ { "command": "open", "input": { "app": "com.example.chat", "platform": "android" } }, { "command": "wait", "input": { "kind": "text", "text": "Inbox", "timeoutMs": 3000 } }, - { "command": "press", "input": { "target": { "kind": "selector", "selector": "label=\"Inbox\" role=button" } } }, - { "command": "press", "input": { "target": { "kind": "selector", "selector": "label=\"Morgan Lee\"" } } }, + { "command": "press", "input": { "selector": "label=\"Inbox\" role=button" } }, + { "command": "press", "input": { "selector": "label=\"Morgan Lee\"" } }, { "command": "fill", "input": { - "target": { "kind": "selector", "selector": "label=\"Message\" role=text-field" }, + "selector": "label=\"Message\" role=text-field", "text": "sent the update" } }, - { "command": "press", "input": { "target": { "kind": "selector", "selector": "label=\"Send\" role=button" } } }, + { "command": "press", "input": { "selector": "label=\"Send\" role=button" } }, { "command": "wait", "input": { "kind": "text", "text": "sent the update", "timeoutMs": 3000 } } ] ``` @@ -168,10 +168,10 @@ Open app -> open action menu -> choose option -> verify { "command": "wait", "input": { "kind": "text", "text": "Home", "timeoutMs": 3000 } }, { "command": "press", - "input": { "target": { "kind": "selector", "selector": "label=\"More actions\" role=button" } } + "input": { "selector": "label=\"More actions\" role=button" } }, { "command": "wait", "input": { "kind": "text", "text": "Scan document", "timeoutMs": 2000 } }, - { "command": "press", "input": { "target": { "kind": "selector", "selector": "label=\"Scan document\"" } } }, + { "command": "press", "input": { "selector": "label=\"Scan document\"" } }, { "command": "wait", "input": { "kind": "text", "text": "Document uploaded", "timeoutMs": 15000 } }, { "command": "is", "input": { "predicate": "visible", "selector": "label=\"Document uploaded\"" } } ] diff --git a/website/docs/docs/quick-start.md b/website/docs/docs/quick-start.md index d44b9377a8..8cd22c6c4f 100644 --- a/website/docs/docs/quick-start.md +++ b/website/docs/docs/quick-start.md @@ -91,19 +91,19 @@ Example batch payload for a known chat flow: ```json [ { "command": "open", "input": { "app": "ChatApp", "platform": "android" } }, - { "command": "click", "input": { "target": { "kind": "selector", "selector": "label=\"Travel chat\"" } } }, + { "command": "click", "input": { "selector": "label=\"Travel chat\"" } }, { "command": "wait", - "input": { "kind": "selector", "selector": "label=\"Message\"", "timeoutMs": 3000 } + "input": { "selector": "label=\"Message\"", "timeoutMs": 3000 } }, { "command": "fill", "input": { - "target": { "kind": "selector", "selector": "label=\"Message\"" }, + "selector": "label=\"Message\"", "text": "Sent the update" } }, - { "command": "press", "input": { "target": { "kind": "selector", "selector": "label=\"Send\"" } } } + { "command": "press", "input": { "selector": "label=\"Send\"" } } ] ```