Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions packages/client/src/__tests__/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -400,12 +400,12 @@ test('structured interaction input keeps UI target separate from deviceTarget',
await runCommand(client, 'get', {
deviceTarget: 'mobile',
format: 'text',
target: { kind: 'ref', ref: '@e1' },
ref: '@e1',
});
await runCommand(client, 'longpress', {
deviceTarget: 'mobile',
durationMs: 800,
target: { kind: 'ref', ref: '@e2' },
ref: '@e2',
});

assert.equal(setup.calls.length, 2);
Expand Down
11 changes: 4 additions & 7 deletions packages/command-surface/src/batch/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,15 @@ const batchCliReader: CliReader = (_positionals, flags) => ({
/**
* The step examples `help batch` prints. A step's `input` is keyed by the command's structured
* field names, which no CLI help text spells out — `help press` documents the positional and the
* flags, not `target: {kind, ref}` — so these lines are the only place the terminal states them.
* flags, not `ref` or `selector` — so these lines are the only place the terminal states them.
* `cli-help-examples.test.ts` reads them back out of the rendered help and runs each `input`
* through its own command's `readInput`, so a renamed field fails there instead of shipping an
* example nobody can run (#2062).
*/
const BATCH_STEP_EXAMPLES: readonly BatchCommandStep[] = [
{ command: 'snapshot', input: { interactiveOnly: true } },
{ command: 'press', input: { target: { kind: 'ref', ref: '@e12' }, settle: true } },
{
command: 'fill',
input: { target: { kind: 'selector', selector: 'id="field-email"' }, text: 'qa@example.com' },
},
{ command: 'press', input: { ref: '@e12', settle: true } },
{ command: 'fill', input: { selector: 'id="field-email"', text: 'qa@example.com' } },
];

/**
Expand All @@ -49,7 +46,7 @@ const BATCH_STEP_EXAMPLES: readonly BatchCommandStep[] = [
*/
function buildBatchCliDetail(): string {
const prose = [
'Each step is {"command":"<name>","input":{...}}, and input is that command\'s structured input object rather than its terminal spelling: a CLI positional becomes a named field (target, text, direction) and a CLI flag becomes a camelCase key (--settle -> "settle":true, -i -> "interactiveOnly":true). There is no positional step form; args, argv, positionals, and flags are not step fields. agent-device help <command> lists arguments in CLI spelling only. The examples below carry the structured spelling; over MCP, the command tool schema states it in full.',
'Each step is {"command":"<name>","input":{...}}, and input is that command\'s structured input object rather than its terminal spelling: a CLI positional becomes a named field (for example ref, label, selector, x, y, text, direction) and a CLI flag becomes a camelCase key (--settle -> "settle":true, -i -> "interactiveOnly":true). There is no positional step form; args, argv, positionals, and flags are not step fields. agent-device help <command> lists arguments in CLI spelling only. The examples below carry the structured spelling; over MCP, the command tool schema states it in full.',
'Steps run serially in one daemon request against the same session, in order. Mutating UI verbs are included (press, click, fill, longpress, scroll, back), which is where the round-trip saving is; --on-error stop halts at the first failing step.',
'The whole batch has a 60-second work budget inside a 90-second client timeout. The running step receives ordinary request cancellation at the budget. When it settles, failure returns COMMAND_FAILED with details.reason daemon_request_timeout and preserves executed, partialResults, and dispatch status. The daemon and other sessions stay available if it responds; a close step may already have ended its session. A completed batch returns success even if it settles just after the deadline. Cancellation does not prove native device work stopped. A step that ignores cancellation beyond the 30-second margin may cause a transport timeout; an unresponsive daemon retains the client liveness-based reset. Split longer sequences into separate requests.',
`Available through batch: ${[...STRUCTURED_BATCH_COMMAND_NAMES].sort().join(', ')}.`,
Expand Down
7 changes: 6 additions & 1 deletion packages/command-surface/src/batch/metadata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,12 @@ export function createBatchCommandMetadata(
'batch',
'Execute multiple commands in one daemon request.',
fields,
{ readInput: (input) => readBatchInput(input, fields) },
{
readInput: (input) => readBatchInput(input, fields),
...(nestedCommands.includes('snapshot')
? { inputExample: { steps: [{ command: 'snapshot', input: { interactiveOnly: true } }] } }
: {}),
},
);
}

Expand Down
31 changes: 3 additions & 28 deletions packages/command-surface/src/cli-grammar/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -151,24 +151,11 @@ export function leaseDeviceSpecInputFromFlags(
return requireLeaseDeviceSpec(flags).spec;
}

export function targetInputFromClientTarget(
target: InteractionTarget | ElementTarget,
): Record<string, unknown> {
if ('ref' in target && target.ref !== undefined) {
return compactRecord({ kind: 'ref', ref: target.ref, label: target.label });
}
if ('selector' in target && target.selector !== undefined) {
return { kind: 'selector', selector: target.selector };
}
const point = target as { x: number; y: number };
return { kind: 'point', x: point.x, y: point.y };
}

export function interactionTargetPositionals(input: InteractionTarget | CommandInput): string[] {
const target = readTargetRecord(input);
const target = input as Record<string, unknown>;
if (typeof target.ref === 'string') return [target.ref, ...optionalTargetLabel(target.label)];
if (typeof target.selector === 'string') return [target.selector];
if (target.kind === 'point' || target.x !== undefined || target.y !== undefined) {
if (target.x !== undefined || target.y !== undefined) {
return [
String(requiredTargetNumber(target.x, 'x')),
String(requiredTargetNumber(target.y, 'y')),
Expand All @@ -178,24 +165,12 @@ export function interactionTargetPositionals(input: InteractionTarget | CommandI
}

export function elementTargetPositionals(input: ElementTarget | CommandInput): string[] {
const target = readTargetRecord(input);
const target = input as Record<string, unknown>;
if (typeof target.ref === 'string') return [target.ref, ...optionalTargetLabel(target.label)];
if (typeof target.selector === 'string') return [target.selector];
throw new AppError('INVALID_ARGS', 'element command requires @ref or selector target');
}

function readTargetRecord(input: unknown): Record<string, unknown> {
if (!input || typeof input !== 'object' || Array.isArray(input)) {
throw new AppError('INVALID_ARGS', 'Expected target object.');
}
const record = input as Record<string, unknown>;
const nestedTarget = record.target;
if (nestedTarget && typeof nestedTarget === 'object' && !Array.isArray(nestedTarget)) {
return nestedTarget as Record<string, unknown>;
}
return record;
}

function requiredTargetNumber(value: unknown, field: string): number {
if (typeof value !== 'number' || !Number.isFinite(value)) {
throw new AppError('INVALID_ARGS', `point target requires numeric ${field}.`);
Expand Down
5 changes: 2 additions & 3 deletions packages/command-surface/src/cli-grammar/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@ type PointInput = {
y?: number;
};

export type CommandInput = Omit<InternalRequestOptions, 'batchSteps' | 'target'> &
Omit<Partial<CliFlags>, 'batchSteps' | 'target'> & {
target?: InternalRequestOptions['target'] | Record<string, unknown>;
export type CommandInput = Omit<InternalRequestOptions, 'batchSteps'> &
Omit<Partial<CliFlags>, 'batchSteps'> & {
action?: string;
amount?: number;
app?: string;
Expand Down
43 changes: 43 additions & 0 deletions packages/command-surface/src/command-contract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,53 @@ export type CommandMetadata<Name extends string, Input> = {
* command prints them after its own text, and no formatter decides this per-command.
*/
parseableOutput?: true;
/**
* The keys of a minimal valid call that a placeholder per required property cannot state, such
* as the one UI target an interaction needs. `commandInputExample` merges them over those.
*/
inputExample?: Readonly<Record<string, unknown>>;
};

export function defineCommandMetadata<Name extends string, Input>(
definition: CommandMetadata<Name, Input>,
): CommandMetadata<Name, Input> {
return definition;
}

/**
* A minimal call for a command: a placeholder for each required property, then its declared
* example keys, which win. A model-facing surface shows it when a call names an argument the tool does not take,
* so a caller that never loaded the schema can retry once.
*/
export function commandInputExample(
metadata: Pick<CommandMetadata<string, unknown>, 'inputSchema' | 'inputExample'>,
): Record<string, unknown> {
return { ...requiredPropertiesExample(metadata.inputSchema), ...metadata.inputExample };
}

function requiredPropertiesExample(schema: JsonSchema): Record<string, unknown> {
return Object.fromEntries(
(schema.required ?? []).map((key) => [key, placeholder(key, schema.properties?.[key])]),
);
}

function placeholder(key: string, schema: JsonSchema | undefined): unknown {
if (schema === undefined) return `<${key}>`;
if (schema.const !== undefined) return schema.const;
if (schema.enum !== undefined && schema.enum.length > 0) return schema.enum[0];
if (schema.oneOf !== undefined && schema.oneOf.length > 0)
return placeholder(key, schema.oneOf[0]);
switch (Array.isArray(schema.type) ? schema.type[0] : schema.type) {
case 'object':
return requiredPropertiesExample(schema);
case 'array':
return schema.items === undefined ? [] : [placeholder(key, schema.items)];
case 'number':
case 'integer':
return schema.minimum ?? 1;
case 'boolean':
return true;
default:
return `<${key}>`;
}
}
73 changes: 73 additions & 0 deletions packages/command-surface/src/command-input-target.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import { expect, test } from 'vitest';
import { AppError } from '@agent-device/kernel/errors';
import { findCommandMetadata } from './command-metadata.ts';

// The UI target is top-level keys, the vocabulary an agent guesses without loading the schema.
test.for([
[{ ref: '@e6' }, { ref: '@e6' }],
[
{ ref: '@e6', label: 'Products' },
{ ref: '@e6', label: 'Products' },
],
[{ selector: 'id=home/products' }, { selector: 'id=home/products' }],
[
{ x: 120, y: 340 },
{ x: 120, y: 340 },
],
] as const)('press reads %j as its UI target', ([input, target]) => {
expect(findCommandMetadata('press').readInput(input)).toStrictEqual(target);
});

test.for([
[{}, /Expected a UI target/],
[{ ref: '@e6', selector: 'id=a' }, /got ref and selector/],
[{ selector: 'id=a', x: 1, y: 2 }, /got selector and x\/y/],
[{ x: 120 }, /Expected x and y together/],
[{ selector: 'id=a', label: 'A' }, /label applies only with ref/],
] as const)('press refuses %j', ([input, message]) => {
expect(() => findCommandMetadata('press').readInput(input)).toThrow(message);
});

// The schema cannot state "exactly one target": a top-level oneOf/anyOf/allOf in a tool input
// schema is rejected by some model clients. The reader enforces it, and its hint names the forms.
test.for([{}, { ref: '@e6', selector: 'id=a' }, { x: 120 }] as const)(
'press answers %j with INVALID_ARGS and the target forms',
(input) => {
try {
findCommandMetadata('press').readInput(input);
expect.unreachable('press should refuse the input');
} catch (error) {
expect(error).toBeInstanceOf(AppError);
expect((error as AppError).code).toBe('INVALID_ARGS');
expect((error as AppError).details?.hint).toMatch(/\{"ref":"@e12"\}|\{"x":120,"y":340\}/);
}
},
);

test.for([
['target', { target: { kind: 'ref', ref: '@e6' } }],
['kind', { kind: 'ref', ref: '@e6' }],
] as const)('the removed %s key answers with the top-level spelling', ([key, input]) => {
const read = () => findCommandMetadata('press').readInput(input);
expect(read).toThrow(AppError);
expect(read).toThrow(new RegExp(`${key} was removed.*\\{"ref":"@e12"\\}`));
});

test('get takes a ref or selector but no point', () => {
const get = findCommandMetadata('get');
expect(get.readInput({ format: 'text', selector: 'id=a' })).toMatchObject({ selector: 'id=a' });
expect(get.inputSchema.properties).not.toHaveProperty('x');
expect(get.inputSchema.properties).not.toHaveProperty('target');
});

// `target` without a UI target field is the deviceTarget alias. An interaction command retires
// it, so the alias must not reappear in its schema as a string a model could write a ref into.
test('interaction schemas advertise top-level target keys and no target', () => {
for (const name of ['press', 'click', 'fill', 'longpress', 'hover'] as const) {
const properties = findCommandMetadata(name).inputSchema.properties ?? {};
expect(Object.keys(properties)).toEqual(
expect.arrayContaining(['ref', 'label', 'selector', 'x', 'y']),
);
expect(properties).not.toHaveProperty('target');
}
});
Loading
Loading