Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/command-output/src/cli-output.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ export async function dispatchCliOutput(
progress: params.progress,
env: params.env,
stderrIsTTY: params.stderrIsTTY,
reader: params.reader,
});
if (output === undefined) return undefined;
return routeResponseWarnings(
Expand Down
16 changes: 16 additions & 0 deletions packages/command-output/src/management/output.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,22 @@ describe('openCliOutput', () => {
});
});

test('leaves the session state directory out of the text a model reads', async () => {
const output = await openCliOutput(
{
session: 'default',
sessionStateDir: '/tmp/agent-device/sessions/cwd_123_default',
identifiers: { session: 'default' },
},
'model',
);

expect(output.text).toBe('Opened: default');
expect(output.data).toMatchObject({
sessionStateDir: '/tmp/agent-device/sessions/cwd_123_default',
});
});

test('keeps internal open timing out of public output data', async () => {
const result: AppOpenResult & { timing: { totalDurationMs: number } } = {
session: 'default',
Expand Down
10 changes: 7 additions & 3 deletions packages/command-output/src/management/output.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import { readCommandMessage } from '@agent-device/kernel/success-text';
import { snapshotCliOutput } from '../capture/output.ts';
import {
type CliOutputFormatter,
type CliOutputFormatterParams,
messageCliOutput,
messageOutput,
resultOutput,
Expand Down Expand Up @@ -87,11 +88,14 @@ async function sessionCliOutput(
return { data, text: JSON.stringify(data, null, 2) };
}

export async function openCliOutput(result: AppOpenResult): Promise<CliOutput> {
export async function openCliOutput(
result: AppOpenResult,
reader: CliOutputFormatterParams['reader'] = 'terminal',
): Promise<CliOutput> {
const { serializeOpenResult } = await import('../output/result-serialization.ts');
const data = serializeOpenResult(result);
const lines = [readCommandMessage(data)].filter((line): line is string => Boolean(line));
if (typeof data.sessionStateDir === 'string') {
if (reader === 'terminal' && typeof data.sessionStateDir === 'string') {
lines.push(`Session state: ${data.sessionStateDir}`);
}
for (const warning of result.warnings ?? []) {
Expand Down Expand Up @@ -243,7 +247,7 @@ export const managementCliOutputFormatters = {
}),
session: resultOutput(sessionCliOutput),
artifacts: resultOutput(artifactsCliOutput),
open: resultOutput(openCliOutput),
open: ({ result, reader }) => openCliOutput(result as AppOpenResult, reader),
close: resultOutput(closeCliOutput),
install: resultOutput(deployCliOutput),
reinstall: resultOutput(deployCliOutput),
Expand Down
8 changes: 8 additions & 0 deletions packages/command-output/src/output-common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,16 @@ export type CliOutputFormatterParams = {
env: EnvMap;
/** Whether the caller's stderr is a terminal; a formatter that colors status markers reads it. */
stderrIsTTY: boolean;
/**
* Who reads the text. `terminal` (the default) is a person or script at the CLI. `model` is
* an agent reading a tool result, which gets no daemon-host state paths it would never open.
*/
reader?: OutputReader;
};

/** Who reads command output: a person or script at the CLI, or an agent reading a tool result. */
export type OutputReader = 'terminal' | 'model';

export type CliOutputFormatter = (
params: CliOutputFormatterParams,
) => CliOutput | Promise<CliOutput>;
Expand Down
37 changes: 34 additions & 3 deletions packages/command-output/src/output/error.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
type NormalizedError,
} from '@agent-device/kernel/errors';
import { readResponseWarnings } from '@agent-device/kernel/success-text';
import type { OutputReader } from '../output-common.ts';
import { formatReplayDivergenceReport } from '@agent-device/ad-replay/divergence';
import { collapseWarningText } from '../output-common.ts';

Expand All @@ -25,6 +26,12 @@ export type HumanErrorView = {
export type HumanErrorOptions = {
/** Append the raw details dump (the `--debug` tier). */
showDetails?: boolean;
/**
* Who reads the text. `terminal` (the default) is a person or script at the CLI. `model` is
* an agent reading a tool result: the typed `details.reason` joins the code and the summary
* stays one line.
*/
reader?: OutputReader;
/** The failing command's synopsis, such as `agent-device fill <target> <text>`. */
synopsis?: string;
};
Expand All @@ -48,7 +55,13 @@ export function formatHumanError(
const candidateViews = readErrorCandidateViews(normalized.details);
const inlineCandidates =
normalized.code === 'AMBIGUOUS_MATCH' ? readInlineCandidates(candidateViews) : undefined;
const { summary, truncated } = formatSummary(normalized, inlineCandidates, options.synopsis);
const forModel = options.reader === 'model';
const { summary, truncated } = formatSummary(
normalized,
inlineCandidates,
options.synopsis,
forModel,
);
return {
summary,
body: formatBody(normalized, {
Expand All @@ -63,6 +76,7 @@ function formatSummary(
normalized: NormalizedError,
inlineCandidates: string[] | undefined,
synopsis: string | undefined,
withReason: boolean,
): { summary: string; truncated: boolean } {
const dispatched = readDispatchDisclosure(normalized.details);
const message = oneLine(normalized.message);
Expand All @@ -71,8 +85,10 @@ function formatSummary(
? `${message.slice(0, SUMMARY_MESSAGE_LIMIT)}…`
: message;
const next = nextStep(normalized, dispatched, inlineCandidates, synopsis);
const reason = withReason ? readReason(normalized) : undefined;
const label = reason ? `${normalized.code}, ${reason}` : normalized.code;
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
const summary = [
`Error (${normalized.code}): ${withFullStop(shown)}`,
`Error (${label}): ${withFullStop(shown)}`,
...(dispatched ? [DISPATCH_SENTENCE[dispatched]] : []),
...(next ? [`Next: ${next}`] : []),
].join(' ');
Expand Down Expand Up @@ -106,9 +122,24 @@ function nextStep(
return parts.length > 0 ? parts.join(' ') : undefined;
}

const REASON_LIMIT = 80;

/** The typed `details.reason`, when the throw site classified the failure, as one bounded line. */
function readReason(normalized: NormalizedError): string | undefined {
const raw = normalized.details?.reason;
if (typeof raw !== 'string') return undefined;
const reason = oneLine(raw);
if (reason.length === 0) return undefined;
return reason.length > REASON_LIMIT ? `${reason.slice(0, REASON_LIMIT)}…` : reason;
}

function formatBody(
normalized: NormalizedError,
options: { fullMessage: boolean; candidateViews: ErrorCandidateView[]; showDetails?: boolean },
options: {
fullMessage: boolean;
candidateViews: ErrorCandidateView[];
showDetails?: boolean;
},
): string[] {
const body: string[] = [];
if (options.fullMessage) body.push(`Message: ${normalized.message}`);
Expand Down
21 changes: 17 additions & 4 deletions packages/command-registry/src/command-text.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@
* description what does it do, and when should I reach for it?
* cliDetail what else does a terminal user need — flags, argument shapes, examples?
* mcpDetail what else does an MCP caller need about this command's behavior?
* searchHint which words would a model searching for this tool type that the text lacks?
Comment thread
thymikee marked this conversation as resolved.
Resolved
*
* `--help`, the command list, the MCP tool description and `explain` are all projections of
* these four, computed where they are rendered rather than written back into the schema. That
* the first four, computed where they are rendered rather than written back into the schema. That
* is deliberate: the field this replaced was authored on some commands and generated on others,
* which is how surfaces silently drifted apart.
* which is how surfaces silently drifted apart. `searchHint` renders nowhere in that text; it
* goes only to the MCP tool's `_meta`.
*
* Because `cliDetail` is the only field carrying terminal vocabulary, and the MCP surface never
* reads it, MCP descriptions stay free of flags and positional syntax structurally rather than
Expand All @@ -27,14 +29,22 @@ export type CommandText = {
cliDetail?: string;
/** Appended to the MCP tool description only: command behavior and input details. */
mcpDetail?: string;
/**
* Space-separated keywords a client's deferred-tool search matches besides the name and
* description: synonyms and task words a model types, such as "tap button" for press.
* Every MCP tool declares one; CLI-only commands need none.
*/
searchHint?: string;
};

/**
* What a command backed by a family facet authors. `description` is absent because the facet's
* metadata already carries it — repeating it here would leave a shadowed copy behind, which is
* exactly the dead-literal problem this file's predecessor had.
*/
export type FacetCommandText = Omit<CommandText, 'description'>;
export type FacetCommandText = Omit<CommandText, 'description' | 'searchHint'> & {
searchHint: string;
};

/** The `--help` body: canonical description plus any terminal-only detail. */
export function helpBody(text: CommandText): string {
Expand All @@ -47,7 +57,10 @@ export function mcpBody(text: Pick<CommandText, 'description' | 'mcpDetail'>): s
}

/** Resolves a facet's authored text against the description its metadata already holds. */
export function resolveFacetText(text: FacetCommandText, description: string): CommandText {
export function resolveFacetText(
text: FacetCommandText,
description: string,
): CommandText & FacetCommandText {
return { ...text, description };
}

Expand Down
10 changes: 5 additions & 5 deletions packages/command-registry/src/descriptors/device-operations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ export const DEVICE_OPERATION_COMMAND_DESCRIPTORS = {
? { ownerFiles: ['packages/command-surface/src/capture/wait.ts'] as const }
Comment thread
thymikee marked this conversation as resolved.
Resolved
: {}),
catalog: { group: 'public' },
frameworkTier: 'core',
frameworkTier: 'loop',
recordsSessionAction: true,
recordingEffect: 'observes-app',
// #1349: a wait's landmark may legitimately be absent when the step
Expand Down Expand Up @@ -253,7 +253,7 @@ export const DEVICE_OPERATION_COMMAND_DESCRIPTORS = {
: {}),
...TARGETED_TOUCH_INTERACTION_TRAITS,
daemon: { ...TARGETED_TOUCH_INTERACTION_TRAITS.daemon, appLease: 'allowed' },
frameworkTier: 'core',
frameworkTier: 'loop',
timeoutPolicy: postActionObservationTimeoutPolicy('fill', PRESERVE_DAEMON_TIMEOUT_POLICY),
postActionObservation: postActionObservation('fill'),
responseDataTransform: FILL_INTERACTION_RESPONSE_DATA_TRANSFORM,
Expand Down Expand Up @@ -309,7 +309,7 @@ export const DEVICE_OPERATION_COMMAND_DESCRIPTORS = {
: {}),
...TARGETED_TOUCH_INTERACTION_TRAITS,
daemon: { ...TARGETED_TOUCH_INTERACTION_TRAITS.daemon, appLease: 'allowed' },
frameworkTier: 'core',
frameworkTier: 'loop',
timeoutPolicy: postActionObservationTimeoutPolicy('press', PRESERVE_DAEMON_TIMEOUT_POLICY),
postActionObservation: postActionObservation('press'),
responseDataTransform: TOUCH_INTERACTION_RESPONSE_DATA_TRANSFORM,
Expand Down Expand Up @@ -380,7 +380,7 @@ export const DEVICE_OPERATION_COMMAND_DESCRIPTORS = {
? { ownerFiles: ['packages/command-surface/src/system/index.ts'] as const }
: {}),
catalog: { group: 'public' },
frameworkTier: 'core',
frameworkTier: 'loop',
// R42 retires this command's capability bucket and its `dispatch` leaf together: admission is
// the owner's `back` fact, and the only execution is the bound operation — the postActionObservation
// timeout trait it kept is admission-independent, like `type` kept its dialog guard.
Expand Down Expand Up @@ -483,7 +483,7 @@ export const DEVICE_OPERATION_COMMAND_DESCRIPTORS = {
? { ownerFiles: ['packages/daemon/src/execution/scroll-runtime.ts'] as const }
: {}),
catalog: { group: 'public' },
frameworkTier: 'core',
frameworkTier: 'loop',
// R53 retires this command's capability bucket and its `dispatch` leaf together: admission is
// the owner's `scrollDirection` fact, and the only execution is the bound operation. `scroll`
// was the last holder of the legacy `dispatch`/`capability` pair, which retires with it.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@ export const SESSION_OPERATION_COMMAND_DESCRIPTORS = [
? { ownerFiles: ['packages/command-surface/src/management/app.ts'] as const }
: {}),
catalog: { group: 'public' },
frameworkTier: 'core',
frameworkTier: 'loop',
recordsSessionAction: true,
recordingEffect: 'mutates-app',
daemon: {
Expand Down
2 changes: 1 addition & 1 deletion packages/command-registry/src/flag-definitions-workflow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [
usageDescription:
'Snapshot: name the affordances merged inside an element (iOS sim); not directly invokable — reach them via the detail screen, labeled children, or coordinates',
inputDescription:
'Name the affordances an element merged away (iOS UIAccessibilityCustomAction, React Native accessibilityActions) — a card whose reply/options controls are not separate elements still lists them here. The names are for PLANNING, not invocation: there is no API to trigger them, so reach the affordance through the element detail screen, through the same control exposed as a labeled element elsewhere, or by coordinates from its rect. iOS simulator only; costs one accessibility round trip per merged element.',
"List the custom actions an element merged away, such as a card's reply control. Not invocable: reach the control via its detail screen, a labeled element elsewhere, or its rect. iOS simulator only; costs one accessibility round trip per merged element.",
projectConfig: true,
recorded: true,
},
Expand Down
4 changes: 2 additions & 2 deletions packages/command-registry/src/registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -443,7 +443,7 @@ export const RAW_COMMAND_DESCRIPTORS = [
? { ownerFiles: ['packages/command-surface/src/capture/snapshot.ts'] as const }
: {}),
catalog: { group: 'public' },
frameworkTier: 'core',
frameworkTier: 'loop',
recordsSessionAction: true,
recordingEffect: 'observes-app',
daemon: { route: 'snapshot', refFrameEffect: 'preserve', appLease: 'allowed' },
Expand Down Expand Up @@ -809,7 +809,7 @@ export function resolveCommandRecordsSessionAction(command: string | undefined):
* `undefined` for a command that never declares one. A tier is declared iff
* `catalog.group === 'public'`; that rule is a runtime pin, not a type rule —
* `src/__tests__/command-descriptor-parity.test.ts` asserts it and the exact
* `'core'` tool set, so a new public command cannot join a framework adapter's
* `'loop'` and `'core'` tool sets, so a new public command cannot join a framework adapter's
* default set silently. Framework adapters (`agent-device/ai-sdk`,
* `@agent-device/eve`) read this to build their default tool set instead of
* hand-listing tool names.
Expand Down
7 changes: 6 additions & 1 deletion packages/command-registry/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,11 @@ export type CommandCatalogGroup = 'public' | 'internal' | 'local-cli';
/**
* Which default tool set a framework adapter (`agent-device/ai-sdk`, the
* planned `@agent-device/eve` package) includes a public command in.
* - `'loop'` — the part of the core set nearly every run calls: launch,
* observe, act, navigate back, wait, scroll. A `'core'` set
* includes it, and the MCP server marks these tools
* always-loaded, so a client that defers tool schemas sends
* them in the first prompt instead of searching for them.
* - `'core'` — the small, curated perceive/act loop a typical tool-calling
* agent needs by default: launch, observe, interact, read,
* verify, wait, navigate, and system-dialog handling.
Expand All @@ -128,7 +133,7 @@ export type CommandCatalogGroup = 'public' | 'internal' | 'local-cli';
* that command's descriptor facets instead of a separate hand-maintained list
* in the adapter code; `resolveCommandFrameworkTier` reads it back.
*/
export type CommandFrameworkTier = 'core' | 'extended';
export type CommandFrameworkTier = 'loop' | 'core' | 'extended';

type CommandCatalogFacetBase = {
/**
Expand Down
23 changes: 22 additions & 1 deletion packages/command-surface/src/batch/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ import { defineCommandFacet, defineCommandFamilyFromFacets } from '../family/typ
import { commonToClientOptions } from '../common-input-fields.ts';
import { createBatchCommandMetadata, type BatchCommandStep, type BatchInput } from './metadata.ts';
import { STRUCTURED_BATCH_COMMAND_NAMES } from '@agent-device/command-registry/batch-policy';
import { COMPOSITE_TIMEOUT_POLICIES } from '@agent-device/command-registry/timeout-policy';

const BATCH_DEADLINE_SECONDS = (COMPOSITE_TIMEOUT_POLICIES.batch.daemonDeadlineMs ?? 0) / 1000;

const batchCommandMetadata = createBatchCommandMetadata();

Expand Down Expand Up @@ -33,11 +36,27 @@ const batchCliReader: CliReader = (_positionals, flags) => ({
* example nobody can run (#2062).
*/
const BATCH_STEP_EXAMPLES: readonly BatchCommandStep[] = [
{ command: 'snapshot', input: { interactiveOnly: true } },
{ command: 'press', input: { ref: '@e12', settle: true } },
{ command: 'press', input: { selector: 'label=Settings', settle: true } },
{ command: 'fill', input: { selector: 'id="field-email"', text: 'qa@example.com' } },
{ command: 'scroll', input: { direction: 'down', until: 'label=Privacy', settle: true } },
{ command: 'back', input: { settle: true } },
{ command: 'wait', input: { kind: 'text', text: 'Saved' } },
{ command: 'snapshot', input: { interactiveOnly: true } },
];

/**
* The MCP description: a model batches a walk it already knows from this text alone, without
* loading each step command's schema. The step examples are the same tested ones `help batch` prints.
*/
const BATCH_MCP_DETAIL = [
'Use it for a known multi-step walk, such as press A, press B, back, instead of one tool call per step.',
"steps is an array of {command, input} objects; input takes the same arguments as that command's own tool (ref, selector, or x and y for targets; settle: true for the settled diff).",
'Steps run serially in one session; onError: "stop" halts at the first failing step, and the result reports the steps that ran.',
`The daemon stops the whole batch after ${BATCH_DEADLINE_SECONDS} seconds; split longer walks.`,
`Example steps, in order (refs such as @e12 come from the last snapshot or settled diff): ${BATCH_STEP_EXAMPLES.map((step) => JSON.stringify(step)).join(', ')}.`,
].join(' ');
Comment thread
thymikee marked this conversation as resolved.
Resolved

/**
* `help batch` documented neither the step shape nor which commands batch accepts, so an agent
* discovered both by trial: `["press @e12"]` and `{"command":"press","args":[...]}` were refused
Expand All @@ -64,6 +83,8 @@ const batchCommandFacet = defineCommandFacet({
effect: 'destructive',
text: {
summary: 'Run multiple commands',
mcpDetail: BATCH_MCP_DETAIL,
searchHint: 'sequence chain several taps',
cliDetail: buildBatchCliDetail(),
},
metadata: batchCommandMetadata,
Expand Down
1 change: 1 addition & 0 deletions packages/command-surface/src/capture/alert.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ export const alertCommandFacet = defineCommandFacet({
effect: 'destructive',
text: {
summary: 'Inspect, accept, or dismiss a platform alert',
searchHint: 'dialog system',
},
metadata: alertCommandMetadata,
run: (client, input) => client.command.alert(input),
Expand Down
1 change: 1 addition & 0 deletions packages/command-surface/src/capture/diff.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ export const diffCommandFacet = defineCommandFacet({
effect: 'write',
text: {
summary: 'Diff snapshot or screenshot',
searchHint: 'visual regression baseline',
cliDetail:
'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; the current image scales proportionally to the baseline width and aligns top-left. Scaled height rounds to the nearest pixel (minimum 1). Extra or missing bottom rows always count as differences, even at threshold 1. The originals remain unchanged. When dimensions differ, normalization metadata reports original, scaled and compared sizes; region coordinates use comparison pixels. Both screenshot inputs are decoded from their bytes, so a baseline or current image may be PNG or JPEG, and the diff image is always PNG. JPEG is lossy, so keep --threshold above 0 whenever either input is JPEG. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines. Use --screen-mask black for a live iOS simulator capture with a black screen outline, and use the same option when creating its baseline. --screen-mask is refused when current.png is supplied; saved images are compared unchanged.',
},
Expand Down
Loading
Loading