From 93debccfee9966ae4015ad6fb39caf9e1896fa77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 11:59:49 +0100 Subject: [PATCH 1/9] feat(recovery): resend reads and pre-send health probes within the command's deadline A remote `/health` gate that finds a daemon there but not ready (a 5xx answer or a probe timeout) now probes again every second for the command's own request budget before it refuses, since nothing was sent while the gate is closed. A connection refused outright still refuses at once. The refusal reports `details.attempts`. A read-only Apple runner command refused with RUNNER_BUSY is now resent until shortly before the request's own deadline instead of a fixed eight attempts, and the final refusal reports `details.attempts`. Mutations are still sent once. The runner host port gains getRequestDeadlineMs to read that deadline. Refs #4522 Co-Authored-By: Claude Opus 5.5 --- .../src/__tests__/index-busy-resend.test.ts | 62 +++++++++++++++- packages/apple-runner/src/host.ts | 6 +- packages/apple-runner/src/index.ts | 70 +++++++++++++++---- ...mon-client-lifecycle-remote-health.test.ts | 65 ++++++++++++----- .../daemon-client/src/__tests__/index.test.ts | 23 +++++- .../daemon-client/src/daemon-client-attach.ts | 13 +++- .../src/daemon-client-lifecycle.ts | 45 ++++++++---- .../daemon-client-remote-health-recovery.ts | 48 +++++++++++++ .../src/daemon-client-transport.ts | 8 ++- .../src/daemon-lifecycle-errors.ts | 7 +- packages/daemon-client/src/index.ts | 10 ++- .../platform-apple/src/core/runner-host.ts | 2 + .../test-kit/src/host-kit-capabilities.ts | 1 + website/docs/docs/commands.md | 4 +- 14 files changed, 301 insertions(+), 63 deletions(-) create mode 100644 packages/daemon-client/src/daemon-client-remote-health-recovery.ts diff --git a/packages/apple-runner/src/__tests__/index-busy-resend.test.ts b/packages/apple-runner/src/__tests__/index-busy-resend.test.ts index 272ef28327..2919473f3c 100644 --- a/packages/apple-runner/src/__tests__/index-busy-resend.test.ts +++ b/packages/apple-runner/src/__tests__/index-busy-resend.test.ts @@ -110,9 +110,67 @@ test('a RUNNER_BUSY resend still ends when the refusals outlast the window', asy assert.ok(error instanceof AppError); assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY'); assert.equal(error.details?.recovery, undefined, 'the raw refusal reaches the caller unwrapped'); + assert.equal(error.details?.attempts, 8); assert.deepEqual(sentCommands(), Array(8).fill('snapshot')); }); +/** Runs the request under a daemon deadline `timeoutMs` from now, as `withRequestDeadline` sets it. */ +function underRequestDeadline(timeoutMs: number): void { + const deadlineMs = Date.now() + timeoutMs; + appleRunnerTestHost.update({ getRequestDeadlineMs: () => deadlineMs }); +} + +test('under a request deadline, read-only commands outlast the fixed RUNNER_BUSY count', async () => { + // Twelve busy answers, four past the fixed eight-attempt window, then the runner answers inside + // the request's 30s deadline. + vi.useFakeTimers(); + underRequestDeadline(30_000); + for (let refusal = 0; refusal < 12; refusal += 1) { + mockExecuteRunnerCommandWithSession.mockRejectedValueOnce(busyRefusal()); + } + mockExecuteRunnerCommandWithSession.mockResolvedValueOnce({ nodes: [], truncated: false }); + + const pending = runAppleRunnerCommand( + IOS_SIMULATOR, + { command: 'snapshot' }, + { requestId: 'req-deadline-recovers' }, + ); + await vi.advanceTimersByTimeAsync(20_000); + + assert.deepEqual(await pending, { nodes: [], truncated: false }); + assert.deepEqual(sentCommands(), Array(13).fill('snapshot')); +}); + +test('a request deadline reached mid-outage ends with the RUNNER_BUSY refusal and its attempt count', async () => { + vi.useFakeTimers(); + underRequestDeadline(10_000); + const sentAt: number[] = []; + mockExecuteRunnerCommandWithSession.mockImplementation(async () => { + sentAt.push(Date.now()); + throw busyRefusal(); + }); + const startedAt = Date.now(); + + const settled = runAppleRunnerCommand( + IOS_SIMULATOR, + { command: 'snapshot' }, + { requestId: 'req-deadline-outage' }, + ).then( + () => undefined, + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(10_000); + const error = await settled; + + assert.ok(error instanceof AppError, `expected the refusal, got ${String(error)}`); + assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY'); + // Delays of 200, 400, 800, then 1000ms put the eleventh send at 8.4s. The window is 9s (10s less + // the 1s reserve), so the next delay is clamped to it and no send follows. + assert.equal(error.details?.attempts, 11); + assert.equal(sentCommands().length, 11); + assert.equal(sentAt.at(-1)! - startedAt, 8_400); +}); + test('read-only transport failures keep the three-attempt resend budget', async () => { mockExecuteRunnerCommandWithSession .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed')) @@ -225,7 +283,9 @@ test('a cancelled request wakes the RUNNER_BUSY delay and reports the cancellati test('a mutating command meets RUNNER_BUSY once, with no status probe and no resend', async () => { // The status-recovery bypass for structured replies reaches mutating commands too: the refusal - // says the tap did not run, so there is nothing to recover and nothing is replayed. + // says the tap did not run, so there is nothing to recover and nothing is replayed, even with a + // request deadline that would let a read resend. + underRequestDeadline(30_000); mockExecuteRunnerCommandWithSession.mockRejectedValueOnce(busyRefusal()); await assert.rejects( diff --git a/packages/apple-runner/src/host.ts b/packages/apple-runner/src/host.ts index 46e6f55012..b8ba04ed9c 100644 --- a/packages/apple-runner/src/host.ts +++ b/packages/apple-runner/src/host.ts @@ -110,7 +110,10 @@ export type AppleRunnerHost = Pick< > & Pick & Pick & - Pick & + Pick< + typeof HostRequest, + 'emitRequestProgress' | 'getRequestDeadlineMs' | 'getRequestSignal' | 'isRequestCanceled' + > & Pick & Pick & Pick & @@ -207,6 +210,7 @@ export const acquireProcessLock = delegate('acquireProcessLock'); export const withProcessLock = delegate('withProcessLock'); export const publishFileSync = delegate('publishFileSync'); export const emitRequestProgress = delegate('emitRequestProgress'); +export const getRequestDeadlineMs = delegate('getRequestDeadlineMs'); export const getRequestSignal = delegate('getRequestSignal'); export const isRequestCanceled = delegate('isRequestCanceled'); export const withKeyedLock = delegate('withKeyedLock'); diff --git a/packages/apple-runner/src/index.ts b/packages/apple-runner/src/index.ts index 9dcac9d16e..81500f8a74 100644 --- a/packages/apple-runner/src/index.ts +++ b/packages/apple-runner/src/index.ts @@ -89,7 +89,14 @@ export { import type * as LegacyXctestDeviceSet from './legacy-xctest-device-set.ts'; import type * as RunnerSequence from './runner-sequence.ts'; -import { retryWithPolicy, emitDiagnostic, hasScopedAppleToolProvider } from './host.ts'; +import { + Deadline, + retryWithPolicy, + emitDiagnostic, + getRequestDeadlineMs, + hasScopedAppleToolProvider, +} from './host.ts'; +import { AppError } from '@agent-device/kernel/errors'; import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; import { ensureRunnerSession, @@ -128,27 +135,45 @@ import { RUNNER_COMMAND_TIMEOUT_MS } from './runner-transport.ts'; /** * Attempts a read-only command may spend on one error class; 1 means no resend. A `RUNNER_BUSY` * refusal is the runner refusing fast on purpose while it drains abandoned XCTest work (#1105), so - * that budget has to outlast the drain. The window is a heuristic, not a measurement: 200ms doubling - * to a 1s cap, no jitter, is 5.4s of delay across eight attempts before round trips, and a drain - * that outlives it still surfaces as `RUNNER_BUSY` (the runner's own `abandonedForSeconds=` marker - * in runner.log is the evidence for tuning it). Transport failures keep the pre-existing three - * attempts. The budget is positional: the error on each attempt sets how many attempts the loop may - * reach in total, so three busy refusals followed by a transport failure resend no further. + * that budget has to outlast the drain. Under a request deadline, busy refusals are resent until + * that deadline, less {@link RUNNER_BUSY_DEADLINE_RESERVE_MS}, so the last refusal (with its attempt + * count) is the answer instead of the deadline's. Without one, the window is a heuristic: 200ms + * doubling to a 1s cap, no jitter, is 5.4s of delay across eight attempts before round trips, and a + * drain that outlives it still surfaces as `RUNNER_BUSY` (the runner's own `abandonedForSeconds=` + * marker in runner.log is the evidence for tuning it). Transport failures keep the pre-existing + * three attempts. The budget is positional: the error on each attempt sets how many attempts the + * loop may reach in total, so three busy refusals followed by a transport failure resend no further. */ const RUNNER_BUSY_RESEND_ATTEMPTS = 8; const TRANSPORT_RESEND_ATTEMPTS = 3; const READ_ONLY_RESEND_POLICY = { - maxAttempts: RUNNER_BUSY_RESEND_ATTEMPTS, baseDelayMs: 200, maxDelayMs: 1_000, jitter: 0, }; +/** One resend delay at its cap: a busy refusal is answered at once, so one round trip fits. */ +const RUNNER_BUSY_DEADLINE_RESERVE_MS = READ_ONLY_RESEND_POLICY.maxDelayMs; -function readOnlyResendBudget(error: unknown): number { - if (isRunnerBusyError(error)) return RUNNER_BUSY_RESEND_ATTEMPTS; +function readOnlyResendBudget(error: unknown, busyAttempts: number): number { + if (isRunnerBusyError(error)) return busyAttempts; return isRetryableRunnerError(error) ? TRANSPORT_RESEND_ATTEMPTS : 1; } +/** The busy resend bound: the request's own deadline when it has one, else the fixed count. */ +function resolveBusyResendWindow(requestId: string | undefined): { + busyAttempts: number; + deadline?: Deadline; +} { + const requestDeadlineMs = getRequestDeadlineMs(requestId); + if (requestDeadlineMs === undefined) return { busyAttempts: RUNNER_BUSY_RESEND_ATTEMPTS }; + return { + busyAttempts: Number.POSITIVE_INFINITY, + deadline: Deadline.fromTimeoutMs( + requestDeadlineMs - Date.now() - RUNNER_BUSY_DEADLINE_RESERVE_MS, + ), + }; +} + export async function runAppleRunnerCommand( device: DeviceInfo, command: RunnerCommand, @@ -161,34 +186,49 @@ export async function runAppleRunnerCommand( if (!isReadOnlyRunnerCommand(runnerCommand)) { return provider.runCommand(device, runnerCommand, options); } - let lastBusyRefusal: unknown; + const { busyAttempts, deadline } = resolveBusyResendWindow(options.requestId); + let lastBusyRefusal: AppError | undefined; + let attempts = 0; try { return await retryWithPolicy( () => { assertRunnerRequestActive(options.requestId); + attempts += 1; return provider.runCommand(device, runnerCommand, options); }, { ...READ_ONLY_RESEND_POLICY, + maxAttempts: busyAttempts, shouldRetry: (error, attempt) => { assertRunnerRequestActive(options.requestId); - if (isRunnerBusyError(error)) lastBusyRefusal = error; - return attempt < readOnlyResendBudget(error); + if (isRunnerBusyError(error) && error instanceof AppError) lastBusyRefusal = error; + return attempt < readOnlyResendBudget(error, busyAttempts); }, }, // The busy window is seconds long, so an abort must wake the delay instead of sleeping it // out and sending one more attempt. - { signal: resolveRunnerRequestSignal(options) }, + { signal: resolveRunnerRequestSignal(options), deadline }, ); } catch (error) { + if (isRunnerBusyError(error) && error instanceof AppError) { + throw withBusyResendAttempts(error, attempts); + } // A caller's deadline (a `wait` poll bounding this capture) that lands mid-window still has an // answer: the runner refused, and that typed refusal is what the caller can act on. Only a // cancelled request reports as a bare cancellation. - if (lastBusyRefusal && callerDeadlineExpired(options)) throw lastBusyRefusal; + if (lastBusyRefusal && callerDeadlineExpired(options)) { + throw withBusyResendAttempts(lastBusyRefusal, attempts); + } throw error; } } +/** The busy refusal the resend ran out on, stating how many times the read was sent. */ +function withBusyResendAttempts(refusal: AppError, attempts: number): AppError { + refusal.details = { ...refusal.details, attempts }; + return refusal; +} + export async function notifyIosRunnerAppRelaunched( device: DeviceInfo, options: AppleRunnerCommandOptions = {}, diff --git a/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts b/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts index 9d8948baa1..ec2c06b487 100644 --- a/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts +++ b/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts @@ -1,6 +1,7 @@ -// A remote connection probes GET /health before it sends anything. When that probe answers nothing -// usable, the refusal is the typed one: the request never left the client, so a caller branches on -// `details.reason` and `details.dispatched` instead of comparing the message. +// A remote connection probes GET /health before it sends anything. While that gate is closed the +// request never left the client, so the gate may re-probe a daemon that is there but not ready for +// the command's own budget, and its refusal is the typed one: a caller branches on `details.reason` +// and `details.dispatched` instead of comparing the message. import assert from 'node:assert/strict'; import http from 'node:http'; @@ -26,17 +27,23 @@ function remotePress(baseUrl: string) { }; } -/** A remote daemon that answers `/health` with `healthStatus` and counts the RPCs it receives. */ -function remoteWithHealthStatus(healthStatus: number): { +/** + * A remote daemon whose `/health` answers each status of `healthStatuses` in turn (the last one from + * then on), and that counts the probes and RPCs it receives. Every RPC answers 500. + */ +function remoteWithHealthStatus(...healthStatuses: number[]): { handler: http.Server; + healthProbes: () => number; rpcSends: () => number; } { + let healthProbes = 0; let rpcSends = 0; const handler = http.createServer((req, res) => { req.resume(); req.on('end', () => { if (req.url?.endsWith('/health')) { - res.statusCode = healthStatus; + res.statusCode = healthStatuses[Math.min(healthProbes, healthStatuses.length - 1)]!; + healthProbes += 1; res.end(); return; } @@ -45,29 +52,49 @@ function remoteWithHealthStatus(healthStatus: number): { res.end(); }); }); - return { handler, rpcSends: () => rpcSends }; + return { handler, healthProbes: () => healthProbes, rpcSends: () => rpcSends }; } -test('a remote press whose health probe answers 5xx refuses with the typed reason and dispatched no', async (t) => { +test('a remote press whose health probe answers 5xx re-probes, then is sent exactly once', async (t) => { if (await skipWhenLoopbackUnavailable(t)) return; - const { handler, rpcSends } = remoteWithHealthStatus(503); + const { handler, healthProbes, rpcSends } = remoteWithHealthStatus(503, 200); const port = await listenOnLoopback(handler); - const baseUrl = `http://127.0.0.1:${port}`; try { - await assert.rejects(sendToDaemon(remotePress(baseUrl)), (error: unknown) => { - assert.ok(error instanceof AppError, `unexpected failure: ${String(error)}`); - assert.equal(error.code, 'COMMAND_FAILED'); - assert.equal(error.details?.reason, REMOTE_DAEMON_UNAVAILABLE_REASON); - assert.equal(error.details?.dispatched, 'no'); - assert.equal(error.details?.daemonBaseUrl, baseUrl); - return true; - }); - assert.equal(rpcSends(), 0, 'an unreachable remote daemon receives no RPC'); + await assert.rejects( + sendToDaemon(remotePress(`http://127.0.0.1:${port}`)), + (error: unknown) => { + assert.ok(error instanceof AppError, `unexpected failure: ${String(error)}`); + assert.notEqual(error.details?.reason, REMOTE_DAEMON_UNAVAILABLE_REASON); + return true; + }, + ); + assert.equal(healthProbes(), 2, 'the gate re-probed once the daemon was back'); + // The press reached the daemon and failed there: a failure after dispatch is never resent. + assert.equal(rpcSends(), 1); } finally { await closeLoopbackServer(handler); } }); +// A connection the host refuses outright found no daemon at all, which a wrong base URL looks like: +// it is refused on the first probe instead of waiting out the command's budget. +test('a remote press whose health probe is refused is refused at once with the typed reason', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const closed = http.createServer(); + const port = await listenOnLoopback(closed); + await closeLoopbackServer(closed); + const baseUrl = `http://127.0.0.1:${port}`; + await assert.rejects(sendToDaemon(remotePress(baseUrl)), (error: unknown) => { + assert.ok(error instanceof AppError, `unexpected failure: ${String(error)}`); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, REMOTE_DAEMON_UNAVAILABLE_REASON); + assert.equal(error.details?.dispatched, 'no'); + assert.equal(error.details?.daemonBaseUrl, baseUrl); + assert.equal(error.details?.attempts, 1); + return true; + }); +}); + // The closest negative: a probe that answers 4xx counts as reachable, so the press is sent and the // unavailable refusal is never stamped on a request that left the client. test('a remote press whose health probe answers 4xx is sent and is not the unavailable refusal', async (t) => { diff --git a/packages/daemon-client/src/__tests__/index.test.ts b/packages/daemon-client/src/__tests__/index.test.ts index e1ada76f63..09ade73f74 100644 --- a/packages/daemon-client/src/__tests__/index.test.ts +++ b/packages/daemon-client/src/__tests__/index.test.ts @@ -1,5 +1,5 @@ import type { RequestProgressEvent } from '@agent-device/contracts/progress'; -import { test } from 'vitest'; +import { test, vi } from 'vitest'; import assert from 'node:assert/strict'; import http from 'node:http'; import net from 'node:net'; @@ -704,11 +704,21 @@ test.each([ }, ); -test('sendToDaemon hints to disconnect when a remote daemon is unavailable', async () => { - const restoreHttpRequest = mockEventHttpRequest(({ res }) => { +test('a remote daemon unavailable for the whole request budget is refused with its probe count and the disconnect hint', async () => { + // `session list` has the 90s default budget: 90 probes a second apart while /health answers 503. + let healthProbes = 0; + let rpcSends = 0; + const restoreHttpRequest = mockEventHttpRequest(({ options, res }) => { + if (options.method === 'GET') healthProbes += 1; + else rpcSends += 1; res.statusCode = 503; res.emit('end'); }); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }); + const stopAdvancing = new AbortController(); + const advancing = (async () => { + while (!stopAdvancing.signal.aborted) await vi.advanceTimersByTimeAsync(1_000); + })(); try { await withRemoteDaemonEnv(async () => { @@ -728,9 +738,16 @@ test('sendToDaemon hints to disconnect when a remote daemon is unavailable', asy assert.ok(thrown instanceof Error); assert.equal((thrown as any).code, 'COMMAND_FAILED'); assert.equal(thrown.message, 'Remote daemon is unavailable'); + assert.equal((thrown as any).details?.dispatched, 'no'); + assert.equal((thrown as any).details?.attempts, 90); assert.match(String((thrown as any).details?.hint ?? ''), /agent-device disconnect/); }); + assert.equal(healthProbes, 90); + assert.equal(rpcSends, 0); } finally { + stopAdvancing.abort(); + await advancing; + vi.useRealTimers(); restoreHttpRequest(); } }); diff --git a/packages/daemon-client/src/daemon-client-attach.ts b/packages/daemon-client/src/daemon-client-attach.ts index 7a066918a9..fd9407f802 100644 --- a/packages/daemon-client/src/daemon-client-attach.ts +++ b/packages/daemon-client/src/daemon-client-attach.ts @@ -1,14 +1,21 @@ import { AppError } from '@agent-device/kernel/errors'; import { ensureManagedDaemon } from './daemon-client-lifecycle.ts'; import { assertDaemonPolicyMatches, observeLocalDaemon } from './daemon-client-startup.ts'; -import type { DaemonClientSettings, EnsuredDaemon } from './daemon-client-lifecycle.ts'; +import type { + DaemonClientSettings, + EnsuredDaemon, + RemoteHealthRecovery, +} from './daemon-client-lifecycle.ts'; import { daemonBindingMismatchError } from './daemon-client-binding.ts'; import { daemonIncompatibleError, daemonNotRunningError } from './daemon-lifecycle-errors.ts'; /** Resolves a managed daemon, or observes and verifies the one an attach client may use. */ -export async function connectToDaemon(settings: DaemonClientSettings): Promise { +export async function connectToDaemon( + settings: DaemonClientSettings, + healthRecovery?: RemoteHealthRecovery, +): Promise { if (settings.remoteBaseUrl || settings.daemonLifecycle !== 'attach') { - return await ensureManagedDaemon(settings); + return await ensureManagedDaemon(settings, healthRecovery); } const stateDir = settings.paths.baseDir; const observed = await observeLocalDaemon(settings, { refineUnreachable: true }).catch( diff --git a/packages/daemon-client/src/daemon-client-lifecycle.ts b/packages/daemon-client/src/daemon-client-lifecycle.ts index b68845aafd..ee33dc56ec 100644 --- a/packages/daemon-client/src/daemon-client-lifecycle.ts +++ b/packages/daemon-client/src/daemon-client-lifecycle.ts @@ -226,9 +226,21 @@ function shouldUseOwnedReplayStateDir( return isOneShotReplayCommand(req.command) && !explicitStateDir && !rawRemoteBaseUrl; } -export async function ensureManagedDaemon(settings: DaemonClientSettings): Promise { +/** + * How long the remote `/health` gate may re-probe a daemon that is there but not ready: the budget + * of the command waiting behind it. Without one, the first failed probe refuses. + */ +export type RemoteHealthRecovery = { + budgetMs: number; + signal: AbortSignal | undefined; +}; + +export async function ensureManagedDaemon( + settings: DaemonClientSettings, + healthRecovery?: RemoteHealthRecovery, +): Promise { if (settings.remoteBaseUrl) { - return await ensureRemoteDaemon(settings); + return await ensureRemoteDaemon(settings, healthRecovery); } const ensured = await ensureLocalDaemon(settings); @@ -244,7 +256,10 @@ async function ensureLocalDaemon(settings: DaemonClientSettings): Promise { +async function ensureRemoteDaemon( + settings: DaemonClientSettings, + healthRecovery: RemoteHealthRecovery | undefined, +): Promise { const remoteInfo: DaemonInfo = { transport: 'http', // Remote mode reuses the auth token as the daemon token so the existing JSON-RPC contract still works. @@ -252,18 +267,20 @@ async function ensureRemoteDaemon(settings: DaemonClientSettings): Promise { + const { info, failedProbe, recovery, hint } = params; + let attempts = 1; + if (recovery && isTransientHealthFailure(failedProbe)) { + const { budgetMs, signal } = recovery; + const deadline = performance.now() + budgetMs; + while (!signal?.aborted) { + const remainingMs = deadline - performance.now(); + if (remainingMs <= 0) break; + await sleep(Math.min(REMOTE_HEALTH_REPROBE_INTERVAL_MS, remainingMs), signal); + const probeMs = deadline - performance.now(); + if (probeMs <= 0 || signal?.aborted) break; + attempts += 1; + const health = await cachedRemoteDaemonHealth(info, probeMs, signal); + if (health.reachable) return health; + } + } + throw remoteDaemonUnavailableError({ daemonBaseUrl: info.baseUrl, attempts, hint }); +} diff --git a/packages/daemon-client/src/daemon-client-transport.ts b/packages/daemon-client/src/daemon-client-transport.ts index db87b7014b..9b509d8f90 100644 --- a/packages/daemon-client/src/daemon-client-transport.ts +++ b/packages/daemon-client/src/daemon-client-transport.ts @@ -965,7 +965,11 @@ function invalidateRemoteDaemonHealth(info: DaemonInfo): void { } } -export async function cachedRemoteDaemonHealth(info: DaemonInfo): Promise { +export async function cachedRemoteDaemonHealth( + info: DaemonInfo, + probeTimeoutMs?: number, + callerSignal?: AbortSignal, +): Promise { if (remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info)) { return await remoteHealthCache.health; } @@ -973,7 +977,7 @@ export async function cachedRemoteDaemonHealth(info: DaemonInfo): Promise { return await withDiagnosticTimer( 'daemon_startup', - async () => await connectToDaemon(settings), + // A remote `/health` gate sends nothing until it opens, so it may wait out an outage for as + // long as this command's own budget. + async () => + await connectToDaemon( + settings, + requestTimeoutMs === undefined + ? undefined + : { budgetMs: requestTimeoutMs, signal: options.signal }, + ), { requestId, session: req.session, diff --git a/packages/platform-apple/src/core/runner-host.ts b/packages/platform-apple/src/core/runner-host.ts index aba97ac67a..de226cbf7d 100644 --- a/packages/platform-apple/src/core/runner-host.ts +++ b/packages/platform-apple/src/core/runner-host.ts @@ -32,6 +32,7 @@ import { parseBooleanLiteral } from '@agent-device/kernel/source-value'; import type { HostEnvironment } from '@agent-device/kernel/source-value'; import { + getRequestDeadlineMs, getRequestSignal, isRequestCanceled, emitRequestProgress, @@ -103,6 +104,7 @@ export const appleRunnerHost: AppleRunnerHost = { parseBooleanLiteral, isRecord, emitRequestProgress, + getRequestDeadlineMs, getRequestSignal, isRequestCanceled, classifyBootFailure, diff --git a/packages/test-kit/src/host-kit-capabilities.ts b/packages/test-kit/src/host-kit-capabilities.ts index 7ef7a6983c..a2775c55f6 100644 --- a/packages/test-kit/src/host-kit-capabilities.ts +++ b/packages/test-kit/src/host-kit-capabilities.ts @@ -45,6 +45,7 @@ export async function loadRealHostKitCapabilities() { withProcessLock: file.withProcessLock, publishFileSync: file.publishFileSync, emitRequestProgress: request.emitRequestProgress, + getRequestDeadlineMs: request.getRequestDeadlineMs, getRequestSignal: request.getRequestSignal, isRequestCanceled: request.isRequestCanceled, runnerRuntimeSlot: runtimeSlots.runtimeSlot, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 888e4b04fe..8007f44be7 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -460,7 +460,7 @@ agent-device alert dismiss - The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators, including Limrun, the XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If a notification, privacy, or location table cannot be read, the runner drops the permission translations and keeps the English labels; the `Open in …?` table is read apart, so its labels survive that, and the permission translations survive it being unreadable. Physical iOS devices, tvOS, and visionOS always use the English labels. On iOS simulators, including Limrun, `alert get` marks the `Open in …?` launch confirmation with `launchConfirmationPrompt: true`; `open` reads that mark to answer the prompt. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. - On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or localized decline copy outside the preferred languages' prompt strings fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline during action or confirmation after the alert is resolved fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. -- iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. A synthesized long press, `scroll`, `swipe` (sent as a planned fling `gesture`), or other planned `gesture` that the event synthesis refuses before its event record is handed to XCTest reports `COMMAND_FAILED` with `synthesized_gesture_not_dispatched` (`dispatched: no`), and so does a planned `gesture` refused while its window or display is resolved (the window cannot be named, or the sole display cannot be measured); one that fails after the record was handed to XCTest (its events may or may not have run) reports `synthesized_gesture_outcome_unknown` (`dispatched: unknown`) and is never repeated through an XCTest fallback, so check the screen with `snapshot` before you retry. Any other refusal, such as a platform without touch input, keeps `UNSUPPORTED_OPERATION`. A long-press step in a press series reports the same reasons; once an earlier step has run, the series is `dispatched: unknown`. `back_swipe_no_change` means the iOS `back` swipe was sent and the screen did not change, and `back_swipe_unverified` means it was sent but may or may not have navigated (both `dispatched: unknown`); `back_swipe_not_dispatched` means it was refused before any touch went down (`dispatched: no`). A `keyboard dismiss` the keyboard would not come down for because it exposes no dismiss key keeps `UNSUPPORTED_OPERATION` and reports `keyboard_dismiss_unavailable` (`dispatched: no`, nothing was tapped); one whose dismiss key was tapped and left the keyboard visible keeps `UNSUPPORTED_OPERATION` with no `reason` and `dispatched: unknown`. A `keyboard enter` that found no return key and no field to submit through keeps `UNSUPPORTED_OPERATION` and reports `keyboard_return_unavailable` (`dispatched: unknown`, because typing the return can fail after XCTest sent it); a sole text field that was found and tapped but would not take the return keeps `UNSUPPORTED_OPERATION` with no `reason`. A press the runner answers that its target cannot perform — today the Action Button on an iPhone or iPad model that has none, since the simulator volume and non-tvOS remote cases are refused before the runner is reached — keeps `UNSUPPORTED_OPERATION` and reports `hardware_control_unavailable` (`dispatched: no`, nothing was pressed). macOS `back` finding no in-app back control reports `in_app_back_control_unavailable` under `COMMAND_FAILED` (`dispatched: no`). See Back above. +- iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); a read-only capture resends it until shortly before the command's own timeout, and `details.attempts` counts the sends; `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. A synthesized long press, `scroll`, `swipe` (sent as a planned fling `gesture`), or other planned `gesture` that the event synthesis refuses before its event record is handed to XCTest reports `COMMAND_FAILED` with `synthesized_gesture_not_dispatched` (`dispatched: no`), and so does a planned `gesture` refused while its window or display is resolved (the window cannot be named, or the sole display cannot be measured); one that fails after the record was handed to XCTest (its events may or may not have run) reports `synthesized_gesture_outcome_unknown` (`dispatched: unknown`) and is never repeated through an XCTest fallback, so check the screen with `snapshot` before you retry. Any other refusal, such as a platform without touch input, keeps `UNSUPPORTED_OPERATION`. A long-press step in a press series reports the same reasons; once an earlier step has run, the series is `dispatched: unknown`. `back_swipe_no_change` means the iOS `back` swipe was sent and the screen did not change, and `back_swipe_unverified` means it was sent but may or may not have navigated (both `dispatched: unknown`); `back_swipe_not_dispatched` means it was refused before any touch went down (`dispatched: no`). A `keyboard dismiss` the keyboard would not come down for because it exposes no dismiss key keeps `UNSUPPORTED_OPERATION` and reports `keyboard_dismiss_unavailable` (`dispatched: no`, nothing was tapped); one whose dismiss key was tapped and left the keyboard visible keeps `UNSUPPORTED_OPERATION` with no `reason` and `dispatched: unknown`. A `keyboard enter` that found no return key and no field to submit through keeps `UNSUPPORTED_OPERATION` and reports `keyboard_return_unavailable` (`dispatched: unknown`, because typing the return can fail after XCTest sent it); a sole text field that was found and tapped but would not take the return keeps `UNSUPPORTED_OPERATION` with no `reason`. A press the runner answers that its target cannot perform — today the Action Button on an iPhone or iPad model that has none, since the simulator volume and non-tvOS remote cases are refused before the runner is reached — keeps `UNSUPPORTED_OPERATION` and reports `hardware_control_unavailable` (`dispatched: no`, nothing was pressed). macOS `back` finding no in-app back control reports `in_app_back_control_unavailable` under `COMMAND_FAILED` (`dispatched: no`). See Back above. - On the Apple XCUITest runner, an alert search that fails, runs out of time, or finds an app alert, Safari dialog, or dismiss-popup window with no actionable controls fails with `error.code: ALERT_UNRESOLVED` and `details.dispatched: no`. `alert get` reports it at once. `accept`, `dismiss`, and `alert wait` retry it within their windows, like absence, and report it, not a timeout, when the last check remained unresolved. Inspect the prompt with `screenshot`, then `snapshot -i`, and `press @ref` on the visible button. - On the Apple XCUITest runner, `alert get` describes a readable alert even when all its buttons are disabled, and lists those disabled buttons. `alert accept` and `alert dismiss` fail with `ALERT_BUTTON_NOT_FOUND`. - If a sheet or permission prompt is visible but `alert` reports none, `alert` did not recognize it. Run `snapshot -i -s ""`, then `press @ref` on the visible button. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. @@ -594,7 +594,7 @@ These refusals carry `dispatched: no` and a typed `error.details.reason`, so bra - `session_app_required`: a per-app setting was requested but the session binds no app and none was named. Run `open `, or name the app on the request for a setting that takes one (`settings clear-app-state com.example.app`). - `app_not_installed`: the named app resolves to no installed app on the target. Run `apps --all` for the exact bundle id or package, or install the app, then retry. - `malformed_request`: the local socket transport refused the request envelope before any handler read it, for example a positional that is not a string. Fix the client that built the request; resending it unchanged fails the same way. -- `remote_daemon_unavailable`: the client's `GET /health` probe of the remote daemon answered nothing usable, so the command did not run, and `details.daemonBaseUrl` names the daemon that failed the probe. The probe before the first send refuses it outright; the probe a restart retry runs after the daemon instance changed refuses it too — there the first request reached the daemon but was refused before execution, so nothing ran either way. Make that daemon reachable, or clear or override `--daemon-base-url` or `AGENT_DEVICE_DAEMON_BASE_URL`; for a saved `connect` connection, run `disconnect` to return to the local daemon, then resend the same command. +- `remote_daemon_unavailable`: the client's `GET /health` probe of the remote daemon answered nothing usable, so the command did not run, and `details.daemonBaseUrl` names the daemon that failed the probe. When that daemon answers a server error or does not answer in time, the probe before the first send probes again every second for the command's own timeout, then refuses; a connection that is refused outright is refused at once. `details.attempts` counts the probes. The probe a restart retry runs after the daemon instance changed refuses it too — there the first request reached the daemon but was refused before execution, so nothing ran either way. Make that daemon reachable, or clear or override `--daemon-base-url` or `AGENT_DEVICE_DAEMON_BASE_URL`; for a saved `connect` connection, run `disconnect` to return to the local daemon, then resend the same command. - `webdriver_route_unsupported`: on a WebDriver-backed device cloud, the driver does not implement the route; the limit is the cloud's driver, not the device. A command with a second route (`orientation` has two) tries it before failing. After an ambiguous failure on that transport, only read-only requests are resent. ## Find (semantic) From 7ce68d3966f4a4863a3d8f31518bb3661650f490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 12:06:12 +0100 Subject: [PATCH 2/9] fix(apple-runner): gate only busy resends on the request deadline Transport resends keep their three attempts near a deadline, and the attempt bound stays finite in retry diagnostics. The docs state that the remote health wait leaves the dispatched command its full timeout. Co-Authored-By: Claude Opus 5.5 --- .../src/__tests__/index-busy-resend.test.ts | 23 ++++++- packages/apple-runner/src/index.ts | 66 +++++++++++-------- .../daemon-client-remote-health-recovery.ts | 8 ++- website/docs/docs/commands.md | 2 +- 4 files changed, 67 insertions(+), 32 deletions(-) diff --git a/packages/apple-runner/src/__tests__/index-busy-resend.test.ts b/packages/apple-runner/src/__tests__/index-busy-resend.test.ts index 2919473f3c..eda145635c 100644 --- a/packages/apple-runner/src/__tests__/index-busy-resend.test.ts +++ b/packages/apple-runner/src/__tests__/index-busy-resend.test.ts @@ -164,8 +164,8 @@ test('a request deadline reached mid-outage ends with the RUNNER_BUSY refusal an assert.ok(error instanceof AppError, `expected the refusal, got ${String(error)}`); assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY'); - // Delays of 200, 400, 800, then 1000ms put the eleventh send at 8.4s. The window is 9s (10s less - // the 1s reserve), so the next delay is clamped to it and no send follows. + // The busy window is 8s (10s less the 2s reserve). Delays of 200, 400, 800, then 1000ms put the + // tenth send at 7.4s, inside it, and the eleventh at 8.4s, past it: that refusal is the answer. assert.equal(error.details?.attempts, 11); assert.equal(sentCommands().length, 11); assert.equal(sentAt.at(-1)! - startedAt, 8_400); @@ -193,6 +193,25 @@ test('read-only transport failures keep the three-attempt resend budget', async assert.equal(sentCommands().filter((command) => command === 'snapshot').length, 3); }); +test('a request deadline too close for a busy resend leaves the transport resends their three attempts', async () => { + underRequestDeadline(1_000); + mockExecuteRunnerCommandWithSession + .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed')) + .mockResolvedValueOnce({ lifecycleState: 'started' }) + .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed')) + .mockResolvedValueOnce({ lifecycleState: 'started' }) + .mockResolvedValueOnce({ nodes: [], truncated: false }); + + const result = await runAppleRunnerCommand( + IOS_SIMULATOR, + { command: 'snapshot' }, + { requestId: 'req-deadline-transport' }, + ); + + assert.deepEqual(result, { nodes: [], truncated: false }); + assert.equal(sentCommands().filter((command) => command === 'snapshot').length, 3); +}); + test('a wait deadline landing mid-window rethrows the last RUNNER_BUSY refusal, not a cancel', async () => { // `wait` bounds each poll with its own abort signal (runWithinWaitDeadline) and keeps the last // typed refusal as the wait's cause. The refusal must therefore survive the deadline: a bare diff --git a/packages/apple-runner/src/index.ts b/packages/apple-runner/src/index.ts index 81500f8a74..916017e4e7 100644 --- a/packages/apple-runner/src/index.ts +++ b/packages/apple-runner/src/index.ts @@ -136,13 +136,13 @@ import { RUNNER_COMMAND_TIMEOUT_MS } from './runner-transport.ts'; * Attempts a read-only command may spend on one error class; 1 means no resend. A `RUNNER_BUSY` * refusal is the runner refusing fast on purpose while it drains abandoned XCTest work (#1105), so * that budget has to outlast the drain. Under a request deadline, busy refusals are resent until - * that deadline, less {@link RUNNER_BUSY_DEADLINE_RESERVE_MS}, so the last refusal (with its attempt - * count) is the answer instead of the deadline's. Without one, the window is a heuristic: 200ms - * doubling to a 1s cap, no jitter, is 5.4s of delay across eight attempts before round trips, and a - * drain that outlives it still surfaces as `RUNNER_BUSY` (the runner's own `abandonedForSeconds=` - * marker in runner.log is the evidence for tuning it). Transport failures keep the pre-existing - * three attempts. The budget is positional: the error on each attempt sets how many attempts the - * loop may reach in total, so three busy refusals followed by a transport failure resend no further. + * {@link RUNNER_BUSY_DEADLINE_RESERVE_MS} before it, so the last refusal (with its attempt count) is + * the answer instead of the deadline's. Without one, the window is a heuristic: 200ms doubling to a + * 1s cap, no jitter, is 5.4s of delay across eight attempts before round trips, and a drain that + * outlives it still surfaces as `RUNNER_BUSY` (the runner's own `abandonedForSeconds=` marker in + * runner.log is the evidence for tuning it). Transport failures keep the pre-existing three attempts + * either way. The budget is positional: the error on each attempt decides whether the loop may send + * again, so three busy refusals followed by a transport failure resend no further. */ const RUNNER_BUSY_RESEND_ATTEMPTS = 8; const TRANSPORT_RESEND_ATTEMPTS = 3; @@ -151,26 +151,40 @@ const READ_ONLY_RESEND_POLICY = { maxDelayMs: 1_000, jitter: 0, }; -/** One resend delay at its cap: a busy refusal is answered at once, so one round trip fits. */ -const RUNNER_BUSY_DEADLINE_RESERVE_MS = READ_ONLY_RESEND_POLICY.maxDelayMs; +/** + * The last busy resend is decided before the window closes and sent one capped delay later, so the + * reserve holds that delay and the refusal's round trip, which the runner answers at once. + */ +const RUNNER_BUSY_DEADLINE_RESERVE_MS = 2 * READ_ONLY_RESEND_POLICY.maxDelayMs; -function readOnlyResendBudget(error: unknown, busyAttempts: number): number { - if (isRunnerBusyError(error)) return busyAttempts; - return isRetryableRunnerError(error) ? TRANSPORT_RESEND_ATTEMPTS : 1; -} +type ReadOnlyResendBudget = { + maxAttempts: number; + resends: (error: unknown, attempt: number) => boolean; +}; -/** The busy resend bound: the request's own deadline when it has one, else the fixed count. */ -function resolveBusyResendWindow(requestId: string | undefined): { - busyAttempts: number; - deadline?: Deadline; -} { +function resolveReadOnlyResendBudget(requestId: string | undefined): ReadOnlyResendBudget { const requestDeadlineMs = getRequestDeadlineMs(requestId); - if (requestDeadlineMs === undefined) return { busyAttempts: RUNNER_BUSY_RESEND_ATTEMPTS }; + const resendsOther = (error: unknown, attempt: number) => + attempt < (isRetryableRunnerError(error) ? TRANSPORT_RESEND_ATTEMPTS : 1); + if (requestDeadlineMs === undefined) { + return { + maxAttempts: RUNNER_BUSY_RESEND_ATTEMPTS, + resends: (error, attempt) => + isRunnerBusyError(error) + ? attempt < RUNNER_BUSY_RESEND_ATTEMPTS + : resendsOther(error, attempt), + }; + } + const windowMs = Math.max(0, requestDeadlineMs - Date.now() - RUNNER_BUSY_DEADLINE_RESERVE_MS); + const busyWindow = Deadline.fromTimeoutMs(windowMs); return { - busyAttempts: Number.POSITIVE_INFINITY, - deadline: Deadline.fromTimeoutMs( - requestDeadlineMs - Date.now() - RUNNER_BUSY_DEADLINE_RESERVE_MS, + // Every delay is at least the base delay, so the window holds no more attempts than this. + maxAttempts: Math.max( + TRANSPORT_RESEND_ATTEMPTS, + Math.ceil(windowMs / READ_ONLY_RESEND_POLICY.baseDelayMs) + 1, ), + resends: (error, attempt) => + isRunnerBusyError(error) ? !busyWindow.isExpired() : resendsOther(error, attempt), }; } @@ -186,7 +200,7 @@ export async function runAppleRunnerCommand( if (!isReadOnlyRunnerCommand(runnerCommand)) { return provider.runCommand(device, runnerCommand, options); } - const { busyAttempts, deadline } = resolveBusyResendWindow(options.requestId); + const budget = resolveReadOnlyResendBudget(options.requestId); let lastBusyRefusal: AppError | undefined; let attempts = 0; try { @@ -198,16 +212,16 @@ export async function runAppleRunnerCommand( }, { ...READ_ONLY_RESEND_POLICY, - maxAttempts: busyAttempts, + maxAttempts: budget.maxAttempts, shouldRetry: (error, attempt) => { assertRunnerRequestActive(options.requestId); if (isRunnerBusyError(error) && error instanceof AppError) lastBusyRefusal = error; - return attempt < readOnlyResendBudget(error, busyAttempts); + return budget.resends(error, attempt); }, }, // The busy window is seconds long, so an abort must wake the delay instead of sleeping it // out and sending one more attempt. - { signal: resolveRunnerRequestSignal(options), deadline }, + { signal: resolveRunnerRequestSignal(options) }, ); } catch (error) { if (isRunnerBusyError(error) && error instanceof AppError) { diff --git a/packages/daemon-client/src/daemon-client-remote-health-recovery.ts b/packages/daemon-client/src/daemon-client-remote-health-recovery.ts index 3211bf0ac7..2861b27847 100644 --- a/packages/daemon-client/src/daemon-client-remote-health-recovery.ts +++ b/packages/daemon-client/src/daemon-client-remote-health-recovery.ts @@ -8,9 +8,9 @@ import { remoteDaemonUnavailableError } from './daemon-lifecycle-errors.ts'; const REMOTE_HEALTH_REPROBE_INTERVAL_MS = 1_000; /** - * Whether a failed probe found an endpoint that is there but not ready: it answered with a server + * Whether a failed probe may have reached an endpoint that is coming back: it answered with a server * error, or it did not answer within the probe's time. A connection that fails outright (refused, no - * such host) has neither, and is refused at once so a wrong base URL never waits out a budget. + * such host) has neither, and is refused at once, so a URL with nothing listening never waits. */ function isTransientHealthFailure(health: RemoteDaemonHealth): boolean { return health.timedOut === true || health.statusCode !== undefined; @@ -20,7 +20,9 @@ function isTransientHealthFailure(health: RemoteDaemonHealth): boolean { * Answers a remote `/health` gate whose first probe failed. When the command has a request budget * and the endpoint is there but not ready, it re-probes until the daemon answers or that budget * ends. The gate stands before the first send, so nothing reached the daemon, and a resend is safe - * for any command, a UI action included. The typed refusal states every probe it made. + * for any command, a UI action included. The wait does not shorten the send's own budget: a command + * dispatched with too little time left could time out after it reached the device. The typed refusal + * states every probe it made. */ export async function recoverRemoteDaemonHealth(params: { info: DaemonInfo; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 8007f44be7..cb789964a2 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -594,7 +594,7 @@ These refusals carry `dispatched: no` and a typed `error.details.reason`, so bra - `session_app_required`: a per-app setting was requested but the session binds no app and none was named. Run `open `, or name the app on the request for a setting that takes one (`settings clear-app-state com.example.app`). - `app_not_installed`: the named app resolves to no installed app on the target. Run `apps --all` for the exact bundle id or package, or install the app, then retry. - `malformed_request`: the local socket transport refused the request envelope before any handler read it, for example a positional that is not a string. Fix the client that built the request; resending it unchanged fails the same way. -- `remote_daemon_unavailable`: the client's `GET /health` probe of the remote daemon answered nothing usable, so the command did not run, and `details.daemonBaseUrl` names the daemon that failed the probe. When that daemon answers a server error or does not answer in time, the probe before the first send probes again every second for the command's own timeout, then refuses; a connection that is refused outright is refused at once. `details.attempts` counts the probes. The probe a restart retry runs after the daemon instance changed refuses it too — there the first request reached the daemon but was refused before execution, so nothing ran either way. Make that daemon reachable, or clear or override `--daemon-base-url` or `AGENT_DEVICE_DAEMON_BASE_URL`; for a saved `connect` connection, run `disconnect` to return to the local daemon, then resend the same command. +- `remote_daemon_unavailable`: the client's `GET /health` probe of the remote daemon answered nothing usable, so the command did not run, and `details.daemonBaseUrl` names the daemon that failed the probe. When the probe before the first send gets a server error or no answer in time, it probes again every second, for up to the command's own timeout, then refuses. A command that gets through still has its full timeout to run. A connection that is refused outright fails at once. `details.attempts` counts the probes. The probe a restart retry runs after the daemon instance changed refuses it too — there the first request reached the daemon but was refused before execution, so nothing ran either way. Make that daemon reachable, or clear or override `--daemon-base-url` or `AGENT_DEVICE_DAEMON_BASE_URL`; for a saved `connect` connection, run `disconnect` to return to the local daemon, then resend the same command. - `webdriver_route_unsupported`: on a WebDriver-backed device cloud, the driver does not implement the route; the limit is the cloud's driver, not the device. A command with a second route (`orientation` has two) tries it before failing. After an ambiguous failure on that transport, only read-only requests are resent. ## Find (semantic) From eed1f7defce86624ceedc487a8c0778f03b789b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 12:20:34 +0100 Subject: [PATCH 3/9] fix(daemon-client): bound each caller's health wait and stop recovery on a refused probe - The first /health probe runs under the command's budget and signal, and the recovery deadline starts with it. - The re-probe loop stops at once when a re-probe is not transient (a refused connection). - Concurrent callers share one in-flight probe, but each applies its own timeout and abort to its wait. The cache moves to daemon-client-remote-health-cache.ts. - remote_daemon_unavailable always carries attempts, and healthRecoveryBudgetMs when the budget ran out. - Tests: busy refusal after the window closed, refused re-probe, shared-probe abort, and stricter dispatched-press assertions. Docs: a closed busy window still answers runner_busy. Co-Authored-By: Claude Opus 5.5 --- .../src/__tests__/index-busy-resend.test.ts | 27 +++- ...mon-client-lifecycle-remote-health.test.ts | 63 ++++++++++ ...daemon-client-remote-health-budget.test.ts | 90 ++++++++++++++ .../daemon-client/src/__tests__/index.test.ts | 50 +------- .../src/daemon-client-lifecycle.ts | 9 +- .../src/daemon-client-remote-health-cache.ts | 116 ++++++++++++++++++ .../daemon-client-remote-health-recovery.ts | 35 +++--- .../src/daemon-client-transport.ts | 90 +++----------- .../src/daemon-lifecycle-errors.ts | 11 +- website/docs/docs/commands.md | 2 +- 10 files changed, 349 insertions(+), 144 deletions(-) create mode 100644 packages/daemon-client/src/__tests__/daemon-client-remote-health-budget.test.ts create mode 100644 packages/daemon-client/src/daemon-client-remote-health-cache.ts diff --git a/packages/apple-runner/src/__tests__/index-busy-resend.test.ts b/packages/apple-runner/src/__tests__/index-busy-resend.test.ts index eda145635c..9843867fad 100644 --- a/packages/apple-runner/src/__tests__/index-busy-resend.test.ts +++ b/packages/apple-runner/src/__tests__/index-busy-resend.test.ts @@ -193,7 +193,32 @@ test('read-only transport failures keep the three-attempt resend budget', async assert.equal(sentCommands().filter((command) => command === 'snapshot').length, 3); }); -test('a request deadline too close for a busy resend leaves the transport resends their three attempts', async () => { +test('a RUNNER_BUSY refusal after the busy window closed is not resent', async () => { + // A 1s request deadline is inside the 2s reserve, so the window is already closed. + underRequestDeadline(1_000); + mockExecuteRunnerCommandWithSession + .mockRejectedValueOnce(busyRefusal()) + .mockResolvedValue({ nodes: [], truncated: false }); + + await assert.rejects( + () => + runAppleRunnerCommand( + IOS_SIMULATOR, + { command: 'snapshot' }, + { requestId: 'req-deadline-window-closed' }, + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY'); + assert.equal(error.details?.dispatched, 'no'); + assert.equal(error.details?.attempts, 1); + return true; + }, + ); + assert.deepEqual(sentCommands(), ['snapshot']); +}); + +test('under a request deadline too close for a busy resend, transport failures keep their three attempts', async () => { underRequestDeadline(1_000); mockExecuteRunnerCommandWithSession .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed')) diff --git a/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts b/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts index ec2c06b487..a49e9cf20d 100644 --- a/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts +++ b/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts @@ -15,6 +15,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { REMOTE_DAEMON_UNAVAILABLE_REASON } from '../daemon-lifecycle-errors.ts'; import { sendToDaemon } from '../index.ts'; +import { cachedRemoteDaemonHealth } from '../daemon-client-transport.ts'; // A press, so the refusal states the fact a caller needs before resending a mutating command. function remotePress(baseUrl: string) { @@ -64,7 +65,9 @@ test('a remote press whose health probe answers 5xx re-probes, then is sent exac sendToDaemon(remotePress(`http://127.0.0.1:${port}`)), (error: unknown) => { assert.ok(error instanceof AppError, `unexpected failure: ${String(error)}`); + assert.equal(error.code, 'COMMAND_FAILED'); assert.notEqual(error.details?.reason, REMOTE_DAEMON_UNAVAILABLE_REASON); + assert.notEqual(error.details?.dispatched, 'no', 'the press left the client'); return true; }, ); @@ -95,6 +98,35 @@ test('a remote press whose health probe is refused is refused at once with the t }); }); +// A daemon that answered 503 and then refuses connections is gone, not restarting: the re-probe +// that is refused ends the wait at once instead of waiting out the command's budget. +test('a remote press re-probing after 5xx stops at the first refused connection', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const handler = http.createServer((req, res) => { + req.resume(); + res.statusCode = 503; + res.end(); + handler.close(); + handler.closeAllConnections(); + }); + const port = await listenOnLoopback(handler); + try { + await assert.rejects( + sendToDaemon(remotePress(`http://127.0.0.1:${port}`)), + (error: unknown) => { + assert.ok(error instanceof AppError, `unexpected failure: ${String(error)}`); + assert.equal(error.details?.reason, REMOTE_DAEMON_UNAVAILABLE_REASON); + assert.equal(error.details?.dispatched, 'no'); + assert.equal(error.details?.attempts, 2); + assert.equal(error.details?.healthRecoveryBudgetMs, undefined, 'the budget was not spent'); + return true; + }, + ); + } finally { + if (handler.listening) await closeLoopbackServer(handler); + } +}); + // The closest negative: a probe that answers 4xx counts as reachable, so the press is sent and the // unavailable refusal is never stamped on a request that left the client. test('a remote press whose health probe answers 4xx is sent and is not the unavailable refusal', async (t) => { @@ -115,3 +147,34 @@ test('a remote press whose health probe answers 4xx is sent and is not the unava await closeLoopbackServer(handler); } }); + +// Concurrent callers share one in-flight probe, and each applies its own bound to it: one caller +// aborting does not end the wait of another that shares the probe. +test('a caller aborting a shared health probe leaves a concurrent caller its answer', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + let probes = 0; + const handler = http.createServer((req, res) => { + req.resume(); + probes += 1; + setTimeout(() => { + res.statusCode = 200; + res.end(JSON.stringify({ instanceId: 'remote-instance' })); + }, 200); + }); + const port = await listenOnLoopback(handler); + const info = { baseUrl: `http://127.0.0.1:${port}`, token: 'shared-probe', pid: 0 }; + try { + const aborting = new AbortController(); + const aborted = cachedRemoteDaemonHealth(info, undefined, aborting.signal); + const unbounded = cachedRemoteDaemonHealth(info); + aborting.abort(); + + assert.equal((await aborted).reachable, false); + const health = await unbounded; + assert.equal(health.reachable, true); + assert.equal(health.instanceId, 'remote-instance'); + assert.equal(probes, 1, 'both callers shared one probe'); + } finally { + await closeLoopbackServer(handler); + } +}); diff --git a/packages/daemon-client/src/__tests__/daemon-client-remote-health-budget.test.ts b/packages/daemon-client/src/__tests__/daemon-client-remote-health-budget.test.ts new file mode 100644 index 0000000000..be3a97db7f --- /dev/null +++ b/packages/daemon-client/src/__tests__/daemon-client-remote-health-budget.test.ts @@ -0,0 +1,90 @@ +// The remote `/health` gate re-probes a daemon that answers 5xx for the command's own budget. This +// drives a whole default budget on fake time: `http.request` answers on the next tick, so no real +// timer or socket is involved. + +import assert from 'node:assert/strict'; +import http from 'node:http'; +import { EventEmitter } from 'node:events'; +import { afterEach, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { sendToDaemon } from '../index.ts'; + +/** Replaces `http.request` with one that answers every request 503, counting them by method. */ +function answerEveryRequest503(): { probes: () => number; rpcs: () => number } { + let probes = 0; + let rpcs = 0; + vi.spyOn(http, 'request').mockImplementation((( + options: { method?: string }, + callback: (res: EventEmitter) => void, + ) => { + const req = Object.assign(new EventEmitter(), { + write: () => {}, + destroy: () => {}, + end: () => { + if (options.method === 'GET') probes += 1; + else rpcs += 1; + const res = Object.assign(new EventEmitter(), { + statusCode: 503, + resume: () => {}, + setEncoding: () => {}, + }); + process.nextTick(() => { + callback(res); + res.emit('end'); + }); + }, + }); + return req; + }) as unknown as typeof http.request); + return { probes: () => probes, rpcs: () => rpcs }; +} + +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +test('a remote daemon answering 5xx for the whole request budget is refused with its probe count and budget', async () => { + const server = answerEveryRequest503(); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }); + let advancing = true; + const advance = (async () => { + while (advancing) { + await new Promise((resolve) => setImmediate(resolve)); + await vi.advanceTimersByTimeAsync(100); + } + })(); + + try { + const outcome = await sendToDaemon( + { + session: 'default', + command: 'session', + positionals: ['list'], + flags: { daemonBaseUrl: 'http://remote-mac.example.test:7777/agent-device' }, + meta: { requestId: 'req-remote-budget' }, + }, + // A non-loopback daemon is trusted only with an auth token. + { authToken: 'remote-secret' }, + ).then( + () => undefined, + (error: unknown) => error, + ); + + assert.ok(outcome instanceof AppError, `unexpected outcome: ${String(outcome)}`); + assert.equal(outcome.code, 'COMMAND_FAILED'); + assert.equal(outcome.details?.reason, 'remote_daemon_unavailable'); + assert.equal(outcome.details?.dispatched, 'no'); + // At most one probe a second fits the 90s budget, and far more than a fixed resend count. Real + // module loading can let fake time run ahead, so the exact count is not asserted. + const attempts = Number(outcome.details?.attempts); + assert.ok(attempts > 8 && attempts <= 90, `unexpected attempts: ${attempts}`); + assert.equal(outcome.details?.healthRecoveryBudgetMs, 90_000); + assert.match(String(outcome.details?.hint ?? ''), /agent-device disconnect/); + assert.equal(server.probes(), attempts); + assert.equal(server.rpcs(), 0); + } finally { + advancing = false; + await advance; + } +}); diff --git a/packages/daemon-client/src/__tests__/index.test.ts b/packages/daemon-client/src/__tests__/index.test.ts index 09ade73f74..3b643d4cd3 100644 --- a/packages/daemon-client/src/__tests__/index.test.ts +++ b/packages/daemon-client/src/__tests__/index.test.ts @@ -1,5 +1,5 @@ import type { RequestProgressEvent } from '@agent-device/contracts/progress'; -import { test, vi } from 'vitest'; +import { test } from 'vitest'; import assert from 'node:assert/strict'; import http from 'node:http'; import net from 'node:net'; @@ -704,54 +704,6 @@ test.each([ }, ); -test('a remote daemon unavailable for the whole request budget is refused with its probe count and the disconnect hint', async () => { - // `session list` has the 90s default budget: 90 probes a second apart while /health answers 503. - let healthProbes = 0; - let rpcSends = 0; - const restoreHttpRequest = mockEventHttpRequest(({ options, res }) => { - if (options.method === 'GET') healthProbes += 1; - else rpcSends += 1; - res.statusCode = 503; - res.emit('end'); - }); - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }); - const stopAdvancing = new AbortController(); - const advancing = (async () => { - while (!stopAdvancing.signal.aborted) await vi.advanceTimersByTimeAsync(1_000); - })(); - - try { - await withRemoteDaemonEnv(async () => { - let thrown: unknown; - try { - await sendToDaemon({ - session: 'default', - command: 'session', - positionals: ['list'], - flags: {}, - meta: { requestId: 'req-remote-unavailable' }, - }); - } catch (error) { - thrown = error; - } - - assert.ok(thrown instanceof Error); - assert.equal((thrown as any).code, 'COMMAND_FAILED'); - assert.equal(thrown.message, 'Remote daemon is unavailable'); - assert.equal((thrown as any).details?.dispatched, 'no'); - assert.equal((thrown as any).details?.attempts, 90); - assert.match(String((thrown as any).details?.hint ?? ''), /agent-device disconnect/); - }); - assert.equal(healthProbes, 90); - assert.equal(rpcSends, 0); - } finally { - stopAdvancing.abort(); - await advancing; - vi.useRealTimers(); - restoreHttpRequest(); - } -}); - test('sendToDaemon sends lease helpers as top-level JSON-RPC methods over HTTP', async () => { const rpcRequests: Record[] = []; const originalHttpRequest = http.request; diff --git a/packages/daemon-client/src/daemon-client-lifecycle.ts b/packages/daemon-client/src/daemon-client-lifecycle.ts index ee33dc56ec..7268be98d0 100644 --- a/packages/daemon-client/src/daemon-client-lifecycle.ts +++ b/packages/daemon-client/src/daemon-client-lifecycle.ts @@ -267,7 +267,13 @@ async function ensureRemoteDaemon( pid: 0, baseUrl: settings.remoteBaseUrl, }; - let health = await cachedRemoteDaemonHealth(remoteInfo); + // The command's budget starts with the first probe, which already runs under it and its signal. + const deadline = performance.now() + (healthRecovery?.budgetMs ?? 0); + let health = await cachedRemoteDaemonHealth( + remoteInfo, + healthRecovery?.budgetMs, + healthRecovery?.signal, + ); if (!health.reachable) { // Loaded on the refusal: this module is not part of what every CLI start pays for. const { recoverRemoteDaemonHealth } = await import('./daemon-client-remote-health-recovery.ts'); @@ -275,6 +281,7 @@ async function ensureRemoteDaemon( info: remoteInfo, failedProbe: health, recovery: healthRecovery, + deadline, hint: 'Fix or unset --daemon-base-url or AGENT_DEVICE_DAEMON_BASE_URL so the client targets a reachable daemon or the local one. If this CLI was connected with connect proxy, run agent-device disconnect to clear the saved connection and return to the local daemon.', }); } diff --git a/packages/daemon-client/src/daemon-client-remote-health-cache.ts b/packages/daemon-client/src/daemon-client-remote-health-cache.ts new file mode 100644 index 0000000000..8389d6afb7 --- /dev/null +++ b/packages/daemon-client/src/daemon-client-remote-health-cache.ts @@ -0,0 +1,116 @@ +import type { DaemonInfo } from './daemon-client-metadata.ts'; + +export type RemoteDaemonHealth = { + reachable: boolean; + statusCode?: number; + service?: string; + version?: string; + rpcProtocolVersion?: number; + instanceId?: string; + hostArch?: string; + /** The daemon behind a proxy, as the proxy's health reported it. */ + upstream?: RemoteDaemonHealthLink; + /** The probe ran out of its time budget before an answer, rather than failing outright. */ + timedOut?: true; +}; + +export type RemoteDaemonHealthLink = Pick< + RemoteDaemonHealth, + 'service' | 'version' | 'rpcProtocolVersion' | 'instanceId' | 'hostArch' +>; + +type RemoteHealthCacheEntry = { + baseUrl: string; + token: string; + pid: number; + health: Promise; +}; + +let remoteHealthCache: RemoteHealthCacheEntry | undefined; + +function matchesRemoteIdentity(entry: RemoteHealthCacheEntry, info: DaemonInfo): boolean { + return entry.baseUrl === info.baseUrl && entry.token === info.token && entry.pid === info.pid; +} + +function isCacheableRemoteHealth(health: RemoteDaemonHealth): boolean { + return ( + health.reachable && !!health.instanceId && (!health.upstream || !!health.upstream.instanceId) + ); +} + +export function cacheRemoteDaemonHealth(info: DaemonInfo, health: RemoteDaemonHealth): void { + if (!isCacheableRemoteHealth(health)) return; + remoteHealthCache = { + baseUrl: info.baseUrl ?? '', + token: info.token, + pid: info.pid, + health: Promise.resolve(health), + }; +} + +export function invalidateRemoteDaemonHealth(info: DaemonInfo): void { + if (remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info)) { + remoteHealthCache = undefined; + } +} + +/** The in-flight or cached `/health` answer for this daemon, starting `probe` when there is none. */ +export function sharedRemoteHealth( + info: DaemonInfo, + probe: () => Promise, +): Promise { + if (remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info)) { + return remoteHealthCache.health; + } + const drop = () => { + if (remoteHealthCache === entry) remoteHealthCache = undefined; + }; + const entry: RemoteHealthCacheEntry = { + baseUrl: info.baseUrl ?? '', + token: info.token, + pid: info.pid, + health: probe().then( + (health) => { + if (!isCacheableRemoteHealth(health)) drop(); + return health; + }, + (error: unknown) => { + drop(); + throw error; + }, + ), + }; + remoteHealthCache = entry; + return entry.health; +} + +/** A caller's view of a shared probe: unanswered once its own time is up, or once it aborts. */ +export async function awaitWithinCallerBound( + health: Promise, + timeoutMs: number | undefined, + signal: AbortSignal | undefined, +): Promise { + if (timeoutMs === undefined && !signal) return await health; + if (signal?.aborted) return { reachable: false }; + if (timeoutMs !== undefined && timeoutMs <= 0) return { reachable: false, timedOut: true }; + return await new Promise((resolve, reject) => { + const timer = + timeoutMs === undefined + ? undefined + : setTimeout(() => settle({ reachable: false, timedOut: true }), Math.ceil(timeoutMs)); + const onAbort = () => settle({ reachable: false }); + signal?.addEventListener('abort', onAbort, { once: true }); + function release(): void { + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + } + function settle(result: RemoteDaemonHealth): void { + release(); + resolve(result); + } + health.then(settle, (error: unknown) => { + release(); + reject(error); + }); + }); +} diff --git a/packages/daemon-client/src/daemon-client-remote-health-recovery.ts b/packages/daemon-client/src/daemon-client-remote-health-recovery.ts index 2861b27847..2a9f38441d 100644 --- a/packages/daemon-client/src/daemon-client-remote-health-recovery.ts +++ b/packages/daemon-client/src/daemon-client-remote-health-recovery.ts @@ -1,7 +1,8 @@ import { sleep } from '@agent-device/host-kit/retry'; import type { RemoteHealthRecovery } from './daemon-client-lifecycle.ts'; import type { DaemonInfo } from './daemon-client-metadata.ts'; -import { cachedRemoteDaemonHealth, type RemoteDaemonHealth } from './daemon-client-transport.ts'; +import type { RemoteDaemonHealth } from './daemon-client-remote-health-cache.ts'; +import { cachedRemoteDaemonHealth } from './daemon-client-transport.ts'; import { remoteDaemonUnavailableError } from './daemon-lifecycle-errors.ts'; /** The pause between `/health` probes while a remote daemon is unavailable. */ @@ -28,23 +29,29 @@ export async function recoverRemoteDaemonHealth(params: { info: DaemonInfo; failedProbe: RemoteDaemonHealth; recovery: RemoteHealthRecovery | undefined; + /** When the command's budget ends, on the `performance.now()` clock: it started with the first probe. */ + deadline: number; hint: string; }): Promise { - const { info, failedProbe, recovery, hint } = params; + const { info, recovery, deadline, hint } = params; let attempts = 1; - if (recovery && isTransientHealthFailure(failedProbe)) { - const { budgetMs, signal } = recovery; - const deadline = performance.now() + budgetMs; - while (!signal?.aborted) { - const remainingMs = deadline - performance.now(); - if (remainingMs <= 0) break; - await sleep(Math.min(REMOTE_HEALTH_REPROBE_INTERVAL_MS, remainingMs), signal); - const probeMs = deadline - performance.now(); - if (probeMs <= 0 || signal?.aborted) break; - attempts += 1; - const health = await cachedRemoteDaemonHealth(info, probeMs, signal); - if (health.reachable) return health; + let probe = params.failedProbe; + while (recovery && isTransientHealthFailure(probe) && !recovery.signal?.aborted) { + const remainingMs = deadline - performance.now(); + if (remainingMs <= 0) { + throw remoteDaemonUnavailableError({ + daemonBaseUrl: info.baseUrl, + attempts, + healthRecoveryBudgetMs: recovery.budgetMs, + hint, + }); } + await sleep(Math.min(REMOTE_HEALTH_REPROBE_INTERVAL_MS, remainingMs), recovery.signal); + const probeMs = deadline - performance.now(); + if (probeMs <= 0 || recovery.signal?.aborted) continue; + attempts += 1; + probe = await cachedRemoteDaemonHealth(info, probeMs, recovery.signal); + if (probe.reachable) return probe; } throw remoteDaemonUnavailableError({ daemonBaseUrl: info.baseUrl, attempts, hint }); } diff --git a/packages/daemon-client/src/daemon-client-transport.ts b/packages/daemon-client/src/daemon-client-transport.ts index 9b509d8f90..f1227909d8 100644 --- a/packages/daemon-client/src/daemon-client-transport.ts +++ b/packages/daemon-client/src/daemon-client-transport.ts @@ -28,6 +28,14 @@ import { refuseOversizedDaemonRequest, } from './daemon-client-rpc.ts'; import { handleRequestTimeout } from './daemon-client-timeout.ts'; +import { + awaitWithinCallerBound, + cacheRemoteDaemonHealth, + invalidateRemoteDaemonHealth, + sharedRemoteHealth, + type RemoteDaemonHealth, + type RemoteDaemonHealthLink, +} from './daemon-client-remote-health-cache.ts'; import { isRemoteDaemon, type DaemonInfo } from './daemon-client-metadata.ts'; import type { HostEnvironment } from '@agent-device/kernel/source-value'; import { readVersion } from '@agent-device/host-kit/version'; @@ -161,25 +169,6 @@ function daemonEndpointUnavailableError(transport: ResolvedDaemonTransport): App ); } -export type RemoteDaemonHealth = { - reachable: boolean; - statusCode?: number; - service?: string; - version?: string; - rpcProtocolVersion?: number; - instanceId?: string; - hostArch?: string; - /** The daemon behind a proxy, as the proxy's health reported it. */ - upstream?: RemoteDaemonHealthLink; - /** The probe ran out of its time budget before an answer, rather than failing outright. */ - timedOut?: true; -}; - -type RemoteDaemonHealthLink = Pick< - RemoteDaemonHealth, - 'service' | 'version' | 'rpcProtocolVersion' | 'instanceId' | 'hostArch' ->; - export async function canConnect( info: DaemonInfo, preference: DaemonTransportPreference, @@ -496,7 +485,7 @@ async function retryAfterRemoteInstanceMismatch( ); if (!health.reachable) { const { remoteDaemonUnavailableError } = await import('./daemon-lifecycle-errors.ts'); - throw remoteDaemonUnavailableError({ daemonBaseUrl: info.baseUrl }); + throw remoteDaemonUnavailableError({ daemonBaseUrl: info.baseUrl, attempts: 1 }); } info.remoteInstanceId = health.instanceId; info.remoteUpstreamInstanceId = health.upstream?.instanceId; @@ -936,63 +925,16 @@ async function sendHttpRequest( }); } -type RemoteHealthCacheEntry = { - baseUrl: string; - token: string; - pid: number; - health: Promise; -}; - -let remoteHealthCache: RemoteHealthCacheEntry | undefined; - -function matchesRemoteIdentity(entry: RemoteHealthCacheEntry, info: DaemonInfo): boolean { - return entry.baseUrl === info.baseUrl && entry.token === info.token && entry.pid === info.pid; -} - -function cacheRemoteDaemonHealth(info: DaemonInfo, health: RemoteDaemonHealth): void { - if (!health.instanceId || (health.upstream && !health.upstream.instanceId)) return; - remoteHealthCache = { - baseUrl: info.baseUrl ?? '', - token: info.token, - pid: info.pid, - health: Promise.resolve(health), - }; -} - -function invalidateRemoteDaemonHealth(info: DaemonInfo): void { - if (remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info)) { - remoteHealthCache = undefined; - } -} - +/** + * The remote `/health` answer, sharing one in-flight probe among concurrent callers. The shared + * probe runs on the default budget; each caller applies its own `probeTimeoutMs` and + * `callerSignal` to its wait, so no caller inherits another's bound or abort. + */ export async function cachedRemoteDaemonHealth( info: DaemonInfo, probeTimeoutMs?: number, callerSignal?: AbortSignal, ): Promise { - if (remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info)) { - return await remoteHealthCache.health; - } - const entry: RemoteHealthCacheEntry = { - baseUrl: info.baseUrl ?? '', - token: info.token, - pid: info.pid, - health: readRemoteDaemonHealth(info, probeTimeoutMs, callerSignal), - }; - remoteHealthCache = entry; - try { - const health = await entry.health; - if ( - (!health.reachable || - !health.instanceId || - (health.upstream && !health.upstream.instanceId)) && - remoteHealthCache === entry - ) { - remoteHealthCache = undefined; - } - return health; - } catch (error) { - if (remoteHealthCache === entry) remoteHealthCache = undefined; - throw error; - } + const shared = sharedRemoteHealth(info, () => readRemoteDaemonHealth(info)); + return await awaitWithinCallerBound(shared, probeTimeoutMs, callerSignal); } diff --git a/packages/daemon-client/src/daemon-lifecycle-errors.ts b/packages/daemon-client/src/daemon-lifecycle-errors.ts index 6ef2fb4d90..640cfceea3 100644 --- a/packages/daemon-client/src/daemon-lifecycle-errors.ts +++ b/packages/daemon-client/src/daemon-lifecycle-errors.ts @@ -66,19 +66,22 @@ export const REMOTE_DAEMON_UNAVAILABLE_REASON = 'remote_daemon_unavailable'; * before running it, and the retry is withheld. So `dispatched: 'no'` states that the command did * not run — what makes a resend safe, for a UI action as much as for a read — and a caller branches * on the reason instead of comparing the message. `attempts` counts the gate's probes, more than - * one when it re-probed within the command's budget. + * one when it re-probed within the command's budget. `healthRecoveryBudgetMs` is present only when + * that budget ran out, which tells a spent budget from a single failed probe. */ export function remoteDaemonUnavailableError(params: { daemonBaseUrl: string | undefined; - attempts?: number; + attempts: number; + healthRecoveryBudgetMs?: number; hint?: string; }): AppError { - const { daemonBaseUrl, attempts, hint } = params; + const { daemonBaseUrl, attempts, healthRecoveryBudgetMs, hint } = params; return new AppError('COMMAND_FAILED', 'Remote daemon is unavailable', { reason: REMOTE_DAEMON_UNAVAILABLE_REASON, dispatched: 'no', daemonBaseUrl, - ...(attempts !== undefined ? { attempts } : {}), + attempts, + ...(healthRecoveryBudgetMs !== undefined ? { healthRecoveryBudgetMs } : {}), ...(hint !== undefined ? { hint } : {}), }); } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index cb789964a2..a576cabe3a 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -460,7 +460,7 @@ agent-device alert dismiss - The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators, including Limrun, the XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If a notification, privacy, or location table cannot be read, the runner drops the permission translations and keeps the English labels; the `Open in …?` table is read apart, so its labels survive that, and the permission translations survive it being unreadable. Physical iOS devices, tvOS, and visionOS always use the English labels. On iOS simulators, including Limrun, `alert get` marks the `Open in …?` launch confirmation with `launchConfirmationPrompt: true`; `open` reads that mark to answer the prompt. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. - On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or localized decline copy outside the preferred languages' prompt strings fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline during action or confirmation after the alert is resolved fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. -- iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); a read-only capture resends it until shortly before the command's own timeout, and `details.attempts` counts the sends; `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. A synthesized long press, `scroll`, `swipe` (sent as a planned fling `gesture`), or other planned `gesture` that the event synthesis refuses before its event record is handed to XCTest reports `COMMAND_FAILED` with `synthesized_gesture_not_dispatched` (`dispatched: no`), and so does a planned `gesture` refused while its window or display is resolved (the window cannot be named, or the sole display cannot be measured); one that fails after the record was handed to XCTest (its events may or may not have run) reports `synthesized_gesture_outcome_unknown` (`dispatched: unknown`) and is never repeated through an XCTest fallback, so check the screen with `snapshot` before you retry. Any other refusal, such as a platform without touch input, keeps `UNSUPPORTED_OPERATION`. A long-press step in a press series reports the same reasons; once an earlier step has run, the series is `dispatched: unknown`. `back_swipe_no_change` means the iOS `back` swipe was sent and the screen did not change, and `back_swipe_unverified` means it was sent but may or may not have navigated (both `dispatched: unknown`); `back_swipe_not_dispatched` means it was refused before any touch went down (`dispatched: no`). A `keyboard dismiss` the keyboard would not come down for because it exposes no dismiss key keeps `UNSUPPORTED_OPERATION` and reports `keyboard_dismiss_unavailable` (`dispatched: no`, nothing was tapped); one whose dismiss key was tapped and left the keyboard visible keeps `UNSUPPORTED_OPERATION` with no `reason` and `dispatched: unknown`. A `keyboard enter` that found no return key and no field to submit through keeps `UNSUPPORTED_OPERATION` and reports `keyboard_return_unavailable` (`dispatched: unknown`, because typing the return can fail after XCTest sent it); a sole text field that was found and tapped but would not take the return keeps `UNSUPPORTED_OPERATION` with no `reason`. A press the runner answers that its target cannot perform — today the Action Button on an iPhone or iPad model that has none, since the simulator volume and non-tvOS remote cases are refused before the runner is reached — keeps `UNSUPPORTED_OPERATION` and reports `hardware_control_unavailable` (`dispatched: no`, nothing was pressed). macOS `back` finding no in-app back control reports `in_app_back_control_unavailable` under `COMMAND_FAILED` (`dispatched: no`). See Back above. +- iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); a read-only capture resends it until shortly before the command's own timeout, and `details.attempts` counts the sends. When that window closes first, the command still fails with `runner_busy` (`dispatched: no`), so retry the read; `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. A synthesized long press, `scroll`, `swipe` (sent as a planned fling `gesture`), or other planned `gesture` that the event synthesis refuses before its event record is handed to XCTest reports `COMMAND_FAILED` with `synthesized_gesture_not_dispatched` (`dispatched: no`), and so does a planned `gesture` refused while its window or display is resolved (the window cannot be named, or the sole display cannot be measured); one that fails after the record was handed to XCTest (its events may or may not have run) reports `synthesized_gesture_outcome_unknown` (`dispatched: unknown`) and is never repeated through an XCTest fallback, so check the screen with `snapshot` before you retry. Any other refusal, such as a platform without touch input, keeps `UNSUPPORTED_OPERATION`. A long-press step in a press series reports the same reasons; once an earlier step has run, the series is `dispatched: unknown`. `back_swipe_no_change` means the iOS `back` swipe was sent and the screen did not change, and `back_swipe_unverified` means it was sent but may or may not have navigated (both `dispatched: unknown`); `back_swipe_not_dispatched` means it was refused before any touch went down (`dispatched: no`). A `keyboard dismiss` the keyboard would not come down for because it exposes no dismiss key keeps `UNSUPPORTED_OPERATION` and reports `keyboard_dismiss_unavailable` (`dispatched: no`, nothing was tapped); one whose dismiss key was tapped and left the keyboard visible keeps `UNSUPPORTED_OPERATION` with no `reason` and `dispatched: unknown`. A `keyboard enter` that found no return key and no field to submit through keeps `UNSUPPORTED_OPERATION` and reports `keyboard_return_unavailable` (`dispatched: unknown`, because typing the return can fail after XCTest sent it); a sole text field that was found and tapped but would not take the return keeps `UNSUPPORTED_OPERATION` with no `reason`. A press the runner answers that its target cannot perform — today the Action Button on an iPhone or iPad model that has none, since the simulator volume and non-tvOS remote cases are refused before the runner is reached — keeps `UNSUPPORTED_OPERATION` and reports `hardware_control_unavailable` (`dispatched: no`, nothing was pressed). macOS `back` finding no in-app back control reports `in_app_back_control_unavailable` under `COMMAND_FAILED` (`dispatched: no`). See Back above. - On the Apple XCUITest runner, an alert search that fails, runs out of time, or finds an app alert, Safari dialog, or dismiss-popup window with no actionable controls fails with `error.code: ALERT_UNRESOLVED` and `details.dispatched: no`. `alert get` reports it at once. `accept`, `dismiss`, and `alert wait` retry it within their windows, like absence, and report it, not a timeout, when the last check remained unresolved. Inspect the prompt with `screenshot`, then `snapshot -i`, and `press @ref` on the visible button. - On the Apple XCUITest runner, `alert get` describes a readable alert even when all its buttons are disabled, and lists those disabled buttons. `alert accept` and `alert dismiss` fail with `ALERT_BUTTON_NOT_FOUND`. - If a sheet or permission prompt is visible but `alert` reports none, `alert` did not recognize it. Run `snapshot -i -s ""`, then `press @ref` on the visible button. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. From ed1937709c3ece77ee225d33e2b0360319a61da3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 12:25:09 +0100 Subject: [PATCH 4/9] test(daemon-client): name the health budget test for its recovery module Co-Authored-By: Claude Opus 5.5 --- ...est.ts => daemon-client-remote-health-recovery-budget.test.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/daemon-client/src/__tests__/{daemon-client-remote-health-budget.test.ts => daemon-client-remote-health-recovery-budget.test.ts} (100%) diff --git a/packages/daemon-client/src/__tests__/daemon-client-remote-health-budget.test.ts b/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts similarity index 100% rename from packages/daemon-client/src/__tests__/daemon-client-remote-health-budget.test.ts rename to packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts From 361c9161e2773b31951520e2eaa78a27f9c37ed9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 12:35:19 +0100 Subject: [PATCH 5/9] fix(daemon-client): keep the health type on the wire surface and simplify the health cache - RemoteDaemonHealth and RemoteDaemonHealthLink are declared in daemon-client-transport.ts again, where the wire-compat manifest expects them. - The remote health cache module loads lazily on the remote path, so the CLI and client entries evaluate no extra module. - The cache shares one in-flight probe among unbounded callers, as on main. A bounded caller reads only a settled answer, or runs its own probe under its own timeout and signal. It never joins another caller's probe, so a stale probe cannot spend its budget, nothing is left unobserved, and an aborted or budgetless caller sends no request. - The budget test asserts 90 probes exactly one second apart on the faked clock. Co-Authored-By: Claude Opus 5.5 --- ...mon-client-lifecycle-remote-health.test.ts | 58 +++++++--- ...ient-remote-health-recovery-budget.test.ts | 26 ++--- .../src/daemon-client-remote-health-cache.ts | 100 ++++++------------ .../daemon-client-remote-health-recovery.ts | 3 +- .../src/daemon-client-transport.ts | 56 ++++++---- 5 files changed, 126 insertions(+), 117 deletions(-) diff --git a/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts b/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts index a49e9cf20d..58383a9b95 100644 --- a/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts +++ b/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts @@ -148,10 +148,12 @@ test('a remote press whose health probe answers 4xx is sent and is not the unava } }); -// Concurrent callers share one in-flight probe, and each applies its own bound to it: one caller -// aborting does not end the wait of another that shares the probe. -test('a caller aborting a shared health probe leaves a concurrent caller its answer', async (t) => { - if (await skipWhenLoopbackUnavailable(t)) return; +/** A remote daemon whose `/health` answers 200 with an instance id after `delayMs`, counting probes. */ +async function slowHealthyRemote(delayMs: number): Promise<{ + handler: http.Server; + info: { baseUrl: string; token: string; pid: number }; + probes: () => number; +}> { let probes = 0; const handler = http.createServer((req, res) => { req.resume(); @@ -159,22 +161,46 @@ test('a caller aborting a shared health probe leaves a concurrent caller its ans setTimeout(() => { res.statusCode = 200; res.end(JSON.stringify({ instanceId: 'remote-instance' })); - }, 200); + }, delayMs); }); const port = await listenOnLoopback(handler); - const info = { baseUrl: `http://127.0.0.1:${port}`, token: 'shared-probe', pid: 0 }; + const info = { baseUrl: `http://127.0.0.1:${port}`, token: `probe-${port}`, pid: 0 }; + return { handler, info, probes: () => probes }; +} + +// A bounded caller never joins a probe another caller started: that probe runs on another budget, +// so waiting on it would spend this caller's budget on a stale request. +test('a bounded health caller runs its own probe instead of joining one in flight', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const remote = await slowHealthyRemote(300); try { - const aborting = new AbortController(); - const aborted = cachedRemoteDaemonHealth(info, undefined, aborting.signal); - const unbounded = cachedRemoteDaemonHealth(info); - aborting.abort(); + const unbounded = cachedRemoteDaemonHealth(remote.info); + const bounded = await cachedRemoteDaemonHealth(remote.info, 50); - assert.equal((await aborted).reachable, false); - const health = await unbounded; - assert.equal(health.reachable, true); - assert.equal(health.instanceId, 'remote-instance'); - assert.equal(probes, 1, 'both callers shared one probe'); + assert.equal(bounded.reachable, false); + assert.equal(bounded.timedOut, true); + assert.equal((await unbounded).reachable, true); + assert.equal(remote.probes(), 2, 'the bounded caller sent its own probe'); + // The settled answer is cached, so a later bounded caller reads it without probing. + assert.equal((await cachedRemoteDaemonHealth(remote.info, 50)).reachable, true); + assert.equal(remote.probes(), 2); } finally { - await closeLoopbackServer(handler); + await closeLoopbackServer(remote.handler); + } +}); + +// A caller already aborted, or with no budget left, sends no request at all. +test('an aborted or budgetless health caller sends no probe', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const remote = await slowHealthyRemote(0); + try { + assert.equal((await cachedRemoteDaemonHealth(remote.info, 0)).reachable, false); + assert.equal( + (await cachedRemoteDaemonHealth(remote.info, undefined, AbortSignal.abort())).reachable, + false, + ); + assert.equal(remote.probes(), 0); + } finally { + await closeLoopbackServer(remote.handler); } }); diff --git a/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts b/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts index be3a97db7f..3ffc631cad 100644 --- a/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts +++ b/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts @@ -9,9 +9,9 @@ import { afterEach, test, vi } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import { sendToDaemon } from '../index.ts'; -/** Replaces `http.request` with one that answers every request 503, counting them by method. */ -function answerEveryRequest503(): { probes: () => number; rpcs: () => number } { - let probes = 0; +/** Replaces `http.request` with one that answers every request 503, recording each probe's time. */ +function answerEveryRequest503(): { probeTimes: number[]; rpcs: () => number } { + const probeTimes: number[] = []; let rpcs = 0; vi.spyOn(http, 'request').mockImplementation((( options: { method?: string }, @@ -21,7 +21,7 @@ function answerEveryRequest503(): { probes: () => number; rpcs: () => number } { write: () => {}, destroy: () => {}, end: () => { - if (options.method === 'GET') probes += 1; + if (options.method === 'GET') probeTimes.push(performance.now()); else rpcs += 1; const res = Object.assign(new EventEmitter(), { statusCode: 503, @@ -36,7 +36,7 @@ function answerEveryRequest503(): { probes: () => number; rpcs: () => number } { }); return req; }) as unknown as typeof http.request); - return { probes: () => probes, rpcs: () => rpcs }; + return { probeTimes, rpcs: () => rpcs }; } afterEach(() => { @@ -48,10 +48,13 @@ test('a remote daemon answering 5xx for the whole request budget is refused with const server = answerEveryRequest503(); vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }); let advancing = true; + // Fake time moves only once real work (module loads, the mocked reply) has gone quiet, and then + // straight to the next timer, so no real latency lands on the faked clock. const advance = (async () => { while (advancing) { - await new Promise((resolve) => setImmediate(resolve)); - await vi.advanceTimersByTimeAsync(100); + for (let turn = 0; turn < 20; turn += 1) + await new Promise((resolve) => setImmediate(resolve)); + if (vi.getTimerCount() > 0) await vi.advanceTimersToNextTimerAsync(); } })(); @@ -75,13 +78,12 @@ test('a remote daemon answering 5xx for the whole request budget is refused with assert.equal(outcome.code, 'COMMAND_FAILED'); assert.equal(outcome.details?.reason, 'remote_daemon_unavailable'); assert.equal(outcome.details?.dispatched, 'no'); - // At most one probe a second fits the 90s budget, and far more than a fixed resend count. Real - // module loading can let fake time run ahead, so the exact count is not asserted. - const attempts = Number(outcome.details?.attempts); - assert.ok(attempts > 8 && attempts <= 90, `unexpected attempts: ${attempts}`); + // One probe a second fills the 90s budget. + assert.equal(outcome.details?.attempts, 90); assert.equal(outcome.details?.healthRecoveryBudgetMs, 90_000); assert.match(String(outcome.details?.hint ?? ''), /agent-device disconnect/); - assert.equal(server.probes(), attempts); + const gaps = server.probeTimes.slice(1).map((at, index) => at - server.probeTimes[index]!); + assert.deepEqual(gaps, Array(89).fill(1_000), 'probes are one second apart'); assert.equal(server.rpcs(), 0); } finally { advancing = false; diff --git a/packages/daemon-client/src/daemon-client-remote-health-cache.ts b/packages/daemon-client/src/daemon-client-remote-health-cache.ts index 8389d6afb7..b7c4e9741e 100644 --- a/packages/daemon-client/src/daemon-client-remote-health-cache.ts +++ b/packages/daemon-client/src/daemon-client-remote-health-cache.ts @@ -1,29 +1,13 @@ import type { DaemonInfo } from './daemon-client-metadata.ts'; - -export type RemoteDaemonHealth = { - reachable: boolean; - statusCode?: number; - service?: string; - version?: string; - rpcProtocolVersion?: number; - instanceId?: string; - hostArch?: string; - /** The daemon behind a proxy, as the proxy's health reported it. */ - upstream?: RemoteDaemonHealthLink; - /** The probe ran out of its time budget before an answer, rather than failing outright. */ - timedOut?: true; -}; - -export type RemoteDaemonHealthLink = Pick< - RemoteDaemonHealth, - 'service' | 'version' | 'rpcProtocolVersion' | 'instanceId' | 'hostArch' ->; +import type { RemoteDaemonHealth } from './daemon-client-transport.ts'; type RemoteHealthCacheEntry = { baseUrl: string; token: string; pid: number; health: Promise; + /** The settled answer, once the probe behind `health` has answered. */ + answer?: RemoteDaemonHealth; }; let remoteHealthCache: RemoteHealthCacheEntry | undefined; @@ -45,6 +29,7 @@ export function cacheRemoteDaemonHealth(info: DaemonInfo, health: RemoteDaemonHe token: info.token, pid: info.pid, health: Promise.resolve(health), + answer: health, }; } @@ -54,63 +39,42 @@ export function invalidateRemoteDaemonHealth(info: DaemonInfo): void { } } -/** The in-flight or cached `/health` answer for this daemon, starting `probe` when there is none. */ -export function sharedRemoteHealth( +/** + * The remote `/health` answer through the process cache. Unbounded callers share one in-flight + * probe. A caller with its own bound (`bounded`) reads only a settled answer and otherwise runs + * `probe` itself, so it never waits on a probe that another caller started under a different + * budget or signal. + */ +export async function cachedRemoteHealth( info: DaemonInfo, + bounded: boolean, probe: () => Promise, ): Promise { - if (remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info)) { - return remoteHealthCache.health; + const cached = + remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info) + ? remoteHealthCache + : undefined; + if (cached?.answer) return cached.answer; + if (bounded) { + const health = await probe(); + cacheRemoteDaemonHealth(info, health); + return health; } - const drop = () => { - if (remoteHealthCache === entry) remoteHealthCache = undefined; - }; + if (cached) return await cached.health; const entry: RemoteHealthCacheEntry = { baseUrl: info.baseUrl ?? '', token: info.token, pid: info.pid, - health: probe().then( - (health) => { - if (!isCacheableRemoteHealth(health)) drop(); - return health; - }, - (error: unknown) => { - drop(); - throw error; - }, - ), + health: probe(), }; remoteHealthCache = entry; - return entry.health; -} - -/** A caller's view of a shared probe: unanswered once its own time is up, or once it aborts. */ -export async function awaitWithinCallerBound( - health: Promise, - timeoutMs: number | undefined, - signal: AbortSignal | undefined, -): Promise { - if (timeoutMs === undefined && !signal) return await health; - if (signal?.aborted) return { reachable: false }; - if (timeoutMs !== undefined && timeoutMs <= 0) return { reachable: false, timedOut: true }; - return await new Promise((resolve, reject) => { - const timer = - timeoutMs === undefined - ? undefined - : setTimeout(() => settle({ reachable: false, timedOut: true }), Math.ceil(timeoutMs)); - const onAbort = () => settle({ reachable: false }); - signal?.addEventListener('abort', onAbort, { once: true }); - function release(): void { - clearTimeout(timer); - signal?.removeEventListener('abort', onAbort); - } - function settle(result: RemoteDaemonHealth): void { - release(); - resolve(result); - } - health.then(settle, (error: unknown) => { - release(); - reject(error); - }); - }); + try { + const health = await entry.health; + if (isCacheableRemoteHealth(health)) entry.answer = health; + else if (remoteHealthCache === entry) remoteHealthCache = undefined; + return health; + } catch (error) { + if (remoteHealthCache === entry) remoteHealthCache = undefined; + throw error; + } } diff --git a/packages/daemon-client/src/daemon-client-remote-health-recovery.ts b/packages/daemon-client/src/daemon-client-remote-health-recovery.ts index 2a9f38441d..658541eeaa 100644 --- a/packages/daemon-client/src/daemon-client-remote-health-recovery.ts +++ b/packages/daemon-client/src/daemon-client-remote-health-recovery.ts @@ -1,8 +1,7 @@ import { sleep } from '@agent-device/host-kit/retry'; import type { RemoteHealthRecovery } from './daemon-client-lifecycle.ts'; import type { DaemonInfo } from './daemon-client-metadata.ts'; -import type { RemoteDaemonHealth } from './daemon-client-remote-health-cache.ts'; -import { cachedRemoteDaemonHealth } from './daemon-client-transport.ts'; +import { cachedRemoteDaemonHealth, type RemoteDaemonHealth } from './daemon-client-transport.ts'; import { remoteDaemonUnavailableError } from './daemon-lifecycle-errors.ts'; /** The pause between `/health` probes while a remote daemon is unavailable. */ diff --git a/packages/daemon-client/src/daemon-client-transport.ts b/packages/daemon-client/src/daemon-client-transport.ts index f1227909d8..c3829f42e8 100644 --- a/packages/daemon-client/src/daemon-client-transport.ts +++ b/packages/daemon-client/src/daemon-client-transport.ts @@ -28,14 +28,6 @@ import { refuseOversizedDaemonRequest, } from './daemon-client-rpc.ts'; import { handleRequestTimeout } from './daemon-client-timeout.ts'; -import { - awaitWithinCallerBound, - cacheRemoteDaemonHealth, - invalidateRemoteDaemonHealth, - sharedRemoteHealth, - type RemoteDaemonHealth, - type RemoteDaemonHealthLink, -} from './daemon-client-remote-health-cache.ts'; import { isRemoteDaemon, type DaemonInfo } from './daemon-client-metadata.ts'; import type { HostEnvironment } from '@agent-device/kernel/source-value'; import { readVersion } from '@agent-device/host-kit/version'; @@ -169,6 +161,25 @@ function daemonEndpointUnavailableError(transport: ResolvedDaemonTransport): App ); } +export type RemoteDaemonHealth = { + reachable: boolean; + statusCode?: number; + service?: string; + version?: string; + rpcProtocolVersion?: number; + instanceId?: string; + hostArch?: string; + /** The daemon behind a proxy, as the proxy's health reported it. */ + upstream?: RemoteDaemonHealthLink; + /** The probe ran out of its time budget before an answer, rather than failing outright. */ + timedOut?: true; +}; + +type RemoteDaemonHealthLink = Pick< + RemoteDaemonHealth, + 'service' | 'version' | 'rpcProtocolVersion' | 'instanceId' | 'hostArch' +>; + export async function canConnect( info: DaemonInfo, preference: DaemonTransportPreference, @@ -457,7 +468,7 @@ async function sendRequestWithFallback( options, ); } - if (isRemoteTransportFailure(error)) invalidateRemoteDaemonHealth(info); + if (isRemoteTransportFailure(error)) await invalidateRemoteDaemonHealth(info); const fallback = chooseAutoFallbackTransport(info, preference, transport); if (!fallback || !isSafeAutoTransportFallbackError(error, transport)) throw error; return await sendRequestWithTransport(info, req, statePaths, timeoutMs, fallback, options); @@ -473,7 +484,7 @@ async function retryAfterRemoteInstanceMismatch( transport: ResolvedDaemonTransport, options: SendRequestOptions, ): Promise { - invalidateRemoteDaemonHealth(info); + await invalidateRemoteDaemonHealth(info); const probeTimeoutMs = remainingRemoteRequestTimeoutMs(req, timeoutMs, deadline); const health = await readRemoteDaemonHealth(info, probeTimeoutMs, options.signal); refuseAbortedRequest(options.signal, req.meta?.requestId); @@ -489,11 +500,11 @@ async function retryAfterRemoteInstanceMismatch( } info.remoteInstanceId = health.instanceId; info.remoteUpstreamInstanceId = health.upstream?.instanceId; - cacheRemoteDaemonHealth(info, health); + (await loadRemoteHealthCache()).cacheRemoteDaemonHealth(info, health); try { return await sendRequestWithTransport(info, req, statePaths, remainingMs, transport, options); } catch (error) { - if (isRemoteTransportFailure(error)) invalidateRemoteDaemonHealth(info); + if (isRemoteTransportFailure(error)) await invalidateRemoteDaemonHealth(info); throw error; } } @@ -925,16 +936,23 @@ async function sendHttpRequest( }); } -/** - * The remote `/health` answer, sharing one in-flight probe among concurrent callers. The shared - * probe runs on the default budget; each caller applies its own `probeTimeoutMs` and - * `callerSignal` to its wait, so no caller inherits another's bound or abort. - */ +// Loaded on the remote path only: a local daemon request never pays for the remote health cache. +const loadRemoteHealthCache = () => import('./daemon-client-remote-health-cache.ts'); + +async function invalidateRemoteDaemonHealth(info: DaemonInfo): Promise { + (await loadRemoteHealthCache()).invalidateRemoteDaemonHealth(info); +} + +/** The remote `/health` answer within this caller's bound, through the process cache. */ export async function cachedRemoteDaemonHealth( info: DaemonInfo, probeTimeoutMs?: number, callerSignal?: AbortSignal, ): Promise { - const shared = sharedRemoteHealth(info, () => readRemoteDaemonHealth(info)); - return await awaitWithinCallerBound(shared, probeTimeoutMs, callerSignal); + const { cachedRemoteHealth } = await loadRemoteHealthCache(); + return await cachedRemoteHealth( + info, + probeTimeoutMs !== undefined || callerSignal !== undefined, + () => readRemoteDaemonHealth(info, probeTimeoutMs, callerSignal), + ); } From 7841ed2f36950ae7b05588c1ec7f0d1c6a99ec9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 12:42:11 +0100 Subject: [PATCH 6/9] fix(daemon-client): invalidate cached health only for a remote daemon A failed local request no longer loads the remote health cache: invalidation returns at once when the daemon has no base URL. A transport test pins both sides. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/daemon-client-transport.test.ts | 40 +++++++++++++++++++ .../src/daemon-client-transport.ts | 2 + 2 files changed, 42 insertions(+) diff --git a/packages/daemon-client/src/__tests__/daemon-client-transport.test.ts b/packages/daemon-client/src/__tests__/daemon-client-transport.test.ts index 4061079136..0d3937dc62 100644 --- a/packages/daemon-client/src/__tests__/daemon-client-transport.test.ts +++ b/packages/daemon-client/src/__tests__/daemon-client-transport.test.ts @@ -30,6 +30,13 @@ import { import { REMOTE_DAEMON_UNAVAILABLE_REASON } from '../daemon-lifecycle-errors.ts'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import { createDaemonProxyServer } from '@agent-device/proxy'; +import { invalidateRemoteDaemonHealth } from '../daemon-client-remote-health-cache.ts'; + +// Calls through to the real cache, so only the invalidation test reads the spy. +vi.mock('../daemon-client-remote-health-cache.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, invalidateRemoteDaemonHealth: vi.fn(actual.invalidateRemoteDaemonHealth) }; +}); function sendWithStaleInstance(port: number, timeoutMs: number) { return sendRequest( @@ -621,3 +628,36 @@ test.for(['timeout', 'reset'] as const)( } }, ); + +// A transport failure drops the cached health of a remote daemon only: a local daemon has no cached +// health, so its failed request neither invalidates nor loads the remote health cache. +test('a transport failure invalidates cached health for a remote daemon only', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const server = http.createServer((req) => req.socket.destroy()); + const port = await listenOnLoopback(server); + const invalidate = vi.mocked(invalidateRemoteDaemonHealth); + invalidate.mockClear(); + const send = (info: Parameters[0]) => + sendRequest( + info, + { token: 'secret', command: 'devices', session: 'default', positionals: [], flags: {} }, + 'http', + resolveDaemonPaths('/tmp/agent-device-health-invalidation-test'), + 2_000, + { host: createTestHostEnvironment(), timeoutRecovery: 'none' }, + ).then( + () => undefined, + (error: unknown) => error, + ); + try { + const local = await send({ token: 'secret', pid: 1, transport: 'http', httpPort: port }); + assert.ok(local instanceof AppError, `unexpected outcome: ${String(local)}`); + assert.equal(local.details?.reason, 'daemon_transport_failure'); + assert.equal(invalidate.mock.calls.length, 0, 'a local failure leaves the remote cache alone'); + + await send({ token: 'secret', pid: 1, baseUrl: `http://127.0.0.1:${port}` }); + assert.equal(invalidate.mock.calls.length, 1); + } finally { + await closeLoopbackServer(server); + } +}); diff --git a/packages/daemon-client/src/daemon-client-transport.ts b/packages/daemon-client/src/daemon-client-transport.ts index c3829f42e8..a12830764c 100644 --- a/packages/daemon-client/src/daemon-client-transport.ts +++ b/packages/daemon-client/src/daemon-client-transport.ts @@ -939,7 +939,9 @@ async function sendHttpRequest( // Loaded on the remote path only: a local daemon request never pays for the remote health cache. const loadRemoteHealthCache = () => import('./daemon-client-remote-health-cache.ts'); +/** Drops a remote daemon's cached health; a local daemon has none, so its failure loads nothing. */ async function invalidateRemoteDaemonHealth(info: DaemonInfo): Promise { + if (!info.baseUrl) return; (await loadRemoteHealthCache()).invalidateRemoteDaemonHealth(info); } From 57ad1ba27ae2f22cecff1d6f7b3ca593bc3042b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 13:06:44 +0100 Subject: [PATCH 7/9] fix(recovery): spend a busy read's deadline only when the request itself is read-only A read-only runner step inside an action (the viewport read before a swipe, the selector query before a press) runs under the action's request id, so the busy window could spend the budget the action needs and turn a fast RUNNER_BUSY into a dispatch with no time left. The daemon now stamps whether each admitted request is read-only (isReadOnlyRequest, from the descriptor's recording effect), host-kit answers getRequestReadOnlyDeadlineMs only for such requests, and the runner reads that through its host port. Reads inside actions keep the 8-attempt window. The remote health cache now holds only settled usable answers; each caller probes under its own bound. Adds an abort-while-re-probing test and documents the probe timing, the wall-time bound and healthRecoveryBudgetMs. Co-Authored-By: Claude Opus 5.5 --- .../src/__tests__/index-busy-resend.test.ts | 59 ++++++++++--- packages/apple-runner/src/host.ts | 7 +- packages/apple-runner/src/index.ts | 12 +-- ...mon-client-lifecycle-remote-health.test.ts | 8 +- ...ient-remote-health-recovery-budget.test.ts | 86 +++++++++++++------ .../src/daemon-client-remote-health-cache.ts | 47 ++-------- .../src/daemon-client-transport.ts | 6 +- .../__tests__/daemon-command-registry.test.ts | 28 ++++++ .../src/core/daemon-command-registry.ts | 13 ++- .../daemon/src/server/http/http-server.ts | 6 +- packages/daemon/src/server/http/transport.ts | 6 +- .../src/internal/request-cancel.test.ts | 23 +++++ .../host-kit/src/internal/request-cancel.ts | 17 +++- packages/host-kit/src/request.ts | 1 + .../platform-apple/src/core/runner-host.ts | 4 +- .../test-kit/src/host-kit-capabilities.ts | 2 +- website/docs/docs/commands.md | 4 +- 17 files changed, 226 insertions(+), 103 deletions(-) diff --git a/packages/apple-runner/src/__tests__/index-busy-resend.test.ts b/packages/apple-runner/src/__tests__/index-busy-resend.test.ts index 9843867fad..bae28deec3 100644 --- a/packages/apple-runner/src/__tests__/index-busy-resend.test.ts +++ b/packages/apple-runner/src/__tests__/index-busy-resend.test.ts @@ -114,17 +114,23 @@ test('a RUNNER_BUSY resend still ends when the refusals outlast the window', asy assert.deepEqual(sentCommands(), Array(8).fill('snapshot')); }); -/** Runs the request under a daemon deadline `timeoutMs` from now, as `withRequestDeadline` sets it. */ -function underRequestDeadline(timeoutMs: number): void { +/** + * Admits `requestId` with a deadline `timeoutMs` from now. The host answers that deadline to a + * resend loop only when the request itself is read-only, as host-kit's + * `getRequestReadOnlyDeadlineMs` does for the fact the daemon stamps at admission. + */ +function admitRequest(requestId: string, timeoutMs: number, readOnly: boolean): string { const deadlineMs = Date.now() + timeoutMs; - appleRunnerTestHost.update({ getRequestDeadlineMs: () => deadlineMs }); + appleRunnerTestHost.update({ + getRequestReadOnlyDeadlineMs: (id) => (id === requestId && readOnly ? deadlineMs : undefined), + }); + return requestId; } test('under a request deadline, read-only commands outlast the fixed RUNNER_BUSY count', async () => { // Twelve busy answers, four past the fixed eight-attempt window, then the runner answers inside // the request's 30s deadline. vi.useFakeTimers(); - underRequestDeadline(30_000); for (let refusal = 0; refusal < 12; refusal += 1) { mockExecuteRunnerCommandWithSession.mockRejectedValueOnce(busyRefusal()); } @@ -133,7 +139,7 @@ test('under a request deadline, read-only commands outlast the fixed RUNNER_BUSY const pending = runAppleRunnerCommand( IOS_SIMULATOR, { command: 'snapshot' }, - { requestId: 'req-deadline-recovers' }, + { requestId: admitRequest('req-deadline-recovers', 30_000, true) }, ); await vi.advanceTimersByTimeAsync(20_000); @@ -143,7 +149,6 @@ test('under a request deadline, read-only commands outlast the fixed RUNNER_BUSY test('a request deadline reached mid-outage ends with the RUNNER_BUSY refusal and its attempt count', async () => { vi.useFakeTimers(); - underRequestDeadline(10_000); const sentAt: number[] = []; mockExecuteRunnerCommandWithSession.mockImplementation(async () => { sentAt.push(Date.now()); @@ -154,7 +159,7 @@ test('a request deadline reached mid-outage ends with the RUNNER_BUSY refusal an const settled = runAppleRunnerCommand( IOS_SIMULATOR, { command: 'snapshot' }, - { requestId: 'req-deadline-outage' }, + { requestId: admitRequest('req-deadline-outage', 10_000, true) }, ).then( () => undefined, (error: unknown) => error, @@ -195,7 +200,6 @@ test('read-only transport failures keep the three-attempt resend budget', async test('a RUNNER_BUSY refusal after the busy window closed is not resent', async () => { // A 1s request deadline is inside the 2s reserve, so the window is already closed. - underRequestDeadline(1_000); mockExecuteRunnerCommandWithSession .mockRejectedValueOnce(busyRefusal()) .mockResolvedValue({ nodes: [], truncated: false }); @@ -205,7 +209,7 @@ test('a RUNNER_BUSY refusal after the busy window closed is not resent', async ( runAppleRunnerCommand( IOS_SIMULATOR, { command: 'snapshot' }, - { requestId: 'req-deadline-window-closed' }, + { requestId: admitRequest('req-deadline-window-closed', 1_000, true) }, ), (error: unknown) => { assert.ok(error instanceof AppError); @@ -219,7 +223,6 @@ test('a RUNNER_BUSY refusal after the busy window closed is not resent', async ( }); test('under a request deadline too close for a busy resend, transport failures keep their three attempts', async () => { - underRequestDeadline(1_000); mockExecuteRunnerCommandWithSession .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed')) .mockResolvedValueOnce({ lifecycleState: 'started' }) @@ -230,7 +233,7 @@ test('under a request deadline too close for a busy resend, transport failures k const result = await runAppleRunnerCommand( IOS_SIMULATOR, { command: 'snapshot' }, - { requestId: 'req-deadline-transport' }, + { requestId: admitRequest('req-deadline-transport', 1_000, true) }, ); assert.deepEqual(result, { nodes: [], truncated: false }); @@ -329,11 +332,15 @@ test('a mutating command meets RUNNER_BUSY once, with no status probe and no res // The status-recovery bypass for structured replies reaches mutating commands too: the refusal // says the tap did not run, so there is nothing to recover and nothing is replayed, even with a // request deadline that would let a read resend. - underRequestDeadline(30_000); mockExecuteRunnerCommandWithSession.mockRejectedValueOnce(busyRefusal()); await assert.rejects( - () => runAppleRunnerCommand(IOS_SIMULATOR, { command: 'tap', x: 120, y: 240 }), + () => + runAppleRunnerCommand( + IOS_SIMULATOR, + { command: 'tap', x: 120, y: 240 }, + { requestId: admitRequest('req-tap-read-only-window', 30_000, true) }, + ), (error: unknown) => { assert.ok(error instanceof AppError); assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY'); @@ -346,6 +353,32 @@ test('a mutating command meets RUNNER_BUSY once, with no status probe and no res assert.equal(mockInvalidateRunnerSession.mock.calls.length, 0); }); +// The viewport read before a swipe runs under the swipe's request. That budget belongs to the swipe +// dispatched after it, so a busy drain keeps the fixed window and refuses fast with nothing sent. +test('a busy read inside a mutating request keeps the fixed window under a 30s deadline', async () => { + vi.useFakeTimers(); + mockExecuteRunnerCommandWithSession.mockImplementation(async () => { + throw busyRefusal(); + }); + + const settled = runAppleRunnerCommand( + IOS_SIMULATOR, + { command: 'gestureViewport', appBundleId: 'com.example.app' }, + { requestId: admitRequest('req-swipe', 30_000, false) }, + ).then( + () => undefined, + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(30_000); + const error = await settled; + + assert.ok(error instanceof AppError, `expected the refusal, got ${String(error)}`); + assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY'); + assert.equal(error.details?.dispatched, 'no'); + assert.equal(error.details?.attempts, 8); + assert.deepEqual(sentCommands(), Array(8).fill('gestureViewport')); +}); + test('a read-only alert get answered ALERT_UNRESOLVED is sent once and reported as itself', async () => { // A structured reply proves the runner answered, so the read resend reserved for RUNNER_BUSY and // transport failures must not spend it: the prompt it saw is reported, never retried as absence. diff --git a/packages/apple-runner/src/host.ts b/packages/apple-runner/src/host.ts index b8ba04ed9c..f4b90cef40 100644 --- a/packages/apple-runner/src/host.ts +++ b/packages/apple-runner/src/host.ts @@ -112,7 +112,10 @@ export type AppleRunnerHost = Pick< Pick & Pick< typeof HostRequest, - 'emitRequestProgress' | 'getRequestDeadlineMs' | 'getRequestSignal' | 'isRequestCanceled' + | 'emitRequestProgress' + | 'getRequestReadOnlyDeadlineMs' + | 'getRequestSignal' + | 'isRequestCanceled' > & Pick & Pick & @@ -210,7 +213,7 @@ export const acquireProcessLock = delegate('acquireProcessLock'); export const withProcessLock = delegate('withProcessLock'); export const publishFileSync = delegate('publishFileSync'); export const emitRequestProgress = delegate('emitRequestProgress'); -export const getRequestDeadlineMs = delegate('getRequestDeadlineMs'); +export const getRequestReadOnlyDeadlineMs = delegate('getRequestReadOnlyDeadlineMs'); export const getRequestSignal = delegate('getRequestSignal'); export const isRequestCanceled = delegate('isRequestCanceled'); export const withKeyedLock = delegate('withKeyedLock'); diff --git a/packages/apple-runner/src/index.ts b/packages/apple-runner/src/index.ts index 916017e4e7..5a0c64b81f 100644 --- a/packages/apple-runner/src/index.ts +++ b/packages/apple-runner/src/index.ts @@ -93,7 +93,7 @@ import { Deadline, retryWithPolicy, emitDiagnostic, - getRequestDeadlineMs, + getRequestReadOnlyDeadlineMs, hasScopedAppleToolProvider, } from './host.ts'; import { AppError } from '@agent-device/kernel/errors'; @@ -135,9 +135,11 @@ import { RUNNER_COMMAND_TIMEOUT_MS } from './runner-transport.ts'; /** * Attempts a read-only command may spend on one error class; 1 means no resend. A `RUNNER_BUSY` * refusal is the runner refusing fast on purpose while it drains abandoned XCTest work (#1105), so - * that budget has to outlast the drain. Under a request deadline, busy refusals are resent until - * {@link RUNNER_BUSY_DEADLINE_RESERVE_MS} before it, so the last refusal (with its attempt count) is - * the answer instead of the deadline's. Without one, the window is a heuristic: 200ms doubling to a + * that budget has to outlast the drain. When the request itself is read-only, busy refusals are + * resent until {@link RUNNER_BUSY_DEADLINE_RESERVE_MS} before its deadline, so the last refusal (with + * its attempt count) is the answer instead of the deadline's. A read inside an action (the viewport + * read before a swipe, the selector query before a press) keeps the fixed window instead, because + * the action dispatched after it needs the rest of the budget. The fixed window is a heuristic: 200ms doubling to a * 1s cap, no jitter, is 5.4s of delay across eight attempts before round trips, and a drain that * outlives it still surfaces as `RUNNER_BUSY` (the runner's own `abandonedForSeconds=` marker in * runner.log is the evidence for tuning it). Transport failures keep the pre-existing three attempts @@ -163,7 +165,7 @@ type ReadOnlyResendBudget = { }; function resolveReadOnlyResendBudget(requestId: string | undefined): ReadOnlyResendBudget { - const requestDeadlineMs = getRequestDeadlineMs(requestId); + const requestDeadlineMs = getRequestReadOnlyDeadlineMs(requestId); const resendsOther = (error: unknown, attempt: number) => attempt < (isRetryableRunnerError(error) ? TRANSPORT_RESEND_ATTEMPTS : 1); if (requestDeadlineMs === undefined) { diff --git a/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts b/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts index 58383a9b95..bfdd34eb85 100644 --- a/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts +++ b/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts @@ -168,9 +168,9 @@ async function slowHealthyRemote(delayMs: number): Promise<{ return { handler, info, probes: () => probes }; } -// A bounded caller never joins a probe another caller started: that probe runs on another budget, -// so waiting on it would spend this caller's budget on a stale request. -test('a bounded health caller runs its own probe instead of joining one in flight', async (t) => { +// Each caller probes under its own bound; only a settled, usable answer is shared through the cache, +// so no caller waits on a probe started under another caller's budget. +test('a health caller probes on its own bound, and a settled answer is cached', async (t) => { if (await skipWhenLoopbackUnavailable(t)) return; const remote = await slowHealthyRemote(300); try { @@ -180,7 +180,7 @@ test('a bounded health caller runs its own probe instead of joining one in fligh assert.equal(bounded.reachable, false); assert.equal(bounded.timedOut, true); assert.equal((await unbounded).reachable, true); - assert.equal(remote.probes(), 2, 'the bounded caller sent its own probe'); + assert.equal(remote.probes(), 2, 'each caller sent its own probe'); // The settled answer is cached, so a later bounded caller reads it without probing. assert.equal((await cachedRemoteDaemonHealth(remote.info, 50)).reachable, true); assert.equal(remote.probes(), 2); diff --git a/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts b/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts index 3ffc631cad..2544568719 100644 --- a/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts +++ b/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts @@ -6,7 +6,7 @@ import assert from 'node:assert/strict'; import http from 'node:http'; import { EventEmitter } from 'node:events'; import { afterEach, test, vi } from 'vitest'; -import { AppError } from '@agent-device/kernel/errors'; +import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; import { sendToDaemon } from '../index.ts'; /** Replaces `http.request` with one that answers every request 503, recording each probe's time. */ @@ -39,6 +39,38 @@ function answerEveryRequest503(): { probeTimes: number[]; rpcs: () => number } { return { probeTimes, rpcs: () => rpcs }; } +/** + * Moves fake time only once real work (module loads, the mocked reply) has gone quiet, and then + * straight to the next timer, so no real latency lands on the faked clock. Returns the stop. + */ +function driveFakeTime(): () => Promise { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }); + let driving = true; + const drive = (async () => { + while (driving) { + for (let turn = 0; turn < 20; turn += 1) { + await new Promise((resolve) => setImmediate(resolve)); + } + if (vi.getTimerCount() > 0) await vi.advanceTimersToNextTimerAsync(); + } + })(); + return async () => { + driving = false; + await drive; + }; +} + +/** `session list`, which has the 90s default budget, against a remote daemon. */ +function remoteSessionList() { + return { + session: 'default', + command: 'session', + positionals: ['list'], + flags: { daemonBaseUrl: 'http://remote-mac.example.test:7777/agent-device' }, + meta: { requestId: 'req-remote-budget' }, + }; +} + afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); @@ -46,30 +78,11 @@ afterEach(() => { test('a remote daemon answering 5xx for the whole request budget is refused with its probe count and budget', async () => { const server = answerEveryRequest503(); - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }); - let advancing = true; - // Fake time moves only once real work (module loads, the mocked reply) has gone quiet, and then - // straight to the next timer, so no real latency lands on the faked clock. - const advance = (async () => { - while (advancing) { - for (let turn = 0; turn < 20; turn += 1) - await new Promise((resolve) => setImmediate(resolve)); - if (vi.getTimerCount() > 0) await vi.advanceTimersToNextTimerAsync(); - } - })(); + const stopDriving = driveFakeTime(); try { - const outcome = await sendToDaemon( - { - session: 'default', - command: 'session', - positionals: ['list'], - flags: { daemonBaseUrl: 'http://remote-mac.example.test:7777/agent-device' }, - meta: { requestId: 'req-remote-budget' }, - }, - // A non-loopback daemon is trusted only with an auth token. - { authToken: 'remote-secret' }, - ).then( + // A non-loopback daemon is trusted only with an auth token. + const outcome = await sendToDaemon(remoteSessionList(), { authToken: 'remote-secret' }).then( () => undefined, (error: unknown) => error, ); @@ -86,7 +99,30 @@ test('a remote daemon answering 5xx for the whole request budget is refused with assert.deepEqual(gaps, Array(89).fill(1_000), 'probes are one second apart'); assert.equal(server.rpcs(), 0); } finally { - advancing = false; - await advance; + await stopDriving(); + } +}); + +test('aborting the request while the gate re-probes stops the probes', async () => { + const aborting = new AbortController(); + const server = answerEveryRequest503(); + const stopDriving = driveFakeTime(); + try { + const outcome = sendToDaemon(remoteSessionList(), { + authToken: 'remote-secret', + signal: aborting.signal, + }).then( + () => undefined, + (error: unknown) => error, + ); + while (server.probeTimes.length < 3) await new Promise((resolve) => setImmediate(resolve)); + aborting.abort(); + + assert.ok(isRequestCanceledError(await outcome), 'the abort answers as a cancellation'); + const probesAtAbort = server.probeTimes.length; + for (let turn = 0; turn < 5; turn += 1) await vi.advanceTimersByTimeAsync(1_000); + assert.equal(server.probeTimes.length, probesAtAbort, 'no probe follows the abort'); + } finally { + await stopDriving(); } }); diff --git a/packages/daemon-client/src/daemon-client-remote-health-cache.ts b/packages/daemon-client/src/daemon-client-remote-health-cache.ts index b7c4e9741e..dc30c3870e 100644 --- a/packages/daemon-client/src/daemon-client-remote-health-cache.ts +++ b/packages/daemon-client/src/daemon-client-remote-health-cache.ts @@ -1,13 +1,12 @@ import type { DaemonInfo } from './daemon-client-metadata.ts'; import type { RemoteDaemonHealth } from './daemon-client-transport.ts'; +/** A settled `/health` answer for one daemon identity. Only usable answers are cached. */ type RemoteHealthCacheEntry = { baseUrl: string; token: string; pid: number; - health: Promise; - /** The settled answer, once the probe behind `health` has answered. */ - answer?: RemoteDaemonHealth; + health: RemoteDaemonHealth; }; let remoteHealthCache: RemoteHealthCacheEntry | undefined; @@ -28,8 +27,7 @@ export function cacheRemoteDaemonHealth(info: DaemonInfo, health: RemoteDaemonHe baseUrl: info.baseUrl ?? '', token: info.token, pid: info.pid, - health: Promise.resolve(health), - answer: health, + health, }; } @@ -39,42 +37,15 @@ export function invalidateRemoteDaemonHealth(info: DaemonInfo): void { } } -/** - * The remote `/health` answer through the process cache. Unbounded callers share one in-flight - * probe. A caller with its own bound (`bounded`) reads only a settled answer and otherwise runs - * `probe` itself, so it never waits on a probe that another caller started under a different - * budget or signal. - */ +/** The cached answer for this daemon, or `probe`'s, which is cached when it is usable. */ export async function cachedRemoteHealth( info: DaemonInfo, - bounded: boolean, probe: () => Promise, ): Promise { - const cached = - remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info) - ? remoteHealthCache - : undefined; - if (cached?.answer) return cached.answer; - if (bounded) { - const health = await probe(); - cacheRemoteDaemonHealth(info, health); - return health; - } - if (cached) return await cached.health; - const entry: RemoteHealthCacheEntry = { - baseUrl: info.baseUrl ?? '', - token: info.token, - pid: info.pid, - health: probe(), - }; - remoteHealthCache = entry; - try { - const health = await entry.health; - if (isCacheableRemoteHealth(health)) entry.answer = health; - else if (remoteHealthCache === entry) remoteHealthCache = undefined; - return health; - } catch (error) { - if (remoteHealthCache === entry) remoteHealthCache = undefined; - throw error; + if (remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info)) { + return remoteHealthCache.health; } + const health = await probe(); + cacheRemoteDaemonHealth(info, health); + return health; } diff --git a/packages/daemon-client/src/daemon-client-transport.ts b/packages/daemon-client/src/daemon-client-transport.ts index a12830764c..1808b54175 100644 --- a/packages/daemon-client/src/daemon-client-transport.ts +++ b/packages/daemon-client/src/daemon-client-transport.ts @@ -952,9 +952,7 @@ export async function cachedRemoteDaemonHealth( callerSignal?: AbortSignal, ): Promise { const { cachedRemoteHealth } = await loadRemoteHealthCache(); - return await cachedRemoteHealth( - info, - probeTimeoutMs !== undefined || callerSignal !== undefined, - () => readRemoteDaemonHealth(info, probeTimeoutMs, callerSignal), + return await cachedRemoteHealth(info, () => + readRemoteDaemonHealth(info, probeTimeoutMs, callerSignal), ); } diff --git a/packages/daemon/src/core/__tests__/daemon-command-registry.test.ts b/packages/daemon/src/core/__tests__/daemon-command-registry.test.ts index 355866dc23..b7fdf63ae6 100644 --- a/packages/daemon/src/core/__tests__/daemon-command-registry.test.ts +++ b/packages/daemon/src/core/__tests__/daemon-command-registry.test.ts @@ -7,6 +7,7 @@ import { getDaemonCommandRoute, getSessionCommandKind, isHumanControlMutation, + isReadOnlyRequest, isLeaseAdmissionExempt, resolveSessionlessLeaseAdmissionExemption, shouldBlockForInvalidRecording, @@ -326,3 +327,30 @@ function makeRequest(command: string, positionals: string[] = []): DaemonRequest flags: {}, }; } + +// The fact the transports stamp on each admitted request: only a request with no action in any of +// its steps may let a resend loop spend its deadline. +test('isReadOnlyRequest marks reads, and never a request that dispatches an action', () => { + const request = (command: string, positionals: string[] = []) => ({ + token: 't', + session: 'default', + command, + positionals, + flags: {}, + }); + for (const [command, positionals] of [ + [PUBLIC_COMMANDS.snapshot, []], + [PUBLIC_COMMANDS.screenshot, []], + [PUBLIC_COMMANDS.alert, ['get']], + ] as const) { + assert.equal(isReadOnlyRequest(request(command, [...positionals])), true, command); + } + for (const [command, positionals] of [ + [PUBLIC_COMMANDS.press, ['@e1']], + [PUBLIC_COMMANDS.swipe, ['up']], + [PUBLIC_COMMANDS.alert, ['accept']], + [PUBLIC_COMMANDS.batch, []], + ] as const) { + assert.equal(isReadOnlyRequest(request(command, [...positionals])), false, command); + } +}); diff --git a/packages/daemon/src/core/daemon-command-registry.ts b/packages/daemon/src/core/daemon-command-registry.ts index 279acd28a5..1821d9ad50 100644 --- a/packages/daemon/src/core/daemon-command-registry.ts +++ b/packages/daemon/src/core/daemon-command-registry.ts @@ -112,12 +112,17 @@ export function shouldGuardAndroidBlockingDialog(command: string): boolean { return getDaemonCommandDescriptor(command)?.androidBlockingDialogGuard === true; } +/** Whether the request only observes the device: no step of it dispatches an action. */ +export function isReadOnlyRequest(req: DaemonWireRequest): boolean { + const recordingEffect = resolveCommandRecordingEffect(req); + if (recordingEffect !== undefined) return recordingEffect === 'observes-app'; + if (getSessionCommandKind(req.command) === 'observability') return true; + return resolveCommandDeviceClaimPolicy(req.command) === 'observe'; +} + export function isHumanControlMutation(req: DaemonRequest): boolean { if (req.command === 'human_control' || req.command === 'lease_heartbeat') return false; - const recordingEffect = resolveCommandRecordingEffect(req); - if (recordingEffect !== undefined) return recordingEffect !== 'observes-app'; - if (getSessionCommandKind(req.command) === 'observability') return false; - return resolveCommandDeviceClaimPolicy(req.command) !== 'observe'; + return !isReadOnlyRequest(req); } export function shouldPreferExplicitDeviceOverExistingSession(req: DaemonRequest): boolean { diff --git a/packages/daemon/src/server/http/http-server.ts b/packages/daemon/src/server/http/http-server.ts index 0149d23aca..ac2ccf843d 100644 --- a/packages/daemon/src/server/http/http-server.ts +++ b/packages/daemon/src/server/http/http-server.ts @@ -16,7 +16,10 @@ import { normalizeError } from '@agent-device/kernel/errors'; import { LEASE_BACKENDS } from '@agent-device/kernel/contracts'; import { isRecord } from '@agent-device/kernel/record'; import type { DaemonInvokeFn, DaemonRequest } from '../../core/daemon-request.ts'; -import { resolveDaemonRequestResultDeadlineMs } from '../../core/daemon-command-registry.ts'; +import { + isReadOnlyRequest, + resolveDaemonRequestResultDeadlineMs, +} from '../../core/daemon-command-registry.ts'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { clearRequestAbortRegistration, @@ -494,6 +497,7 @@ function beginRpcRequestLifetime(params: { requestId = resolveRequestTrackingId(daemonRequest.meta?.requestId, rpcId); abortRegistration = registerRequestAbort(requestId, undefined, { resultDeadlineMs: resolveDaemonRequestResultDeadlineMs(daemonRequest), + readOnly: isReadOnlyRequest(daemonRequest), }); return { ...daemonRequest, diff --git a/packages/daemon/src/server/http/transport.ts b/packages/daemon/src/server/http/transport.ts index 12c5f82a7f..8755e6911e 100644 --- a/packages/daemon/src/server/http/transport.ts +++ b/packages/daemon/src/server/http/transport.ts @@ -11,7 +11,10 @@ import { AsyncResource } from 'node:async_hooks'; import net from 'node:net'; import type { Server as HttpServer } from 'node:http'; import type { DaemonInvokeFn, DaemonRequest } from '../../core/daemon-request.ts'; -import { resolveDaemonRequestResultDeadlineMs } from '../../core/daemon-command-registry.ts'; +import { + isReadOnlyRequest, + resolveDaemonRequestResultDeadlineMs, +} from '../../core/daemon-command-registry.ts'; import { clearRequestAbortRegistration, isRequestCanceled, @@ -91,6 +94,7 @@ export function createSocketServer(handleRequest: DaemonInvokeFn): DaemonServer }; requestAbortRegistration = registerRequestAbort(requestIdForCleanup, undefined, { resultDeadlineMs: resolveDaemonRequestResultDeadlineMs(req), + readOnly: isReadOnlyRequest(req), }); activeRequestIds.add(requestIdForCleanup); if (isRequestCanceled(requestIdForCleanup)) { diff --git a/packages/host-kit/src/internal/request-cancel.test.ts b/packages/host-kit/src/internal/request-cancel.test.ts index 2af4c76b41..d85e6cd5ad 100644 --- a/packages/host-kit/src/internal/request-cancel.test.ts +++ b/packages/host-kit/src/internal/request-cancel.test.ts @@ -13,6 +13,7 @@ import { registerRequestAbort, resolveRequestTrackingId, getRequestDeadlineMs, + getRequestReadOnlyDeadlineMs, getRequestSignal, throwIfRequestCanceled, withRequestDeadline, @@ -427,3 +428,25 @@ test('a request runs under the earlier of its execution and result deadlines', a } assert.equal(getRequestDeadlineMs('unregistered-request'), undefined); }); + +test('a resend loop gets the request deadline only for a read-only request, and an attempt inherits it', () => { + const deadlineMs = Date.now() + 30_000; + const read = registerRequestAbort('req-read', undefined, { + resultDeadlineMs: deadlineMs, + readOnly: true, + }); + const action = registerRequestAbort('req-action', undefined, { + resultDeadlineMs: deadlineMs, + readOnly: false, + }); + const step = registerRequestAbort('req-read-step', 'req-read'); + try { + assert.equal(getRequestReadOnlyDeadlineMs('req-read'), deadlineMs); + assert.equal(getRequestReadOnlyDeadlineMs('req-read-step'), deadlineMs); + // A read inside an action leaves the action's budget to the dispatch after it. + assert.equal(getRequestDeadlineMs('req-action'), deadlineMs); + assert.equal(getRequestReadOnlyDeadlineMs('req-action'), undefined); + } finally { + for (const registration of [step, action, read]) clearRequestAbortRegistration(registration); + } +}); diff --git a/packages/host-kit/src/internal/request-cancel.ts b/packages/host-kit/src/internal/request-cancel.ts index 74b82611af..c8f130174d 100644 --- a/packages/host-kit/src/internal/request-cancel.ts +++ b/packages/host-kit/src/internal/request-cancel.ts @@ -23,6 +23,11 @@ type RequestAbortController = AbortController & { * registered under a parent inherits the parent's, because it runs inside the parent's envelope. */ resultDeadlineMs?: number; + /** + * Whether the admitted request only reads: no step of it dispatches an action, so a resend loop + * inside it may spend its deadline. An attempt registered under a parent inherits the parent's. + */ + readOnly?: boolean; }; const REQUEST_CANCEL_SLOT = Symbol('requestCancel'); @@ -81,7 +86,7 @@ function evictOldestSetEntries(set: Set): void { export function registerRequestAbort( requestId: string | undefined, parentRequestId?: string, - admission?: Readonly<{ resultDeadlineMs?: number }>, + admission?: Readonly<{ resultDeadlineMs?: number; readOnly?: boolean }>, ): RequestAbortRegistration | undefined { if (!requestId) return undefined; if (requestAbortControllers().has(requestId)) { @@ -96,6 +101,7 @@ export function registerRequestAbort( controller.resultDeadlineMs = parentRequestId ? parent?.resultDeadlineMs : admission?.resultDeadlineMs; + controller.readOnly = parentRequestId ? parent?.readOnly : admission?.readOnly; requestAbortControllers().set(requestId, controller); if (canceledRequestIds().has(requestId)) { controller.abort(createRequestCanceledError()); @@ -171,6 +177,15 @@ export function getRequestDeadlineMs(requestId: string | undefined): number | un return deadlines.length > 0 ? Math.min(...deadlines) : undefined; } +/** + * The deadline a resend loop may spend, which is the request's deadline only when the request is + * read-only. A read inside an action leaves that budget to the action dispatched after it. + */ +export function getRequestReadOnlyDeadlineMs(requestId: string | undefined): number | undefined { + if (!requestId || requestAbortControllers().get(requestId)?.readOnly !== true) return undefined; + return getRequestDeadlineMs(requestId); +} + export function throwIfRequestCanceled(requestId: string | undefined): void { getRequestSignal(requestId)?.throwIfAborted(); if (isRequestCanceled(requestId)) { diff --git a/packages/host-kit/src/request.ts b/packages/host-kit/src/request.ts index 3193812c8d..5fa591bf95 100644 --- a/packages/host-kit/src/request.ts +++ b/packages/host-kit/src/request.ts @@ -5,6 +5,7 @@ export { getRequestDeadlineMs, getRequestSignal, getRequestExecutionDeadlineMs, + getRequestReadOnlyDeadlineMs, getRequestResultDeadlineMs, isRequestCanceled, markRequestCanceled, diff --git a/packages/platform-apple/src/core/runner-host.ts b/packages/platform-apple/src/core/runner-host.ts index de226cbf7d..0c87712e3d 100644 --- a/packages/platform-apple/src/core/runner-host.ts +++ b/packages/platform-apple/src/core/runner-host.ts @@ -32,7 +32,7 @@ import { parseBooleanLiteral } from '@agent-device/kernel/source-value'; import type { HostEnvironment } from '@agent-device/kernel/source-value'; import { - getRequestDeadlineMs, + getRequestReadOnlyDeadlineMs, getRequestSignal, isRequestCanceled, emitRequestProgress, @@ -104,7 +104,7 @@ export const appleRunnerHost: AppleRunnerHost = { parseBooleanLiteral, isRecord, emitRequestProgress, - getRequestDeadlineMs, + getRequestReadOnlyDeadlineMs, getRequestSignal, isRequestCanceled, classifyBootFailure, diff --git a/packages/test-kit/src/host-kit-capabilities.ts b/packages/test-kit/src/host-kit-capabilities.ts index a2775c55f6..a2f82efce8 100644 --- a/packages/test-kit/src/host-kit-capabilities.ts +++ b/packages/test-kit/src/host-kit-capabilities.ts @@ -45,7 +45,7 @@ export async function loadRealHostKitCapabilities() { withProcessLock: file.withProcessLock, publishFileSync: file.publishFileSync, emitRequestProgress: request.emitRequestProgress, - getRequestDeadlineMs: request.getRequestDeadlineMs, + getRequestReadOnlyDeadlineMs: request.getRequestReadOnlyDeadlineMs, getRequestSignal: request.getRequestSignal, isRequestCanceled: request.isRequestCanceled, runnerRuntimeSlot: runtimeSlots.runtimeSlot, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index a576cabe3a..74436cec1e 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -460,7 +460,7 @@ agent-device alert dismiss - The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators, including Limrun, the XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If a notification, privacy, or location table cannot be read, the runner drops the permission translations and keeps the English labels; the `Open in …?` table is read apart, so its labels survive that, and the permission translations survive it being unreadable. Physical iOS devices, tvOS, and visionOS always use the English labels. On iOS simulators, including Limrun, `alert get` marks the `Open in …?` launch confirmation with `launchConfirmationPrompt: true`; `open` reads that mark to answer the prompt. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. - On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or localized decline copy outside the preferred languages' prompt strings fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline during action or confirmation after the alert is resolved fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. -- iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); a read-only capture resends it until shortly before the command's own timeout, and `details.attempts` counts the sends. When that window closes first, the command still fails with `runner_busy` (`dispatched: no`), so retry the read; `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. A synthesized long press, `scroll`, `swipe` (sent as a planned fling `gesture`), or other planned `gesture` that the event synthesis refuses before its event record is handed to XCTest reports `COMMAND_FAILED` with `synthesized_gesture_not_dispatched` (`dispatched: no`), and so does a planned `gesture` refused while its window or display is resolved (the window cannot be named, or the sole display cannot be measured); one that fails after the record was handed to XCTest (its events may or may not have run) reports `synthesized_gesture_outcome_unknown` (`dispatched: unknown`) and is never repeated through an XCTest fallback, so check the screen with `snapshot` before you retry. Any other refusal, such as a platform without touch input, keeps `UNSUPPORTED_OPERATION`. A long-press step in a press series reports the same reasons; once an earlier step has run, the series is `dispatched: unknown`. `back_swipe_no_change` means the iOS `back` swipe was sent and the screen did not change, and `back_swipe_unverified` means it was sent but may or may not have navigated (both `dispatched: unknown`); `back_swipe_not_dispatched` means it was refused before any touch went down (`dispatched: no`). A `keyboard dismiss` the keyboard would not come down for because it exposes no dismiss key keeps `UNSUPPORTED_OPERATION` and reports `keyboard_dismiss_unavailable` (`dispatched: no`, nothing was tapped); one whose dismiss key was tapped and left the keyboard visible keeps `UNSUPPORTED_OPERATION` with no `reason` and `dispatched: unknown`. A `keyboard enter` that found no return key and no field to submit through keeps `UNSUPPORTED_OPERATION` and reports `keyboard_return_unavailable` (`dispatched: unknown`, because typing the return can fail after XCTest sent it); a sole text field that was found and tapped but would not take the return keeps `UNSUPPORTED_OPERATION` with no `reason`. A press the runner answers that its target cannot perform — today the Action Button on an iPhone or iPad model that has none, since the simulator volume and non-tvOS remote cases are refused before the runner is reached — keeps `UNSUPPORTED_OPERATION` and reports `hardware_control_unavailable` (`dispatched: no`, nothing was pressed). macOS `back` finding no in-app back control reports `in_app_back_control_unavailable` under `COMMAND_FAILED` (`dispatched: no`). See Back above. +- iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); a read-only command resends it until shortly before its own timeout, and `details.attempts` counts the sends. A read inside an action, such as the viewport read before a swipe, resends it at most 8 times, because the action needs the rest of its timeout. When that window closes first, the command still fails with `runner_busy` (`dispatched: no`), so retry the read; `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. A synthesized long press, `scroll`, `swipe` (sent as a planned fling `gesture`), or other planned `gesture` that the event synthesis refuses before its event record is handed to XCTest reports `COMMAND_FAILED` with `synthesized_gesture_not_dispatched` (`dispatched: no`), and so does a planned `gesture` refused while its window or display is resolved (the window cannot be named, or the sole display cannot be measured); one that fails after the record was handed to XCTest (its events may or may not have run) reports `synthesized_gesture_outcome_unknown` (`dispatched: unknown`) and is never repeated through an XCTest fallback, so check the screen with `snapshot` before you retry. Any other refusal, such as a platform without touch input, keeps `UNSUPPORTED_OPERATION`. A long-press step in a press series reports the same reasons; once an earlier step has run, the series is `dispatched: unknown`. `back_swipe_no_change` means the iOS `back` swipe was sent and the screen did not change, and `back_swipe_unverified` means it was sent but may or may not have navigated (both `dispatched: unknown`); `back_swipe_not_dispatched` means it was refused before any touch went down (`dispatched: no`). A `keyboard dismiss` the keyboard would not come down for because it exposes no dismiss key keeps `UNSUPPORTED_OPERATION` and reports `keyboard_dismiss_unavailable` (`dispatched: no`, nothing was tapped); one whose dismiss key was tapped and left the keyboard visible keeps `UNSUPPORTED_OPERATION` with no `reason` and `dispatched: unknown`. A `keyboard enter` that found no return key and no field to submit through keeps `UNSUPPORTED_OPERATION` and reports `keyboard_return_unavailable` (`dispatched: unknown`, because typing the return can fail after XCTest sent it); a sole text field that was found and tapped but would not take the return keeps `UNSUPPORTED_OPERATION` with no `reason`. A press the runner answers that its target cannot perform — today the Action Button on an iPhone or iPad model that has none, since the simulator volume and non-tvOS remote cases are refused before the runner is reached — keeps `UNSUPPORTED_OPERATION` and reports `hardware_control_unavailable` (`dispatched: no`, nothing was pressed). macOS `back` finding no in-app back control reports `in_app_back_control_unavailable` under `COMMAND_FAILED` (`dispatched: no`). See Back above. - On the Apple XCUITest runner, an alert search that fails, runs out of time, or finds an app alert, Safari dialog, or dismiss-popup window with no actionable controls fails with `error.code: ALERT_UNRESOLVED` and `details.dispatched: no`. `alert get` reports it at once. `accept`, `dismiss`, and `alert wait` retry it within their windows, like absence, and report it, not a timeout, when the last check remained unresolved. Inspect the prompt with `screenshot`, then `snapshot -i`, and `press @ref` on the visible button. - On the Apple XCUITest runner, `alert get` describes a readable alert even when all its buttons are disabled, and lists those disabled buttons. `alert accept` and `alert dismiss` fail with `ALERT_BUTTON_NOT_FOUND`. - If a sheet or permission prompt is visible but `alert` reports none, `alert` did not recognize it. Run `snapshot -i -s ""`, then `press @ref` on the visible button. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. @@ -594,7 +594,7 @@ These refusals carry `dispatched: no` and a typed `error.details.reason`, so bra - `session_app_required`: a per-app setting was requested but the session binds no app and none was named. Run `open `, or name the app on the request for a setting that takes one (`settings clear-app-state com.example.app`). - `app_not_installed`: the named app resolves to no installed app on the target. Run `apps --all` for the exact bundle id or package, or install the app, then retry. - `malformed_request`: the local socket transport refused the request envelope before any handler read it, for example a positional that is not a string. Fix the client that built the request; resending it unchanged fails the same way. -- `remote_daemon_unavailable`: the client's `GET /health` probe of the remote daemon answered nothing usable, so the command did not run, and `details.daemonBaseUrl` names the daemon that failed the probe. When the probe before the first send gets a server error or no answer in time, it probes again every second, for up to the command's own timeout, then refuses. A command that gets through still has its full timeout to run. A connection that is refused outright fails at once. `details.attempts` counts the probes. The probe a restart retry runs after the daemon instance changed refuses it too — there the first request reached the daemon but was refused before execution, so nothing ran either way. Make that daemon reachable, or clear or override `--daemon-base-url` or `AGENT_DEVICE_DAEMON_BASE_URL`; for a saved `connect` connection, run `disconnect` to return to the local daemon, then resend the same command. +- `remote_daemon_unavailable`: the client's `GET /health` probe of the remote daemon answered nothing usable, so the command did not run, and `details.daemonBaseUrl` names the daemon that failed the probe. When the probe before the first send gets a server error or no answer in time, it waits a second and probes again, for up to the command's own timeout, then refuses. A probe that gets no answer takes up to 3 seconds before that wait. A command that gets through then runs with its full timeout, so the whole request can take up to about twice that timeout. A connection that is refused outright fails at once. `details.attempts` counts the probes, and `details.healthRecoveryBudgetMs` is set to the timeout when the probes used all of it. The probe a restart retry runs after the daemon instance changed refuses it too — there the first request reached the daemon but was refused before execution, so nothing ran either way. Make that daemon reachable, or clear or override `--daemon-base-url` or `AGENT_DEVICE_DAEMON_BASE_URL`; for a saved `connect` connection, run `disconnect` to return to the local daemon, then resend the same command. - `webdriver_route_unsupported`: on a WebDriver-backed device cloud, the driver does not implement the route; the limit is the cloud's driver, not the device. A command with a second route (`orientation` has two) tries it before failing. After an ambiguous failure on that transport, only read-only requests are resent. ## Find (semantic) From 8fb891c8d0fdb58c7cd68e9ac2475d5163ee9556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 13:21:40 +0100 Subject: [PATCH 8/9] fix(recovery): classify read-only requests by declared effect and guard stale health writes - isReadOnlyRequest no longer counts observability commands (logs start/stop/mark, logs clear --restart, audio probe start/stop) as reads, since they change state; isHumanControlMutation keeps its original classification. - A /health answer that lands after an invalidation is returned but not cached. - Wording: a busy read inside an action is sent at most 8 times in total; the budget test's wait loop is bounded. Co-Authored-By: Claude Opus 5.5 --- .../src/__tests__/index-busy-resend.test.ts | 3 +- .../daemon-client-remote-health-cache.test.ts | 37 +++++++++++++++++++ ...ient-remote-health-recovery-budget.test.ts | 9 ++++- .../src/daemon-client-remote-health-cache.ts | 11 +++++- .../__tests__/daemon-command-registry.test.ts | 12 +++++- .../src/core/daemon-command-registry.ts | 13 +++++-- website/docs/docs/commands.md | 2 +- 7 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 packages/daemon-client/src/__tests__/daemon-client-remote-health-cache.test.ts diff --git a/packages/apple-runner/src/__tests__/index-busy-resend.test.ts b/packages/apple-runner/src/__tests__/index-busy-resend.test.ts index bae28deec3..e19668bcdc 100644 --- a/packages/apple-runner/src/__tests__/index-busy-resend.test.ts +++ b/packages/apple-runner/src/__tests__/index-busy-resend.test.ts @@ -354,7 +354,8 @@ test('a mutating command meets RUNNER_BUSY once, with no status probe and no res }); // The viewport read before a swipe runs under the swipe's request. That budget belongs to the swipe -// dispatched after it, so a busy drain keeps the fixed window and refuses fast with nothing sent. +// dispatched after it, so a busy drain keeps the fixed window: the read is sent 8 times (about 5.4 s +// of delay), then RUNNER_BUSY is returned and the swipe is never dispatched. test('a busy read inside a mutating request keeps the fixed window under a 30s deadline', async () => { vi.useFakeTimers(); mockExecuteRunnerCommandWithSession.mockImplementation(async () => { diff --git a/packages/daemon-client/src/__tests__/daemon-client-remote-health-cache.test.ts b/packages/daemon-client/src/__tests__/daemon-client-remote-health-cache.test.ts new file mode 100644 index 0000000000..5be44596db --- /dev/null +++ b/packages/daemon-client/src/__tests__/daemon-client-remote-health-cache.test.ts @@ -0,0 +1,37 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + cachedRemoteHealth, + invalidateRemoteDaemonHealth, +} from '../daemon-client-remote-health-cache.ts'; +import type { RemoteDaemonHealth } from '../daemon-client-transport.ts'; + +const HEALTHY: RemoteDaemonHealth = { reachable: true, instanceId: 'remote-instance' }; + +// An invalidation that lands while a probe is in flight wins: the probe's answer is returned to its +// caller but not cached, so the next caller probes again instead of reading a stale answer. +test('a probe answer that lands after an invalidation is not cached', async () => { + const info = { baseUrl: 'http://remote.example.test:7777', token: 'cache-test', pid: 0 }; + let answer: (health: RemoteDaemonHealth) => void = () => {}; + const inFlight = cachedRemoteHealth( + info, + () => new Promise((resolve) => (answer = resolve)), + ); + invalidateRemoteDaemonHealth(info); + answer(HEALTHY); + assert.deepEqual(await inFlight, HEALTHY); + + let probes = 0; + const next = await cachedRemoteHealth(info, async () => { + probes += 1; + return HEALTHY; + }); + assert.deepEqual(next, HEALTHY); + assert.equal(probes, 1, 'the stale answer was not cached'); + // The closest negative: with no invalidation in flight, that usable answer is cached. + await cachedRemoteHealth(info, async () => { + probes += 1; + return HEALTHY; + }); + assert.equal(probes, 1); +}); diff --git a/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts b/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts index 2544568719..c7b0a32862 100644 --- a/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts +++ b/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts @@ -115,7 +115,14 @@ test('aborting the request while the gate re-probes stops the probes', async () () => undefined, (error: unknown) => error, ); - while (server.probeTimes.length < 3) await new Promise((resolve) => setImmediate(resolve)); + // Bounded, so a gate that never re-probes fails here instead of hanging the test. + for (let turn = 0; turn < 10_000 && server.probeTimes.length < 3; turn += 1) { + await new Promise((resolve) => setImmediate(resolve)); + } + assert.ok( + server.probeTimes.length >= 3, + `the gate re-probed ${server.probeTimes.length} times`, + ); aborting.abort(); assert.ok(isRequestCanceledError(await outcome), 'the abort answers as a cancellation'); diff --git a/packages/daemon-client/src/daemon-client-remote-health-cache.ts b/packages/daemon-client/src/daemon-client-remote-health-cache.ts index dc30c3870e..72c1b10334 100644 --- a/packages/daemon-client/src/daemon-client-remote-health-cache.ts +++ b/packages/daemon-client/src/daemon-client-remote-health-cache.ts @@ -10,6 +10,8 @@ type RemoteHealthCacheEntry = { }; let remoteHealthCache: RemoteHealthCacheEntry | undefined; +/** Bumped by every invalidation, so a probe that started before one cannot cache its answer. */ +let remoteHealthGeneration = 0; function matchesRemoteIdentity(entry: RemoteHealthCacheEntry, info: DaemonInfo): boolean { return entry.baseUrl === info.baseUrl && entry.token === info.token && entry.pid === info.pid; @@ -32,12 +34,16 @@ export function cacheRemoteDaemonHealth(info: DaemonInfo, health: RemoteDaemonHe } export function invalidateRemoteDaemonHealth(info: DaemonInfo): void { + remoteHealthGeneration += 1; if (remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info)) { remoteHealthCache = undefined; } } -/** The cached answer for this daemon, or `probe`'s, which is cached when it is usable. */ +/** + * The cached answer for this daemon, or `probe`'s, which is cached when it is usable and no + * invalidation landed while it was in flight. + */ export async function cachedRemoteHealth( info: DaemonInfo, probe: () => Promise, @@ -45,7 +51,8 @@ export async function cachedRemoteHealth( if (remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info)) { return remoteHealthCache.health; } + const generation = remoteHealthGeneration; const health = await probe(); - cacheRemoteDaemonHealth(info, health); + if (generation === remoteHealthGeneration) cacheRemoteDaemonHealth(info, health); return health; } diff --git a/packages/daemon/src/core/__tests__/daemon-command-registry.test.ts b/packages/daemon/src/core/__tests__/daemon-command-registry.test.ts index b7fdf63ae6..6923ad850e 100644 --- a/packages/daemon/src/core/__tests__/daemon-command-registry.test.ts +++ b/packages/daemon/src/core/__tests__/daemon-command-registry.test.ts @@ -342,6 +342,7 @@ test('isReadOnlyRequest marks reads, and never a request that dispatches an acti [PUBLIC_COMMANDS.snapshot, []], [PUBLIC_COMMANDS.screenshot, []], [PUBLIC_COMMANDS.alert, ['get']], + [PUBLIC_COMMANDS.appState, []], ] as const) { assert.equal(isReadOnlyRequest(request(command, [...positionals])), true, command); } @@ -350,7 +351,16 @@ test('isReadOnlyRequest marks reads, and never a request that dispatches an acti [PUBLIC_COMMANDS.swipe, ['up']], [PUBLIC_COMMANDS.alert, ['accept']], [PUBLIC_COMMANDS.batch, []], + // Observability commands that change state: never read-only, though not human-control mutations. + [PUBLIC_COMMANDS.logs, ['clear', '--restart']], + [PUBLIC_COMMANDS.logs, ['start']], + [PUBLIC_COMMANDS.logs, ['stop']], + [PUBLIC_COMMANDS.logs, ['mark']], + [PUBLIC_COMMANDS.audio, ['probe', 'start']], + [PUBLIC_COMMANDS.audio, ['probe', 'stop']], ] as const) { - assert.equal(isReadOnlyRequest(request(command, [...positionals])), false, command); + const req = request(command, [...positionals]); + assert.equal(isReadOnlyRequest(req), false, `${command} ${positionals.join(' ')}`); } + assert.equal(isHumanControlMutation(request(PUBLIC_COMMANDS.logs, ['start']) as never), false); }); diff --git a/packages/daemon/src/core/daemon-command-registry.ts b/packages/daemon/src/core/daemon-command-registry.ts index 1821d9ad50..4c5e6ecbc5 100644 --- a/packages/daemon/src/core/daemon-command-registry.ts +++ b/packages/daemon/src/core/daemon-command-registry.ts @@ -112,17 +112,24 @@ export function shouldGuardAndroidBlockingDialog(command: string): boolean { return getDaemonCommandDescriptor(command)?.androidBlockingDialogGuard === true; } -/** Whether the request only observes the device: no step of it dispatches an action. */ +/** + * Whether the request only observes the device: no step of it dispatches an action or changes state. + * It reads the descriptor's declared effect: a recorded command's recording effect, otherwise an + * observe-only device claim. An observability command (logs, network, audio capture) can change + * state, so it is never read-only here. + */ export function isReadOnlyRequest(req: DaemonWireRequest): boolean { const recordingEffect = resolveCommandRecordingEffect(req); if (recordingEffect !== undefined) return recordingEffect === 'observes-app'; - if (getSessionCommandKind(req.command) === 'observability') return true; return resolveCommandDeviceClaimPolicy(req.command) === 'observe'; } export function isHumanControlMutation(req: DaemonRequest): boolean { if (req.command === 'human_control' || req.command === 'lease_heartbeat') return false; - return !isReadOnlyRequest(req); + const recordingEffect = resolveCommandRecordingEffect(req); + if (recordingEffect !== undefined) return recordingEffect !== 'observes-app'; + if (getSessionCommandKind(req.command) === 'observability') return false; + return resolveCommandDeviceClaimPolicy(req.command) !== 'observe'; } export function shouldPreferExplicitDeviceOverExistingSession(req: DaemonRequest): boolean { diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 74436cec1e..57404b2d18 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -460,7 +460,7 @@ agent-device alert dismiss - The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators, including Limrun, the XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If a notification, privacy, or location table cannot be read, the runner drops the permission translations and keeps the English labels; the `Open in …?` table is read apart, so its labels survive that, and the permission translations survive it being unreadable. Physical iOS devices, tvOS, and visionOS always use the English labels. On iOS simulators, including Limrun, `alert get` marks the `Open in …?` launch confirmation with `launchConfirmationPrompt: true`; `open` reads that mark to answer the prompt. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. - On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or localized decline copy outside the preferred languages' prompt strings fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline during action or confirmation after the alert is resolved fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. -- iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); a read-only command resends it until shortly before its own timeout, and `details.attempts` counts the sends. A read inside an action, such as the viewport read before a swipe, resends it at most 8 times, because the action needs the rest of its timeout. When that window closes first, the command still fails with `runner_busy` (`dispatched: no`), so retry the read; `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. A synthesized long press, `scroll`, `swipe` (sent as a planned fling `gesture`), or other planned `gesture` that the event synthesis refuses before its event record is handed to XCTest reports `COMMAND_FAILED` with `synthesized_gesture_not_dispatched` (`dispatched: no`), and so does a planned `gesture` refused while its window or display is resolved (the window cannot be named, or the sole display cannot be measured); one that fails after the record was handed to XCTest (its events may or may not have run) reports `synthesized_gesture_outcome_unknown` (`dispatched: unknown`) and is never repeated through an XCTest fallback, so check the screen with `snapshot` before you retry. Any other refusal, such as a platform without touch input, keeps `UNSUPPORTED_OPERATION`. A long-press step in a press series reports the same reasons; once an earlier step has run, the series is `dispatched: unknown`. `back_swipe_no_change` means the iOS `back` swipe was sent and the screen did not change, and `back_swipe_unverified` means it was sent but may or may not have navigated (both `dispatched: unknown`); `back_swipe_not_dispatched` means it was refused before any touch went down (`dispatched: no`). A `keyboard dismiss` the keyboard would not come down for because it exposes no dismiss key keeps `UNSUPPORTED_OPERATION` and reports `keyboard_dismiss_unavailable` (`dispatched: no`, nothing was tapped); one whose dismiss key was tapped and left the keyboard visible keeps `UNSUPPORTED_OPERATION` with no `reason` and `dispatched: unknown`. A `keyboard enter` that found no return key and no field to submit through keeps `UNSUPPORTED_OPERATION` and reports `keyboard_return_unavailable` (`dispatched: unknown`, because typing the return can fail after XCTest sent it); a sole text field that was found and tapped but would not take the return keeps `UNSUPPORTED_OPERATION` with no `reason`. A press the runner answers that its target cannot perform — today the Action Button on an iPhone or iPad model that has none, since the simulator volume and non-tvOS remote cases are refused before the runner is reached — keeps `UNSUPPORTED_OPERATION` and reports `hardware_control_unavailable` (`dispatched: no`, nothing was pressed). macOS `back` finding no in-app back control reports `in_app_back_control_unavailable` under `COMMAND_FAILED` (`dispatched: no`). See Back above. +- iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); a read-only command resends it until shortly before its own timeout, and `details.attempts` counts the sends. A read inside an action, such as the viewport read before a swipe, sends it at most 8 times in total, as `details.attempts` reports, because the action needs the rest of its timeout. When that window closes first, the command still fails with `runner_busy` (`dispatched: no`), so retry the read; `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. A synthesized long press, `scroll`, `swipe` (sent as a planned fling `gesture`), or other planned `gesture` that the event synthesis refuses before its event record is handed to XCTest reports `COMMAND_FAILED` with `synthesized_gesture_not_dispatched` (`dispatched: no`), and so does a planned `gesture` refused while its window or display is resolved (the window cannot be named, or the sole display cannot be measured); one that fails after the record was handed to XCTest (its events may or may not have run) reports `synthesized_gesture_outcome_unknown` (`dispatched: unknown`) and is never repeated through an XCTest fallback, so check the screen with `snapshot` before you retry. Any other refusal, such as a platform without touch input, keeps `UNSUPPORTED_OPERATION`. A long-press step in a press series reports the same reasons; once an earlier step has run, the series is `dispatched: unknown`. `back_swipe_no_change` means the iOS `back` swipe was sent and the screen did not change, and `back_swipe_unverified` means it was sent but may or may not have navigated (both `dispatched: unknown`); `back_swipe_not_dispatched` means it was refused before any touch went down (`dispatched: no`). A `keyboard dismiss` the keyboard would not come down for because it exposes no dismiss key keeps `UNSUPPORTED_OPERATION` and reports `keyboard_dismiss_unavailable` (`dispatched: no`, nothing was tapped); one whose dismiss key was tapped and left the keyboard visible keeps `UNSUPPORTED_OPERATION` with no `reason` and `dispatched: unknown`. A `keyboard enter` that found no return key and no field to submit through keeps `UNSUPPORTED_OPERATION` and reports `keyboard_return_unavailable` (`dispatched: unknown`, because typing the return can fail after XCTest sent it); a sole text field that was found and tapped but would not take the return keeps `UNSUPPORTED_OPERATION` with no `reason`. A press the runner answers that its target cannot perform — today the Action Button on an iPhone or iPad model that has none, since the simulator volume and non-tvOS remote cases are refused before the runner is reached — keeps `UNSUPPORTED_OPERATION` and reports `hardware_control_unavailable` (`dispatched: no`, nothing was pressed). macOS `back` finding no in-app back control reports `in_app_back_control_unavailable` under `COMMAND_FAILED` (`dispatched: no`). See Back above. - On the Apple XCUITest runner, an alert search that fails, runs out of time, or finds an app alert, Safari dialog, or dismiss-popup window with no actionable controls fails with `error.code: ALERT_UNRESOLVED` and `details.dispatched: no`. `alert get` reports it at once. `accept`, `dismiss`, and `alert wait` retry it within their windows, like absence, and report it, not a timeout, when the last check remained unresolved. Inspect the prompt with `screenshot`, then `snapshot -i`, and `press @ref` on the visible button. - On the Apple XCUITest runner, `alert get` describes a readable alert even when all its buttons are disabled, and lists those disabled buttons. `alert accept` and `alert dismiss` fail with `ALERT_BUTTON_NOT_FOUND`. - If a sheet or permission prompt is visible but `alert` reports none, `alert` did not recognize it. Run `snapshot -i -s ""`, then `press @ref` on the visible button. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. From 0e2a0c68749b44b13a6c644909d5642ecdc06bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 13:27:06 +0100 Subject: [PATCH 9/9] docs(daemon): state that observability commands follow their declared effect Co-Authored-By: Claude Opus 5.5 --- packages/daemon/src/core/daemon-command-registry.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/daemon/src/core/daemon-command-registry.ts b/packages/daemon/src/core/daemon-command-registry.ts index 4c5e6ecbc5..1a018a32e4 100644 --- a/packages/daemon/src/core/daemon-command-registry.ts +++ b/packages/daemon/src/core/daemon-command-registry.ts @@ -115,8 +115,8 @@ export function shouldGuardAndroidBlockingDialog(command: string): boolean { /** * Whether the request only observes the device: no step of it dispatches an action or changes state. * It reads the descriptor's declared effect: a recorded command's recording effect, otherwise an - * observe-only device claim. An observability command (logs, network, audio capture) can change - * state, so it is never read-only here. + * observe-only device claim. Observability commands get no special case: one that declares + * `observes-app` (such as `perf`) is read-only, and one that starts, stops or clears a capture is not. */ export function isReadOnlyRequest(req: DaemonWireRequest): boolean { const recordingEffect = resolveCommandRecordingEffect(req);