diff --git a/packages/apple-runner/src/__tests__/index-busy-resend.test.ts b/packages/apple-runner/src/__tests__/index-busy-resend.test.ts index 272ef28327..e19668bcdc 100644 --- a/packages/apple-runner/src/__tests__/index-busy-resend.test.ts +++ b/packages/apple-runner/src/__tests__/index-busy-resend.test.ts @@ -110,9 +110,72 @@ test('a RUNNER_BUSY resend still ends when the refusals outlast the window', asy assert.ok(error instanceof AppError); assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY'); assert.equal(error.details?.recovery, undefined, 'the raw refusal reaches the caller unwrapped'); + assert.equal(error.details?.attempts, 8); assert.deepEqual(sentCommands(), Array(8).fill('snapshot')); }); +/** + * Admits `requestId` with a deadline `timeoutMs` from now. The host answers that deadline to a + * resend loop only when the request itself is read-only, as host-kit's + * `getRequestReadOnlyDeadlineMs` does for the fact the daemon stamps at admission. + */ +function admitRequest(requestId: string, timeoutMs: number, readOnly: boolean): string { + const deadlineMs = Date.now() + timeoutMs; + appleRunnerTestHost.update({ + getRequestReadOnlyDeadlineMs: (id) => (id === requestId && readOnly ? deadlineMs : undefined), + }); + return requestId; +} + +test('under a request deadline, read-only commands outlast the fixed RUNNER_BUSY count', async () => { + // Twelve busy answers, four past the fixed eight-attempt window, then the runner answers inside + // the request's 30s deadline. + vi.useFakeTimers(); + for (let refusal = 0; refusal < 12; refusal += 1) { + mockExecuteRunnerCommandWithSession.mockRejectedValueOnce(busyRefusal()); + } + mockExecuteRunnerCommandWithSession.mockResolvedValueOnce({ nodes: [], truncated: false }); + + const pending = runAppleRunnerCommand( + IOS_SIMULATOR, + { command: 'snapshot' }, + { requestId: admitRequest('req-deadline-recovers', 30_000, true) }, + ); + await vi.advanceTimersByTimeAsync(20_000); + + assert.deepEqual(await pending, { nodes: [], truncated: false }); + assert.deepEqual(sentCommands(), Array(13).fill('snapshot')); +}); + +test('a request deadline reached mid-outage ends with the RUNNER_BUSY refusal and its attempt count', async () => { + vi.useFakeTimers(); + const sentAt: number[] = []; + mockExecuteRunnerCommandWithSession.mockImplementation(async () => { + sentAt.push(Date.now()); + throw busyRefusal(); + }); + const startedAt = Date.now(); + + const settled = runAppleRunnerCommand( + IOS_SIMULATOR, + { command: 'snapshot' }, + { requestId: admitRequest('req-deadline-outage', 10_000, true) }, + ).then( + () => undefined, + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(10_000); + const error = await settled; + + assert.ok(error instanceof AppError, `expected the refusal, got ${String(error)}`); + assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY'); + // The busy window is 8s (10s less the 2s reserve). Delays of 200, 400, 800, then 1000ms put the + // tenth send at 7.4s, inside it, and the eleventh at 8.4s, past it: that refusal is the answer. + assert.equal(error.details?.attempts, 11); + assert.equal(sentCommands().length, 11); + assert.equal(sentAt.at(-1)! - startedAt, 8_400); +}); + test('read-only transport failures keep the three-attempt resend budget', async () => { mockExecuteRunnerCommandWithSession .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed')) @@ -135,6 +198,48 @@ test('read-only transport failures keep the three-attempt resend budget', async assert.equal(sentCommands().filter((command) => command === 'snapshot').length, 3); }); +test('a RUNNER_BUSY refusal after the busy window closed is not resent', async () => { + // A 1s request deadline is inside the 2s reserve, so the window is already closed. + mockExecuteRunnerCommandWithSession + .mockRejectedValueOnce(busyRefusal()) + .mockResolvedValue({ nodes: [], truncated: false }); + + await assert.rejects( + () => + runAppleRunnerCommand( + IOS_SIMULATOR, + { command: 'snapshot' }, + { requestId: admitRequest('req-deadline-window-closed', 1_000, true) }, + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY'); + assert.equal(error.details?.dispatched, 'no'); + assert.equal(error.details?.attempts, 1); + return true; + }, + ); + assert.deepEqual(sentCommands(), ['snapshot']); +}); + +test('under a request deadline too close for a busy resend, transport failures keep their three attempts', async () => { + mockExecuteRunnerCommandWithSession + .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed')) + .mockResolvedValueOnce({ lifecycleState: 'started' }) + .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed')) + .mockResolvedValueOnce({ lifecycleState: 'started' }) + .mockResolvedValueOnce({ nodes: [], truncated: false }); + + const result = await runAppleRunnerCommand( + IOS_SIMULATOR, + { command: 'snapshot' }, + { requestId: admitRequest('req-deadline-transport', 1_000, true) }, + ); + + assert.deepEqual(result, { nodes: [], truncated: false }); + assert.equal(sentCommands().filter((command) => command === 'snapshot').length, 3); +}); + test('a wait deadline landing mid-window rethrows the last RUNNER_BUSY refusal, not a cancel', async () => { // `wait` bounds each poll with its own abort signal (runWithinWaitDeadline) and keeps the last // typed refusal as the wait's cause. The refusal must therefore survive the deadline: a bare @@ -225,11 +330,17 @@ test('a cancelled request wakes the RUNNER_BUSY delay and reports the cancellati test('a mutating command meets RUNNER_BUSY once, with no status probe and no resend', async () => { // The status-recovery bypass for structured replies reaches mutating commands too: the refusal - // says the tap did not run, so there is nothing to recover and nothing is replayed. + // says the tap did not run, so there is nothing to recover and nothing is replayed, even with a + // request deadline that would let a read resend. mockExecuteRunnerCommandWithSession.mockRejectedValueOnce(busyRefusal()); await assert.rejects( - () => runAppleRunnerCommand(IOS_SIMULATOR, { command: 'tap', x: 120, y: 240 }), + () => + runAppleRunnerCommand( + IOS_SIMULATOR, + { command: 'tap', x: 120, y: 240 }, + { requestId: admitRequest('req-tap-read-only-window', 30_000, true) }, + ), (error: unknown) => { assert.ok(error instanceof AppError); assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY'); @@ -242,6 +353,33 @@ test('a mutating command meets RUNNER_BUSY once, with no status probe and no res assert.equal(mockInvalidateRunnerSession.mock.calls.length, 0); }); +// The viewport read before a swipe runs under the swipe's request. That budget belongs to the swipe +// dispatched after it, so a busy drain keeps the fixed window: the read is sent 8 times (about 5.4 s +// of delay), then RUNNER_BUSY is returned and the swipe is never dispatched. +test('a busy read inside a mutating request keeps the fixed window under a 30s deadline', async () => { + vi.useFakeTimers(); + mockExecuteRunnerCommandWithSession.mockImplementation(async () => { + throw busyRefusal(); + }); + + const settled = runAppleRunnerCommand( + IOS_SIMULATOR, + { command: 'gestureViewport', appBundleId: 'com.example.app' }, + { requestId: admitRequest('req-swipe', 30_000, false) }, + ).then( + () => undefined, + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(30_000); + const error = await settled; + + assert.ok(error instanceof AppError, `expected the refusal, got ${String(error)}`); + assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY'); + assert.equal(error.details?.dispatched, 'no'); + assert.equal(error.details?.attempts, 8); + assert.deepEqual(sentCommands(), Array(8).fill('gestureViewport')); +}); + test('a read-only alert get answered ALERT_UNRESOLVED is sent once and reported as itself', async () => { // A structured reply proves the runner answered, so the read resend reserved for RUNNER_BUSY and // transport failures must not spend it: the prompt it saw is reported, never retried as absence. diff --git a/packages/apple-runner/src/host.ts b/packages/apple-runner/src/host.ts index 46e6f55012..f4b90cef40 100644 --- a/packages/apple-runner/src/host.ts +++ b/packages/apple-runner/src/host.ts @@ -110,7 +110,13 @@ export type AppleRunnerHost = Pick< > & Pick & Pick & - Pick & + Pick< + typeof HostRequest, + | 'emitRequestProgress' + | 'getRequestReadOnlyDeadlineMs' + | 'getRequestSignal' + | 'isRequestCanceled' + > & Pick & Pick & Pick & @@ -207,6 +213,7 @@ export const acquireProcessLock = delegate('acquireProcessLock'); export const withProcessLock = delegate('withProcessLock'); export const publishFileSync = delegate('publishFileSync'); export const emitRequestProgress = delegate('emitRequestProgress'); +export const getRequestReadOnlyDeadlineMs = delegate('getRequestReadOnlyDeadlineMs'); export const getRequestSignal = delegate('getRequestSignal'); export const isRequestCanceled = delegate('isRequestCanceled'); export const withKeyedLock = delegate('withKeyedLock'); diff --git a/packages/apple-runner/src/index.ts b/packages/apple-runner/src/index.ts index 9dcac9d16e..5a0c64b81f 100644 --- a/packages/apple-runner/src/index.ts +++ b/packages/apple-runner/src/index.ts @@ -89,7 +89,14 @@ export { import type * as LegacyXctestDeviceSet from './legacy-xctest-device-set.ts'; import type * as RunnerSequence from './runner-sequence.ts'; -import { retryWithPolicy, emitDiagnostic, hasScopedAppleToolProvider } from './host.ts'; +import { + Deadline, + retryWithPolicy, + emitDiagnostic, + getRequestReadOnlyDeadlineMs, + hasScopedAppleToolProvider, +} from './host.ts'; +import { AppError } from '@agent-device/kernel/errors'; import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; import { ensureRunnerSession, @@ -128,25 +135,59 @@ import { RUNNER_COMMAND_TIMEOUT_MS } from './runner-transport.ts'; /** * Attempts a read-only command may spend on one error class; 1 means no resend. A `RUNNER_BUSY` * refusal is the runner refusing fast on purpose while it drains abandoned XCTest work (#1105), so - * that budget has to outlast the drain. The window is a heuristic, not a measurement: 200ms doubling - * to a 1s cap, no jitter, is 5.4s of delay across eight attempts before round trips, and a drain - * that outlives it still surfaces as `RUNNER_BUSY` (the runner's own `abandonedForSeconds=` marker - * in runner.log is the evidence for tuning it). Transport failures keep the pre-existing three - * attempts. The budget is positional: the error on each attempt sets how many attempts the loop may - * reach in total, so three busy refusals followed by a transport failure resend no further. + * that budget has to outlast the drain. When the request itself is read-only, busy refusals are + * resent until {@link RUNNER_BUSY_DEADLINE_RESERVE_MS} before its deadline, so the last refusal (with + * its attempt count) is the answer instead of the deadline's. A read inside an action (the viewport + * read before a swipe, the selector query before a press) keeps the fixed window instead, because + * the action dispatched after it needs the rest of the budget. The fixed window is a heuristic: 200ms doubling to a + * 1s cap, no jitter, is 5.4s of delay across eight attempts before round trips, and a drain that + * outlives it still surfaces as `RUNNER_BUSY` (the runner's own `abandonedForSeconds=` marker in + * runner.log is the evidence for tuning it). Transport failures keep the pre-existing three attempts + * either way. The budget is positional: the error on each attempt decides whether the loop may send + * again, so three busy refusals followed by a transport failure resend no further. */ const RUNNER_BUSY_RESEND_ATTEMPTS = 8; const TRANSPORT_RESEND_ATTEMPTS = 3; const READ_ONLY_RESEND_POLICY = { - maxAttempts: RUNNER_BUSY_RESEND_ATTEMPTS, baseDelayMs: 200, maxDelayMs: 1_000, jitter: 0, }; +/** + * The last busy resend is decided before the window closes and sent one capped delay later, so the + * reserve holds that delay and the refusal's round trip, which the runner answers at once. + */ +const RUNNER_BUSY_DEADLINE_RESERVE_MS = 2 * READ_ONLY_RESEND_POLICY.maxDelayMs; -function readOnlyResendBudget(error: unknown): number { - if (isRunnerBusyError(error)) return RUNNER_BUSY_RESEND_ATTEMPTS; - return isRetryableRunnerError(error) ? TRANSPORT_RESEND_ATTEMPTS : 1; +type ReadOnlyResendBudget = { + maxAttempts: number; + resends: (error: unknown, attempt: number) => boolean; +}; + +function resolveReadOnlyResendBudget(requestId: string | undefined): ReadOnlyResendBudget { + const requestDeadlineMs = getRequestReadOnlyDeadlineMs(requestId); + const resendsOther = (error: unknown, attempt: number) => + attempt < (isRetryableRunnerError(error) ? TRANSPORT_RESEND_ATTEMPTS : 1); + if (requestDeadlineMs === undefined) { + return { + maxAttempts: RUNNER_BUSY_RESEND_ATTEMPTS, + resends: (error, attempt) => + isRunnerBusyError(error) + ? attempt < RUNNER_BUSY_RESEND_ATTEMPTS + : resendsOther(error, attempt), + }; + } + const windowMs = Math.max(0, requestDeadlineMs - Date.now() - RUNNER_BUSY_DEADLINE_RESERVE_MS); + const busyWindow = Deadline.fromTimeoutMs(windowMs); + return { + // Every delay is at least the base delay, so the window holds no more attempts than this. + maxAttempts: Math.max( + TRANSPORT_RESEND_ATTEMPTS, + Math.ceil(windowMs / READ_ONLY_RESEND_POLICY.baseDelayMs) + 1, + ), + resends: (error, attempt) => + isRunnerBusyError(error) ? !busyWindow.isExpired() : resendsOther(error, attempt), + }; } export async function runAppleRunnerCommand( @@ -161,19 +202,23 @@ export async function runAppleRunnerCommand( if (!isReadOnlyRunnerCommand(runnerCommand)) { return provider.runCommand(device, runnerCommand, options); } - let lastBusyRefusal: unknown; + const budget = resolveReadOnlyResendBudget(options.requestId); + let lastBusyRefusal: AppError | undefined; + let attempts = 0; try { return await retryWithPolicy( () => { assertRunnerRequestActive(options.requestId); + attempts += 1; return provider.runCommand(device, runnerCommand, options); }, { ...READ_ONLY_RESEND_POLICY, + maxAttempts: budget.maxAttempts, shouldRetry: (error, attempt) => { assertRunnerRequestActive(options.requestId); - if (isRunnerBusyError(error)) lastBusyRefusal = error; - return attempt < readOnlyResendBudget(error); + if (isRunnerBusyError(error) && error instanceof AppError) lastBusyRefusal = error; + return budget.resends(error, attempt); }, }, // The busy window is seconds long, so an abort must wake the delay instead of sleeping it @@ -181,14 +226,25 @@ export async function runAppleRunnerCommand( { signal: resolveRunnerRequestSignal(options) }, ); } catch (error) { + if (isRunnerBusyError(error) && error instanceof AppError) { + throw withBusyResendAttempts(error, attempts); + } // A caller's deadline (a `wait` poll bounding this capture) that lands mid-window still has an // answer: the runner refused, and that typed refusal is what the caller can act on. Only a // cancelled request reports as a bare cancellation. - if (lastBusyRefusal && callerDeadlineExpired(options)) throw lastBusyRefusal; + if (lastBusyRefusal && callerDeadlineExpired(options)) { + throw withBusyResendAttempts(lastBusyRefusal, attempts); + } throw error; } } +/** The busy refusal the resend ran out on, stating how many times the read was sent. */ +function withBusyResendAttempts(refusal: AppError, attempts: number): AppError { + refusal.details = { ...refusal.details, attempts }; + return refusal; +} + export async function notifyIosRunnerAppRelaunched( device: DeviceInfo, options: AppleRunnerCommandOptions = {}, diff --git a/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts b/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts index 9d8948baa1..bfdd34eb85 100644 --- a/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts +++ b/packages/daemon-client/src/__tests__/daemon-client-lifecycle-remote-health.test.ts @@ -1,6 +1,7 @@ -// A remote connection probes GET /health before it sends anything. When that probe answers nothing -// usable, the refusal is the typed one: the request never left the client, so a caller branches on -// `details.reason` and `details.dispatched` instead of comparing the message. +// A remote connection probes GET /health before it sends anything. While that gate is closed the +// request never left the client, so the gate may re-probe a daemon that is there but not ready for +// the command's own budget, and its refusal is the typed one: a caller branches on `details.reason` +// and `details.dispatched` instead of comparing the message. import assert from 'node:assert/strict'; import http from 'node:http'; @@ -14,6 +15,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { REMOTE_DAEMON_UNAVAILABLE_REASON } from '../daemon-lifecycle-errors.ts'; import { sendToDaemon } from '../index.ts'; +import { cachedRemoteDaemonHealth } from '../daemon-client-transport.ts'; // A press, so the refusal states the fact a caller needs before resending a mutating command. function remotePress(baseUrl: string) { @@ -26,17 +28,23 @@ function remotePress(baseUrl: string) { }; } -/** A remote daemon that answers `/health` with `healthStatus` and counts the RPCs it receives. */ -function remoteWithHealthStatus(healthStatus: number): { +/** + * A remote daemon whose `/health` answers each status of `healthStatuses` in turn (the last one from + * then on), and that counts the probes and RPCs it receives. Every RPC answers 500. + */ +function remoteWithHealthStatus(...healthStatuses: number[]): { handler: http.Server; + healthProbes: () => number; rpcSends: () => number; } { + let healthProbes = 0; let rpcSends = 0; const handler = http.createServer((req, res) => { req.resume(); req.on('end', () => { if (req.url?.endsWith('/health')) { - res.statusCode = healthStatus; + res.statusCode = healthStatuses[Math.min(healthProbes, healthStatuses.length - 1)]!; + healthProbes += 1; res.end(); return; } @@ -45,29 +53,80 @@ function remoteWithHealthStatus(healthStatus: number): { res.end(); }); }); - return { handler, rpcSends: () => rpcSends }; + return { handler, healthProbes: () => healthProbes, rpcSends: () => rpcSends }; } -test('a remote press whose health probe answers 5xx refuses with the typed reason and dispatched no', async (t) => { +test('a remote press whose health probe answers 5xx re-probes, then is sent exactly once', async (t) => { if (await skipWhenLoopbackUnavailable(t)) return; - const { handler, rpcSends } = remoteWithHealthStatus(503); + const { handler, healthProbes, rpcSends } = remoteWithHealthStatus(503, 200); const port = await listenOnLoopback(handler); - const baseUrl = `http://127.0.0.1:${port}`; try { - await assert.rejects(sendToDaemon(remotePress(baseUrl)), (error: unknown) => { - assert.ok(error instanceof AppError, `unexpected failure: ${String(error)}`); - assert.equal(error.code, 'COMMAND_FAILED'); - assert.equal(error.details?.reason, REMOTE_DAEMON_UNAVAILABLE_REASON); - assert.equal(error.details?.dispatched, 'no'); - assert.equal(error.details?.daemonBaseUrl, baseUrl); - return true; - }); - assert.equal(rpcSends(), 0, 'an unreachable remote daemon receives no RPC'); + await assert.rejects( + sendToDaemon(remotePress(`http://127.0.0.1:${port}`)), + (error: unknown) => { + assert.ok(error instanceof AppError, `unexpected failure: ${String(error)}`); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.notEqual(error.details?.reason, REMOTE_DAEMON_UNAVAILABLE_REASON); + assert.notEqual(error.details?.dispatched, 'no', 'the press left the client'); + return true; + }, + ); + assert.equal(healthProbes(), 2, 'the gate re-probed once the daemon was back'); + // The press reached the daemon and failed there: a failure after dispatch is never resent. + assert.equal(rpcSends(), 1); } finally { await closeLoopbackServer(handler); } }); +// A connection the host refuses outright found no daemon at all, which a wrong base URL looks like: +// it is refused on the first probe instead of waiting out the command's budget. +test('a remote press whose health probe is refused is refused at once with the typed reason', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const closed = http.createServer(); + const port = await listenOnLoopback(closed); + await closeLoopbackServer(closed); + const baseUrl = `http://127.0.0.1:${port}`; + await assert.rejects(sendToDaemon(remotePress(baseUrl)), (error: unknown) => { + assert.ok(error instanceof AppError, `unexpected failure: ${String(error)}`); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, REMOTE_DAEMON_UNAVAILABLE_REASON); + assert.equal(error.details?.dispatched, 'no'); + assert.equal(error.details?.daemonBaseUrl, baseUrl); + assert.equal(error.details?.attempts, 1); + return true; + }); +}); + +// A daemon that answered 503 and then refuses connections is gone, not restarting: the re-probe +// that is refused ends the wait at once instead of waiting out the command's budget. +test('a remote press re-probing after 5xx stops at the first refused connection', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const handler = http.createServer((req, res) => { + req.resume(); + res.statusCode = 503; + res.end(); + handler.close(); + handler.closeAllConnections(); + }); + const port = await listenOnLoopback(handler); + try { + await assert.rejects( + sendToDaemon(remotePress(`http://127.0.0.1:${port}`)), + (error: unknown) => { + assert.ok(error instanceof AppError, `unexpected failure: ${String(error)}`); + assert.equal(error.details?.reason, REMOTE_DAEMON_UNAVAILABLE_REASON); + assert.equal(error.details?.dispatched, 'no'); + assert.equal(error.details?.attempts, 2); + assert.equal(error.details?.healthRecoveryBudgetMs, undefined, 'the budget was not spent'); + return true; + }, + ); + } finally { + if (handler.listening) await closeLoopbackServer(handler); + } +}); + // The closest negative: a probe that answers 4xx counts as reachable, so the press is sent and the // unavailable refusal is never stamped on a request that left the client. test('a remote press whose health probe answers 4xx is sent and is not the unavailable refusal', async (t) => { @@ -88,3 +147,60 @@ test('a remote press whose health probe answers 4xx is sent and is not the unava await closeLoopbackServer(handler); } }); + +/** A remote daemon whose `/health` answers 200 with an instance id after `delayMs`, counting probes. */ +async function slowHealthyRemote(delayMs: number): Promise<{ + handler: http.Server; + info: { baseUrl: string; token: string; pid: number }; + probes: () => number; +}> { + let probes = 0; + const handler = http.createServer((req, res) => { + req.resume(); + probes += 1; + setTimeout(() => { + res.statusCode = 200; + res.end(JSON.stringify({ instanceId: 'remote-instance' })); + }, delayMs); + }); + const port = await listenOnLoopback(handler); + const info = { baseUrl: `http://127.0.0.1:${port}`, token: `probe-${port}`, pid: 0 }; + return { handler, info, probes: () => probes }; +} + +// Each caller probes under its own bound; only a settled, usable answer is shared through the cache, +// so no caller waits on a probe started under another caller's budget. +test('a health caller probes on its own bound, and a settled answer is cached', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const remote = await slowHealthyRemote(300); + try { + const unbounded = cachedRemoteDaemonHealth(remote.info); + const bounded = await cachedRemoteDaemonHealth(remote.info, 50); + + assert.equal(bounded.reachable, false); + assert.equal(bounded.timedOut, true); + assert.equal((await unbounded).reachable, true); + assert.equal(remote.probes(), 2, 'each caller sent its own probe'); + // The settled answer is cached, so a later bounded caller reads it without probing. + assert.equal((await cachedRemoteDaemonHealth(remote.info, 50)).reachable, true); + assert.equal(remote.probes(), 2); + } finally { + await closeLoopbackServer(remote.handler); + } +}); + +// A caller already aborted, or with no budget left, sends no request at all. +test('an aborted or budgetless health caller sends no probe', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const remote = await slowHealthyRemote(0); + try { + assert.equal((await cachedRemoteDaemonHealth(remote.info, 0)).reachable, false); + assert.equal( + (await cachedRemoteDaemonHealth(remote.info, undefined, AbortSignal.abort())).reachable, + false, + ); + assert.equal(remote.probes(), 0); + } finally { + await closeLoopbackServer(remote.handler); + } +}); diff --git a/packages/daemon-client/src/__tests__/daemon-client-remote-health-cache.test.ts b/packages/daemon-client/src/__tests__/daemon-client-remote-health-cache.test.ts new file mode 100644 index 0000000000..5be44596db --- /dev/null +++ b/packages/daemon-client/src/__tests__/daemon-client-remote-health-cache.test.ts @@ -0,0 +1,37 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + cachedRemoteHealth, + invalidateRemoteDaemonHealth, +} from '../daemon-client-remote-health-cache.ts'; +import type { RemoteDaemonHealth } from '../daemon-client-transport.ts'; + +const HEALTHY: RemoteDaemonHealth = { reachable: true, instanceId: 'remote-instance' }; + +// An invalidation that lands while a probe is in flight wins: the probe's answer is returned to its +// caller but not cached, so the next caller probes again instead of reading a stale answer. +test('a probe answer that lands after an invalidation is not cached', async () => { + const info = { baseUrl: 'http://remote.example.test:7777', token: 'cache-test', pid: 0 }; + let answer: (health: RemoteDaemonHealth) => void = () => {}; + const inFlight = cachedRemoteHealth( + info, + () => new Promise((resolve) => (answer = resolve)), + ); + invalidateRemoteDaemonHealth(info); + answer(HEALTHY); + assert.deepEqual(await inFlight, HEALTHY); + + let probes = 0; + const next = await cachedRemoteHealth(info, async () => { + probes += 1; + return HEALTHY; + }); + assert.deepEqual(next, HEALTHY); + assert.equal(probes, 1, 'the stale answer was not cached'); + // The closest negative: with no invalidation in flight, that usable answer is cached. + await cachedRemoteHealth(info, async () => { + probes += 1; + return HEALTHY; + }); + assert.equal(probes, 1); +}); diff --git a/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts b/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts new file mode 100644 index 0000000000..c7b0a32862 --- /dev/null +++ b/packages/daemon-client/src/__tests__/daemon-client-remote-health-recovery-budget.test.ts @@ -0,0 +1,135 @@ +// The remote `/health` gate re-probes a daemon that answers 5xx for the command's own budget. This +// drives a whole default budget on fake time: `http.request` answers on the next tick, so no real +// timer or socket is involved. + +import assert from 'node:assert/strict'; +import http from 'node:http'; +import { EventEmitter } from 'node:events'; +import { afterEach, test, vi } from 'vitest'; +import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; +import { sendToDaemon } from '../index.ts'; + +/** Replaces `http.request` with one that answers every request 503, recording each probe's time. */ +function answerEveryRequest503(): { probeTimes: number[]; rpcs: () => number } { + const probeTimes: number[] = []; + let rpcs = 0; + vi.spyOn(http, 'request').mockImplementation((( + options: { method?: string }, + callback: (res: EventEmitter) => void, + ) => { + const req = Object.assign(new EventEmitter(), { + write: () => {}, + destroy: () => {}, + end: () => { + if (options.method === 'GET') probeTimes.push(performance.now()); + else rpcs += 1; + const res = Object.assign(new EventEmitter(), { + statusCode: 503, + resume: () => {}, + setEncoding: () => {}, + }); + process.nextTick(() => { + callback(res); + res.emit('end'); + }); + }, + }); + return req; + }) as unknown as typeof http.request); + return { probeTimes, rpcs: () => rpcs }; +} + +/** + * Moves fake time only once real work (module loads, the mocked reply) has gone quiet, and then + * straight to the next timer, so no real latency lands on the faked clock. Returns the stop. + */ +function driveFakeTime(): () => Promise { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }); + let driving = true; + const drive = (async () => { + while (driving) { + for (let turn = 0; turn < 20; turn += 1) { + await new Promise((resolve) => setImmediate(resolve)); + } + if (vi.getTimerCount() > 0) await vi.advanceTimersToNextTimerAsync(); + } + })(); + return async () => { + driving = false; + await drive; + }; +} + +/** `session list`, which has the 90s default budget, against a remote daemon. */ +function remoteSessionList() { + return { + session: 'default', + command: 'session', + positionals: ['list'], + flags: { daemonBaseUrl: 'http://remote-mac.example.test:7777/agent-device' }, + meta: { requestId: 'req-remote-budget' }, + }; +} + +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +test('a remote daemon answering 5xx for the whole request budget is refused with its probe count and budget', async () => { + const server = answerEveryRequest503(); + const stopDriving = driveFakeTime(); + + try { + // A non-loopback daemon is trusted only with an auth token. + const outcome = await sendToDaemon(remoteSessionList(), { authToken: 'remote-secret' }).then( + () => undefined, + (error: unknown) => error, + ); + + assert.ok(outcome instanceof AppError, `unexpected outcome: ${String(outcome)}`); + assert.equal(outcome.code, 'COMMAND_FAILED'); + assert.equal(outcome.details?.reason, 'remote_daemon_unavailable'); + assert.equal(outcome.details?.dispatched, 'no'); + // One probe a second fills the 90s budget. + assert.equal(outcome.details?.attempts, 90); + assert.equal(outcome.details?.healthRecoveryBudgetMs, 90_000); + assert.match(String(outcome.details?.hint ?? ''), /agent-device disconnect/); + const gaps = server.probeTimes.slice(1).map((at, index) => at - server.probeTimes[index]!); + assert.deepEqual(gaps, Array(89).fill(1_000), 'probes are one second apart'); + assert.equal(server.rpcs(), 0); + } finally { + await stopDriving(); + } +}); + +test('aborting the request while the gate re-probes stops the probes', async () => { + const aborting = new AbortController(); + const server = answerEveryRequest503(); + const stopDriving = driveFakeTime(); + try { + const outcome = sendToDaemon(remoteSessionList(), { + authToken: 'remote-secret', + signal: aborting.signal, + }).then( + () => undefined, + (error: unknown) => error, + ); + // Bounded, so a gate that never re-probes fails here instead of hanging the test. + for (let turn = 0; turn < 10_000 && server.probeTimes.length < 3; turn += 1) { + await new Promise((resolve) => setImmediate(resolve)); + } + assert.ok( + server.probeTimes.length >= 3, + `the gate re-probed ${server.probeTimes.length} times`, + ); + aborting.abort(); + + assert.ok(isRequestCanceledError(await outcome), 'the abort answers as a cancellation'); + const probesAtAbort = server.probeTimes.length; + for (let turn = 0; turn < 5; turn += 1) await vi.advanceTimersByTimeAsync(1_000); + assert.equal(server.probeTimes.length, probesAtAbort, 'no probe follows the abort'); + } finally { + await stopDriving(); + } +}); diff --git a/packages/daemon-client/src/__tests__/daemon-client-transport.test.ts b/packages/daemon-client/src/__tests__/daemon-client-transport.test.ts index 4061079136..0d3937dc62 100644 --- a/packages/daemon-client/src/__tests__/daemon-client-transport.test.ts +++ b/packages/daemon-client/src/__tests__/daemon-client-transport.test.ts @@ -30,6 +30,13 @@ import { import { REMOTE_DAEMON_UNAVAILABLE_REASON } from '../daemon-lifecycle-errors.ts'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import { createDaemonProxyServer } from '@agent-device/proxy'; +import { invalidateRemoteDaemonHealth } from '../daemon-client-remote-health-cache.ts'; + +// Calls through to the real cache, so only the invalidation test reads the spy. +vi.mock('../daemon-client-remote-health-cache.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, invalidateRemoteDaemonHealth: vi.fn(actual.invalidateRemoteDaemonHealth) }; +}); function sendWithStaleInstance(port: number, timeoutMs: number) { return sendRequest( @@ -621,3 +628,36 @@ test.for(['timeout', 'reset'] as const)( } }, ); + +// A transport failure drops the cached health of a remote daemon only: a local daemon has no cached +// health, so its failed request neither invalidates nor loads the remote health cache. +test('a transport failure invalidates cached health for a remote daemon only', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const server = http.createServer((req) => req.socket.destroy()); + const port = await listenOnLoopback(server); + const invalidate = vi.mocked(invalidateRemoteDaemonHealth); + invalidate.mockClear(); + const send = (info: Parameters[0]) => + sendRequest( + info, + { token: 'secret', command: 'devices', session: 'default', positionals: [], flags: {} }, + 'http', + resolveDaemonPaths('/tmp/agent-device-health-invalidation-test'), + 2_000, + { host: createTestHostEnvironment(), timeoutRecovery: 'none' }, + ).then( + () => undefined, + (error: unknown) => error, + ); + try { + const local = await send({ token: 'secret', pid: 1, transport: 'http', httpPort: port }); + assert.ok(local instanceof AppError, `unexpected outcome: ${String(local)}`); + assert.equal(local.details?.reason, 'daemon_transport_failure'); + assert.equal(invalidate.mock.calls.length, 0, 'a local failure leaves the remote cache alone'); + + await send({ token: 'secret', pid: 1, baseUrl: `http://127.0.0.1:${port}` }); + assert.equal(invalidate.mock.calls.length, 1); + } finally { + await closeLoopbackServer(server); + } +}); diff --git a/packages/daemon-client/src/__tests__/index.test.ts b/packages/daemon-client/src/__tests__/index.test.ts index e1ada76f63..3b643d4cd3 100644 --- a/packages/daemon-client/src/__tests__/index.test.ts +++ b/packages/daemon-client/src/__tests__/index.test.ts @@ -704,37 +704,6 @@ test.each([ }, ); -test('sendToDaemon hints to disconnect when a remote daemon is unavailable', async () => { - const restoreHttpRequest = mockEventHttpRequest(({ res }) => { - res.statusCode = 503; - res.emit('end'); - }); - - try { - await withRemoteDaemonEnv(async () => { - let thrown: unknown; - try { - await sendToDaemon({ - session: 'default', - command: 'session', - positionals: ['list'], - flags: {}, - meta: { requestId: 'req-remote-unavailable' }, - }); - } catch (error) { - thrown = error; - } - - assert.ok(thrown instanceof Error); - assert.equal((thrown as any).code, 'COMMAND_FAILED'); - assert.equal(thrown.message, 'Remote daemon is unavailable'); - assert.match(String((thrown as any).details?.hint ?? ''), /agent-device disconnect/); - }); - } finally { - restoreHttpRequest(); - } -}); - test('sendToDaemon sends lease helpers as top-level JSON-RPC methods over HTTP', async () => { const rpcRequests: Record[] = []; const originalHttpRequest = http.request; diff --git a/packages/daemon-client/src/daemon-client-attach.ts b/packages/daemon-client/src/daemon-client-attach.ts index 7a066918a9..fd9407f802 100644 --- a/packages/daemon-client/src/daemon-client-attach.ts +++ b/packages/daemon-client/src/daemon-client-attach.ts @@ -1,14 +1,21 @@ import { AppError } from '@agent-device/kernel/errors'; import { ensureManagedDaemon } from './daemon-client-lifecycle.ts'; import { assertDaemonPolicyMatches, observeLocalDaemon } from './daemon-client-startup.ts'; -import type { DaemonClientSettings, EnsuredDaemon } from './daemon-client-lifecycle.ts'; +import type { + DaemonClientSettings, + EnsuredDaemon, + RemoteHealthRecovery, +} from './daemon-client-lifecycle.ts'; import { daemonBindingMismatchError } from './daemon-client-binding.ts'; import { daemonIncompatibleError, daemonNotRunningError } from './daemon-lifecycle-errors.ts'; /** Resolves a managed daemon, or observes and verifies the one an attach client may use. */ -export async function connectToDaemon(settings: DaemonClientSettings): Promise { +export async function connectToDaemon( + settings: DaemonClientSettings, + healthRecovery?: RemoteHealthRecovery, +): Promise { if (settings.remoteBaseUrl || settings.daemonLifecycle !== 'attach') { - return await ensureManagedDaemon(settings); + return await ensureManagedDaemon(settings, healthRecovery); } const stateDir = settings.paths.baseDir; const observed = await observeLocalDaemon(settings, { refineUnreachable: true }).catch( diff --git a/packages/daemon-client/src/daemon-client-lifecycle.ts b/packages/daemon-client/src/daemon-client-lifecycle.ts index b68845aafd..7268be98d0 100644 --- a/packages/daemon-client/src/daemon-client-lifecycle.ts +++ b/packages/daemon-client/src/daemon-client-lifecycle.ts @@ -226,9 +226,21 @@ function shouldUseOwnedReplayStateDir( return isOneShotReplayCommand(req.command) && !explicitStateDir && !rawRemoteBaseUrl; } -export async function ensureManagedDaemon(settings: DaemonClientSettings): Promise { +/** + * How long the remote `/health` gate may re-probe a daemon that is there but not ready: the budget + * of the command waiting behind it. Without one, the first failed probe refuses. + */ +export type RemoteHealthRecovery = { + budgetMs: number; + signal: AbortSignal | undefined; +}; + +export async function ensureManagedDaemon( + settings: DaemonClientSettings, + healthRecovery?: RemoteHealthRecovery, +): Promise { if (settings.remoteBaseUrl) { - return await ensureRemoteDaemon(settings); + return await ensureRemoteDaemon(settings, healthRecovery); } const ensured = await ensureLocalDaemon(settings); @@ -244,7 +256,10 @@ async function ensureLocalDaemon(settings: DaemonClientSettings): Promise { +async function ensureRemoteDaemon( + settings: DaemonClientSettings, + healthRecovery: RemoteHealthRecovery | undefined, +): Promise { const remoteInfo: DaemonInfo = { transport: 'http', // Remote mode reuses the auth token as the daemon token so the existing JSON-RPC contract still works. @@ -252,18 +267,27 @@ async function ensureRemoteDaemon(settings: DaemonClientSettings): Promise Promise, +): Promise { + if (remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info)) { + return remoteHealthCache.health; + } + const generation = remoteHealthGeneration; + const health = await probe(); + if (generation === remoteHealthGeneration) cacheRemoteDaemonHealth(info, health); + return health; +} diff --git a/packages/daemon-client/src/daemon-client-remote-health-recovery.ts b/packages/daemon-client/src/daemon-client-remote-health-recovery.ts new file mode 100644 index 0000000000..658541eeaa --- /dev/null +++ b/packages/daemon-client/src/daemon-client-remote-health-recovery.ts @@ -0,0 +1,56 @@ +import { sleep } from '@agent-device/host-kit/retry'; +import type { RemoteHealthRecovery } from './daemon-client-lifecycle.ts'; +import type { DaemonInfo } from './daemon-client-metadata.ts'; +import { cachedRemoteDaemonHealth, type RemoteDaemonHealth } from './daemon-client-transport.ts'; +import { remoteDaemonUnavailableError } from './daemon-lifecycle-errors.ts'; + +/** The pause between `/health` probes while a remote daemon is unavailable. */ +const REMOTE_HEALTH_REPROBE_INTERVAL_MS = 1_000; + +/** + * Whether a failed probe may have reached an endpoint that is coming back: it answered with a server + * error, or it did not answer within the probe's time. A connection that fails outright (refused, no + * such host) has neither, and is refused at once, so a URL with nothing listening never waits. + */ +function isTransientHealthFailure(health: RemoteDaemonHealth): boolean { + return health.timedOut === true || health.statusCode !== undefined; +} + +/** + * Answers a remote `/health` gate whose first probe failed. When the command has a request budget + * and the endpoint is there but not ready, it re-probes until the daemon answers or that budget + * ends. The gate stands before the first send, so nothing reached the daemon, and a resend is safe + * for any command, a UI action included. The wait does not shorten the send's own budget: a command + * dispatched with too little time left could time out after it reached the device. The typed refusal + * states every probe it made. + */ +export async function recoverRemoteDaemonHealth(params: { + info: DaemonInfo; + failedProbe: RemoteDaemonHealth; + recovery: RemoteHealthRecovery | undefined; + /** When the command's budget ends, on the `performance.now()` clock: it started with the first probe. */ + deadline: number; + hint: string; +}): Promise { + const { info, recovery, deadline, hint } = params; + let attempts = 1; + let probe = params.failedProbe; + while (recovery && isTransientHealthFailure(probe) && !recovery.signal?.aborted) { + const remainingMs = deadline - performance.now(); + if (remainingMs <= 0) { + throw remoteDaemonUnavailableError({ + daemonBaseUrl: info.baseUrl, + attempts, + healthRecoveryBudgetMs: recovery.budgetMs, + hint, + }); + } + await sleep(Math.min(REMOTE_HEALTH_REPROBE_INTERVAL_MS, remainingMs), recovery.signal); + const probeMs = deadline - performance.now(); + if (probeMs <= 0 || recovery.signal?.aborted) continue; + attempts += 1; + probe = await cachedRemoteDaemonHealth(info, probeMs, recovery.signal); + if (probe.reachable) return probe; + } + throw remoteDaemonUnavailableError({ daemonBaseUrl: info.baseUrl, attempts, hint }); +} diff --git a/packages/daemon-client/src/daemon-client-transport.ts b/packages/daemon-client/src/daemon-client-transport.ts index db87b7014b..1808b54175 100644 --- a/packages/daemon-client/src/daemon-client-transport.ts +++ b/packages/daemon-client/src/daemon-client-transport.ts @@ -468,7 +468,7 @@ async function sendRequestWithFallback( options, ); } - if (isRemoteTransportFailure(error)) invalidateRemoteDaemonHealth(info); + if (isRemoteTransportFailure(error)) await invalidateRemoteDaemonHealth(info); const fallback = chooseAutoFallbackTransport(info, preference, transport); if (!fallback || !isSafeAutoTransportFallbackError(error, transport)) throw error; return await sendRequestWithTransport(info, req, statePaths, timeoutMs, fallback, options); @@ -484,7 +484,7 @@ async function retryAfterRemoteInstanceMismatch( transport: ResolvedDaemonTransport, options: SendRequestOptions, ): Promise { - invalidateRemoteDaemonHealth(info); + await invalidateRemoteDaemonHealth(info); const probeTimeoutMs = remainingRemoteRequestTimeoutMs(req, timeoutMs, deadline); const health = await readRemoteDaemonHealth(info, probeTimeoutMs, options.signal); refuseAbortedRequest(options.signal, req.meta?.requestId); @@ -496,15 +496,15 @@ async function retryAfterRemoteInstanceMismatch( ); if (!health.reachable) { const { remoteDaemonUnavailableError } = await import('./daemon-lifecycle-errors.ts'); - throw remoteDaemonUnavailableError({ daemonBaseUrl: info.baseUrl }); + throw remoteDaemonUnavailableError({ daemonBaseUrl: info.baseUrl, attempts: 1 }); } info.remoteInstanceId = health.instanceId; info.remoteUpstreamInstanceId = health.upstream?.instanceId; - cacheRemoteDaemonHealth(info, health); + (await loadRemoteHealthCache()).cacheRemoteDaemonHealth(info, health); try { return await sendRequestWithTransport(info, req, statePaths, remainingMs, transport, options); } catch (error) { - if (isRemoteTransportFailure(error)) invalidateRemoteDaemonHealth(info); + if (isRemoteTransportFailure(error)) await invalidateRemoteDaemonHealth(info); throw error; } } @@ -936,59 +936,23 @@ async function sendHttpRequest( }); } -type RemoteHealthCacheEntry = { - baseUrl: string; - token: string; - pid: number; - health: Promise; -}; - -let remoteHealthCache: RemoteHealthCacheEntry | undefined; - -function matchesRemoteIdentity(entry: RemoteHealthCacheEntry, info: DaemonInfo): boolean { - return entry.baseUrl === info.baseUrl && entry.token === info.token && entry.pid === info.pid; -} - -function cacheRemoteDaemonHealth(info: DaemonInfo, health: RemoteDaemonHealth): void { - if (!health.instanceId || (health.upstream && !health.upstream.instanceId)) return; - remoteHealthCache = { - baseUrl: info.baseUrl ?? '', - token: info.token, - pid: info.pid, - health: Promise.resolve(health), - }; -} +// Loaded on the remote path only: a local daemon request never pays for the remote health cache. +const loadRemoteHealthCache = () => import('./daemon-client-remote-health-cache.ts'); -function invalidateRemoteDaemonHealth(info: DaemonInfo): void { - if (remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info)) { - remoteHealthCache = undefined; - } +/** Drops a remote daemon's cached health; a local daemon has none, so its failure loads nothing. */ +async function invalidateRemoteDaemonHealth(info: DaemonInfo): Promise { + if (!info.baseUrl) return; + (await loadRemoteHealthCache()).invalidateRemoteDaemonHealth(info); } -export async function cachedRemoteDaemonHealth(info: DaemonInfo): Promise { - if (remoteHealthCache && matchesRemoteIdentity(remoteHealthCache, info)) { - return await remoteHealthCache.health; - } - const entry: RemoteHealthCacheEntry = { - baseUrl: info.baseUrl ?? '', - token: info.token, - pid: info.pid, - health: readRemoteDaemonHealth(info), - }; - remoteHealthCache = entry; - try { - const health = await entry.health; - if ( - (!health.reachable || - !health.instanceId || - (health.upstream && !health.upstream.instanceId)) && - remoteHealthCache === entry - ) { - remoteHealthCache = undefined; - } - return health; - } catch (error) { - if (remoteHealthCache === entry) remoteHealthCache = undefined; - throw error; - } +/** The remote `/health` answer within this caller's bound, through the process cache. */ +export async function cachedRemoteDaemonHealth( + info: DaemonInfo, + probeTimeoutMs?: number, + callerSignal?: AbortSignal, +): Promise { + const { cachedRemoteHealth } = await loadRemoteHealthCache(); + return await cachedRemoteHealth(info, () => + readRemoteDaemonHealth(info, probeTimeoutMs, callerSignal), + ); } diff --git a/packages/daemon-client/src/daemon-lifecycle-errors.ts b/packages/daemon-client/src/daemon-lifecycle-errors.ts index 9c7a111e33..640cfceea3 100644 --- a/packages/daemon-client/src/daemon-lifecycle-errors.ts +++ b/packages/daemon-client/src/daemon-lifecycle-errors.ts @@ -65,17 +65,23 @@ export const REMOTE_DAEMON_UNAVAILABLE_REASON = 'remote_daemon_unavailable'; * first request did reach the daemon, but it refused it with the instance-mismatch precondition * before running it, and the retry is withheld. So `dispatched: 'no'` states that the command did * not run — what makes a resend safe, for a UI action as much as for a read — and a caller branches - * on the reason instead of comparing the message. + * on the reason instead of comparing the message. `attempts` counts the gate's probes, more than + * one when it re-probed within the command's budget. `healthRecoveryBudgetMs` is present only when + * that budget ran out, which tells a spent budget from a single failed probe. */ export function remoteDaemonUnavailableError(params: { daemonBaseUrl: string | undefined; + attempts: number; + healthRecoveryBudgetMs?: number; hint?: string; }): AppError { - const { daemonBaseUrl, hint } = params; + const { daemonBaseUrl, attempts, healthRecoveryBudgetMs, hint } = params; return new AppError('COMMAND_FAILED', 'Remote daemon is unavailable', { reason: REMOTE_DAEMON_UNAVAILABLE_REASON, dispatched: 'no', daemonBaseUrl, + attempts, + ...(healthRecoveryBudgetMs !== undefined ? { healthRecoveryBudgetMs } : {}), ...(hint !== undefined ? { hint } : {}), }); } diff --git a/packages/daemon-client/src/index.ts b/packages/daemon-client/src/index.ts index f43dd2c360..366c7c67d4 100644 --- a/packages/daemon-client/src/index.ts +++ b/packages/daemon-client/src/index.ts @@ -102,7 +102,15 @@ async function sendToDaemonOnHost( const daemon = await cancellation.guard(async () => { return await withDiagnosticTimer( 'daemon_startup', - async () => await connectToDaemon(settings), + // A remote `/health` gate sends nothing until it opens, so it may wait out an outage for as + // long as this command's own budget. + async () => + await connectToDaemon( + settings, + requestTimeoutMs === undefined + ? undefined + : { budgetMs: requestTimeoutMs, signal: options.signal }, + ), { requestId, session: req.session, diff --git a/packages/daemon/src/core/__tests__/daemon-command-registry.test.ts b/packages/daemon/src/core/__tests__/daemon-command-registry.test.ts index 355866dc23..6923ad850e 100644 --- a/packages/daemon/src/core/__tests__/daemon-command-registry.test.ts +++ b/packages/daemon/src/core/__tests__/daemon-command-registry.test.ts @@ -7,6 +7,7 @@ import { getDaemonCommandRoute, getSessionCommandKind, isHumanControlMutation, + isReadOnlyRequest, isLeaseAdmissionExempt, resolveSessionlessLeaseAdmissionExemption, shouldBlockForInvalidRecording, @@ -326,3 +327,40 @@ function makeRequest(command: string, positionals: string[] = []): DaemonRequest flags: {}, }; } + +// The fact the transports stamp on each admitted request: only a request with no action in any of +// its steps may let a resend loop spend its deadline. +test('isReadOnlyRequest marks reads, and never a request that dispatches an action', () => { + const request = (command: string, positionals: string[] = []) => ({ + token: 't', + session: 'default', + command, + positionals, + flags: {}, + }); + for (const [command, positionals] of [ + [PUBLIC_COMMANDS.snapshot, []], + [PUBLIC_COMMANDS.screenshot, []], + [PUBLIC_COMMANDS.alert, ['get']], + [PUBLIC_COMMANDS.appState, []], + ] as const) { + assert.equal(isReadOnlyRequest(request(command, [...positionals])), true, command); + } + for (const [command, positionals] of [ + [PUBLIC_COMMANDS.press, ['@e1']], + [PUBLIC_COMMANDS.swipe, ['up']], + [PUBLIC_COMMANDS.alert, ['accept']], + [PUBLIC_COMMANDS.batch, []], + // Observability commands that change state: never read-only, though not human-control mutations. + [PUBLIC_COMMANDS.logs, ['clear', '--restart']], + [PUBLIC_COMMANDS.logs, ['start']], + [PUBLIC_COMMANDS.logs, ['stop']], + [PUBLIC_COMMANDS.logs, ['mark']], + [PUBLIC_COMMANDS.audio, ['probe', 'start']], + [PUBLIC_COMMANDS.audio, ['probe', 'stop']], + ] as const) { + const req = request(command, [...positionals]); + assert.equal(isReadOnlyRequest(req), false, `${command} ${positionals.join(' ')}`); + } + assert.equal(isHumanControlMutation(request(PUBLIC_COMMANDS.logs, ['start']) as never), false); +}); diff --git a/packages/daemon/src/core/daemon-command-registry.ts b/packages/daemon/src/core/daemon-command-registry.ts index 279acd28a5..1a018a32e4 100644 --- a/packages/daemon/src/core/daemon-command-registry.ts +++ b/packages/daemon/src/core/daemon-command-registry.ts @@ -112,6 +112,18 @@ export function shouldGuardAndroidBlockingDialog(command: string): boolean { return getDaemonCommandDescriptor(command)?.androidBlockingDialogGuard === true; } +/** + * Whether the request only observes the device: no step of it dispatches an action or changes state. + * It reads the descriptor's declared effect: a recorded command's recording effect, otherwise an + * observe-only device claim. Observability commands get no special case: one that declares + * `observes-app` (such as `perf`) is read-only, and one that starts, stops or clears a capture is not. + */ +export function isReadOnlyRequest(req: DaemonWireRequest): boolean { + const recordingEffect = resolveCommandRecordingEffect(req); + if (recordingEffect !== undefined) return recordingEffect === 'observes-app'; + return resolveCommandDeviceClaimPolicy(req.command) === 'observe'; +} + export function isHumanControlMutation(req: DaemonRequest): boolean { if (req.command === 'human_control' || req.command === 'lease_heartbeat') return false; const recordingEffect = resolveCommandRecordingEffect(req); diff --git a/packages/daemon/src/server/http/http-server.ts b/packages/daemon/src/server/http/http-server.ts index 0149d23aca..ac2ccf843d 100644 --- a/packages/daemon/src/server/http/http-server.ts +++ b/packages/daemon/src/server/http/http-server.ts @@ -16,7 +16,10 @@ import { normalizeError } from '@agent-device/kernel/errors'; import { LEASE_BACKENDS } from '@agent-device/kernel/contracts'; import { isRecord } from '@agent-device/kernel/record'; import type { DaemonInvokeFn, DaemonRequest } from '../../core/daemon-request.ts'; -import { resolveDaemonRequestResultDeadlineMs } from '../../core/daemon-command-registry.ts'; +import { + isReadOnlyRequest, + resolveDaemonRequestResultDeadlineMs, +} from '../../core/daemon-command-registry.ts'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { clearRequestAbortRegistration, @@ -494,6 +497,7 @@ function beginRpcRequestLifetime(params: { requestId = resolveRequestTrackingId(daemonRequest.meta?.requestId, rpcId); abortRegistration = registerRequestAbort(requestId, undefined, { resultDeadlineMs: resolveDaemonRequestResultDeadlineMs(daemonRequest), + readOnly: isReadOnlyRequest(daemonRequest), }); return { ...daemonRequest, diff --git a/packages/daemon/src/server/http/transport.ts b/packages/daemon/src/server/http/transport.ts index 12c5f82a7f..8755e6911e 100644 --- a/packages/daemon/src/server/http/transport.ts +++ b/packages/daemon/src/server/http/transport.ts @@ -11,7 +11,10 @@ import { AsyncResource } from 'node:async_hooks'; import net from 'node:net'; import type { Server as HttpServer } from 'node:http'; import type { DaemonInvokeFn, DaemonRequest } from '../../core/daemon-request.ts'; -import { resolveDaemonRequestResultDeadlineMs } from '../../core/daemon-command-registry.ts'; +import { + isReadOnlyRequest, + resolveDaemonRequestResultDeadlineMs, +} from '../../core/daemon-command-registry.ts'; import { clearRequestAbortRegistration, isRequestCanceled, @@ -91,6 +94,7 @@ export function createSocketServer(handleRequest: DaemonInvokeFn): DaemonServer }; requestAbortRegistration = registerRequestAbort(requestIdForCleanup, undefined, { resultDeadlineMs: resolveDaemonRequestResultDeadlineMs(req), + readOnly: isReadOnlyRequest(req), }); activeRequestIds.add(requestIdForCleanup); if (isRequestCanceled(requestIdForCleanup)) { diff --git a/packages/host-kit/src/internal/request-cancel.test.ts b/packages/host-kit/src/internal/request-cancel.test.ts index 2af4c76b41..d85e6cd5ad 100644 --- a/packages/host-kit/src/internal/request-cancel.test.ts +++ b/packages/host-kit/src/internal/request-cancel.test.ts @@ -13,6 +13,7 @@ import { registerRequestAbort, resolveRequestTrackingId, getRequestDeadlineMs, + getRequestReadOnlyDeadlineMs, getRequestSignal, throwIfRequestCanceled, withRequestDeadline, @@ -427,3 +428,25 @@ test('a request runs under the earlier of its execution and result deadlines', a } assert.equal(getRequestDeadlineMs('unregistered-request'), undefined); }); + +test('a resend loop gets the request deadline only for a read-only request, and an attempt inherits it', () => { + const deadlineMs = Date.now() + 30_000; + const read = registerRequestAbort('req-read', undefined, { + resultDeadlineMs: deadlineMs, + readOnly: true, + }); + const action = registerRequestAbort('req-action', undefined, { + resultDeadlineMs: deadlineMs, + readOnly: false, + }); + const step = registerRequestAbort('req-read-step', 'req-read'); + try { + assert.equal(getRequestReadOnlyDeadlineMs('req-read'), deadlineMs); + assert.equal(getRequestReadOnlyDeadlineMs('req-read-step'), deadlineMs); + // A read inside an action leaves the action's budget to the dispatch after it. + assert.equal(getRequestDeadlineMs('req-action'), deadlineMs); + assert.equal(getRequestReadOnlyDeadlineMs('req-action'), undefined); + } finally { + for (const registration of [step, action, read]) clearRequestAbortRegistration(registration); + } +}); diff --git a/packages/host-kit/src/internal/request-cancel.ts b/packages/host-kit/src/internal/request-cancel.ts index 74b82611af..c8f130174d 100644 --- a/packages/host-kit/src/internal/request-cancel.ts +++ b/packages/host-kit/src/internal/request-cancel.ts @@ -23,6 +23,11 @@ type RequestAbortController = AbortController & { * registered under a parent inherits the parent's, because it runs inside the parent's envelope. */ resultDeadlineMs?: number; + /** + * Whether the admitted request only reads: no step of it dispatches an action, so a resend loop + * inside it may spend its deadline. An attempt registered under a parent inherits the parent's. + */ + readOnly?: boolean; }; const REQUEST_CANCEL_SLOT = Symbol('requestCancel'); @@ -81,7 +86,7 @@ function evictOldestSetEntries(set: Set): void { export function registerRequestAbort( requestId: string | undefined, parentRequestId?: string, - admission?: Readonly<{ resultDeadlineMs?: number }>, + admission?: Readonly<{ resultDeadlineMs?: number; readOnly?: boolean }>, ): RequestAbortRegistration | undefined { if (!requestId) return undefined; if (requestAbortControllers().has(requestId)) { @@ -96,6 +101,7 @@ export function registerRequestAbort( controller.resultDeadlineMs = parentRequestId ? parent?.resultDeadlineMs : admission?.resultDeadlineMs; + controller.readOnly = parentRequestId ? parent?.readOnly : admission?.readOnly; requestAbortControllers().set(requestId, controller); if (canceledRequestIds().has(requestId)) { controller.abort(createRequestCanceledError()); @@ -171,6 +177,15 @@ export function getRequestDeadlineMs(requestId: string | undefined): number | un return deadlines.length > 0 ? Math.min(...deadlines) : undefined; } +/** + * The deadline a resend loop may spend, which is the request's deadline only when the request is + * read-only. A read inside an action leaves that budget to the action dispatched after it. + */ +export function getRequestReadOnlyDeadlineMs(requestId: string | undefined): number | undefined { + if (!requestId || requestAbortControllers().get(requestId)?.readOnly !== true) return undefined; + return getRequestDeadlineMs(requestId); +} + export function throwIfRequestCanceled(requestId: string | undefined): void { getRequestSignal(requestId)?.throwIfAborted(); if (isRequestCanceled(requestId)) { diff --git a/packages/host-kit/src/request.ts b/packages/host-kit/src/request.ts index 3193812c8d..5fa591bf95 100644 --- a/packages/host-kit/src/request.ts +++ b/packages/host-kit/src/request.ts @@ -5,6 +5,7 @@ export { getRequestDeadlineMs, getRequestSignal, getRequestExecutionDeadlineMs, + getRequestReadOnlyDeadlineMs, getRequestResultDeadlineMs, isRequestCanceled, markRequestCanceled, diff --git a/packages/platform-apple/src/core/runner-host.ts b/packages/platform-apple/src/core/runner-host.ts index aba97ac67a..0c87712e3d 100644 --- a/packages/platform-apple/src/core/runner-host.ts +++ b/packages/platform-apple/src/core/runner-host.ts @@ -32,6 +32,7 @@ import { parseBooleanLiteral } from '@agent-device/kernel/source-value'; import type { HostEnvironment } from '@agent-device/kernel/source-value'; import { + getRequestReadOnlyDeadlineMs, getRequestSignal, isRequestCanceled, emitRequestProgress, @@ -103,6 +104,7 @@ export const appleRunnerHost: AppleRunnerHost = { parseBooleanLiteral, isRecord, emitRequestProgress, + getRequestReadOnlyDeadlineMs, getRequestSignal, isRequestCanceled, classifyBootFailure, diff --git a/packages/test-kit/src/host-kit-capabilities.ts b/packages/test-kit/src/host-kit-capabilities.ts index 7ef7a6983c..a2f82efce8 100644 --- a/packages/test-kit/src/host-kit-capabilities.ts +++ b/packages/test-kit/src/host-kit-capabilities.ts @@ -45,6 +45,7 @@ export async function loadRealHostKitCapabilities() { withProcessLock: file.withProcessLock, publishFileSync: file.publishFileSync, emitRequestProgress: request.emitRequestProgress, + getRequestReadOnlyDeadlineMs: request.getRequestReadOnlyDeadlineMs, getRequestSignal: request.getRequestSignal, isRequestCanceled: request.isRequestCanceled, runnerRuntimeSlot: runtimeSlots.runtimeSlot, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 888e4b04fe..57404b2d18 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -460,7 +460,7 @@ agent-device alert dismiss - The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators, including Limrun, the XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If a notification, privacy, or location table cannot be read, the runner drops the permission translations and keeps the English labels; the `Open in …?` table is read apart, so its labels survive that, and the permission translations survive it being unreadable. Physical iOS devices, tvOS, and visionOS always use the English labels. On iOS simulators, including Limrun, `alert get` marks the `Open in …?` launch confirmation with `launchConfirmationPrompt: true`; `open` reads that mark to answer the prompt. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. - On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or localized decline copy outside the preferred languages' prompt strings fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline during action or confirmation after the alert is resolved fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. -- iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. A synthesized long press, `scroll`, `swipe` (sent as a planned fling `gesture`), or other planned `gesture` that the event synthesis refuses before its event record is handed to XCTest reports `COMMAND_FAILED` with `synthesized_gesture_not_dispatched` (`dispatched: no`), and so does a planned `gesture` refused while its window or display is resolved (the window cannot be named, or the sole display cannot be measured); one that fails after the record was handed to XCTest (its events may or may not have run) reports `synthesized_gesture_outcome_unknown` (`dispatched: unknown`) and is never repeated through an XCTest fallback, so check the screen with `snapshot` before you retry. Any other refusal, such as a platform without touch input, keeps `UNSUPPORTED_OPERATION`. A long-press step in a press series reports the same reasons; once an earlier step has run, the series is `dispatched: unknown`. `back_swipe_no_change` means the iOS `back` swipe was sent and the screen did not change, and `back_swipe_unverified` means it was sent but may or may not have navigated (both `dispatched: unknown`); `back_swipe_not_dispatched` means it was refused before any touch went down (`dispatched: no`). A `keyboard dismiss` the keyboard would not come down for because it exposes no dismiss key keeps `UNSUPPORTED_OPERATION` and reports `keyboard_dismiss_unavailable` (`dispatched: no`, nothing was tapped); one whose dismiss key was tapped and left the keyboard visible keeps `UNSUPPORTED_OPERATION` with no `reason` and `dispatched: unknown`. A `keyboard enter` that found no return key and no field to submit through keeps `UNSUPPORTED_OPERATION` and reports `keyboard_return_unavailable` (`dispatched: unknown`, because typing the return can fail after XCTest sent it); a sole text field that was found and tapped but would not take the return keeps `UNSUPPORTED_OPERATION` with no `reason`. A press the runner answers that its target cannot perform — today the Action Button on an iPhone or iPad model that has none, since the simulator volume and non-tvOS remote cases are refused before the runner is reached — keeps `UNSUPPORTED_OPERATION` and reports `hardware_control_unavailable` (`dispatched: no`, nothing was pressed). macOS `back` finding no in-app back control reports `in_app_back_control_unavailable` under `COMMAND_FAILED` (`dispatched: no`). See Back above. +- iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); a read-only command resends it until shortly before its own timeout, and `details.attempts` counts the sends. A read inside an action, such as the viewport read before a swipe, sends it at most 8 times in total, as `details.attempts` reports, because the action needs the rest of its timeout. When that window closes first, the command still fails with `runner_busy` (`dispatched: no`), so retry the read; `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. A synthesized long press, `scroll`, `swipe` (sent as a planned fling `gesture`), or other planned `gesture` that the event synthesis refuses before its event record is handed to XCTest reports `COMMAND_FAILED` with `synthesized_gesture_not_dispatched` (`dispatched: no`), and so does a planned `gesture` refused while its window or display is resolved (the window cannot be named, or the sole display cannot be measured); one that fails after the record was handed to XCTest (its events may or may not have run) reports `synthesized_gesture_outcome_unknown` (`dispatched: unknown`) and is never repeated through an XCTest fallback, so check the screen with `snapshot` before you retry. Any other refusal, such as a platform without touch input, keeps `UNSUPPORTED_OPERATION`. A long-press step in a press series reports the same reasons; once an earlier step has run, the series is `dispatched: unknown`. `back_swipe_no_change` means the iOS `back` swipe was sent and the screen did not change, and `back_swipe_unverified` means it was sent but may or may not have navigated (both `dispatched: unknown`); `back_swipe_not_dispatched` means it was refused before any touch went down (`dispatched: no`). A `keyboard dismiss` the keyboard would not come down for because it exposes no dismiss key keeps `UNSUPPORTED_OPERATION` and reports `keyboard_dismiss_unavailable` (`dispatched: no`, nothing was tapped); one whose dismiss key was tapped and left the keyboard visible keeps `UNSUPPORTED_OPERATION` with no `reason` and `dispatched: unknown`. A `keyboard enter` that found no return key and no field to submit through keeps `UNSUPPORTED_OPERATION` and reports `keyboard_return_unavailable` (`dispatched: unknown`, because typing the return can fail after XCTest sent it); a sole text field that was found and tapped but would not take the return keeps `UNSUPPORTED_OPERATION` with no `reason`. A press the runner answers that its target cannot perform — today the Action Button on an iPhone or iPad model that has none, since the simulator volume and non-tvOS remote cases are refused before the runner is reached — keeps `UNSUPPORTED_OPERATION` and reports `hardware_control_unavailable` (`dispatched: no`, nothing was pressed). macOS `back` finding no in-app back control reports `in_app_back_control_unavailable` under `COMMAND_FAILED` (`dispatched: no`). See Back above. - On the Apple XCUITest runner, an alert search that fails, runs out of time, or finds an app alert, Safari dialog, or dismiss-popup window with no actionable controls fails with `error.code: ALERT_UNRESOLVED` and `details.dispatched: no`. `alert get` reports it at once. `accept`, `dismiss`, and `alert wait` retry it within their windows, like absence, and report it, not a timeout, when the last check remained unresolved. Inspect the prompt with `screenshot`, then `snapshot -i`, and `press @ref` on the visible button. - On the Apple XCUITest runner, `alert get` describes a readable alert even when all its buttons are disabled, and lists those disabled buttons. `alert accept` and `alert dismiss` fail with `ALERT_BUTTON_NOT_FOUND`. - If a sheet or permission prompt is visible but `alert` reports none, `alert` did not recognize it. Run `snapshot -i -s ""`, then `press @ref` on the visible button. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. @@ -594,7 +594,7 @@ These refusals carry `dispatched: no` and a typed `error.details.reason`, so bra - `session_app_required`: a per-app setting was requested but the session binds no app and none was named. Run `open `, or name the app on the request for a setting that takes one (`settings clear-app-state com.example.app`). - `app_not_installed`: the named app resolves to no installed app on the target. Run `apps --all` for the exact bundle id or package, or install the app, then retry. - `malformed_request`: the local socket transport refused the request envelope before any handler read it, for example a positional that is not a string. Fix the client that built the request; resending it unchanged fails the same way. -- `remote_daemon_unavailable`: the client's `GET /health` probe of the remote daemon answered nothing usable, so the command did not run, and `details.daemonBaseUrl` names the daemon that failed the probe. The probe before the first send refuses it outright; the probe a restart retry runs after the daemon instance changed refuses it too — there the first request reached the daemon but was refused before execution, so nothing ran either way. Make that daemon reachable, or clear or override `--daemon-base-url` or `AGENT_DEVICE_DAEMON_BASE_URL`; for a saved `connect` connection, run `disconnect` to return to the local daemon, then resend the same command. +- `remote_daemon_unavailable`: the client's `GET /health` probe of the remote daemon answered nothing usable, so the command did not run, and `details.daemonBaseUrl` names the daemon that failed the probe. When the probe before the first send gets a server error or no answer in time, it waits a second and probes again, for up to the command's own timeout, then refuses. A probe that gets no answer takes up to 3 seconds before that wait. A command that gets through then runs with its full timeout, so the whole request can take up to about twice that timeout. A connection that is refused outright fails at once. `details.attempts` counts the probes, and `details.healthRecoveryBudgetMs` is set to the timeout when the probes used all of it. The probe a restart retry runs after the daemon instance changed refuses it too — there the first request reached the daemon but was refused before execution, so nothing ran either way. Make that daemon reachable, or clear or override `--daemon-base-url` or `AGENT_DEVICE_DAEMON_BASE_URL`; for a saved `connect` connection, run `disconnect` to return to the local daemon, then resend the same command. - `webdriver_route_unsupported`: on a WebDriver-backed device cloud, the driver does not implement the route; the limit is the cloud's driver, not the device. A command with a second route (`orientation` has two) tries it before failing. After an ambiguous failure on that transport, only read-only requests are resent. ## Find (semantic)