Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 140 additions & 2 deletions packages/apple-runner/src/__tests__/index-busy-resend.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,9 +110,72 @@ test('a RUNNER_BUSY resend still ends when the refusals outlast the window', asy
assert.ok(error instanceof AppError);
assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY');
assert.equal(error.details?.recovery, undefined, 'the raw refusal reaches the caller unwrapped');
assert.equal(error.details?.attempts, 8);
assert.deepEqual(sentCommands(), Array<string>(8).fill('snapshot'));
});

/**
* Admits `requestId` with a deadline `timeoutMs` from now. The host answers that deadline to a
* resend loop only when the request itself is read-only, as host-kit's
* `getRequestReadOnlyDeadlineMs` does for the fact the daemon stamps at admission.
*/
function admitRequest(requestId: string, timeoutMs: number, readOnly: boolean): string {
const deadlineMs = Date.now() + timeoutMs;
appleRunnerTestHost.update({
getRequestReadOnlyDeadlineMs: (id) => (id === requestId && readOnly ? deadlineMs : undefined),
});
return requestId;
}

test('under a request deadline, read-only commands outlast the fixed RUNNER_BUSY count', async () => {
// Twelve busy answers, four past the fixed eight-attempt window, then the runner answers inside
// the request's 30s deadline.
vi.useFakeTimers();
for (let refusal = 0; refusal < 12; refusal += 1) {
mockExecuteRunnerCommandWithSession.mockRejectedValueOnce(busyRefusal());
}
mockExecuteRunnerCommandWithSession.mockResolvedValueOnce({ nodes: [], truncated: false });

const pending = runAppleRunnerCommand(
IOS_SIMULATOR,
{ command: 'snapshot' },
{ requestId: admitRequest('req-deadline-recovers', 30_000, true) },
);
await vi.advanceTimersByTimeAsync(20_000);

assert.deepEqual(await pending, { nodes: [], truncated: false });
assert.deepEqual(sentCommands(), Array<string>(13).fill('snapshot'));
});

test('a request deadline reached mid-outage ends with the RUNNER_BUSY refusal and its attempt count', async () => {
vi.useFakeTimers();
const sentAt: number[] = [];
mockExecuteRunnerCommandWithSession.mockImplementation(async () => {
sentAt.push(Date.now());
throw busyRefusal();
});
const startedAt = Date.now();

const settled = runAppleRunnerCommand(
IOS_SIMULATOR,
{ command: 'snapshot' },
{ requestId: admitRequest('req-deadline-outage', 10_000, true) },
).then(
() => undefined,
(error: unknown) => error,
);
await vi.advanceTimersByTimeAsync(10_000);
const error = await settled;

assert.ok(error instanceof AppError, `expected the refusal, got ${String(error)}`);
assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY');
// The busy window is 8s (10s less the 2s reserve). Delays of 200, 400, 800, then 1000ms put the
// tenth send at 7.4s, inside it, and the eleventh at 8.4s, past it: that refusal is the answer.
assert.equal(error.details?.attempts, 11);
assert.equal(sentCommands().length, 11);
assert.equal(sentAt.at(-1)! - startedAt, 8_400);
});

test('read-only transport failures keep the three-attempt resend budget', async () => {
mockExecuteRunnerCommandWithSession
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
Expand All @@ -135,6 +198,48 @@ test('read-only transport failures keep the three-attempt resend budget', async
assert.equal(sentCommands().filter((command) => command === 'snapshot').length, 3);
});

test('a RUNNER_BUSY refusal after the busy window closed is not resent', async () => {
// A 1s request deadline is inside the 2s reserve, so the window is already closed.
mockExecuteRunnerCommandWithSession
.mockRejectedValueOnce(busyRefusal())
.mockResolvedValue({ nodes: [], truncated: false });

await assert.rejects(
() =>
runAppleRunnerCommand(
IOS_SIMULATOR,
{ command: 'snapshot' },
{ requestId: admitRequest('req-deadline-window-closed', 1_000, true) },
),
(error: unknown) => {
assert.ok(error instanceof AppError);
assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY');
assert.equal(error.details?.dispatched, 'no');
assert.equal(error.details?.attempts, 1);
return true;
},
);
assert.deepEqual(sentCommands(), ['snapshot']);
});

test('under a request deadline too close for a busy resend, transport failures keep their three attempts', async () => {
mockExecuteRunnerCommandWithSession
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
.mockResolvedValueOnce({ lifecycleState: 'started' })
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
.mockResolvedValueOnce({ lifecycleState: 'started' })
.mockResolvedValueOnce({ nodes: [], truncated: false });

const result = await runAppleRunnerCommand(
IOS_SIMULATOR,
{ command: 'snapshot' },
{ requestId: admitRequest('req-deadline-transport', 1_000, true) },
);

assert.deepEqual(result, { nodes: [], truncated: false });
assert.equal(sentCommands().filter((command) => command === 'snapshot').length, 3);
});

test('a wait deadline landing mid-window rethrows the last RUNNER_BUSY refusal, not a cancel', async () => {
// `wait` bounds each poll with its own abort signal (runWithinWaitDeadline) and keeps the last
// typed refusal as the wait's cause. The refusal must therefore survive the deadline: a bare
Expand Down Expand Up @@ -225,11 +330,17 @@ test('a cancelled request wakes the RUNNER_BUSY delay and reports the cancellati

test('a mutating command meets RUNNER_BUSY once, with no status probe and no resend', async () => {
// The status-recovery bypass for structured replies reaches mutating commands too: the refusal
// says the tap did not run, so there is nothing to recover and nothing is replayed.
// says the tap did not run, so there is nothing to recover and nothing is replayed, even with a
// request deadline that would let a read resend.
mockExecuteRunnerCommandWithSession.mockRejectedValueOnce(busyRefusal());

await assert.rejects(
() => runAppleRunnerCommand(IOS_SIMULATOR, { command: 'tap', x: 120, y: 240 }),
() =>
runAppleRunnerCommand(
IOS_SIMULATOR,
{ command: 'tap', x: 120, y: 240 },
{ requestId: admitRequest('req-tap-read-only-window', 30_000, true) },
),
(error: unknown) => {
assert.ok(error instanceof AppError);
assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY');
Expand All @@ -242,6 +353,33 @@ test('a mutating command meets RUNNER_BUSY once, with no status probe and no res
assert.equal(mockInvalidateRunnerSession.mock.calls.length, 0);
});

// The viewport read before a swipe runs under the swipe's request. That budget belongs to the swipe
// dispatched after it, so a busy drain keeps the fixed window: the read is sent 8 times (about 5.4 s
// of delay), then RUNNER_BUSY is returned and the swipe is never dispatched.
test('a busy read inside a mutating request keeps the fixed window under a 30s deadline', async () => {
vi.useFakeTimers();
mockExecuteRunnerCommandWithSession.mockImplementation(async () => {
throw busyRefusal();
});

const settled = runAppleRunnerCommand(
IOS_SIMULATOR,
{ command: 'gestureViewport', appBundleId: 'com.example.app' },
{ requestId: admitRequest('req-swipe', 30_000, false) },
).then(
() => undefined,
(error: unknown) => error,
);
await vi.advanceTimersByTimeAsync(30_000);
const error = await settled;

assert.ok(error instanceof AppError, `expected the refusal, got ${String(error)}`);
assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY');
assert.equal(error.details?.dispatched, 'no');
assert.equal(error.details?.attempts, 8);
assert.deepEqual(sentCommands(), Array<string>(8).fill('gestureViewport'));
});

test('a read-only alert get answered ALERT_UNRESOLVED is sent once and reported as itself', async () => {
// A structured reply proves the runner answered, so the read resend reserved for RUNNER_BUSY and
// transport failures must not spend it: the prompt it saw is reported, never retried as absence.
Expand Down
9 changes: 8 additions & 1 deletion packages/apple-runner/src/host.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,13 @@ export type AppleRunnerHost = Pick<
> &
Pick<typeof HostVersion, 'findProjectRoot' | 'readVersion'> &
Pick<typeof HostFile, 'acquireProcessLock' | 'withProcessLock' | 'publishFileSync'> &
Pick<typeof HostRequest, 'emitRequestProgress' | 'getRequestSignal' | 'isRequestCanceled'> &
Pick<
typeof HostRequest,
| 'emitRequestProgress'
| 'getRequestReadOnlyDeadlineMs'
| 'getRequestSignal'
| 'isRequestCanceled'
> &
Pick<typeof KernelKeyedLock, 'withKeyedLock'> &
Pick<typeof KernelRecord, 'isRecord'> &
Pick<typeof KernelSourceValue, 'parseBooleanLiteral'> &
Expand Down Expand Up @@ -207,6 +213,7 @@ export const acquireProcessLock = delegate('acquireProcessLock');
export const withProcessLock = delegate('withProcessLock');
export const publishFileSync = delegate('publishFileSync');
export const emitRequestProgress = delegate('emitRequestProgress');
export const getRequestReadOnlyDeadlineMs = delegate('getRequestReadOnlyDeadlineMs');
export const getRequestSignal = delegate('getRequestSignal');
export const isRequestCanceled = delegate('isRequestCanceled');
export const withKeyedLock = delegate('withKeyedLock');
Expand Down
86 changes: 71 additions & 15 deletions packages/apple-runner/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,14 @@ export {

import type * as LegacyXctestDeviceSet from './legacy-xctest-device-set.ts';
import type * as RunnerSequence from './runner-sequence.ts';
import { retryWithPolicy, emitDiagnostic, hasScopedAppleToolProvider } from './host.ts';
import {
Deadline,
retryWithPolicy,
emitDiagnostic,
getRequestReadOnlyDeadlineMs,
hasScopedAppleToolProvider,
} from './host.ts';
import { AppError } from '@agent-device/kernel/errors';
import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device';
import {
ensureRunnerSession,
Expand Down Expand Up @@ -128,25 +135,59 @@ import { RUNNER_COMMAND_TIMEOUT_MS } from './runner-transport.ts';
/**
* Attempts a read-only command may spend on one error class; 1 means no resend. A `RUNNER_BUSY`
* refusal is the runner refusing fast on purpose while it drains abandoned XCTest work (#1105), so
* that budget has to outlast the drain. The window is a heuristic, not a measurement: 200ms doubling
* to a 1s cap, no jitter, is 5.4s of delay across eight attempts before round trips, and a drain
* that outlives it still surfaces as `RUNNER_BUSY` (the runner's own `abandonedForSeconds=` marker
* in runner.log is the evidence for tuning it). Transport failures keep the pre-existing three
* attempts. The budget is positional: the error on each attempt sets how many attempts the loop may
* reach in total, so three busy refusals followed by a transport failure resend no further.
* that budget has to outlast the drain. When the request itself is read-only, busy refusals are
* resent until {@link RUNNER_BUSY_DEADLINE_RESERVE_MS} before its deadline, so the last refusal (with
* its attempt count) is the answer instead of the deadline's. A read inside an action (the viewport
* read before a swipe, the selector query before a press) keeps the fixed window instead, because
* the action dispatched after it needs the rest of the budget. The fixed window is a heuristic: 200ms doubling to a
* 1s cap, no jitter, is 5.4s of delay across eight attempts before round trips, and a drain that
* outlives it still surfaces as `RUNNER_BUSY` (the runner's own `abandonedForSeconds=` marker in
* runner.log is the evidence for tuning it). Transport failures keep the pre-existing three attempts
* either way. The budget is positional: the error on each attempt decides whether the loop may send
* again, so three busy refusals followed by a transport failure resend no further.
*/
const RUNNER_BUSY_RESEND_ATTEMPTS = 8;
const TRANSPORT_RESEND_ATTEMPTS = 3;
const READ_ONLY_RESEND_POLICY = {
maxAttempts: RUNNER_BUSY_RESEND_ATTEMPTS,
baseDelayMs: 200,
maxDelayMs: 1_000,
jitter: 0,
};
/**
* The last busy resend is decided before the window closes and sent one capped delay later, so the
* reserve holds that delay and the refusal's round trip, which the runner answers at once.
*/
const RUNNER_BUSY_DEADLINE_RESERVE_MS = 2 * READ_ONLY_RESEND_POLICY.maxDelayMs;

function readOnlyResendBudget(error: unknown): number {
if (isRunnerBusyError(error)) return RUNNER_BUSY_RESEND_ATTEMPTS;
return isRetryableRunnerError(error) ? TRANSPORT_RESEND_ATTEMPTS : 1;
type ReadOnlyResendBudget = {
maxAttempts: number;
resends: (error: unknown, attempt: number) => boolean;
};

function resolveReadOnlyResendBudget(requestId: string | undefined): ReadOnlyResendBudget {
const requestDeadlineMs = getRequestReadOnlyDeadlineMs(requestId);
const resendsOther = (error: unknown, attempt: number) =>
attempt < (isRetryableRunnerError(error) ? TRANSPORT_RESEND_ATTEMPTS : 1);
if (requestDeadlineMs === undefined) {
return {
maxAttempts: RUNNER_BUSY_RESEND_ATTEMPTS,
resends: (error, attempt) =>
isRunnerBusyError(error)
? attempt < RUNNER_BUSY_RESEND_ATTEMPTS
: resendsOther(error, attempt),
};
}
const windowMs = Math.max(0, requestDeadlineMs - Date.now() - RUNNER_BUSY_DEADLINE_RESERVE_MS);
const busyWindow = Deadline.fromTimeoutMs(windowMs);
return {
// Every delay is at least the base delay, so the window holds no more attempts than this.
maxAttempts: Math.max(
TRANSPORT_RESEND_ATTEMPTS,
Math.ceil(windowMs / READ_ONLY_RESEND_POLICY.baseDelayMs) + 1,
),
resends: (error, attempt) =>
isRunnerBusyError(error) ? !busyWindow.isExpired() : resendsOther(error, attempt),
};
}

export async function runAppleRunnerCommand(
Expand All @@ -161,34 +202,49 @@ export async function runAppleRunnerCommand(
if (!isReadOnlyRunnerCommand(runnerCommand)) {
return provider.runCommand(device, runnerCommand, options);
}
let lastBusyRefusal: unknown;
const budget = resolveReadOnlyResendBudget(options.requestId);
let lastBusyRefusal: AppError | undefined;
let attempts = 0;
try {
return await retryWithPolicy(
() => {
assertRunnerRequestActive(options.requestId);
attempts += 1;
return provider.runCommand(device, runnerCommand, options);
},
{
...READ_ONLY_RESEND_POLICY,
maxAttempts: budget.maxAttempts,
shouldRetry: (error, attempt) => {
assertRunnerRequestActive(options.requestId);
if (isRunnerBusyError(error)) lastBusyRefusal = error;
return attempt < readOnlyResendBudget(error);
if (isRunnerBusyError(error) && error instanceof AppError) lastBusyRefusal = error;
return budget.resends(error, attempt);
},
},
// The busy window is seconds long, so an abort must wake the delay instead of sleeping it
// out and sending one more attempt.
{ signal: resolveRunnerRequestSignal(options) },
);
} catch (error) {
if (isRunnerBusyError(error) && error instanceof AppError) {
throw withBusyResendAttempts(error, attempts);
}
// A caller's deadline (a `wait` poll bounding this capture) that lands mid-window still has an
// answer: the runner refused, and that typed refusal is what the caller can act on. Only a
// cancelled request reports as a bare cancellation.
if (lastBusyRefusal && callerDeadlineExpired(options)) throw lastBusyRefusal;
if (lastBusyRefusal && callerDeadlineExpired(options)) {
throw withBusyResendAttempts(lastBusyRefusal, attempts);
}
throw error;
}
}

/** The busy refusal the resend ran out on, stating how many times the read was sent. */
function withBusyResendAttempts(refusal: AppError, attempts: number): AppError {
refusal.details = { ...refusal.details, attempts };
return refusal;
}

export async function notifyIosRunnerAppRelaunched(
device: DeviceInfo,
options: AppleRunnerCommandOptions = {},
Expand Down
Loading
Loading