diff --git a/packages/host-kit/src/internal/transport-phases.test.ts b/packages/host-kit/src/internal/transport-phases.test.ts new file mode 100644 index 0000000000..68865afe46 --- /dev/null +++ b/packages/host-kit/src/internal/transport-phases.test.ts @@ -0,0 +1,218 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import type { ClientRequest } from 'node:http'; +import { afterEach, beforeEach, onTestFinished, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { + createTransferPhaseWatchdog, + isTransferPhaseTimeoutError, + observeClientRequestPhases, + observeCompletedRequestWrites, + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + TRANSFER_CONNECT_TIMEOUT_MS, + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, +} from './transport.ts'; + +beforeEach(() => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +function watchPhases(callerSignal?: AbortSignal) { + const watchdog = createTransferPhaseWatchdog(callerSignal); + onTestFinished(() => watchdog.dispose()); + return watchdog; +} + +test('the three transfer ceilings are the adopted policy values', () => { + assert.equal(TRANSFER_CONNECT_TIMEOUT_MS, 30_000); + assert.equal(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, 30_000); + assert.equal(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, 120_000); +}); + +test('a body phase restarts its ceiling on every byte that moves', async () => { + const watchdog = watchPhases(); + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + for (const _ of [0, 1, 2]) { + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - 20_000); + watchdog.touchBody(); + } + assert.equal(watchdog.signal.aborted, false); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 1); + assert.equal(watchdog.signal.aborted, true); + assertPhaseDetails(watchdog.signal.reason, 'response-body', 120_000); +}); + +test('a phase that carries no bytes is not extended by progress reported in another phase', async () => { + const watchdog = watchPhases(); + watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 10_000); + watchdog.touchBody(); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + assertPhaseDetails(watchdog.signal.reason, 'response-headers', 30_000); +}); + +test('arming the next phase stops counting the previous one', async () => { + const watchdog = watchPhases(); + watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS - 1_000); + watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1_000); + assert.equal(watchdog.signal.aborted, false); + await vi.advanceTimersByTimeAsync(1_000); + assertPhaseDetails(watchdog.signal.reason, 'response-headers', 30_000); +}); + +test('an enclosing deadline wins and is never extended by a phase timer', async () => { + const caller = new AbortController(); + const reason = new Error('preparation aggregate expired'); + setTimeout(() => caller.abort(reason), 5_000); + const watchdog = watchPhases(caller.signal); + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(5_001); + assert.equal(watchdog.signal.aborted, true); + assert.equal(watchdog.signal.reason, reason); + assert.equal(isTransferPhaseTimeoutError(watchdog.signal.reason), false); +}); + +test('a phase expiry is its own typed evidence and never caller cancellation', async () => { + const caller = new AbortController(); + const watchdog = watchPhases(caller.signal); + watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS + 1); + assertPhaseDetails(watchdog.signal.reason, 'connect', 30_000); + assert.equal(caller.signal.aborted, false); +}); + +test('a disposed watchdog stops watching', async () => { + const watchdog = watchPhases(); + watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + watchdog.dispose(); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS * 3); + assert.equal(watchdog.signal.aborted, false); +}); + +test('a phase is never armed after the attempt is already over', async () => { + const watchdog = watchPhases(); + watchdog.signal.throwIfAborted(); + const caller = new AbortController(); + caller.abort(new Error('gone')); + const late = watchPhases(caller.signal); + late.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + assert.equal(vi.getTimerCount(), 0); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS * 2); + assert.equal(late.signal.reason.message, 'gone'); +}); + +class FakeRequest extends EventEmitter { + destroyed = false; + readonly callbacks: unknown[] = []; + write(...args: unknown[]): boolean { + this.callbacks.push(args.find((arg) => typeof arg === 'function')); + return true; + } + complete(index: number): void { + const callback = this.callbacks[index]; + assert.ok(typeof callback === 'function', `write ${index} has no callback`); + callback(null); + } +} + +function createObservedRequest() { + const fake = new FakeRequest(); + const watchdog = watchPhases(); + watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + const req = fake as unknown as ClientRequest; + const phases = observeClientRequestPhases(req, watchdog); + const stopObservingWrites = observeCompletedRequestWrites(req, watchdog.signal, () => + watchdog.touchBody(), + ); + const emit = (event: string, ...args: unknown[]) => fake.emit(event, ...args); + const socket = (state: { connecting: boolean; encrypted?: boolean }) => + Object.assign(new EventEmitter(), state); + return { fake, req, watchdog, phases, stopObservingWrites, emit, socket }; +} + +test('a fresh socket keeps the connect ceiling until it connects, even after the body flushed', async () => { + const { watchdog, emit, socket } = createObservedRequest(); + const fresh = socket({ connecting: true }); + emit('socket', fresh); + emit('finish'); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS + 1); + assertPhaseDetails(watchdog.signal.reason, 'connect', 30_000); +}); + +test('the header wait starts only after both the flush and the connection', async () => { + const { watchdog, emit, socket } = createObservedRequest(); + const fresh = socket({ connecting: true }); + emit('socket', fresh); + emit('finish'); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS - 1_000); + fresh.emit('connect'); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1); + assert.equal(watchdog.signal.aborted, false); + await vi.advanceTimersByTimeAsync(1); + assertPhaseDetails(watchdog.signal.reason, 'response-headers', 30_000); +}); + +test('a TLS socket is established by secureConnect, not connect', async () => { + const { watchdog, emit, socket } = createObservedRequest(); + const tls = socket({ connecting: true, encrypted: true }); + emit('socket', tls); + tls.emit('connect'); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS + 1); + assertPhaseDetails(watchdog.signal.reason, 'connect', 30_000); +}); + +test('a pooled socket is already connected and times the body write as inactivity', async () => { + const { fake, req, watchdog, emit, socket } = createObservedRequest(); + emit('socket', socket({ connecting: false })); + req.write(Buffer.from('body')); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - 1_000); + fake.complete(0); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - 1); + assert.equal(watchdog.signal.aborted, false); + await vi.advanceTimersByTimeAsync(1); + assertPhaseDetails(watchdog.signal.reason, 'request-body', 120_000); +}); + +test('once the response started, later request events do not overwrite its phase', async () => { + const { fake, req, watchdog, phases, emit, socket } = createObservedRequest(); + const fresh = socket({ connecting: true }); + emit('socket', fresh); + req.write(Buffer.from('body')); + emit('response'); + assert.equal(phases.responseStarted, true); + fresh.emit('connect'); + emit('finish'); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - 1_000); + fake.complete(0); + await vi.advanceTimersByTimeAsync(1_001); + assertPhaseDetails(watchdog.signal.reason, 'response-body', 120_000); +}); + +test('disposing the phase observer removes only its own listeners', () => { + const { fake, phases, stopObservingWrites, emit, socket } = createObservedRequest(); + const fresh = socket({ connecting: true }); + emit('socket', fresh); + phases.dispose(); + assert.equal(fake.listenerCount('socket'), 0); + assert.equal(fake.listenerCount('finish'), 0); + assert.equal(fresh.listenerCount('connect'), 0); + assert.equal(fake.listenerCount('response'), 1, "the write owner's listener is its own"); + stopObservingWrites(); + assert.equal(fake.listenerCount('response'), 0); +}); + +function assertPhaseDetails(reason: unknown, phase: string, timeoutMs: number): void { + assert.ok(reason instanceof AppError, `phase expiry must be an AppError, got ${String(reason)}`); + assert.equal(isTransferPhaseTimeoutError(reason), true); + assert.deepEqual(reason.details, { + reason: 'transfer_phase_timeout', + transferPhase: phase, + timeoutMs, + }); +} diff --git a/packages/host-kit/src/internal/transport.ts b/packages/host-kit/src/internal/transport.ts index 8e4a1351b1..d698985b1e 100644 --- a/packages/host-kit/src/internal/transport.ts +++ b/packages/host-kit/src/internal/transport.ts @@ -1,6 +1,7 @@ import crypto from 'node:crypto'; import { types } from 'node:util'; import type { ClientRequest, IncomingMessage } from 'node:http'; +import type { Socket } from 'node:net'; import { AppError } from '@agent-device/kernel/errors'; export function consumeTextLines( @@ -89,13 +90,9 @@ export function timingSafeStringEqual(a: string, b: string): boolean { } /** - * Reports each non-empty `req.write` chunk that Node completes without an error: its bytes reached - * the local kernel, not the peer. A queued write, `drain`, and a completion after the response - * started, after `signal` aborted, after a destroy or after disposal are not reported. - * - * Attach it in the tick that creates the request, before anything writes to it. The wrapper keeps - * the arguments, `this`, the backpressure result and one call of the caller's callback with its - * error. Disposing removes its listener and restores `write` only while this wrapper is installed. + * Reports successful nonempty write completions as local-kernel acceptance, never peer receipt. + * Attach at construction. Late response/abort/destroy/dispose completions do not count. + * Disposal restores this wrapper only; caller arguments, callback and backpressure are preserved. */ export function observeCompletedRequestWrites( req: ClientRequest, @@ -128,6 +125,7 @@ export function observeCompletedRequestWrites( return Reflect.apply(write, this, args); }; req.write = observedWrite; + // Stop counting writes before the request's response callback runs. req.prependListener('response', onResponse); return () => { observing = false; @@ -137,3 +135,143 @@ export function observeCompletedRequestWrites( else Reflect.deleteProperty(req, 'write'); }; } + +export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; + +/** Header wait begins after connection and outgoing flush. */ +export const TRANSFER_RESPONSE_HEADER_TIMEOUT_MS = 30_000; + +export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; + +export type TransferPhase = + | 'connect' + | 'request-body' + | 'response-headers' + | 'response-body' + | 'preflight' + | 'finalize'; + +const BODY_PHASES: ReadonlySet = new Set(['request-body', 'response-body']); + +const TRANSFER_PHASE_TIMEOUT_REASON = 'transfer_phase_timeout'; + +/** Sequential phases under the earliest caller/aggregate deadline; only body progress rearms. */ +export type TransferPhaseWatchdog = { + signal: AbortSignal; + arm(phase: TransferPhase, timeoutMs: number): void; + touchBody(): void; + dispose(): void; +}; + +export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): TransferPhaseWatchdog { + const phase = new AbortController(); + const signal = callerSignal ? AbortSignal.any([callerSignal, phase.signal]) : phase.signal; + let armed: { phase: TransferPhase; timeoutMs: number } | undefined; + let timer: ReturnType | undefined; + const rearm = () => { + if (timer !== undefined) clearTimeout(timer); + timer = undefined; + const current = armed; + if (!current) return; + timer = setTimeout(() => { + timer = undefined; + phase.abort(createTransferPhaseTimeoutError(current.phase, current.timeoutMs)); + }, current.timeoutMs); + timer.unref(); + }; + return { + signal, + arm(nextPhase, timeoutMs) { + if (signal.aborted) return; + armed = { phase: nextPhase, timeoutMs }; + rearm(); + }, + touchBody() { + if (!armed || !BODY_PHASES.has(armed.phase)) return; + rearm(); + }, + dispose() { + armed = undefined; + if (timer !== undefined) clearTimeout(timer); + timer = undefined; + }, + }; +} + +function createTransferPhaseTimeoutError(phase: TransferPhase, timeoutMs: number): AppError { + return new AppError('COMMAND_FAILED', `Artifact transfer ${phase} timed out`, { + reason: TRANSFER_PHASE_TIMEOUT_REASON, + transferPhase: phase, + timeoutMs, + }); +} + +export function isTransferPhaseTimeoutError(error: unknown): boolean { + return error instanceof AppError && error.details?.reason === TRANSFER_PHASE_TIMEOUT_REASON; +} + +export type ClientRequestPhases = { + readonly responseStarted: boolean; + dispose(): void; +}; + +/** + * Attach at construction after arming connect. Headers need connection and request finish. + * TLS needs secureConnect; pooled sockets are connected. Buffered bytes are not progress. + * The caller reports body bytes; response start retires request events. Dispose at settlement. + */ +export function observeClientRequestPhases( + req: ClientRequest, + watchdog: TransferPhaseWatchdog, +): ClientRequestPhases { + let established = false; + let flushed = false; + let responseStarted = false; + let socket: Socket | undefined; + let connectEvent: 'connect' | 'secureConnect' | undefined; + const advance = () => { + if (responseStarted || watchdog.signal.aborted || !established) return; + if (flushed) { + watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + } else { + watchdog.arm('request-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + } + }; + const onConnected = () => { + established = true; + advance(); + }; + const onSocket = (next: Socket) => { + if (!next.connecting) { + established = true; + advance(); + return; + } + socket = next; + const secured = (next as { encrypted?: boolean }).encrypted === true; + connectEvent = secured ? 'secureConnect' : 'connect'; + next.once(connectEvent, onConnected); + }; + const onFinish = () => { + flushed = true; + advance(); + }; + const onResponse = () => { + responseStarted = true; + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + }; + req.on('socket', onSocket); + req.on('finish', onFinish); + req.prependListener('response', onResponse); + return { + get responseStarted() { + return responseStarted; + }, + dispose() { + req.off('socket', onSocket); + req.off('finish', onFinish); + req.off('response', onResponse); + if (socket && connectEvent) socket.off(connectEvent, onConnected); + }, + }; +} diff --git a/packages/host-kit/src/transport.ts b/packages/host-kit/src/transport.ts index 6b55289d46..16202f2ff3 100644 --- a/packages/host-kit/src/transport.ts +++ b/packages/host-kit/src/transport.ts @@ -1,9 +1,13 @@ export { consumeTextLines, + createTransferPhaseWatchdog, + isTransferPhaseTimeoutError, loadNodeHttpRequester, + observeClientRequestPhases, observeCompletedRequestWrites, readNodeHttpRequestBody, readNodeHttpResponseBody, timingSafeStringEqual, + TRANSFER_CONNECT_TIMEOUT_MS, type NodeHttpRequester, } from './internal/transport.ts'; diff --git a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts index ed1cd9a47a..c7a01db9dd 100644 --- a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts +++ b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts @@ -1,5 +1,6 @@ import { afterEach, assert, test, vi } from 'vitest'; -import { isRetryableUploadStreamError, isTransferPhaseTimeoutError } from '../upload-stream.ts'; +import { isTransferPhaseTimeoutError } from '@agent-device/host-kit/transport'; +import { isRetryableUploadStreamError } from '../upload-stream.ts'; import { LEGACY_UPLOAD_REQUEST, TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, diff --git a/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts b/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts deleted file mode 100644 index 788c7e3ce5..0000000000 --- a/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts +++ /dev/null @@ -1,151 +0,0 @@ -import assert from 'node:assert/strict'; -import { afterEach, test, vi } from 'vitest'; -import { AppError } from '@agent-device/kernel/errors'; -import { - createTransferPhaseWatchdog, - isTransferPhaseTimeoutError, - TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, - TRANSFER_CONNECT_TIMEOUT_MS, - TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, -} from '../upload-stream.ts'; - -afterEach(() => { - vi.useRealTimers(); -}); - -test('the three transfer ceilings are the adopted policy values', () => { - assert.equal(TRANSFER_CONNECT_TIMEOUT_MS, 30_000); - assert.equal(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, 30_000); - assert.equal(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, 120_000); -}); - -test('a body phase restarts its ceiling on every byte that moves', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); - const watchdog = createTransferPhaseWatchdog(); - try { - watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); - for (const _ of [0, 1, 2]) { - await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - 20_000); - watchdog.touchBody(); - } - assert.equal(watchdog.signal.aborted, false); - await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 1); - assert.equal(watchdog.signal.aborted, true); - assertPhaseDetails( - watchdog.signal.reason, - 'response-body', - TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, - ); - } finally { - watchdog.dispose(); - } -}); - -test('a phase that carries no bytes is not extended by progress reported in another phase', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); - const watchdog = createTransferPhaseWatchdog(); - try { - watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); - await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 10_000); - watchdog.touchBody(); - await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); - assertPhaseDetails( - watchdog.signal.reason, - 'response-headers', - TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, - ); - } finally { - watchdog.dispose(); - } -}); - -test('arming the next phase stops counting the previous one', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); - const watchdog = createTransferPhaseWatchdog(); - try { - watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); - await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS - 1_000); - watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); - await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1_000); - assert.equal(watchdog.signal.aborted, false); - await vi.advanceTimersByTimeAsync(1_000); - assertPhaseDetails( - watchdog.signal.reason, - 'response-headers', - TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, - ); - } finally { - watchdog.dispose(); - } -}); - -test('an enclosing deadline wins and is never extended by a phase timer', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); - const caller = new AbortController(); - const reason = new Error('preparation aggregate expired'); - setTimeout(() => caller.abort(reason), 5_000); - const watchdog = createTransferPhaseWatchdog(caller.signal); - try { - watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); - await vi.advanceTimersByTimeAsync(5_001); - assert.equal(watchdog.signal.aborted, true); - assert.equal(watchdog.signal.reason, reason); - assert.equal(isTransferPhaseTimeoutError(watchdog.signal.reason), false); - } finally { - watchdog.dispose(); - } -}); - -test('a phase expiry is its own typed evidence and never caller cancellation', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); - const caller = new AbortController(); - const watchdog = createTransferPhaseWatchdog(caller.signal); - try { - watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); - await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS + 1); - assertPhaseDetails(watchdog.signal.reason, 'connect', TRANSFER_CONNECT_TIMEOUT_MS); - assert.equal(caller.signal.aborted, false); - } finally { - watchdog.dispose(); - } -}); - -test('a disposed watchdog stops watching', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); - const watchdog = createTransferPhaseWatchdog(); - watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); - watchdog.dispose(); - await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS * 3); - assert.equal(watchdog.signal.aborted, false); -}); - -test('a phase is never armed after the attempt is already over', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); - const watchdog = createTransferPhaseWatchdog(); - try { - watchdog.signal.throwIfAborted(); - const caller = new AbortController(); - caller.abort(new Error('gone')); - const late = createTransferPhaseWatchdog(caller.signal); - try { - late.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); - assert.equal(vi.getTimerCount(), 0); - await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS * 2); - assert.equal(late.signal.reason.message, 'gone'); - } finally { - late.dispose(); - } - } finally { - watchdog.dispose(); - } -}); - -function assertPhaseDetails(reason: unknown, phase: string, timeoutMs: number): void { - assert.ok(reason instanceof AppError, `phase expiry must be an AppError, got ${String(reason)}`); - assert.equal(isTransferPhaseTimeoutError(reason), true); - assert.deepEqual(reason.details, { - reason: 'transfer_phase_timeout', - transferPhase: phase, - timeoutMs, - }); -} diff --git a/packages/remote/src/upload-client.ts b/packages/remote/src/upload-client.ts index 62291f5fe1..b390acbbf4 100644 --- a/packages/remote/src/upload-client.ts +++ b/packages/remote/src/upload-client.ts @@ -7,13 +7,15 @@ import { } from '@agent-device/contracts/daemon-http'; import { prepareUploadArtifact, type PreparedUploadArtifact } from './upload-client-artifact.ts'; import { - createTransferPhaseWatchdog, isRetryableUploadStreamError, - isTransferPhaseTimeoutError, streamFileToHttpRequest, withUploadPreparationDeadline, } from './upload-stream.ts'; import type { UploadProgressSink } from './upload-progress.ts'; +import { + createTransferPhaseWatchdog, + isTransferPhaseTimeoutError, +} from '@agent-device/host-kit/transport'; /** Ceiling for the small JSON `preflight` and `finalize` calls that bracket an upload. */ const UPLOAD_CONTROL_PLANE_TIMEOUT_MS = 30_000; diff --git a/packages/remote/src/upload-stream.ts b/packages/remote/src/upload-stream.ts index 8b112650ac..ce9e439515 100644 --- a/packages/remote/src/upload-stream.ts +++ b/packages/remote/src/upload-stream.ts @@ -5,9 +5,12 @@ import { pipeline } from 'node:stream/promises'; import { addAbortSignal } from 'node:stream'; import { AppError } from '@agent-device/kernel/errors'; import { + createTransferPhaseWatchdog, loadNodeHttpRequester, + observeClientRequestPhases, observeCompletedRequestWrites, readNodeHttpResponseBody, + TRANSFER_CONNECT_TIMEOUT_MS, } from '@agent-device/host-kit/transport'; import { createUploadProgressTransform, @@ -15,81 +18,6 @@ import { type UploadProgressStage, } from './upload-progress.ts'; -export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; - -/** Header wait starts after connection and outgoing flush, never while the body progresses. */ -export const TRANSFER_RESPONSE_HEADER_TIMEOUT_MS = 30_000; - -export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; - -export type TransferPhase = - | 'connect' - | 'request-body' - | 'response-headers' - | 'response-body' - | 'preflight' - | 'finalize'; - -const BODY_PHASES: ReadonlySet = new Set(['request-body', 'response-body']); - -const TRANSFER_PHASE_TIMEOUT_REASON = 'transfer_phase_timeout'; - -/** One sequential phase under the earliest caller/aggregate deadline; progress resets body phases only. */ -export type TransferPhaseWatchdog = { - signal: AbortSignal; - arm(phase: TransferPhase, timeoutMs: number): void; - touchBody(): void; - /** Stops the armed timer. Call it once the attempt is settled. */ - dispose(): void; -}; - -export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): TransferPhaseWatchdog { - const phase = new AbortController(); - const signal = callerSignal ? AbortSignal.any([callerSignal, phase.signal]) : phase.signal; - let armed: { phase: TransferPhase; timeoutMs: number } | undefined; - let timer: ReturnType | undefined; - const rearm = () => { - if (timer !== undefined) clearTimeout(timer); - const current = armed; - if (!current) return; - timer = setTimeout(() => { - timer = undefined; - phase.abort(createTransferPhaseTimeoutError(current.phase, current.timeoutMs)); - }, current.timeoutMs); - timer.unref(); - }; - return { - signal, - arm(nextPhase, timeoutMs) { - if (signal.aborted) return; - armed = { phase: nextPhase, timeoutMs }; - rearm(); - }, - touchBody() { - if (!armed || !BODY_PHASES.has(armed.phase)) return; - rearm(); - }, - dispose() { - armed = undefined; - if (timer !== undefined) clearTimeout(timer); - timer = undefined; - }, - }; -} - -function createTransferPhaseTimeoutError(phase: TransferPhase, timeoutMs: number): AppError { - return new AppError('COMMAND_FAILED', `Artifact transfer ${phase} timed out`, { - reason: TRANSFER_PHASE_TIMEOUT_REASON, - transferPhase: phase, - timeoutMs, - }); -} - -/** Whether `error` is a transfer phase's own expiry, which no fallback or retry may restart. */ -export function isTransferPhaseTimeoutError(error: unknown): boolean { - return error instanceof AppError && error.details?.reason === TRANSFER_PHASE_TIMEOUT_REASON; -} - const UPLOAD_PREPARATION_TIMEOUT_MS = 600_000; export async function withUploadPreparationDeadline( @@ -144,7 +72,7 @@ export async function streamFileToHttpRequest(options: { errorMessage: string; errorHint?: string; retryable?: boolean; - /** Every attempt and hop shares the enclosing aggregate/caller signal. */ + /** All hops share the enclosing aggregate/caller signal. */ signal: AbortSignal; progress?: UploadStreamProgressOptions; }): Promise { @@ -182,31 +110,13 @@ async function streamFileToHttpRequestAttempt(options: { let upload: Promise | undefined; let uploadSettled = false; - let responseReceived = false; return await new Promise((resolve, reject) => { - let outgoingEnded = false; - let transportEstablished = false; - let bodyFlushed = false; - const settle = (result: UploadStreamResponse) => { - stopObservingWrites(); - watchdog.dispose(); - resolve(result); - }; const fail = (error: unknown) => { + phases.dispose(); stopObservingWrites(); watchdog.dispose(); - // The queued pipeline has to settle before the join in `finally`. req.destroy(); - reject(error); - }; - const advanceOutgoingPhase = () => { - if (responseReceived || watchdog.signal.aborted) return; - if (!transportEstablished) return; - if (bodyFlushed) { - watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); - } else { - watchdog.arm('request-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); - } + reject(watchdog.signal.aborted ? watchdog.signal.reason : error); }; const req = transport.request( { @@ -221,43 +131,18 @@ async function streamFileToHttpRequestAttempt(options: { signal: watchdog.signal, }, (res) => { - responseReceived = true; - watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); addAbortSignal(watchdog.signal, res); - void handleUploadResponse(res) - .then((result) => settle(result)) - .catch((error: unknown) => - fail(watchdog.signal.aborted ? watchdog.signal.reason : error), - ); + void handleUploadResponse(res).then(resolve).catch(fail); }, ); - req.on('socket', (socket) => { - if (socket.connecting) { - const secured = (socket as { encrypted?: boolean }).encrypted === true; - socket.once(secured ? 'secureConnect' : 'connect', () => { - transportEstablished = true; - advanceOutgoingPhase(); - }); - } else { - transportEstablished = true; - advanceOutgoingPhase(); - } - }); - req.on('finish', () => { - bodyFlushed = true; - advanceOutgoingPhase(); - }); + const phases = observeClientRequestPhases(req, watchdog); const stopObservingWrites = observeCompletedRequestWrites(req, watchdog.signal, () => watchdog.touchBody(), ); req.on('error', (err) => { - if (responseReceived) return; - if (watchdog.signal.aborted) { - fail(watchdog.signal.reason); - return; - } + if (phases.responseStarted) return; fail( new AppError( 'COMMAND_FAILED', @@ -285,32 +170,18 @@ async function streamFileToHttpRequestAttempt(options: { })().catch((err: unknown) => { // After a response, only this attempt's own teardown ends the pipeline, and the server's // verdict stays the outcome: a 413 read completely is a 413, not a transport break. - if (responseReceived) return; - req.destroy(); - if (watchdog.signal.aborted) { - fail(watchdog.signal.reason); - return; - } + if (phases.responseStarted) return; const error = err instanceof Error ? err : new Error(String(err)); fail(new AppError('COMMAND_FAILED', 'Failed to read local artifact', {}, error)); }); - const endOutgoingWork = (): Promise => { - if (!outgoingEnded) { - outgoingEnded = true; - // The response is consumed, so a destroy cannot cut it short; it settles a pipeline blocked - // on a peer that stopped reading. A flushed pipeline keeps its pooled socket. - if (!uploadSettled) req.destroy(); - } - return upload!.then( - () => undefined, - () => undefined, - ); - }; - async function handleUploadResponse(res: IncomingMessage): Promise { const body = await readNodeHttpResponseBody(res, () => watchdog.touchBody()); - await endOutgoingWork(); + // The response is consumed, so a destroy cannot cut it short; it settles a pipeline blocked + // on a peer that stopped reading. A flushed pipeline keeps its pooled socket. + if (!uploadSettled) req.destroy(); + await upload!.catch(() => undefined); + phases.dispose(); stopObservingWrites(); watchdog.dispose(); diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index 98ffb5dd36..6abd0a754c 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -182,7 +182,7 @@ "packages/remote/src/upload-stream.ts#parseNonNegativeIntegerHeader": "sha256:5cbe9c6994459d673d988a51407e65556e28175b187ce0a9d27cbb6331962834", "packages/remote/src/upload-stream.ts#parseUploadResumeOffset": "sha256:297184ea22a6dc1cda9aee17034122ae0ea2f43d0355647f6fcf02f3f4b5ca0e", "packages/remote/src/upload-stream.ts#streamFileToHttpRequest": "sha256:c2b5e1369c8b1c9420036e23163d21e43fa2ac8b889c6a8f028cc6095055a16f", - "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:1a4b994c3ac2cf89e01e9c56526a5c9a0356fb8de425a248d062bd1cd67686b8", + "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:23ba867eb3ec61b3d95a094715d974951eae1c9e3ebd4b1cb335ed71ec113259", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_HEADER": "sha256:d2bb3f4846470188a0fc1724271da0f0bcbdf2337bd6bf331508142cadcb65d8", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_UPSTREAM_INSTANCE_HEADER": "sha256:4564c2ce437d30d7a823e3326f4375c0750ba8d42bec22b4d02e8eb7f7db2973", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_MISMATCH_HEADER": "sha256:a197edaf7f9fa94a6aab269856aa852da3204cc586f5fd23198689eb11908b61", @@ -496,8 +496,8 @@ }, { "declaration": "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt", - "digest": "sha256:1a4b994c3ac2cf89e01e9c56526a5c9a0356fb8de425a248d062bd1cd67686b8", - "rationale": "#4441 keeps each attempt under its sequential phase watchdog and corrects its transport lifecycle: the connect ceiling now ends at actual socket or TLS-handshake establishment, body progress counts only transport writes, and an early response reads the accepted answer to its end, then destroys and joins the owned file/request pipeline and retires this attempt's phase watchdog before resolving or starting the redirect/resume attempt, so a completed hop's timers never reach the next hop. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own sequencing and deadlines moved, so a protocol-2 peer parses exactly what it parsed before. Request-body progress is now each non-empty request write that completes without an error, which Node reports once the bytes reach the local kernel, instead of drain, which fires only after a write returned false. The write wrapper passes every argument and the backpressure result through, so the bytes and framing on the wire do not change. That completed-write observer is host-kit's transport owner, which also stops reporting after the response starts, the signal aborts or the request is destroyed; no request changes. The response-header ceiling starts at connected request finish, without waiting for or rearming after source-file close. This changes local deadline timing only; URL, method, credentials, body, redirects, resume headers and response framing are unchanged." + "digest": "sha256:23ba867eb3ec61b3d95a094715d974951eae1c9e3ebd4b1cb335ed71ec113259", + "rationale": "#4441 keeps each attempt under its sequential phase watchdog and corrects its transport lifecycle: the connect ceiling now ends at actual socket or TLS-handshake establishment, body progress counts only transport writes, and an early response reads the accepted answer to its end, then destroys and joins the owned file/request pipeline and retires this attempt's phase watchdog before resolving or starting the redirect/resume attempt, so a completed hop's timers never reach the next hop. The outgoing-request phase observation now comes from @agent-device/host-kit/transport's observeClientRequestPhases, which listens to the same socket, TLS-handshake and finish events and is removed when the attempt settles. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own sequencing and deadlines moved, so a protocol-2 peer parses exactly what it parsed before. Request-body progress is now each non-empty request write that completes without an error, which Node reports once the bytes reach the local kernel, instead of drain, which fires only after a write returned false. The write wrapper passes every argument and the backpressure result through, so the bytes and framing on the wire do not change. That completed-write observer is host-kit's transport owner, which also stops reporting after the response starts, the signal aborts or the request is destroyed; no request changes. The attempt composes that write owner with the phase observer once per request and disposes both when it settles. Duplicate success disposal and abort-reason mapping are consolidated without changing precedence. The sole response-path pipeline join remains before outcome or next hop; URL, method, authentication, payloads and framing remain unchanged." }, { "declaration": "packages/remote/src/daemon-artifacts.ts#resolveMaterializedArtifactPath",