diff --git a/packages/host-kit/src/internal/transport-writes.test.ts b/packages/host-kit/src/internal/transport-writes.test.ts new file mode 100644 index 0000000000..62737827a4 --- /dev/null +++ b/packages/host-kit/src/internal/transport-writes.test.ts @@ -0,0 +1,150 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { runInNewContext } from 'node:vm'; +import type { ClientRequest } from 'node:http'; +import { test } from 'vitest'; +import { observeCompletedRequestWrites } from '../transport.ts'; + +class FakeRequest extends EventEmitter { + destroyed = false; + accepts = true; + readonly writes: { self: unknown; args: unknown[] }[] = []; + write(this: FakeRequest, ...args: unknown[]): boolean { + this.writes.push({ self: this, args }); + return this.accepts; + } + complete(index: number, error: Error | null = null): void { + const callback = this.writes[index]?.args.find((arg) => typeof arg === 'function'); + assert.ok(typeof callback === 'function', `write ${index} has no callback`); + callback(error); + } +} + +type Observed = ReturnType; + +function observe(prepare?: (fake: FakeRequest) => void) { + const fake = new FakeRequest(); + prepare?.(fake); + const caller = new AbortController(); + const completed = { count: 0 }; + const req = fake as unknown as ClientRequest; + const dispose = observeCompletedRequestWrites(req, caller.signal, () => (completed.count += 1)); + return { fake, req, caller, completed, dispose }; +} + +test('only a non-empty write that completes without an error is reported', () => { + const { fake, req, completed, dispose } = observe(); + req.write(Buffer.from('queued')); + req.write('', () => {}); + fake.complete(1); + req.write(new Uint8Array(0), () => {}); + fake.complete(2); + req.write('failed', () => {}); + fake.complete(3, new Error('EPIPE')); + for (const [text, encoding] of [ + ['a', 'hex'], + ['zz', 'hex'], + ['zz', 'HEX'], + ['=', 'base64'], + ['=', 'base64url'], + ] as const) { + req.write(text, encoding as BufferEncoding, () => {}); + fake.complete(fake.writes.length - 1); + } + assert.equal( + completed.count, + 0, + 'a queued, empty, failed or encoded-empty write is not progress', + ); + fake.complete(0); + assert.equal(completed.count, 1); + dispose(); +}); + +test.each([ + ['utf8', 'é'], + ['hex', 'ff'], + ['base64', 'YQ=='], + ['base64url', 'YQ'], +] as const)('a successful %s string write is reported', (encoding, text) => { + const { fake, req, completed, dispose } = observe(); + req.write(text, encoding); + fake.complete(0); + assert.equal(completed.count, 1); + dispose(); +}); + +test('a Uint8Array from another realm is reported', () => { + const { fake, req, completed, dispose } = observe(); + const chunk = runInNewContext('new Uint8Array([1])') as Uint8Array; + assert.equal(chunk instanceof Uint8Array, false); + req.write(chunk); + fake.complete(0); + assert.equal(completed.count, 1); + dispose(); +}); + +test('the observed write keeps every argument, this, the result and one callback', () => { + const { fake, req, dispose } = observe(); + const calls: unknown[] = []; + const error = new Error('EPIPE'); + fake.accepts = false; + assert.equal( + req.write('a', 'latin1', (cause) => calls.push(cause)), + false, + ); + fake.accepts = true; + assert.equal( + req.write('b', (cause) => calls.push(cause)), + true, + ); + assert.equal(req.write('c'), true); + assert.deepEqual( + fake.writes.map(({ self, args }) => [self === fake, ...args.map((arg) => typeof arg)]), + [ + [true, 'string', 'string', 'function'], + [true, 'string', 'function'], + [true, 'string', 'undefined', 'function'], + ], + ); + assert.equal(fake.writes[0]?.args[1], 'latin1'); + fake.complete(0, error); + fake.complete(1); + fake.complete(2); + assert.deepEqual(calls, [error, null]); + dispose(); +}); + +const endings: [string, (observed: Observed) => unknown][] = [ + ['the response started', ({ fake }) => fake.emit('response')], + ['the signal aborted', ({ caller }) => caller.abort()], + ['the request was destroyed', ({ fake }) => (fake.destroyed = true)], + ['the observer was disposed', ({ dispose }) => dispose()], +]; + +test.each(endings)('a completion after %s is not reported', (_, end) => { + const observed = observe(); + const calls: unknown[] = []; + observed.req.write(Buffer.from('late'), (cause) => calls.push(cause)); + end(observed); + observed.fake.complete(0); + assert.equal(observed.completed.count, 0); + assert.deepEqual(calls, [null], 'the caller still hears its own completion'); + observed.dispose(); +}); + +test('disposing removes the listener and restores only the write it wrapped', () => { + const inherited = observe(); + inherited.dispose(); + assert.equal(inherited.fake.listenerCount('response'), 0); + assert.equal(Object.hasOwn(inherited.fake, 'write'), false, 'the inherited write is back'); + const ownWrite = () => true; + const owned = observe((fake) => Object.assign(fake, { write: ownWrite })); + owned.dispose(); + assert.equal(owned.fake.write, ownWrite, 'an own write is put back'); + const wrapped = observe(); + const foreign = () => true; + Object.assign(wrapped.fake, { write: foreign }); + wrapped.dispose(); + assert.equal(wrapped.fake.write, foreign, 'a later wrapper stays installed'); +}); diff --git a/packages/host-kit/src/internal/transport.test.ts b/packages/host-kit/src/internal/transport.test.ts index 3ae23547d4..0cc966a8a8 100644 --- a/packages/host-kit/src/internal/transport.test.ts +++ b/packages/host-kit/src/internal/transport.test.ts @@ -44,6 +44,13 @@ test('loadNodeHttpRequester selects the protocol module and keeps request mutabl test('readNodeHttpResponseBody decodes the complete response stream', async () => { const response = Readable.from(['hello', Buffer.from(' world')]) as unknown as IncomingMessage; assert.equal(await readNodeHttpResponseBody(response), 'hello world'); + const observed: string[] = []; + const progressing = Readable.from(['hello', Buffer.from(' world')]) as unknown as IncomingMessage; + assert.equal( + await readNodeHttpResponseBody(progressing, (chunk) => observed.push(chunk)), + 'hello world', + ); + assert.deepEqual(observed, ['hello', ' world']); }); test('readNodeHttpRequestBody returns bytes and preserves the caller error message at the limit', async () => { diff --git a/packages/host-kit/src/internal/transport.ts b/packages/host-kit/src/internal/transport.ts index 13fa052b69..8e4a1351b1 100644 --- a/packages/host-kit/src/internal/transport.ts +++ b/packages/host-kit/src/internal/transport.ts @@ -1,5 +1,6 @@ import crypto from 'node:crypto'; -import type { IncomingMessage } from 'node:http'; +import { types } from 'node:util'; +import type { ClientRequest, IncomingMessage } from 'node:http'; import { AppError } from '@agent-device/kernel/errors'; export function consumeTextLines( @@ -42,11 +43,15 @@ export async function loadNodeHttpRequester(protocol: string): Promise { +export function readNodeHttpResponseBody( + res: IncomingMessage, + onChunk?: (chunk: string) => void, +): Promise { return new Promise((resolve, reject) => { let body = ''; res.setEncoding('utf8'); - res.on('data', (chunk) => { + res.on('data', (chunk: string) => { + onChunk?.(chunk); body += chunk; }); res.on('end', () => resolve(body)); @@ -82,3 +87,53 @@ export function timingSafeStringEqual(a: string, b: string): boolean { const hashB = crypto.createHash('sha256').update(b).digest(); return crypto.timingSafeEqual(hashA, hashB); } + +/** + * Reports each non-empty `req.write` chunk that Node completes without an error: its bytes reached + * the local kernel, not the peer. A queued write, `drain`, and a completion after the response + * started, after `signal` aborted, after a destroy or after disposal are not reported. + * + * Attach it in the tick that creates the request, before anything writes to it. The wrapper keeps + * the arguments, `this`, the backpressure result and one call of the caller's callback with its + * error. Disposing removes its listener and restores `write` only while this wrapper is installed. + */ +export function observeCompletedRequestWrites( + req: ClientRequest, + signal: AbortSignal, + onCompleted: () => void, +): () => void { + const ownWrite = Object.getOwnPropertyDescriptor(req, 'write'); + const write = req.write; + let observing = true; + const onResponse = () => { + observing = false; + }; + const observedWrite = function (this: ClientRequest, ...args: unknown[]): boolean { + const [chunk, encoding] = args; + const callbackIndex = typeof encoding === 'function' ? 1 : 2; + const callback = args[callbackIndex]; + const encodingName = typeof encoding === 'string' ? encoding.toLowerCase() : undefined; + const hasBytes = + typeof chunk === 'string' + ? encodingName === 'hex' || encodingName === 'base64' || encodingName === 'base64url' + ? Buffer.from(chunk, encodingName).byteLength > 0 + : chunk.length > 0 + : types.isUint8Array(chunk) && chunk.byteLength > 0; + if (observing && hasBytes) { + args[callbackIndex] = (error?: Error | null) => { + if (!error && observing && !req.destroyed && !signal.aborted) onCompleted(); + if (typeof callback === 'function') callback(error); + }; + } + return Reflect.apply(write, this, args); + }; + req.write = observedWrite; + req.prependListener('response', onResponse); + return () => { + observing = false; + req.off('response', onResponse); + if (req.write !== observedWrite) return; + if (ownWrite) Object.defineProperty(req, 'write', ownWrite); + else Reflect.deleteProperty(req, 'write'); + }; +} diff --git a/packages/host-kit/src/transport.ts b/packages/host-kit/src/transport.ts index a2d250af38..6b55289d46 100644 --- a/packages/host-kit/src/transport.ts +++ b/packages/host-kit/src/transport.ts @@ -1,6 +1,7 @@ export { consumeTextLines, loadNodeHttpRequester, + observeCompletedRequestWrites, readNodeHttpRequestBody, readNodeHttpResponseBody, timingSafeStringEqual, diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts new file mode 100644 index 0000000000..83dd4dfc2c --- /dev/null +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -0,0 +1,609 @@ +import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; +import fs from 'node:fs'; +import http from 'node:http'; +import https from 'node:https'; +import net from 'node:net'; +import path from 'node:path'; +import { PassThrough } from 'node:stream'; +import tls from 'node:tls'; +import { setTimeout as realSetTimeout, clearTimeout as realClearTimeout } from 'node:timers'; +import { beforeEach, onTestFinished, vi } from 'vitest'; +import { closeLoopbackServer, listenOnLoopback, mkdtempForTestSync } from '@agent-device/test-kit'; +import { uploadArtifact } from '../../upload-client.ts'; +import { type UploadStreamResponse, streamFileToHttpRequest } from '../../upload-stream.ts'; +export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; +export const TRANSFER_RESPONSE_HEADER_TIMEOUT_MS = 30_000; +export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; + +export const PREFLIGHT_REQUEST = 'POST /upload/preflight'; +export const LEGACY_UPLOAD_REQUEST = 'POST /upload'; + +const BUDGET_STEP_MS = 5_000; +const RELEASE_QUOTA_BYTES = 256 * 1024; + +export type Settles = { readonly settled: boolean }; + +const ioWaiters = new Map<() => void, () => void>(); +let ioDeadline: number; +beforeEach(({ task }) => { + assert.ok(task.timeout > 100, 'transfer diagnostics require a finite test timeout'); + ioDeadline = performance.now() + task.timeout - 100; + onTestFinished(() => { + for (const release of ioWaiters.values()) release(); + ioWaiters.clear(); + }); +}); + +function noteIo(): void { + for (const wake of ioWaiters.keys()) wake(); +} + +/** Charges no fake time; diagnostics fire 100ms before the declared test timeout. */ +export async function untilIo( + ready: () => boolean, + label = 'I/O prerequisite', + snapshot: () => unknown = () => ({}), +): Promise { + if (ready()) return; + const unmet = new Error('untilIo: the I/O this wait needs never happened before its test ended'); + await new Promise((resolve, reject) => { + const finish = (error?: unknown) => { + realClearTimeout(timer); + ioWaiters.delete(wake); + if (error) reject(error); + else resolve(); + }; + const wake = () => { + try { + if (ready()) finish(); + } catch (error) { + finish(error); + } + }; + const timer = realSetTimeout( + () => { + const error = new Error(`${label}: real I/O deadline; ${JSON.stringify(snapshot())}`, { + cause: unmet, + }); + finish(error); + }, + Math.max(0, ioDeadline - performance.now()), + ); + ioWaiters.set(wake, () => finish(unmet)); + }); +} + +export function useTransferClock(): void { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); +} + +export type ObservedOutcome = Promise<{ value: T | undefined; error: unknown }> & Settles; + +export function observeOutcome(work: Promise): ObservedOutcome { + let settled = false; + const outcome = work.then( + (value) => ({ value, error: undefined as unknown }), + (error: unknown) => ({ value: undefined as T | undefined, error }), + ); + void outcome.then(() => { + settled = true; + noteIo(); + }); + return Object.defineProperty(outcome, 'settled', { get: () => settled }) as ObservedOutcome; +} + +export function startUpload(options: { + baseUrl: string; + artifactPath: string; + signal?: AbortSignal; +}): ObservedOutcome { + return observeOutcome( + uploadArtifact({ + env: process.env, + localPath: options.artifactPath, + baseUrl: options.baseUrl, + token: 'daemon-token', + ...(options.signal ? { signal: options.signal } : {}), + }), + ); +} + +export function startUploadAttempt(options: { + url: string; + artifactPath: string; + method?: 'POST' | 'PUT'; + headers?: Record; + retryable?: boolean; + signal?: AbortSignal; +}): ObservedOutcome { + return observeOutcome( + streamFileToHttpRequest({ + url: new URL(options.url), + method: options.method ?? 'POST', + headers: options.headers ?? { authorization: 'Bearer daemon-token' }, + payloadPath: options.artifactPath, + errorMessage: 'Failed to upload artifact to remote daemon', + ...(options.retryable === undefined ? {} : { retryable: options.retryable }), + signal: options.signal ?? new AbortController().signal, + }), + ); +} + +export async function spendBudget(ms: number, transfer: Settles): Promise { + for (let elapsed = 0; elapsed < ms && !transfer.settled; elapsed += BUDGET_STEP_MS) { + await vi.advanceTimersByTimeAsync(BUDGET_STEP_MS); + await pumpRealWork(); + } +} + +export async function spendProgressWindows( + daemon: ParkedUploadDaemon, + upload: Settles, + options: { windows: number; windowMs: number; writes: OutgoingWrites }, +): Promise<{ progressedInEveryWindow: boolean; budgetMs: number }> { + await untilIo( + () => upload.settled || (options.writes.completed > 0 && daemon.receivedBytes > 0), + 'first body progress', + () => ({ + writes: options.writes, + receivedBytes: daemon.receivedBytes, + requests: daemon.requests, + settled: upload.settled, + }), + ); + let progressedInEveryWindow = true; + let spent = 0; + for (let window = 0; window < options.windows; window += 1) { + const bodyInFlight = !daemon.bodyEnded; + const before = { bytes: daemon.receivedBytes, completed: options.writes.completed }; + daemon.releaseAllParked(); + const grew = bodyInFlight + ? await awaitBodyProgress(daemon, options.writes, before, upload) + : true; + await vi.advanceTimersByTimeAsync(options.windowMs); + spent += options.windowMs; + if (upload.settled) break; + if (!grew) progressedInEveryWindow = false; + } + return { progressedInEveryWindow, budgetMs: spent }; +} + +export async function awaitBodyProgress( + daemon: { readonly receivedBytes: number; releaseAllParked(): void }, + writes: OutgoingWrites, + before: { bytes: number; completed: number }, + transfer: Settles, +): Promise { + const progressed = () => + writes.completed > before.completed && daemon.receivedBytes > before.bytes; + await untilIo( + () => { + if (progressed() || transfer.settled) return true; + daemon.releaseAllParked(); + return false; + }, + 'next body progress', + () => ({ writes, receivedBytes: daemon.receivedBytes, before, settled: transfer.settled }), + ); + return progressed(); +} + +export type OutgoingWrites = { + readonly falseReturns: number; + /** Successful write callbacks mean local-kernel acceptance, not peer receipt. */ + readonly completed: number; + readonly drains: number; + readonly lastCompletedAt: number | undefined; + readonly finished: boolean; + readonly destroyed: boolean; + readonly responseChunkRead: boolean; +}; + +export function recordOutgoingWrites(): OutgoingWrites { + let currentRequest: http.ClientRequest | undefined; + const log: { -readonly [K in keyof OutgoingWrites]: OutgoingWrites[K] } = { + falseReturns: 0, + completed: 0, + drains: 0, + finished: false, + get destroyed() { + return currentRequest?.destroyed ?? false; + }, + lastCompletedAt: undefined, + responseChunkRead: false, + }; + const observe = (req: http.ClientRequest): http.ClientRequest => { + currentRequest = req; + log.finished = false; + log.lastCompletedAt = undefined; + log.responseChunkRead = false; + const write = req.write; + req.write = function (this: http.ClientRequest, ...writeArgs: unknown[]): boolean { + const callbackIndex = typeof writeArgs[1] === 'function' ? 1 : 2; + const callback = writeArgs[callbackIndex]; + writeArgs[callbackIndex] = (error?: Error | null) => { + if (!error) { + log.completed += 1; + log.lastCompletedAt = vi.getMockedSystemTime()?.getTime(); + } + if (typeof callback === 'function') callback(error); + noteIo(); + }; + const accepted: boolean = Reflect.apply(write, this, writeArgs); + if (!accepted) log.falseReturns += 1; + noteIo(); + return accepted; + }; + req.on('response', (res) => { + res.once('data', () => { + log.responseChunkRead = true; + noteIo(); + }); + }); + req.on('drain', () => { + log.drains += 1; + noteIo(); + }); + req.on('finish', () => { + log.finished = true; + noteIo(); + }); + return req; + }; + const httpRequest = http.request; + vi.spyOn(http, 'request').mockImplementation((...args) => + observe(Reflect.apply(httpRequest, http, args) as http.ClientRequest), + ); + const httpsRequest = https.request; + vi.spyOn(https, 'request').mockImplementation((...args) => + observe(Reflect.apply(httpsRequest, https, args) as http.ClientRequest), + ); + return log; +} + +export async function spendSinceLastWrite( + ms: number, + writes: OutgoingWrites, + transfer: Settles, +): Promise { + await untilIo(() => writes.lastCompletedAt !== undefined || transfer.settled); + while (!transfer.settled) { + const remaining = (writes.lastCompletedAt ?? 0) + ms - controlledNow(); + if (remaining <= 0) return; + await vi.advanceTimersByTimeAsync(Math.min(remaining, BUDGET_STEP_MS)); + await pumpRealWork(); + } +} + +function controlledNow(): number { + const now = vi.getMockedSystemTime(); + assert.ok(now, 'the transfer clock is installed'); + return now.getTime(); +} + +export async function settleAfterBudget( + transfer: Settles, + snapshot: () => unknown = () => ({ settled: transfer.settled }), +): Promise { + for (let round = 0; round < 200 && !transfer.settled; round += 1) { + await vi.advanceTimersByTimeAsync(100); + await pumpRealWork(5); + } + await untilIo(() => transfer.settled, 'settlement after phase expiry', snapshot); +} + +/** Holds upload reads selected by start offset; metadata hashing remains real. */ +export function holdArtifactSourceOpen(bytes = 0): { + readonly source: PassThrough; + readonly destroyedAtNextOpen: boolean | undefined; +} { + const source = new PassThrough(); + onTestFinished(() => { + source.destroy(); + }); + if (bytes > 0) source.write(Buffer.alloc(bytes, 'x')); + const open = fs.createReadStream; + const held = { source, destroyedAtNextOpen: undefined as boolean | undefined }; + let opened = false; + vi.spyOn(fs, 'createReadStream').mockImplementation((...args) => { + const options = args[1]; + if (options && typeof options === 'object' && typeof options.start === 'number') { + if (!opened) { + opened = true; + return source as unknown as fs.ReadStream; + } + held.destroyedAtNextOpen ??= source.destroyed; + } + return Reflect.apply(open, fs, args); + }); + return held; +} + +export async function pumpRealWork(rounds = 40): Promise { + for (let round = 0; round < rounds; round += 1) { + await new Promise((resolve) => setImmediate(resolve)); + } +} + +export function createArtifactFile(bytes: number, fileName = 'app.apk'): string { + const dir = mkdtempForTestSync('agent-device-upload-phases-'); + const artifactPath = path.join(dir, fileName); + fs.writeFileSync(artifactPath, Buffer.alloc(bytes, 'x')); + return artifactPath; +} + +async function listenUntimed(server: net.Server): Promise { + if (server instanceof http.Server) { + server.requestTimeout = 0; + server.headersTimeout = 0; + server.keepAliveTimeout = 0; + } + server.unref(); + onTestFinished(() => closeLoopbackServer(server)); + return await listenOnLoopback(server); +} + +function refusedPreflight(req: http.IncomingMessage, res: http.ServerResponse): boolean { + if (`${req.method} ${req.url}` !== PREFLIGHT_REQUEST) return false; + req.resume(); + req.on('end', () => { + res.writeHead(404).end('not found'); + }); + return true; +} + +export async function startParkedUploadDaemon( + options: { + answerFirstLeg?: { status: number; headers?: Record }; + } = {}, +): Promise { + const requests: string[] = []; + const parked: http.IncomingMessage[] = []; + let receivedBytes = 0; + let bytesTakenSinceRelease = 0; + let parkedAtQuota: http.IncomingMessage | undefined; + let bodyEnded = false; + let uploadLegs = 0; + let newestParkedLeg: http.IncomingMessage | undefined; + let readingToEnd = false; + const server = http.createServer((req, res) => { + const key = `${req.method} ${req.url}`; + requests.push(key); + noteIo(); + if (refusedPreflight(req, res)) return; + if (key === LEGACY_UPLOAD_REQUEST) { + uploadLegs += 1; + req.on('error', () => {}); + const answer = options.answerFirstLeg; + if (answer && uploadLegs === 1) { + res.writeHead(answer.status, answer.headers); + res.end(); + req.pause(); + return; + } + newestParkedLeg = req; + req.on('data', (chunk: Buffer) => { + receivedBytes += chunk.length; + bytesTakenSinceRelease += chunk.length; + if (!readingToEnd) { + req.pause(); + if (bytesTakenSinceRelease < RELEASE_QUOTA_BYTES) parked.push(req); + else parkedAtQuota = req; + } + noteIo(); + }); + req.on('end', () => { + bodyEnded = true; + noteIo(); + }); + return; + } + req.resume(); + req.on('end', () => { + res.statusCode = 500; + res.end('only the upload leg the test set up is expected'); + }); + }); + const port = await listenUntimed(server); + return { + baseUrl: `http://127.0.0.1:${port}`, + requests, + get receivedBytes() { + return receivedBytes; + }, + get bodyEnded() { + return bodyEnded; + }, + releaseAllParked() { + bytesTakenSinceRelease = 0; + const resumable = parked.splice(0); + if (parkedAtQuota) { + resumable.push(parkedAtQuota); + parkedAtQuota = undefined; + } + for (const entry of resumable) entry.resume(); + }, + readToEnd() { + readingToEnd = true; + for (const entry of parked.splice(0)) entry.resume(); + parkedAtQuota?.resume(); + parkedAtQuota = undefined; + }, + resetParkedLeg() { + assert.ok(newestParkedLeg, 'no parked leg to reset'); + newestParkedLeg.socket.resetAndDestroy(); + }, + }; +} + +export type ParkedUploadDaemon = { + readonly baseUrl: string; + readonly requests: readonly string[]; + readonly receivedBytes: number; + readonly bodyEnded: boolean; + releaseAllParked(): void; + readToEnd(): void; + resetParkedLeg(): void; +}; + +export async function startAnswerUploadDaemon(options: { + respond(req: http.IncomingMessage): { + status: number; + headers?: Record; + body: string; + }; +}) { + const requests: string[] = []; + const server = http.createServer((req, res) => { + requests.push(`${req.method} ${req.url}`); + req.on('error', () => {}); + const answer = options.respond(req); + res.writeHead(answer.status, answer.headers); + res.end(answer.body); + req.pause(); + }); + const port = await listenUntimed(server); + return { + baseUrl: `http://127.0.0.1:${port}`, + requests, + }; +} + +export async function startSilentTcpOrigin() { + const server = net.createServer((socket) => { + socket.pause(); + socket.on('error', () => {}); + }); + const port = await listenUntimed(server); + return { port }; +} + +export async function startHalfAnsweredOrigin(options: { firstChunk: string }) { + const server = http.createServer((req, res) => { + req.resume(); + res.writeHead(200, { 'content-type': 'text/plain' }); + res.write(options.firstChunk); + }); + const port = await listenUntimed(server); + return { baseUrl: `http://127.0.0.1:${port}` }; +} + +export async function startSlowReadingUploadDaemon(options: { status: number; body: string }) { + const parked: http.IncomingMessage[] = []; + let receivedBytes = 0; + let bodyEnded = false; + const server = http.createServer((req, res) => { + if (refusedPreflight(req, res)) return; + req.on('data', (chunk: Buffer) => { + receivedBytes += chunk.length; + req.pause(); + parked.push(req); + noteIo(); + }); + req.on('end', () => { + bodyEnded = true; + res.writeHead(options.status).end(options.body); + noteIo(); + }); + }); + const port = await listenUntimed(server); + return { + baseUrl: `http://127.0.0.1:${port}`, + get receivedBytes() { + return receivedBytes; + }, + get bodyEnded() { + return bodyEnded; + }, + releaseAllParked() { + for (const entry of parked.splice(0)) entry.resume(); + }, + }; +} + +export async function startConnectedStalledTlsOrigin() { + const server = tls.createServer( + { key: readRepoFixture('localhost-key.pem'), cert: readRepoFixture('localhost-cert.pem') }, + (socket) => { + socket.on('data', () => socket.pause()); + socket.on('error', () => {}); + }, + ); + const port = await listenUntimed(server); + return { port }; +} + +/** Restores the global agent even when later origin setup fails. */ +export function trustSelfSignedLocalhostCertificate(): void { + const savedCa = https.globalAgent.options.ca; + https.globalAgent.options.ca = readRepoFixture('localhost-cert.pem'); + onTestFinished(() => { + https.globalAgent.options.ca = savedCa; + }); +} + +export async function startUploadLegsDaemon( + behaviors: readonly { + readAllBody: boolean; + respond?: () => { status: number; headers?: Record; body: string }; + }[], +) { + const requests: string[] = []; + let legs = 0; + let connections = 0; + const server = http.createServer((req, res) => { + requests.push(`${req.method} ${req.url}`); + noteIo(); + const behavior = behaviors[legs]; + if (!behavior) { + res.statusCode = 500; + res.end(`unexpected upload leg ${legs + 1}`); + return; + } + legs += 1; + const answer = () => { + if (!behavior.respond) return; + const reply = behavior.respond(); + res.writeHead(reply.status, reply.headers); + res.end(reply.body); + }; + if (behavior.readAllBody) { + req.resume(); + req.on('end', answer); + return; + } + req.once('data', () => req.pause()); + answer(); + }); + server.on('connection', () => { + connections += 1; + }); + const port = await listenUntimed(server); + return { + baseUrl: `http://127.0.0.1:${port}`, + requests, + get connections() { + return connections; + }, + }; +} + +function readRepoFixture(fileName: string): Buffer { + return fs.readFileSync( + new URL( + `../../../../../test/integration/provider-scenarios/fixtures/${fileName}`, + import.meta.url, + ), + ); +} + +export function assertDetailedError( + error: unknown, + details: Record, +): asserts error is AppError { + assert.ok(error instanceof AppError, `expected AppError, got ${String(error)}`); + for (const [key, value] of Object.entries(details)) { + assert.equal(error.details?.[key], value, `${key} of ${JSON.stringify(error.details)}`); + } +} diff --git a/packages/remote/src/__tests__/upload-client-cancellation.test.ts b/packages/remote/src/__tests__/upload-client-cancellation.test.ts index b42aed13e5..573e8c2911 100644 --- a/packages/remote/src/__tests__/upload-client-cancellation.test.ts +++ b/packages/remote/src/__tests__/upload-client-cancellation.test.ts @@ -5,26 +5,191 @@ // abort signal — a rejection the caller swallows would keep streaming a full app bundle to a device // nobody owns. -import { afterEach, test } from 'vitest'; +import { afterEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import fs from 'node:fs'; import http, { type IncomingMessage, type ServerResponse } from 'node:http'; import path from 'node:path'; +import { PassThrough } from 'node:stream'; +import { AppError } from '@agent-device/kernel/errors'; import { once } from 'node:events'; import { uploadArtifact } from '../upload-client.ts'; import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { observeOutcome } from './support/upload-transfer-fixtures.ts'; const TEST_TOKEN = 'agent-device-upload-cancel-token'; const tempDirs: string[] = []; afterEach(async () => { + vi.restoreAllMocks(); + vi.useRealTimers(); for (const dir of tempDirs) { await fs.promises.rm(dir, { recursive: true, force: true }).catch(() => {}); } tempDirs.length = 0; }); +test('cancellation refuses preparation before it reads a missing artifact', async () => { + const artifactPath = createTempFile('missing.apk', 'payload'); + fs.unlinkSync(artifactPath); + const control = new AbortController(); + const reason = new Error('request owner stopped'); + control.abort(reason); + const fetch = vi.spyOn(globalThis, 'fetch'); + await assert.rejects( + uploadArtifact({ + env: process.env, + localPath: artifactPath, + baseUrl: 'http://127.0.0.1:1', + token: TEST_TOKEN, + signal: control.signal, + }), + (error: unknown) => error === reason, + ); + assert.equal(fetch.mock.calls.length, 0); +}); + +test('a standalone upload expires its preparation aggregate during hashing without starting a request', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const artifactPath = createTempFile('app.apk', 'payload'); + const hashSource = new PassThrough(); + vi.spyOn(fs, 'createReadStream').mockReturnValue(hashSource as unknown as fs.ReadStream); + const fetch = vi.spyOn(globalThis, 'fetch'); + const outcome = uploadArtifact({ + env: process.env, + localPath: artifactPath, + baseUrl: 'http://127.0.0.1:1', + token: TEST_TOKEN, + }).catch((error: unknown) => error); + try { + await vi.advanceTimersByTimeAsync(600_001); + const error = await outcome; + assert.ok(error instanceof AppError); + assert.deepEqual(error.details, { + reason: 'artifact_preparation_deadline_exceeded', + transferPhase: 'preparation', + timeoutMs: 600_000, + }); + assert.equal(hashSource.destroyed, true); + assert.equal(fetch.mock.calls.length, 0); + } finally { + hashSource.destroy(); + await outcome; + } +}); + +test.each([ + 'preflight', + 'finalize', + 'retry-preflight', + 'retry-finalize', + 'cancel-preflight', + 'cancel-finalize', + 'prose-preflight', + 'prose-finalize', +])('control request %s distinguishes a deadline from matching error prose', async (stage) => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const caller = new AbortController(); + const artifactPath = createTempFile('app.apk', 'control-plane-payload'); + const retry = stage.startsWith('retry-'); + const prose = stage.startsWith('prose-'); + const canceled = stage.startsWith('cancel-'); + const callerReason = new Error('lease ended during control request'); + const phase = stage.endsWith('preflight') ? 'preflight' : 'finalize'; + const events: string[] = []; + const routes: string[] = []; + let uploads = 0; + const server = await startServer(async (req, res) => { + routes.push(`${req.method} ${req.url}`); + if (retry && ++uploads === 1) { + req.destroy(); + return; + } + await readRequestBody(req); + sendJson(res, { ok: true, uploadId: 'control-upload' }); + }); + let entered = () => {}; + const stalled = new Promise((resolve) => { + entered = resolve; + }); + let stalledSignal: AbortSignal | undefined; + let preflights = 0; + let refuseStall: ((error: Error) => void) | undefined; + const fetch = vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { + const route = new URL(String(input)).pathname; + if (route.endsWith('/preflight')) preflights += 1; + if (route.endsWith('/' + phase) && (!retry || preflights === 2)) { + const signal = init?.signal; + assert.ok(signal); + stalledSignal = signal; + entered(); + if (prose) throw new Error(`Artifact transfer ${phase} timed out`); + return await new Promise((_resolve, reject) => { + refuseStall = reject; + signal.addEventListener('abort', () => reject(signal.reason), { once: true }); + }); + } + return Response.json({ + ok: true, + uploadId: 'control-upload', + upload: { url: `${server.baseUrl}/ticket`, headers: {} }, + }); + }); + const outcome = observeOutcome( + uploadArtifact({ + env: process.env, + localPath: artifactPath, + baseUrl: server.baseUrl, + token: TEST_TOKEN, + signal: caller.signal, + onProgress: (event) => events.push(event.type), + }), + ); + try { + await stalled; + if (prose) { + const result = await outcome; + assert.equal(result.error, undefined); + assert.equal(result.value, 'control-upload'); + assert.ok(routes.includes('POST /upload')); + assert.equal(stalledSignal?.aborted, false); + return; + } + await vi.advanceTimersByTimeAsync(canceled ? 10_000 : 29_999); + assert.equal(stalledSignal?.aborted, false); + assert.equal(outcome.settled, false); + if (canceled) caller.abort(callerReason); + await vi.advanceTimersByTimeAsync(canceled ? 0 : 1); + assert.equal(stalledSignal?.aborted, true); + assert.equal(outcome.settled, true, 'expiry settles without starting another transfer'); + const result = await outcome; + assert.equal(result.value, undefined); + if (canceled) assert.equal(result.error, callerReason); + else { + assert.ok(result.error instanceof AppError); + assert.deepEqual(result.error.details, { + reason: 'transfer_phase_timeout', + transferPhase: phase, + timeoutMs: 30_000, + }); + } + assert.equal(caller.signal.aborted, canceled); + assert.equal(events.includes('fallback'), false); + const expectedTicketLegs = Number(retry) + Number(phase === 'finalize'); + assert.deepEqual( + routes, + Array.from({ length: expectedTicketLegs }, () => 'PUT /ticket'), + ); + assert.equal(fetch.mock.calls.length, 1 + expectedTicketLegs); + } finally { + caller.abort(new Error('fixture cleanup')); + refuseStall?.(new Error('fixture cleanup')); + await outcome; + await server.close(); + } +}); + test('an aborted signal ends a legacy upload mid-stream instead of finishing the bytes', async () => { // Two megabytes is well past what a paused read lets through: the assertions below are about the // stream stopping, and a smaller payload keeps that off the CPU in a loaded lane. @@ -76,6 +241,59 @@ test('an aborted signal ends a legacy upload mid-stream instead of finishing the } }); +test('a cancel during the retried direct upload ends the upload without announcing a fallback', async () => { + const artifactPath = createTempFile('app.apk', 'retried-direct-payload'); + const control = new AbortController(); + const reason = new Error('request owner stopped'); + const requests: string[] = []; + const events: string[] = []; + let ticketLegs = 0; + const server = await startServer(async (req, res) => { + requests.push(`${req.method} ${req.url}`); + if (req.url === '/upload/preflight') { + await readRequestBody(req); + sendJson(res, { + ok: true, + uploadId: 'retried-ticket', + upload: { url: `${server.baseUrl}/ticket`, headers: {} }, + }); + return; + } + if (req.url === '/ticket') { + ticketLegs += 1; + // Leg one breaks as a retryable transport failure; the caller cancels during leg two. + if (ticketLegs === 1) req.destroy(); + else control.abort(reason); + return; + } + res.statusCode = 404; + res.end('not found'); + }); + + try { + await assert.rejects( + uploadArtifact({ + env: process.env, + localPath: artifactPath, + baseUrl: server.baseUrl, + token: TEST_TOKEN, + signal: control.signal, + onProgress: (event) => events.push(event.type), + }), + (error: unknown) => error === reason, + ); + assert.equal(events.includes('fallback'), false, events.join(',')); + assert.deepEqual(requests, [ + 'POST /upload/preflight', + 'PUT /ticket', + 'POST /upload/preflight', + 'PUT /ticket', + ]); + } finally { + await server.close(); + } +}); + test('an aborted signal before preflight refuses to ask the daemon for a ticket', async () => { const artifactPath = createTempFile('app.apk', 'payload'); const control = new AbortController(); diff --git a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts new file mode 100644 index 0000000000..ed1cd9a47a --- /dev/null +++ b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts @@ -0,0 +1,375 @@ +import { afterEach, assert, test, vi } from 'vitest'; +import { isRetryableUploadStreamError, isTransferPhaseTimeoutError } from '../upload-stream.ts'; +import { + LEGACY_UPLOAD_REQUEST, + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + TRANSFER_CONNECT_TIMEOUT_MS, + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + type ParkedUploadDaemon, + assertDetailedError, + awaitBodyProgress, + createArtifactFile, + holdArtifactSourceOpen, + recordOutgoingWrites, + settleAfterBudget, + spendBudget, + spendSinceLastWrite, + startAnswerUploadDaemon, + startSilentTcpOrigin, + startConnectedStalledTlsOrigin, + startHalfAnsweredOrigin, + startParkedUploadDaemon, + startSlowReadingUploadDaemon, + startUploadAttempt, + startUploadLegsDaemon, + trustSelfSignedLocalhostCertificate, + untilIo, + useTransferClock, +} from './support/upload-transfer-fixtures.ts'; + +const MEBIBYTE = 1024 * 1024; + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +test('an early 413 from a peer that stops reading is consumed and its owned work is ended', async () => { + useTransferClock(); + const daemon = await startAnswerUploadDaemon({ + respond: () => ({ status: 413, body: 'payload too large' }), + }); + const writes = recordOutgoingWrites(); + const { source } = holdArtifactSourceOpen(MEBIBYTE); + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(4 * MEBIBYTE), + }); + const atSettle = attempt.then(() => ({ destroyed: source.destroyed, finished: writes.finished })); + await settleAfterBudget(attempt); + assert.deepEqual(await atSettle, { destroyed: true, finished: false }); + const result = await attempt; + assert.equal(result.error, undefined, `got ${String(result.error)}`); + assert.equal(result.value?.statusCode, 413); + assert.equal(result.value?.body, 'payload too large'); + assert.deepEqual(daemon.requests, [LEGACY_UPLOAD_REQUEST]); +}); + +test('a resume response ends the stalled hop before the next attempt is dialed', async () => { + useTransferClock(); + const resumeOffset = MEBIBYTE; + const daemon = await startAnswerUploadDaemon({ + respond: () => ({ + status: 308, + headers: { 'x-upload-offset': String(resumeOffset) }, + body: '', + }), + }); + const held = holdArtifactSourceOpen(MEBIBYTE); + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(4 * MEBIBYTE), + }); + await settleAfterBudget(attempt); + const result = await attempt; + assertDetailedError(result.error, { offset: resumeOffset, previousOffset: resumeOffset }); + assert.equal(daemon.requests.filter((entry) => entry === LEGACY_UPLOAD_REQUEST).length, 2); + assert.equal(held.destroyedAtNextOpen, true, 'hop one was ended before hop two opened'); +}); + +test('a resumed hop that makes real progress past the first hop ceiling keeps its own retryable reset', async () => { + useTransferClock(); + const progressWindowMs = 30_000; + assert.ok(progressWindowMs < TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + const artifactBytes = 32 * MEBIBYTE; + const writes = recordOutgoingWrites(); + const daemon = await startParkedUploadDaemon({ + answerFirstLeg: { status: 308, headers: { 'x-upload-offset': String(MEBIBYTE) } }, + }); + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(artifactBytes), + method: 'POST', + retryable: true, + }); + await untilIo(() => hopTwoTookBytes(daemon) || attempt.settled); + assert.ok(hopTwoTookBytes(daemon), 'the resumed hop was dialed and took bytes'); + for (let window = 0; window < 6; window += 1) { + const before = { bytes: daemon.receivedBytes, completed: writes.completed }; + const progressed = await awaitBodyProgress(daemon, writes, before, attempt); + assert.equal(progressed, true, `window ${window}`); + await vi.advanceTimersByTimeAsync(progressWindowMs); + assert.equal(attempt.settled, false); + assert.equal(daemon.bodyEnded, false); + } + assert.ok(6 * progressWindowMs > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + daemon.resetParkedLeg(); + await settleAfterBudget(attempt); + const result = await attempt; + assert.equal(result.value, undefined); + assert.equal(isTransferPhaseTimeoutError(result.error), false, JSON.stringify(result.error)); + assert.equal(isRetryableUploadStreamError(result.error), true); + assert.equal(daemon.requests.filter((entry) => entry === LEGACY_UPLOAD_REQUEST).length, 2); +}); + +function hopTwoTookBytes(daemon: ParkedUploadDaemon): boolean { + return ( + daemon.requests.filter((entry) => entry === LEGACY_UPLOAD_REQUEST).length === 2 && + daemon.receivedBytes > 0 + ); +} + +test('a redirect response ends the stalled hop before the next hop is dialed', async () => { + useTransferClock(); + const daemon = await startAnswerUploadDaemon({ + respond: (req) => + req.url === '/moved' + ? { status: 413, body: 'moved and refused' } + : { status: 307, headers: { location: '/moved' }, body: '' }, + }); + const held = holdArtifactSourceOpen(MEBIBYTE); + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(4 * MEBIBYTE), + }); + await settleAfterBudget(attempt); + const result = await attempt; + assert.equal(result.error, undefined, `got ${String(result.error)}`); + assert.equal(result.value?.statusCode, 413); + assert.equal(result.value?.body, 'moved and refused'); + assert.deepEqual(daemon.requests, ['POST /upload', 'POST /moved']); + assert.equal(held.destroyedAtNextOpen, true, 'the redirected-from hop ended before the next'); +}); + +test('a response body that stops mid-answer expires the response-body phase after what was read', async () => { + useTransferClock(); + const writes = recordOutgoingWrites(); + const origin = await startHalfAnsweredOrigin({ firstChunk: 'first chunk' }); + const attempt = startUploadAttempt({ + url: `${origin.baseUrl}/upload`, + artifactPath: createArtifactFile(64), + }); + await untilIo(() => attempt.settled || writes.responseChunkRead); + assert.equal(writes.responseChunkRead, true); + await spendBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); + await settleAfterBudget(attempt); + const result = await attempt; + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'response-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); +}); + +test('a connected HTTP upload with an unfinished source expires body inactivity, not connect', async () => { + useTransferClock(); + const writes = recordOutgoingWrites(); + const origin = await startSilentTcpOrigin(); + holdArtifactSourceOpen(1024); + const attempt = startUploadAttempt({ + url: `http://127.0.0.1:${origin.port}/upload`, + artifactPath: createArtifactFile(8 * 1024), + }); + await spendSinceLastWrite(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, writes, attempt); + assert.equal(attempt.settled, false); + await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, attempt); + assert.equal(writes.finished, false, 'the source still holds the body open'); + await settleAfterBudget(attempt); + const result = await attempt; + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'request-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); +}); + +test('a TLS handshake that never completes expires connect even with the whole body buffered', async () => { + useTransferClock(); + const origin = await startSilentTcpOrigin(); + const attempt = startUploadAttempt({ + url: `https://127.0.0.1:${origin.port}/upload`, + artifactPath: createArtifactFile(4 * MEBIBYTE), + }); + await spendBudget(TRANSFER_CONNECT_TIMEOUT_MS + 5_000, attempt); + await settleAfterBudget(attempt); + const result = await attempt; + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'connect', + timeoutMs: TRANSFER_CONNECT_TIMEOUT_MS, + }); +}); + +test('a completed TLS handshake with an unfinished source expires body inactivity, not connect', async () => { + useTransferClock(); + trustSelfSignedLocalhostCertificate(); + const origin = await startConnectedStalledTlsOrigin(); + const writes = recordOutgoingWrites(); + holdArtifactSourceOpen(1024); + const attempt = startUploadAttempt({ + url: `https://127.0.0.1:${origin.port}/upload`, + artifactPath: createArtifactFile(8 * 1024), + }); + await spendSinceLastWrite(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, writes, attempt); + assert.equal(attempt.settled, false, 'secureConnect ended the connect phase'); + await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, attempt); + assert.equal(writes.finished, false, 'the source still holds the body open'); + await settleAfterBudget(attempt); + const result = await attempt; + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'request-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); +}); + +test('a zero-length artifact on a peer that never answers waits on its header budget', async () => { + useTransferClock(); + const writes = recordOutgoingWrites(); + const origin = await startSilentTcpOrigin(); + const attempt = startUploadAttempt({ + url: `http://127.0.0.1:${origin.port}/upload`, + artifactPath: createArtifactFile(0), + }); + const snapshot = () => ({ writes, settled: attempt.settled }); + await untilIo(() => attempt.settled || writes.finished, 'empty artifact flush', snapshot); + assert.equal(writes.finished, true); + await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 5_000, attempt); + assert.equal(attempt.settled, false); + await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, attempt); + await settleAfterBudget(attempt, snapshot); + const result = await attempt; + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + }); + assert.equal(writes.completed, 0); +}); + +test('an upload the peer keeps taking finishes while every phase ceiling is unexpired', async () => { + useTransferClock(); + const writes = recordOutgoingWrites(); + const daemon = await startSlowReadingUploadDaemon({ + status: 200, + body: JSON.stringify({ ok: true, uploadId: 'joined-upload' }), + }); + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(MEBIBYTE), + }); + let spent = 0; + while ( + !attempt.settled && + spent < TRANSFER_CONNECT_TIMEOUT_MS + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + ) { + const bytesBefore = daemon.receivedBytes; + daemon.releaseAllParked(); + if (!writes.finished) { + await vi.advanceTimersByTimeAsync(1_000); + spent += 1_000; + } + await untilIo( + () => attempt.settled || daemon.receivedBytes > bytesBefore, + 'slow upload next chunk', + () => ({ writes, receivedBytes: daemon.receivedBytes, settled: attempt.settled }), + ); + } + const result = await attempt; + assert.equal(result.error, undefined, `upload failed: ${String(result.error)}`); + assert.equal(result.value?.statusCode, 200); + assert.ok( + spent < TRANSFER_CONNECT_TIMEOUT_MS + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + `finished in ${spent}ms of budget, inside both ceilings a progressing upload must not exceed`, + ); + assert.equal(daemon.bodyEnded, true); + assert.equal(daemon.receivedBytes, MEBIBYTE); +}, 30_000); + +test('a caller cancel during a stalled connect stays caller cancellation, distinct from the connect expiry', async () => { + useTransferClock(); + const control = new AbortController(); + const origin = await startSilentTcpOrigin(); + const canceled = startUploadAttempt({ + url: `https://127.0.0.1:${origin.port}/upload`, + artifactPath: createArtifactFile(64 * 1024), + signal: control.signal, + }); + await spendBudget(5_000, canceled); + control.abort(); + const result = await canceled; + assert.equal(result.value, undefined); + assert.ok(result.error instanceof DOMException, `got ${String(result.error)}`); + assert.equal((result.error as DOMException).name, 'AbortError'); + assert.equal(control.signal.aborted, true); +}); + +test('an enclosing aggregate deadline ends a redirect hop with its own reason, not a phase timeout', async () => { + useTransferClock(); + const aggregate = new AbortController(); + const daemon = await startUploadLegsDaemon([ + { + readAllBody: false, + respond: () => ({ status: 307, headers: { location: '/moved' }, body: '' }), + }, + { readAllBody: true }, + ]); + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(64 * 1024), + signal: aggregate.signal, + }); + await untilIo(() => attempt.settled || daemon.requests.includes('POST /moved')); + assert.deepEqual(daemon.requests, ['POST /upload', 'POST /moved']); + assert.equal(attempt.settled, false); + const reason = new Error('preparation aggregate expired'); + aggregate.abort(reason); + await settleAfterBudget(attempt); + const result = await attempt; + assert.equal(result.value, undefined); + assert.equal(result.error, reason); + assert.equal(isTransferPhaseTimeoutError(result.error), false); +}); + +test('an attempt on a reused connection is already past its connect phase', async () => { + useTransferClock(); + const writes = recordOutgoingWrites(); + const daemon = await startUploadLegsDaemon([ + { + readAllBody: true, + respond: () => ({ status: 200, body: JSON.stringify({ ok: true, uploadId: 'pooled' }) }), + }, + { readAllBody: true }, + ]); + const first = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(64 * 1024), + }); + const firstResult = await first; + assert.equal(firstResult.error, undefined, `leg one failed: ${String(firstResult.error)}`); + const second = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(64 * 1024), + }); + const snapshot = () => ({ writes, requests: daemon.requests, settled: second.settled }); + await untilIo( + () => (daemon.requests.length === 2 && writes.finished) || second.settled, + 'reused request flush', + snapshot, + ); + await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, second); + await settleAfterBudget(second, snapshot); + const result = await second; + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + }); + assert.equal(daemon.connections, 1, 'the second attempt reused the pooled socket'); +}); diff --git a/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts b/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts new file mode 100644 index 0000000000..788c7e3ce5 --- /dev/null +++ b/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts @@ -0,0 +1,151 @@ +import assert from 'node:assert/strict'; +import { afterEach, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { + createTransferPhaseWatchdog, + isTransferPhaseTimeoutError, + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + TRANSFER_CONNECT_TIMEOUT_MS, + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, +} from '../upload-stream.ts'; + +afterEach(() => { + vi.useRealTimers(); +}); + +test('the three transfer ceilings are the adopted policy values', () => { + assert.equal(TRANSFER_CONNECT_TIMEOUT_MS, 30_000); + assert.equal(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, 30_000); + assert.equal(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, 120_000); +}); + +test('a body phase restarts its ceiling on every byte that moves', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const watchdog = createTransferPhaseWatchdog(); + try { + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + for (const _ of [0, 1, 2]) { + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - 20_000); + watchdog.touchBody(); + } + assert.equal(watchdog.signal.aborted, false); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 1); + assert.equal(watchdog.signal.aborted, true); + assertPhaseDetails( + watchdog.signal.reason, + 'response-body', + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + ); + } finally { + watchdog.dispose(); + } +}); + +test('a phase that carries no bytes is not extended by progress reported in another phase', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const watchdog = createTransferPhaseWatchdog(); + try { + watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 10_000); + watchdog.touchBody(); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + assertPhaseDetails( + watchdog.signal.reason, + 'response-headers', + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + ); + } finally { + watchdog.dispose(); + } +}); + +test('arming the next phase stops counting the previous one', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const watchdog = createTransferPhaseWatchdog(); + try { + watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS - 1_000); + watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1_000); + assert.equal(watchdog.signal.aborted, false); + await vi.advanceTimersByTimeAsync(1_000); + assertPhaseDetails( + watchdog.signal.reason, + 'response-headers', + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + ); + } finally { + watchdog.dispose(); + } +}); + +test('an enclosing deadline wins and is never extended by a phase timer', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const caller = new AbortController(); + const reason = new Error('preparation aggregate expired'); + setTimeout(() => caller.abort(reason), 5_000); + const watchdog = createTransferPhaseWatchdog(caller.signal); + try { + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(5_001); + assert.equal(watchdog.signal.aborted, true); + assert.equal(watchdog.signal.reason, reason); + assert.equal(isTransferPhaseTimeoutError(watchdog.signal.reason), false); + } finally { + watchdog.dispose(); + } +}); + +test('a phase expiry is its own typed evidence and never caller cancellation', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const caller = new AbortController(); + const watchdog = createTransferPhaseWatchdog(caller.signal); + try { + watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS + 1); + assertPhaseDetails(watchdog.signal.reason, 'connect', TRANSFER_CONNECT_TIMEOUT_MS); + assert.equal(caller.signal.aborted, false); + } finally { + watchdog.dispose(); + } +}); + +test('a disposed watchdog stops watching', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const watchdog = createTransferPhaseWatchdog(); + watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + watchdog.dispose(); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS * 3); + assert.equal(watchdog.signal.aborted, false); +}); + +test('a phase is never armed after the attempt is already over', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const watchdog = createTransferPhaseWatchdog(); + try { + watchdog.signal.throwIfAborted(); + const caller = new AbortController(); + caller.abort(new Error('gone')); + const late = createTransferPhaseWatchdog(caller.signal); + try { + late.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + assert.equal(vi.getTimerCount(), 0); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS * 2); + assert.equal(late.signal.reason.message, 'gone'); + } finally { + late.dispose(); + } + } finally { + watchdog.dispose(); + } +}); + +function assertPhaseDetails(reason: unknown, phase: string, timeoutMs: number): void { + assert.ok(reason instanceof AppError, `phase expiry must be an AppError, got ${String(reason)}`); + assert.equal(isTransferPhaseTimeoutError(reason), true); + assert.deepEqual(reason.details, { + reason: 'transfer_phase_timeout', + transferPhase: phase, + timeoutMs, + }); +} diff --git a/packages/remote/src/__tests__/upload-stream-phases.test.ts b/packages/remote/src/__tests__/upload-stream-phases.test.ts new file mode 100644 index 0000000000..d668f97cd6 --- /dev/null +++ b/packages/remote/src/__tests__/upload-stream-phases.test.ts @@ -0,0 +1,198 @@ +import fs from 'node:fs'; +import { afterEach, assert, onTestFinished, test, vi } from 'vitest'; +import { + LEGACY_UPLOAD_REQUEST, + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + TRANSFER_CONNECT_TIMEOUT_MS, + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + PREFLIGHT_REQUEST, + assertDetailedError, + awaitBodyProgress, + createArtifactFile, + holdArtifactSourceOpen, + pumpRealWork, + recordOutgoingWrites, + settleAfterBudget, + spendBudget, + spendProgressWindows, + spendSinceLastWrite, + startParkedUploadDaemon, + startUpload, + startUploadAttempt, + untilIo, + useTransferClock, +} from './support/upload-transfer-fixtures.ts'; + +const MEBIBYTE = 1024 * 1024; + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +test('a body that keeps moving outlives the body-inactivity ceiling', async () => { + useTransferClock(); + const writes = recordOutgoingWrites(); + const daemon = await startParkedUploadDaemon(); + const upload = startUpload({ + baseUrl: daemon.baseUrl, + artifactPath: createArtifactFile(16 * MEBIBYTE), + }); + const progress = await spendProgressWindows(daemon, upload, { + windows: 8, + windowMs: TRANSFER_CONNECT_TIMEOUT_MS, + writes, + }); + assert.equal(progress.progressedInEveryWindow, true); + assert.ok( + progress.budgetMs > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + `the transfer stayed alive past the body-inactivity ceiling, at ${progress.budgetMs}ms of budget`, + ); + assert.deepEqual(daemon.requests, [PREFLIGHT_REQUEST, LEGACY_UPLOAD_REQUEST]); +}); + +test('an unfinished source with no new bytes expires the request-body phase on its own budget', async () => { + useTransferClock(); + const writes = recordOutgoingWrites(); + const daemon = await startParkedUploadDaemon(); + holdArtifactSourceOpen(1024); + const upload = startUpload({ + baseUrl: daemon.baseUrl, + artifactPath: createArtifactFile(8 * 1024), + }); + await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, upload); + assert.equal(writes.finished, false, 'the source still holds the body open'); + await settleAfterBudget(upload); + const result = await upload; + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'request-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); + assert.deepEqual(daemon.requests, [PREFLIGHT_REQUEST, LEGACY_UPLOAD_REQUEST]); + assert.ok(daemon.receivedBytes > 0, 'the attempt had a body in flight to stall'); +}); + +test('a body admitted past four connect ceilings still waits only its own header budget', async () => { + useTransferClock(); + const writes = recordOutgoingWrites(); + const daemon = await startParkedUploadDaemon(); + const upload = startUpload({ + baseUrl: daemon.baseUrl, + artifactPath: createArtifactFile(16 * MEBIBYTE), + }); + const progress = await spendProgressWindows(daemon, upload, { + windows: 5, + windowMs: TRANSFER_CONNECT_TIMEOUT_MS, + writes, + }); + assert.equal(progress.progressedInEveryWindow, true); + assert.ok(progress.budgetMs > 4 * TRANSFER_CONNECT_TIMEOUT_MS, `${progress.budgetMs}ms`); + assert.equal(writes.finished, false, 'the body was still being written'); + assert.equal(daemon.bodyEnded, false); + assert.equal(upload.settled, false); + const close = fs.close; + let sourceClosed: Promise | undefined; + const closeSpy = vi.spyOn(fs, 'close').mockImplementation((fd, callback) => { + assert.ok(callback); + const release = vi.fn().mockImplementationOnce(() => { + sourceClosed = new Promise((resolve) => { + close(fd, (error) => { + callback(error); + resolve(); + }); + }); + return sourceClosed; + }); + onTestFinished(release); + setTimeout(release, 20_000); + }); + daemon.readToEnd(); + const snapshot = () => ({ + writes, + bodyEnded: daemon.bodyEnded, + receivedBytes: daemon.receivedBytes, + requests: daemon.requests, + settled: upload.settled, + }); + await untilIo( + () => (writes.finished && daemon.bodyEnded) || upload.settled, + 'header proof flush', + snapshot, + ); + assert.equal(writes.finished && daemon.bodyEnded, true, 'the whole body reached the peer'); + assert.equal(closeSpy.mock.calls.length, 1, 'source close is held after the outgoing flush'); + assert.equal(fs.fstatSync(closeSpy.mock.calls[0]![0]).size, 16 * MEBIBYTE); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1); + assert.ok(sourceClosed, 'source close was released before the header deadline'); + await sourceClosed; + await pumpRealWork(); + assert.equal(upload.settled, false, 'the header wait counts from the flush'); + await vi.advanceTimersByTimeAsync(1); + await pumpRealWork(); + assert.equal(writes.destroyed, true, 'source close cannot reset the header deadline'); + await settleAfterBudget(upload, snapshot); + const result = await upload; + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + }); +}); + +test('writes below the high-water mark keep the body budget on each completed write', async () => { + useTransferClock(); + const { source } = holdArtifactSourceOpen(); + const writes = recordOutgoingWrites(); + const daemon = await startParkedUploadDaemon(); + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(64 * 1024), + }); + for (let window = 0; window < 6; window += 1) { + const before = { bytes: daemon.receivedBytes, completed: writes.completed }; + source.write(Buffer.alloc(1024, 'x')); + const progressed = await awaitBodyProgress(daemon, writes, before, attempt); + assert.equal(progressed, true, `window ${window}`); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS); + assert.equal(attempt.settled, false, `window ${window}`); + } + assert.ok(6 * TRANSFER_CONNECT_TIMEOUT_MS > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + assert.equal(writes.drains, 0); + assert.equal(writes.falseReturns, 0); + await vi.advanceTimersByTimeAsync( + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - TRANSFER_CONNECT_TIMEOUT_MS - 1, + ); + await pumpRealWork(); + assert.equal(attempt.settled, false); + await vi.advanceTimersByTimeAsync(1); + await settleAfterBudget(attempt); + assertDetailedError((await attempt).error, { + reason: 'transfer_phase_timeout', + transferPhase: 'request-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); +}); + +test('the caller ending the upload is reported as cancellation, not as an expired phase', async () => { + useTransferClock(); + const writes = recordOutgoingWrites(); + const daemon = await startParkedUploadDaemon(); + const control = new AbortController(); + const upload = startUpload({ + baseUrl: daemon.baseUrl, + artifactPath: createArtifactFile(8 * MEBIBYTE), + signal: control.signal, + }); + await untilIo(() => upload.settled || (writes.completed > 0 && daemon.receivedBytes > 0)); + assert.equal(upload.settled, false); + assert.ok(writes.completed > 0 && daemon.receivedBytes > 0); + await spendBudget(5_000, upload); + control.abort(); + const result = await upload; + assert.equal(result.value, undefined); + assert.ok(result.error instanceof DOMException, `got ${String(result.error)}`); + assert.equal((result.error as DOMException).name, 'AbortError'); +}); diff --git a/packages/remote/src/__tests__/upload-stream.test.ts b/packages/remote/src/__tests__/upload-stream.test.ts index 92a4ebf1a7..6a148c483b 100644 --- a/packages/remote/src/__tests__/upload-stream.test.ts +++ b/packages/remote/src/__tests__/upload-stream.test.ts @@ -48,7 +48,7 @@ test('an upload refuses a redirect to another origin and never sends it the toke method: 'POST', headers: { authorization: 'Bearer daemon-token', 'transfer-encoding': 'chunked' }, payloadPath, - timeoutMessage: 'timed out', + signal: new AbortController().signal, errorMessage: 'failed', }), (error: unknown) => diff --git a/packages/remote/src/daemon-artifacts.ts b/packages/remote/src/daemon-artifacts.ts index f85b624aee..226980ad16 100644 --- a/packages/remote/src/daemon-artifacts.ts +++ b/packages/remote/src/daemon-artifacts.ts @@ -15,6 +15,7 @@ import { } from '@agent-device/capture-kit/recording-output-path'; import { parsePerfRuntimeRequest } from '@agent-device/capture-kit/perf-runtime-plan'; import { uploadArtifact } from './upload-client.ts'; +import { withUploadPreparationDeadline } from './upload-stream.ts'; import { createStderrUploadProgressReporter, type UploadProgressSink } from './upload-progress.ts'; // Mirrors `DEFAULT_TEST_ARTIFACTS_ROOT` in `packages/replay-test/src/internal/session-test-artifacts.ts` @@ -41,6 +42,22 @@ export async function prepareRemoteRequestArtifacts( info: DaemonArtifactEndpoint, signal: AbortSignal, host: HostEnvironment, +): Promise { + if (!isRemoteDaemon(info)) { + return await prepareRemoteRequestArtifactsWithinBudget(req, info, signal, host); + } + return await withUploadPreparationDeadline( + signal, + async (preparationSignal) => + await prepareRemoteRequestArtifactsWithinBudget(req, info, preparationSignal, host), + ); +} + +async function prepareRemoteRequestArtifactsWithinBudget( + req: Omit, + info: DaemonArtifactEndpoint, + signal: AbortSignal, + host: HostEnvironment, ): Promise { const positionals = [...(req.positionals ?? [])]; let flags = req.flags ? { ...req.flags } : undefined; diff --git a/packages/remote/src/upload-client-artifact.ts b/packages/remote/src/upload-client-artifact.ts index d4de4a4f9c..b037d7e59e 100644 --- a/packages/remote/src/upload-client-artifact.ts +++ b/packages/remote/src/upload-client-artifact.ts @@ -26,7 +26,9 @@ export async function prepareUploadArtifact( localPath: string, requestedPlatform: string | undefined, env: Readonly, + signal?: AbortSignal, ): Promise { + signal?.throwIfAborted(); const stat = fs.statSync(localPath); const fileName = path.basename(localPath); const isDirectory = stat.isDirectory(); @@ -35,8 +37,9 @@ export async function prepareUploadArtifact( const cleanupPaths: string[] = []; try { const payloadPath = isDirectory - ? await createGzipTarArchive(localPath, cleanupPaths, env) + ? await createGzipTarArchive(localPath, cleanupPaths, env, signal) : localPath; + signal?.throwIfAborted(); const payloadStat = fs.statSync(payloadPath); return { @@ -45,7 +48,7 @@ export async function prepareUploadArtifact( artifactType: isDirectory ? 'app-bundle' : 'file', platform, contentType: isDirectory ? 'application/gzip' : DEFAULT_CONTENT_TYPE, - sha256: await computeFileHash(payloadPath), + sha256: await computeFileHash(payloadPath, signal), sizeBytes: payloadStat.size, cleanup: () => cleanupUploadPaths(cleanupPaths), }; @@ -59,6 +62,7 @@ async function createGzipTarArchive( localPath: string, cleanupPaths: string[], env: Readonly, + signal?: AbortSignal, ): Promise { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), `agent-device-upload-${randomUUID()}-`)); cleanupPaths.push(tempDir); @@ -67,6 +71,7 @@ async function createGzipTarArchive( 'tar', ['czf', archivePath, '-C', path.dirname(localPath), path.basename(localPath)], { + signal, env: { ...env, COPYFILE_DISABLE: '1', @@ -97,7 +102,7 @@ function cleanupUploadPaths(cleanupPaths: string[]): void { } } -async function computeFileHash(localPath: string): Promise { +async function computeFileHash(localPath: string, signal?: AbortSignal): Promise { const hash = createHash(ARTIFACT_HASH_ALGORITHM); const sink = new Writable({ write(chunk: Buffer, _encoding, callback) { @@ -105,7 +110,8 @@ async function computeFileHash(localPath: string): Promise { callback(); }, }); - await pipeline(fs.createReadStream(localPath), sink).catch((error: unknown) => { + await pipeline(fs.createReadStream(localPath), sink, { signal }).catch((error: unknown) => { + signal?.throwIfAborted(); throw new AppError( 'COMMAND_FAILED', 'Failed to read local artifact', diff --git a/packages/remote/src/upload-client.ts b/packages/remote/src/upload-client.ts index a684db0d5a..62291f5fe1 100644 --- a/packages/remote/src/upload-client.ts +++ b/packages/remote/src/upload-client.ts @@ -6,10 +6,18 @@ import { buildDaemonHttpLeaseHeaders, } from '@agent-device/contracts/daemon-http'; import { prepareUploadArtifact, type PreparedUploadArtifact } from './upload-client-artifact.ts'; -import { isRetryableUploadStreamError, streamFileToHttpRequest } from './upload-stream.ts'; +import { + createTransferPhaseWatchdog, + isRetryableUploadStreamError, + isTransferPhaseTimeoutError, + streamFileToHttpRequest, + withUploadPreparationDeadline, +} from './upload-stream.ts'; import type { UploadProgressSink } from './upload-progress.ts'; -const UPLOAD_PREFLIGHT_TIMEOUT_MS = 30 * 1000; +/** Ceiling for the small JSON `preflight` and `finalize` calls that bracket an upload. */ +const UPLOAD_CONTROL_PLANE_TIMEOUT_MS = 30_000; + const ARTIFACT_HASH_ALGORITHM = 'sha256'; type UploadArtifactOptions = { @@ -57,7 +65,21 @@ type UploadPreflightResult = }; export async function uploadArtifact(options: UploadArtifactOptions): Promise { - const prepared = await prepareUploadArtifact(options.localPath, options.platform, options.env); + return await withUploadPreparationDeadline( + options.signal, + async (signal) => await uploadArtifactWithinBudget({ ...options, signal }), + ); +} + +async function uploadArtifactWithinBudget( + options: UploadArtifactOptions & { signal: AbortSignal }, +): Promise { + const prepared = await prepareUploadArtifact( + options.localPath, + options.platform, + options.env, + options.signal, + ); const normalizedBase = options.baseUrl.endsWith('/') ? options.baseUrl : `${options.baseUrl}/`; const uploadAttemptId = randomUUID(); @@ -123,7 +145,7 @@ async function tryDirectUploadWithResume(options: { preflight: Extract; uploadAttemptId: string; onProgress?: UploadProgressSink; - signal?: AbortSignal; + signal: AbortSignal; }): Promise { const uploadOnce = async ( preflight: Extract, @@ -143,7 +165,9 @@ async function tryDirectUploadWithResume(options: { } catch (error) { // A canceled upload resumes for no reason: the thing it was uploaded for is already gone, and // re-preflighting would ask the daemon for a fresh ticket to send bytes nobody is waiting on. - if (options.signal?.aborted) throw error; + if (options.signal.aborted) throw error; + // An expired phase is this client's own deadline, never a transport break to retry. + if (isTransferPhaseTimeoutError(error)) throw error; if (!shouldRetryDirectUpload(error)) return undefined; return await retryDirectUpload(options, uploadOnce); } @@ -164,7 +188,7 @@ async function retryDirectUpload( artifact: PreparedUploadArtifact; uploadAttemptId: string; onProgress?: UploadProgressSink; - signal?: AbortSignal; + signal: AbortSignal; }, uploadOnce: ( preflight: Extract, @@ -184,7 +208,9 @@ async function retryDirectUpload( if (retryPreflight?.kind === 'direct-upload') { try { return await uploadOnce(retryPreflight); - } catch { + } catch (error) { + options.signal.throwIfAborted(); + if (isTransferPhaseTimeoutError(error)) throw error; return undefined; } } @@ -202,7 +228,7 @@ async function uploadLegacyArtifact(options: { leaseId?: string; artifact: PreparedUploadArtifact; onProgress?: UploadProgressSink; - signal?: AbortSignal; + signal: AbortSignal; }): Promise { const { normalizedBase, token, leaseId, artifact } = options; const uploadUrl = new URL('upload', normalizedBase); @@ -222,8 +248,6 @@ async function uploadLegacyArtifact(options: { method: 'POST', headers, payloadPath: artifact.payloadPath, - timeoutMessage: 'Artifact upload timed out', - timeoutHint: 'The upload to the remote daemon exceeded the 5-minute timeout.', errorMessage: 'Failed to upload artifact to remote daemon', errorHint: 'Verify the remote daemon is reachable and supports artifact uploads.', signal: options.signal, @@ -264,43 +288,62 @@ async function requestUploadPreflight(options: { buildDaemonHttpLeaseHeaders(options.leaseId), ); - const response = await fetch(preflightUrl, { - method: 'POST', - // The daemon never redirects this route. A redirect would carry x-agent-device-token, which - // fetch keeps on a cross-origin hop, so any redirect fails the preflight instead. - redirect: 'error', - headers, - signal: combineUploadSignals(options.signal, UPLOAD_PREFLIGHT_TIMEOUT_MS), - body: JSON.stringify({ - uploadAttemptId: options.uploadAttemptId, - sha256: options.artifact.sha256, - fileName: options.artifact.fileName, - sizeBytes: options.artifact.sizeBytes, - artifactType: options.artifact.artifactType, - ...(options.artifact.platform ? { platform: options.artifact.platform } : {}), - contentType: options.artifact.contentType, - }), - }).catch((error: unknown) => { + const result = await requestUploadControl( + preflightUrl, + { + headers, + body: JSON.stringify({ + uploadAttemptId: options.uploadAttemptId, + sha256: options.artifact.sha256, + fileName: options.artifact.fileName, + sizeBytes: options.artifact.sizeBytes, + artifactType: options.artifact.artifactType, + ...(options.artifact.platform ? { platform: options.artifact.platform } : {}), + contentType: options.artifact.contentType, + }), + }, + 'preflight', + options.signal, + ).catch((error: unknown) => { // A failed preflight is ordinary here — the caller falls back to the legacy upload. A canceled // one is not: there is nothing to fall back to when the lease that paid for this upload is gone. - if (options.signal?.aborted) throw error; + if (options.signal?.aborted || isTransferPhaseTimeoutError(error)) throw error; return undefined; }); - if (!response?.ok) { + if (!result?.response.ok) { return undefined; } - return parseUploadPreflightResult(await response.json().catch(() => undefined)); + return parseUploadPreflightResult(result.body); } -/** - * The signal one upload request runs under: the caller's cancellation and the request's own - * timeout, whichever fires first. - */ -function combineUploadSignals(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal { - const timeout = AbortSignal.timeout(timeoutMs); - return signal ? AbortSignal.any([signal, timeout]) : timeout; +async function requestUploadControl( + url: URL, + init: { headers: Record; body: string }, + phase: 'preflight' | 'finalize', + callerSignal?: AbortSignal, +): Promise<{ response: Response; body: unknown }> { + const watchdog = createTransferPhaseWatchdog(callerSignal); + watchdog.signal.throwIfAborted(); + watchdog.arm(phase, UPLOAD_CONTROL_PLANE_TIMEOUT_MS); + try { + // Redirects could forward the daemon token, so these control requests never follow them. + const response = await fetch(url, { + ...init, + method: 'POST', + redirect: 'error', + signal: watchdog.signal, + }); + const body: unknown = response.ok ? await response.json().catch(() => undefined) : undefined; + watchdog.signal.throwIfAborted(); + return { response, body }; + } catch (error) { + watchdog.signal.throwIfAborted(); + throw error; + } finally { + watchdog.dispose(); + } } function parseUploadPreflightResult(value: unknown): UploadPreflightResult | undefined { @@ -347,7 +390,7 @@ async function uploadDirectArtifact( leaseId?: string; artifact: PreparedUploadArtifact; onProgress?: UploadProgressSink; - signal?: AbortSignal; + signal: AbortSignal; }, ticket: Extract, ): Promise { @@ -363,8 +406,6 @@ async function uploadDirectArtifact( ? { ...ticket.headers, ...buildDaemonHttpLeaseHeaders(options.leaseId) } : ticket.headers, payloadPath: artifact.payloadPath, - timeoutMessage: 'Direct artifact upload timed out', - timeoutHint: 'The direct upload ticket did not accept the artifact within the timeout.', errorMessage: 'Failed to upload artifact with direct upload ticket', retryable: true, signal, @@ -400,28 +441,29 @@ async function finalizeDirectUpload(options: { buildDaemonHttpLeaseHeaders(options.leaseId), ); - const response = await fetch(finalizeUrl, { - method: 'POST', - // As with preflight: the route never redirects, and a redirect must not carry the token. - redirect: 'error', - headers, - signal: combineUploadSignals(options.signal, UPLOAD_PREFLIGHT_TIMEOUT_MS), - body: JSON.stringify({ uploadId: options.uploadId }), - }).catch((error) => { + const result = await requestUploadControl( + finalizeUrl, + { + headers, + body: JSON.stringify({ uploadId: options.uploadId }), + }, + 'finalize', + options.signal, + ).catch((error: unknown) => { // As with preflight: a canceled finalize is not a transport failure to report, it is the caller // saying this upload is no longer worth finishing. - if (options.signal?.aborted) throw error; + if (options.signal?.aborted || isTransferPhaseTimeoutError(error)) throw error; throw new AppError('COMMAND_FAILED', 'Failed to finalize direct artifact upload', {}, error); }); - if (!response.ok) { + if (!result.response.ok) { throw new AppError('COMMAND_FAILED', 'Direct artifact upload finalize failed', { - status: response.status, - statusText: response.statusText, + status: result.response.status, + statusText: result.response.statusText, }); } - const parsed = (await response.json().catch(() => undefined)) as UploadResponse | undefined; + const parsed = result.body as UploadResponse | undefined; if (!parsed?.ok || !parsed.uploadId) { throw new AppError('COMMAND_FAILED', 'Invalid upload finalize response'); } diff --git a/packages/remote/src/upload-stream.ts b/packages/remote/src/upload-stream.ts index fdaf1fc018..8b112650ac 100644 --- a/packages/remote/src/upload-stream.ts +++ b/packages/remote/src/upload-stream.ts @@ -1,16 +1,126 @@ import fs from 'node:fs'; -import type { IncomingHttpHeaders } from 'node:http'; +import type { IncomingHttpHeaders, IncomingMessage } from 'node:http'; import path from 'node:path'; import { pipeline } from 'node:stream/promises'; +import { addAbortSignal } from 'node:stream'; import { AppError } from '@agent-device/kernel/errors'; -import { loadNodeHttpRequester, readNodeHttpResponseBody } from '@agent-device/host-kit/transport'; +import { + loadNodeHttpRequester, + observeCompletedRequestWrites, + readNodeHttpResponseBody, +} from '@agent-device/host-kit/transport'; import { createUploadProgressTransform, type UploadProgressSink, type UploadProgressStage, } from './upload-progress.ts'; -const UPLOAD_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes +export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; + +/** Header wait starts after connection and outgoing flush, never while the body progresses. */ +export const TRANSFER_RESPONSE_HEADER_TIMEOUT_MS = 30_000; + +export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; + +export type TransferPhase = + | 'connect' + | 'request-body' + | 'response-headers' + | 'response-body' + | 'preflight' + | 'finalize'; + +const BODY_PHASES: ReadonlySet = new Set(['request-body', 'response-body']); + +const TRANSFER_PHASE_TIMEOUT_REASON = 'transfer_phase_timeout'; + +/** One sequential phase under the earliest caller/aggregate deadline; progress resets body phases only. */ +export type TransferPhaseWatchdog = { + signal: AbortSignal; + arm(phase: TransferPhase, timeoutMs: number): void; + touchBody(): void; + /** Stops the armed timer. Call it once the attempt is settled. */ + dispose(): void; +}; + +export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): TransferPhaseWatchdog { + const phase = new AbortController(); + const signal = callerSignal ? AbortSignal.any([callerSignal, phase.signal]) : phase.signal; + let armed: { phase: TransferPhase; timeoutMs: number } | undefined; + let timer: ReturnType | undefined; + const rearm = () => { + if (timer !== undefined) clearTimeout(timer); + const current = armed; + if (!current) return; + timer = setTimeout(() => { + timer = undefined; + phase.abort(createTransferPhaseTimeoutError(current.phase, current.timeoutMs)); + }, current.timeoutMs); + timer.unref(); + }; + return { + signal, + arm(nextPhase, timeoutMs) { + if (signal.aborted) return; + armed = { phase: nextPhase, timeoutMs }; + rearm(); + }, + touchBody() { + if (!armed || !BODY_PHASES.has(armed.phase)) return; + rearm(); + }, + dispose() { + armed = undefined; + if (timer !== undefined) clearTimeout(timer); + timer = undefined; + }, + }; +} + +function createTransferPhaseTimeoutError(phase: TransferPhase, timeoutMs: number): AppError { + return new AppError('COMMAND_FAILED', `Artifact transfer ${phase} timed out`, { + reason: TRANSFER_PHASE_TIMEOUT_REASON, + transferPhase: phase, + timeoutMs, + }); +} + +/** Whether `error` is a transfer phase's own expiry, which no fallback or retry may restart. */ +export function isTransferPhaseTimeoutError(error: unknown): boolean { + return error instanceof AppError && error.details?.reason === TRANSFER_PHASE_TIMEOUT_REASON; +} + +const UPLOAD_PREPARATION_TIMEOUT_MS = 600_000; + +export async function withUploadPreparationDeadline( + callerSignal: AbortSignal | undefined, + prepare: (signal: AbortSignal) => Promise, +): Promise { + const deadline = new AbortController(); + const signal = callerSignal ? AbortSignal.any([callerSignal, deadline.signal]) : deadline.signal; + signal.throwIfAborted(); + const timer = setTimeout(() => { + deadline.abort( + new AppError('COMMAND_FAILED', 'Artifact preparation deadline exceeded', { + reason: 'artifact_preparation_deadline_exceeded', + transferPhase: 'preparation', + timeoutMs: UPLOAD_PREPARATION_TIMEOUT_MS, + }), + ); + }, UPLOAD_PREPARATION_TIMEOUT_MS); + timer.unref(); + try { + const result = await prepare(signal); + signal.throwIfAborted(); + return result; + } catch (error) { + signal.throwIfAborted(); + throw error; + } finally { + clearTimeout(timer); + } +} + const MAX_UPLOAD_REDIRECTS = 5; const UPLOAD_STREAM_RETRYABLE_DETAIL = 'uploadStreamRetryable'; @@ -31,21 +141,14 @@ export async function streamFileToHttpRequest(options: { method: 'POST' | 'PUT'; headers: Record; payloadPath: string; - timeoutMessage: string; - timeoutHint?: string; errorMessage: string; errorHint?: string; retryable?: boolean; - /** Ends this request when the work the upload serves is over; see `uploadArtifact`. */ - signal?: AbortSignal; + /** Every attempt and hop shares the enclosing aggregate/caller signal. */ + signal: AbortSignal; progress?: UploadStreamProgressOptions; }): Promise { - return await streamFileToHttpRequestAttempt({ - ...options, - url: options.url, - redirectCount: 0, - startOffset: 0, - }); + return await streamFileToHttpRequestAttempt({ ...options, redirectCount: 0, startOffset: 0 }); } export function isRetryableUploadStreamError(error: unknown): boolean { @@ -57,16 +160,15 @@ async function streamFileToHttpRequestAttempt(options: { method: 'POST' | 'PUT'; headers: Record; payloadPath: string; - timeoutMessage: string; - timeoutHint?: string; errorMessage: string; errorHint?: string; retryable?: boolean; - signal?: AbortSignal; + signal: AbortSignal; redirectCount: number; startOffset: number; progress?: UploadStreamProgressOptions; }): Promise { + options.signal.throwIfAborted(); // This module sits in the CLI's eager import closure via the remote-artifact // upload client, but only a remote daemon ever uploads, so the HTTP stack // loads here rather than at import time. @@ -75,8 +177,37 @@ async function streamFileToHttpRequestAttempt(options: { const headers = buildUploadRequestHeaders(options.headers, options.startOffset, payloadSize); emitUploadAttemptStarted(options.progress, options.startOffset, payloadSize); - return await new Promise((resolve, reject) => { - let responseReceived = false; + const watchdog = createTransferPhaseWatchdog(options.signal); + watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + + let upload: Promise | undefined; + let uploadSettled = false; + let responseReceived = false; + return await new Promise((resolve, reject) => { + let outgoingEnded = false; + let transportEstablished = false; + let bodyFlushed = false; + const settle = (result: UploadStreamResponse) => { + stopObservingWrites(); + watchdog.dispose(); + resolve(result); + }; + const fail = (error: unknown) => { + stopObservingWrites(); + watchdog.dispose(); + // The queued pipeline has to settle before the join in `finally`. + req.destroy(); + reject(error); + }; + const advanceOutgoingPhase = () => { + if (responseReceived || watchdog.signal.aborted) return; + if (!transportEstablished) return; + if (bodyFlushed) { + watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + } else { + watchdog.arm('request-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + } + }; const req = transport.request( { protocol: options.url.protocol, @@ -87,110 +218,47 @@ async function streamFileToHttpRequestAttempt(options: { headers, // Aborting destroys the request mid-stream, which is the only way to stop bytes that are // already piped at a device this client no longer holds the lease on. - ...(options.signal ? { signal: options.signal } : {}), + signal: watchdog.signal, }, (res) => { responseReceived = true; - void readNodeHttpResponseBody(res) - .then((body) => { - clearTimeout(timeout); - const statusCode = res.statusCode ?? 500; - const location = res.headers.location; - if (location && isUploadRedirectStatus(statusCode)) { - if (options.redirectCount >= MAX_UPLOAD_REDIRECTS) { - reject( - new AppError('COMMAND_FAILED', 'Artifact upload exceeded redirect limit', { - maxRedirects: MAX_UPLOAD_REDIRECTS, - url: options.url.toString(), - }), - ); - return; - } - const redirectedUrl = new URL(location, options.url); - // The request headers carry the daemon token or a signed ticket, and the body is the - // artifact: neither may follow a hop to an origin the caller did not choose. - if (redirectedUrl.origin !== options.url.origin) { - reject( - new AppError( - 'COMMAND_FAILED', - 'Artifact upload refused a cross-origin redirect', - { - reason: 'cross-origin-redirect', - url: options.url.toString(), - redirectOrigin: redirectedUrl.origin, - }, - ), - ); - return; - } - void streamFileToHttpRequestAttempt({ - ...options, - url: redirectedUrl, - redirectCount: options.redirectCount + 1, - }).then(resolve, reject); - return; - } - - const resumeOffset = isUploadResumeStatus(statusCode) - ? parseUploadResumeOffset(res.headers, payloadSize) - : undefined; - if (resumeOffset !== undefined) { - if (resumeOffset >= payloadSize) { - resolve({ - statusCode: 200, - statusMessage: 'Upload already complete', - body: '', - }); - return; - } - if (resumeOffset <= options.startOffset) { - reject( - new AppError('COMMAND_FAILED', 'Artifact upload resume did not advance', { - offset: resumeOffset, - previousOffset: options.startOffset, - url: options.url.toString(), - }), - ); - return; - } - void streamFileToHttpRequestAttempt({ - ...options, - startOffset: resumeOffset, - }).then(resolve, reject); - return; - } - - resolve({ - statusCode, - statusMessage: res.statusMessage, - body, - }); - }) - .catch(reject); + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + addAbortSignal(watchdog.signal, res); + void handleUploadResponse(res) + .then((result) => settle(result)) + .catch((error: unknown) => + fail(watchdog.signal.aborted ? watchdog.signal.reason : error), + ); }, ); - const timeout = setTimeout(() => { - req.destroy(); - reject( - new AppError('COMMAND_FAILED', options.timeoutMessage, { - timeoutMs: UPLOAD_TIMEOUT_MS, - ...(options.timeoutHint ? { hint: options.timeoutHint } : {}), - ...retryableUploadStreamDetail(options), - }), - ); - }, UPLOAD_TIMEOUT_MS); + req.on('socket', (socket) => { + if (socket.connecting) { + const secured = (socket as { encrypted?: boolean }).encrypted === true; + socket.once(secured ? 'secureConnect' : 'connect', () => { + transportEstablished = true; + advanceOutgoingPhase(); + }); + } else { + transportEstablished = true; + advanceOutgoingPhase(); + } + }); + req.on('finish', () => { + bodyFlushed = true; + advanceOutgoingPhase(); + }); + const stopObservingWrites = observeCompletedRequestWrites(req, watchdog.signal, () => + watchdog.touchBody(), + ); req.on('error', (err) => { if (responseReceived) return; - clearTimeout(timeout); - // A caller that canceled owns this outcome: the signal's reason is the answer, and wrapping it - // as an ordinary transport failure would let a retry path treat "we stopped" as "it broke". - if (options.signal?.aborted) { - reject(options.signal.reason); + if (watchdog.signal.aborted) { + fail(watchdog.signal.reason); return; } - reject( + fail( new AppError( 'COMMAND_FAILED', options.errorMessage, @@ -202,24 +270,101 @@ async function streamFileToHttpRequestAttempt(options: { ), ); }); - req.on('close', () => clearTimeout(timeout)); - - void pipeline( - fs.createReadStream(options.payloadPath, { start: options.startOffset }), - createUploadProgressTransform({ - stage: options.progress?.stage ?? 'legacy', - fileName: options.progress?.fileName ?? path.basename(options.payloadPath), - startOffset: options.startOffset, - totalBytes: payloadSize, - onProgress: options.progress?.onProgress, - }), - req, - ).catch((err: unknown) => { + + const fileStream = fs.createReadStream(options.payloadPath, { start: options.startOffset }); + const progressTransform = createUploadProgressTransform({ + stage: options.progress?.stage ?? 'legacy', + fileName: options.progress?.fileName ?? path.basename(options.payloadPath), + startOffset: options.startOffset, + totalBytes: payloadSize, + onProgress: options.progress?.onProgress, + }); + upload = (async () => { + await pipeline(fileStream, progressTransform, req, { signal: watchdog.signal }); + uploadSettled = true; + })().catch((err: unknown) => { + // After a response, only this attempt's own teardown ends the pipeline, and the server's + // verdict stays the outcome: a 413 read completely is a 413, not a transport break. if (responseReceived) return; req.destroy(); + if (watchdog.signal.aborted) { + fail(watchdog.signal.reason); + return; + } const error = err instanceof Error ? err : new Error(String(err)); - reject(new AppError('COMMAND_FAILED', 'Failed to read local artifact', {}, error)); + fail(new AppError('COMMAND_FAILED', 'Failed to read local artifact', {}, error)); }); + + const endOutgoingWork = (): Promise => { + if (!outgoingEnded) { + outgoingEnded = true; + // The response is consumed, so a destroy cannot cut it short; it settles a pipeline blocked + // on a peer that stopped reading. A flushed pipeline keeps its pooled socket. + if (!uploadSettled) req.destroy(); + } + return upload!.then( + () => undefined, + () => undefined, + ); + }; + + async function handleUploadResponse(res: IncomingMessage): Promise { + const body = await readNodeHttpResponseBody(res, () => watchdog.touchBody()); + await endOutgoingWork(); + stopObservingWrites(); + watchdog.dispose(); + + const statusCode = res.statusCode ?? 500; + const location = res.headers.location; + if (location && isUploadRedirectStatus(statusCode)) { + if (options.redirectCount >= MAX_UPLOAD_REDIRECTS) { + throw new AppError('COMMAND_FAILED', 'Artifact upload exceeded redirect limit', { + maxRedirects: MAX_UPLOAD_REDIRECTS, + url: options.url.toString(), + }); + } + const redirectedUrl = new URL(location, options.url); + // The request headers carry the daemon token or a signed ticket, and the body is the + // artifact: neither may follow a hop to an origin the caller did not choose. + if (redirectedUrl.origin !== options.url.origin) { + throw new AppError('COMMAND_FAILED', 'Artifact upload refused a cross-origin redirect', { + reason: 'cross-origin-redirect', + url: options.url.toString(), + redirectOrigin: redirectedUrl.origin, + }); + } + return await streamFileToHttpRequestAttempt({ + ...options, + url: redirectedUrl, + redirectCount: options.redirectCount + 1, + }); + } + + const resumeOffset = isUploadResumeStatus(statusCode) + ? parseUploadResumeOffset(res.headers, payloadSize) + : undefined; + if (resumeOffset !== undefined) { + if (resumeOffset >= payloadSize) { + return { statusCode: 200, statusMessage: 'Upload already complete', body: '' }; + } + if (resumeOffset <= options.startOffset) { + throw new AppError('COMMAND_FAILED', 'Artifact upload resume did not advance', { + offset: resumeOffset, + previousOffset: options.startOffset, + url: options.url.toString(), + }); + } + return await streamFileToHttpRequestAttempt({ + ...options, + startOffset: resumeOffset, + }); + } + + return { statusCode, statusMessage: res.statusMessage, body }; + } + }).finally(async () => { + watchdog.dispose(); + await upload; }); } diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index e464f3f929..98ffb5dd36 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -164,14 +164,15 @@ "packages/remote/src/upload-client.ts#UploadPreflightResponse": "sha256:196a1d9ef41a1aec192589fa7ccb30cf71d971a005349d5d73eed42e6ab21327", "packages/remote/src/upload-client.ts#UploadPreflightResult": "sha256:bd8fc6d617c0630325455fd64d0fe319daaced198ee6745e4673a8ace3df460d", "packages/remote/src/upload-client.ts#UploadResponse": "sha256:499e01466b5e0e03413f8d3f1986f5af6bc120704c50a3747d84de422b3f0314", - "packages/remote/src/upload-client.ts#finalizeDirectUpload": "sha256:8cd172a169aa5e54695d9185fa259d93c71b82938455398b83e7e4e214578df2", + "packages/remote/src/upload-client.ts#finalizeDirectUpload": "sha256:ca3fa667ab2a60fb8af095bddeacb4875dcd2d12db783585a0fb045db35da212", "packages/remote/src/upload-client.ts#isStringRecord": "sha256:b6d6b571b90baf0976d9c347c4a45a4363a4df5ad184ea1087cb3b9febd4b624", "packages/remote/src/upload-client.ts#parseUploadPreflightResult": "sha256:7ce3f43b45c03df8b7f2690432beedc7a537c1bba540961716862f8a632d5623", - "packages/remote/src/upload-client.ts#requestUploadPreflight": "sha256:ed644d7e59c4703e3b7a05dda769b3f0554dc481598ba01906a131c543d68ffc", + "packages/remote/src/upload-client.ts#requestUploadControl": "sha256:5689da0c0cd18d0293e5bb05ddc0d8a07d1bca9ab1d94f8ca185b03bc5b8d758", + "packages/remote/src/upload-client.ts#requestUploadPreflight": "sha256:6976942601980c80ef069a0e1cbd7058840d0384e9e87c48f5dfd8d1c774703c", "packages/remote/src/upload-client.ts#shouldRetryDirectUpload": "sha256:29059f076ff7f2b9540ad15408be48792b2d63a601ef2b9fb2d54f5576d49bbd", - "packages/remote/src/upload-client.ts#tryDirectUploadWithResume": "sha256:db1042b6b8f6969add9480f30a8649d83eecf60af38e80f00a4e789fcf925f97", - "packages/remote/src/upload-client.ts#uploadDirectArtifact": "sha256:3edb622eabee720085a544a4d4e6fdd74c19ec25d3e14e7f9f06ea10d550c1c7", - "packages/remote/src/upload-client.ts#uploadLegacyArtifact": "sha256:3acabacbdea5b7da7f3d8e6e87496728eb3b1df122c490751a9edabc1e9c9adc", + "packages/remote/src/upload-client.ts#tryDirectUploadWithResume": "sha256:cc5d33f38e047cf3be4bd412b64958c7495eb9a45e58eefadaa00eefa0cb6b53", + "packages/remote/src/upload-client.ts#uploadDirectArtifact": "sha256:93a36a9d3dea88064569517f3cdc34f27b33df6bf5f0c0ecc9efc29b47f24b77", + "packages/remote/src/upload-client.ts#uploadLegacyArtifact": "sha256:1546c696134b47bfaaaffcf95deb176cccdce243818b658ae0494dafb57fafb6", "packages/remote/src/upload-stream.ts#MAX_UPLOAD_REDIRECTS": "sha256:12ff46dfe33a1dce95bdb79c90c17a3ba4f3919e7dc4b3a1b04590fff430f79f", "packages/remote/src/upload-stream.ts#UploadStreamResponse": "sha256:724e284c2a6bd7b4b3a49b1d8c61c8923d6cc24777c3d76224a69d190e56adce", "packages/remote/src/upload-stream.ts#buildUploadRequestHeaders": "sha256:827576b13a9f3e6cea445119a6a29b2ac5eaa23b66436cf810b2d09765c82a4c", @@ -180,8 +181,8 @@ "packages/remote/src/upload-stream.ts#isUploadResumeStatus": "sha256:9f730ccd1c3401c326f8df67d1fd8243fef757357a38a1f53736df2b90bd8fdc", "packages/remote/src/upload-stream.ts#parseNonNegativeIntegerHeader": "sha256:5cbe9c6994459d673d988a51407e65556e28175b187ce0a9d27cbb6331962834", "packages/remote/src/upload-stream.ts#parseUploadResumeOffset": "sha256:297184ea22a6dc1cda9aee17034122ae0ea2f43d0355647f6fcf02f3f4b5ca0e", - "packages/remote/src/upload-stream.ts#streamFileToHttpRequest": "sha256:ce07ea33a275e06e4cceaf0c75a079bd0c7938f36df6817407dbb1cc8c4ff900", - "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:b49e17dab9a638f3e547af01717aad52264e20334d62489608bf2c7f9a238838", + "packages/remote/src/upload-stream.ts#streamFileToHttpRequest": "sha256:c2b5e1369c8b1c9420036e23163d21e43fa2ac8b889c6a8f028cc6095055a16f", + "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:1a4b994c3ac2cf89e01e9c56526a5c9a0356fb8de425a248d062bd1cd67686b8", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_HEADER": "sha256:d2bb3f4846470188a0fc1724271da0f0bcbdf2337bd6bf331508142cadcb65d8", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_UPSTREAM_INSTANCE_HEADER": "sha256:4564c2ce437d30d7a823e3326f4375c0750ba8d42bec22b4d02e8eb7f7db2973", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_MISMATCH_HEADER": "sha256:a197edaf7f9fa94a6aab269856aa852da3204cc586f5fd23198689eb11908b61", @@ -330,8 +331,8 @@ }, { "declaration": "packages/remote/src/upload-stream.ts#streamFileToHttpRequest", - "digest": "sha256:ce07ea33a275e06e4cceaf0c75a079bd0c7938f36df6817407dbb1cc8c4ff900", - "rationale": "#2946 widens the options with an optional abort signal handed to node:http.request. The request line, headers built by buildUploadRequestHeaders, and piped payload are unchanged, so an older daemon's 200/308 handling is unaffected." + "digest": "sha256:c2b5e1369c8b1c9420036e23163d21e43fa2ac8b889c6a8f028cc6095055a16f", + "rationale": "#4441 replaces the one 5-minute timer over a whole upload with the preparation aggregate that bounds it, and hands that aggregate's signal to the attempt. The request a peer parses is unchanged: same route, method, the headers buildUploadRequestHeaders produces, the piped payload and the 200/308 handling. A progressing upload may now consume the shared ten-minute aggregate instead of the former five-minute per-attempt ceiling. Connection, header and body-inactivity phase limits stop stalled work sooner; retries and resumes never restart the aggregate. A protocol-2 daemon or proxy parses the same requests, while client-side timeout and retry timing deliberately change. Its only callers run inside uploadArtifact's aggregate, so it no longer opens a second copy of that deadline: it requires the aggregate's signal and drops the timeout message and hint that the outer deadline always preempted." }, { "declaration": "packages/contracts/src/daemon-http.ts#DaemonHealthPayload", @@ -435,18 +436,18 @@ }, { "declaration": "packages/remote/src/upload-client.ts#uploadDirectArtifact", - "digest": "sha256:3edb622eabee720085a544a4d4e6fdd74c19ec25d3e14e7f9f06ea10d550c1c7", - "rationale": "Clients name a held agent-device lease in an optional x-agent-device-lease-id request header on upload, artifact download and request diagnostics HTTP calls, so a gateway can route them. Released daemons ignore unknown request headers and the proxy forwards only its allowlist, so routes, bodies and responses parse as before; with no lease held the request is unchanged (ADR 0006)." + "digest": "sha256:93a36a9d3dea88064569517f3cdc34f27b33df6bf5f0c0ecc9efc29b47f24b77", + "rationale": "Clients name a held agent-device lease in an optional x-agent-device-lease-id request header on upload, artifact download and request diagnostics HTTP calls, so a gateway can route them. Released daemons ignore unknown request headers and the proxy forwards only its allowlist, so routes, bodies and responses parse as before; with no lease held the request is unchanged (ADR 0006). #4441 also stops passing a timeout message and hint that no caller could see, and requires the preparation aggregate's signal. The ticket PUT keeps its route, method, headers and body." }, { "declaration": "packages/remote/src/upload-client.ts#tryDirectUploadWithResume", - "digest": "sha256:db1042b6b8f6969add9480f30a8649d83eecf60af38e80f00a4e789fcf925f97", - "rationale": "Clients name a held agent-device lease in an optional x-agent-device-lease-id request header on upload, artifact download and request diagnostics HTTP calls, so a gateway can route them. Released daemons ignore unknown request headers and the proxy forwards only its allowlist, so routes, bodies and responses parse as before; with no lease held the request is unchanged (ADR 0006)." + "digest": "sha256:cc5d33f38e047cf3be4bd412b64958c7495eb9a45e58eefadaa00eefa0cb6b53", + "rationale": "#4441 ends the direct-upload sequence when a transfer phase's own budget expires, instead of re-preflighting for a fresh ticket or falling back, because a second attempt would be permission that budget already refused. Preflight, ticket PUT and finalize requests are unchanged in route, method, headers and body, so a protocol-2 daemon parses everything it parsed before; it just receives no further legs once the caller's aggregate has expired. Its signal is now required, because the preparation aggregate is always the one passed down, and it is read directly; no request changes." }, { "declaration": "packages/remote/src/upload-client.ts#uploadLegacyArtifact", - "digest": "sha256:3acabacbdea5b7da7f3d8e6e87496728eb3b1df122c490751a9edabc1e9c9adc", - "rationale": "Clients name a held agent-device lease in an optional x-agent-device-lease-id request header on upload, artifact download and request diagnostics HTTP calls, so a gateway can route them. Released daemons ignore unknown request headers and the proxy forwards only its allowlist, so routes, bodies and responses parse as before; with no lease held the request is unchanged (ADR 0006)." + "digest": "sha256:1546c696134b47bfaaaffcf95deb176cccdce243818b658ae0494dafb57fafb6", + "rationale": "#4441 drops the hint that named the removed 5-minute timeout, and an expired transfer phase is no longer treated as a transport break, so this route is not re-entered by a fallback after the client's own budget ended the attempt. The POST /upload request itself is unchanged: same method, headers, chunked body and response parsing. A protocol-2 daemon sees the identical request in every case that completes, and no further request in a stalled one. The unreachable timeout message and hint are no longer passed, and the aggregate signal is required." }, { "declaration": "packages/remote/src/artifact-download.ts#downloadRemoteArtifactFromUrl", @@ -495,8 +496,8 @@ }, { "declaration": "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt", - "digest": "sha256:b49e17dab9a638f3e547af01717aad52264e20334d62489608bf2c7f9a238838", - "rationale": "The upload client refuses a redirect to another origin, including a 308 that carries a Location, instead of re-sending the daemon token or ticket headers and the artifact there. Same-origin redirects, the Location-less 308 resume contract, and the redirect limit are unchanged. A released daemon or proxy answers its upload routes on its own origin, so it parses the same requests. A direct-upload ticket that redirects cross-origin fails as a non-retryable direct upload, and uploadArtifact then takes the legacy /upload route on the daemon." + "digest": "sha256:1a4b994c3ac2cf89e01e9c56526a5c9a0356fb8de425a248d062bd1cd67686b8", + "rationale": "#4441 keeps each attempt under its sequential phase watchdog and corrects its transport lifecycle: the connect ceiling now ends at actual socket or TLS-handshake establishment, body progress counts only transport writes, and an early response reads the accepted answer to its end, then destroys and joins the owned file/request pipeline and retires this attempt's phase watchdog before resolving or starting the redirect/resume attempt, so a completed hop's timers never reach the next hop. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own sequencing and deadlines moved, so a protocol-2 peer parses exactly what it parsed before. Request-body progress is now each non-empty request write that completes without an error, which Node reports once the bytes reach the local kernel, instead of drain, which fires only after a write returned false. The write wrapper passes every argument and the backpressure result through, so the bytes and framing on the wire do not change. That completed-write observer is host-kit's transport owner, which also stops reporting after the response starts, the signal aborts or the request is destroyed; no request changes. The response-header ceiling starts at connected request finish, without waiting for or rearming after source-file close. This changes local deadline timing only; URL, method, credentials, body, redirects, resume headers and response framing are unchanged." }, { "declaration": "packages/remote/src/daemon-artifacts.ts#resolveMaterializedArtifactPath", @@ -508,15 +509,20 @@ "digest": "sha256:eed6cb154db2b67ffe5f058ef0570fec247a5d2a43af2d42952dee0b08870a87", "rationale": "The /rpc body limit is now DAEMON_REQUEST_MAX_BYTES (64 MiB), the one request limit the socket transport, the proxy default, and client writers share, raised from 1 MiB. A larger limit refuses nothing a released client sends; a new client refuses a request over 64 MiB before sending it, and a released daemon still answers anything between 1 MiB and 64 MiB with its own -32700 refusal, as before." }, + { + "declaration": "packages/remote/src/upload-client.ts#requestUploadControl", + "digest": "sha256:5689da0c0cd18d0293e5bb05ddc0d8a07d1bca9ab1d94f8ca185b03bc5b8d758", + "rationale": "Lists the shared control Fetch/JSON owner so changes to method, headers, body or redirect forwarding remain wire-gated. Existing preflight/finalize routes and protocol-2 payloads are preserved; the original 30-second ceiling now retains typed local phase expiry and stops later legs. Non-success status handling is unchanged." + }, { "declaration": "packages/remote/src/upload-client.ts#requestUploadPreflight", - "digest": "sha256:ed644d7e59c4703e3b7a05dda769b3f0554dc481598ba01906a131c543d68ffc", - "rationale": "The request now uses fetch with redirect: 'error', so no redirect is followed: fetch keeps x-agent-device-token on a cross-origin hop. The request body, headers, and route are unchanged. A released daemon or proxy never redirects this route, so it parses the same requests. A redirected preflight falls back to the legacy /upload route, as any failed preflight does." + "digest": "sha256:6976942601980c80ef069a0e1cbd7058840d0384e9e87c48f5dfd8d1c774703c", + "rationale": "#4441 preserves the 30-second control-request ceiling but reports its typed phase timeout instead of retrying or falling back after expiry. The shared Fetch/JSON owner retains route, method, authentication, request body, redirect refusal and successful response parsing, so protocol-2 peers parse the same messages; only the local timeout outcome stops subsequent upload legs." }, { "declaration": "packages/remote/src/upload-client.ts#finalizeDirectUpload", - "digest": "sha256:8cd172a169aa5e54695d9185fa259d93c71b82938455398b83e7e4e214578df2", - "rationale": "The request now uses fetch with redirect: 'error', so no redirect is followed: fetch keeps x-agent-device-token on a cross-origin hop. The request body, headers, and route are unchanged. A released daemon or proxy never redirects this route, so it parses the same requests. A redirected finalize fails with COMMAND_FAILED, and uploadArtifact then takes the legacy /upload route." + "digest": "sha256:ca3fa667ab2a60fb8af095bddeacb4875dcd2d12db783585a0fb045db35da212", + "rationale": "#4441 preserves the 30-second control-request ceiling but reports its typed phase timeout instead of retrying or falling back after expiry. The shared Fetch/JSON owner retains route, method, authentication, request body, redirect refusal and successful response parsing, so protocol-2 peers parse the same messages; only the local timeout outcome stops subsequent upload legs." }, { "declaration": "packages/daemon/src/server/http/http-rpc-methods.ts#toInstallFromSourceDaemonRequest", diff --git a/test/wire-compat/surface.ts b/test/wire-compat/surface.ts index a434384d33..61d685487b 100644 --- a/test/wire-compat/surface.ts +++ b/test/wire-compat/surface.ts @@ -393,6 +393,7 @@ export const WIRE_SURFACE: readonly WireSurfaceGroup[] = [ 'parseUploadPreflightResult', 'isStringRecord', 'requestUploadPreflight', + 'requestUploadControl', 'uploadDirectArtifact', 'tryDirectUploadWithResume', 'shouldRetryDirectUpload', diff --git a/test/wire-compat/wire-mutations.test.ts b/test/wire-compat/wire-mutations.test.ts index 92340a37c4..a88c4e027a 100644 --- a/test/wire-compat/wire-mutations.test.ts +++ b/test/wire-compat/wire-mutations.test.ts @@ -49,6 +49,13 @@ type WireMutation = { }; const MUTATIONS: readonly WireMutation[] = [ + { + breakClass: 'upload control: Fetch drops authentication headers and the JSON body', + file: 'packages/remote/src/upload-client.ts', + name: 'requestUploadControl', + from: '...init,', + to: '', + }, { breakClass: 'upload ticket: the response wrapper drops the configured origin', file: 'packages/proxy/src/daemon-proxy.ts',