From b2f7d146c3f7a07da4160f56a8265b1e78c52c14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 16:29:15 +0100 Subject: [PATCH 01/27] fix(remote): bound artifact preparation and propagate cancellation --- .../upload-client-cancellation.test.ts | 71 ++++++++++++++++++- packages/remote/src/daemon-artifacts.ts | 17 +++++ packages/remote/src/upload-client-artifact.ts | 14 ++-- packages/remote/src/upload-client.ts | 17 ++++- .../remote/src/upload-preparation-deadline.ts | 34 +++++++++ packages/remote/src/upload-stream.ts | 54 +++++++------- 6 files changed, 175 insertions(+), 32 deletions(-) create mode 100644 packages/remote/src/upload-preparation-deadline.ts diff --git a/packages/remote/src/__tests__/upload-client-cancellation.test.ts b/packages/remote/src/__tests__/upload-client-cancellation.test.ts index b42aed13e5..c5ebb6f4af 100644 --- a/packages/remote/src/__tests__/upload-client-cancellation.test.ts +++ b/packages/remote/src/__tests__/upload-client-cancellation.test.ts @@ -5,7 +5,7 @@ // abort signal — a rejection the caller swallows would keep streaming a full app bundle to a device // nobody owns. -import { afterEach, test } from 'vitest'; +import { afterEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import fs from 'node:fs'; @@ -19,12 +19,81 @@ const TEST_TOKEN = 'agent-device-upload-cancel-token'; const tempDirs: string[] = []; afterEach(async () => { + vi.restoreAllMocks(); + vi.useRealTimers(); for (const dir of tempDirs) { await fs.promises.rm(dir, { recursive: true, force: true }).catch(() => {}); } tempDirs.length = 0; }); +test('cancellation refuses preparation before it reads a missing artifact', async () => { + const artifactPath = createTempFile('missing.apk', 'payload'); + fs.unlinkSync(artifactPath); + const control = new AbortController(); + const reason = new Error('request owner stopped'); + control.abort(reason); + const fetch = vi.spyOn(globalThis, 'fetch'); + await assert.rejects( + uploadArtifact({ + env: process.env, + localPath: artifactPath, + baseUrl: 'http://127.0.0.1:1', + token: TEST_TOKEN, + signal: control.signal, + }), + (error: unknown) => error === reason, + ); + assert.equal(fetch.mock.calls.length, 0); +}); + +test('a standalone upload expires its preparation aggregate without starting a fallback', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const artifactPath = createTempFile('app.apk', 'payload'); + let preflightEntered = () => {}; + const entered = new Promise((resolve) => { + preflightEntered = resolve; + }); + let preflightSignal: AbortSignal | undefined; + let refusePreflight: ((reason: Error) => void) | undefined; + const fetch = vi.spyOn(globalThis, 'fetch').mockImplementation(async (_url, init) => { + const signal = init?.signal; + assert.ok(signal); + preflightSignal = signal; + preflightEntered(); + return await new Promise((_resolve, reject) => { + refusePreflight = reject; + signal.addEventListener('abort', () => reject(signal.reason), { once: true }); + }); + }); + const outcome = uploadArtifact({ + env: process.env, + localPath: artifactPath, + baseUrl: 'http://127.0.0.1:1', + token: TEST_TOKEN, + }).then( + (value) => ({ value, error: undefined }), + (error: unknown) => ({ value: undefined, error }), + ); + try { + await entered; + await vi.advanceTimersByTimeAsync(600_001); + assert.equal(preflightSignal?.aborted, true); + const result = await outcome; + assert.equal(result.value, undefined); + assert.ok(result.error && typeof result.error === 'object' && 'details' in result.error); + assert.deepEqual(result.error.details, { + reason: 'artifact_preparation_deadline_exceeded', + transferPhase: 'preparation', + timeoutMs: 600_000, + }); + assert.equal(fetch.mock.calls.length, 1); + } finally { + refusePreflight?.(new Error('fixture cleanup')); + await outcome; + } +}); + test('an aborted signal ends a legacy upload mid-stream instead of finishing the bytes', async () => { // Two megabytes is well past what a paused read lets through: the assertions below are about the // stream stopping, and a smaller payload keeps that off the CPU in a loaded lane. diff --git a/packages/remote/src/daemon-artifacts.ts b/packages/remote/src/daemon-artifacts.ts index f85b624aee..b57d4db170 100644 --- a/packages/remote/src/daemon-artifacts.ts +++ b/packages/remote/src/daemon-artifacts.ts @@ -15,6 +15,7 @@ import { } from '@agent-device/capture-kit/recording-output-path'; import { parsePerfRuntimeRequest } from '@agent-device/capture-kit/perf-runtime-plan'; import { uploadArtifact } from './upload-client.ts'; +import { withUploadPreparationDeadline } from './upload-preparation-deadline.ts'; import { createStderrUploadProgressReporter, type UploadProgressSink } from './upload-progress.ts'; // Mirrors `DEFAULT_TEST_ARTIFACTS_ROOT` in `packages/replay-test/src/internal/session-test-artifacts.ts` @@ -41,6 +42,22 @@ export async function prepareRemoteRequestArtifacts( info: DaemonArtifactEndpoint, signal: AbortSignal, host: HostEnvironment, +): Promise { + if (!isRemoteDaemon(info)) { + return await prepareRemoteRequestArtifactsWithinBudget(req, info, signal, host); + } + return await withUploadPreparationDeadline( + signal, + async (preparationSignal) => + await prepareRemoteRequestArtifactsWithinBudget(req, info, preparationSignal, host), + ); +} + +async function prepareRemoteRequestArtifactsWithinBudget( + req: Omit, + info: DaemonArtifactEndpoint, + signal: AbortSignal, + host: HostEnvironment, ): Promise { const positionals = [...(req.positionals ?? [])]; let flags = req.flags ? { ...req.flags } : undefined; diff --git a/packages/remote/src/upload-client-artifact.ts b/packages/remote/src/upload-client-artifact.ts index d4de4a4f9c..b037d7e59e 100644 --- a/packages/remote/src/upload-client-artifact.ts +++ b/packages/remote/src/upload-client-artifact.ts @@ -26,7 +26,9 @@ export async function prepareUploadArtifact( localPath: string, requestedPlatform: string | undefined, env: Readonly, + signal?: AbortSignal, ): Promise { + signal?.throwIfAborted(); const stat = fs.statSync(localPath); const fileName = path.basename(localPath); const isDirectory = stat.isDirectory(); @@ -35,8 +37,9 @@ export async function prepareUploadArtifact( const cleanupPaths: string[] = []; try { const payloadPath = isDirectory - ? await createGzipTarArchive(localPath, cleanupPaths, env) + ? await createGzipTarArchive(localPath, cleanupPaths, env, signal) : localPath; + signal?.throwIfAborted(); const payloadStat = fs.statSync(payloadPath); return { @@ -45,7 +48,7 @@ export async function prepareUploadArtifact( artifactType: isDirectory ? 'app-bundle' : 'file', platform, contentType: isDirectory ? 'application/gzip' : DEFAULT_CONTENT_TYPE, - sha256: await computeFileHash(payloadPath), + sha256: await computeFileHash(payloadPath, signal), sizeBytes: payloadStat.size, cleanup: () => cleanupUploadPaths(cleanupPaths), }; @@ -59,6 +62,7 @@ async function createGzipTarArchive( localPath: string, cleanupPaths: string[], env: Readonly, + signal?: AbortSignal, ): Promise { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), `agent-device-upload-${randomUUID()}-`)); cleanupPaths.push(tempDir); @@ -67,6 +71,7 @@ async function createGzipTarArchive( 'tar', ['czf', archivePath, '-C', path.dirname(localPath), path.basename(localPath)], { + signal, env: { ...env, COPYFILE_DISABLE: '1', @@ -97,7 +102,7 @@ function cleanupUploadPaths(cleanupPaths: string[]): void { } } -async function computeFileHash(localPath: string): Promise { +async function computeFileHash(localPath: string, signal?: AbortSignal): Promise { const hash = createHash(ARTIFACT_HASH_ALGORITHM); const sink = new Writable({ write(chunk: Buffer, _encoding, callback) { @@ -105,7 +110,8 @@ async function computeFileHash(localPath: string): Promise { callback(); }, }); - await pipeline(fs.createReadStream(localPath), sink).catch((error: unknown) => { + await pipeline(fs.createReadStream(localPath), sink, { signal }).catch((error: unknown) => { + signal?.throwIfAborted(); throw new AppError( 'COMMAND_FAILED', 'Failed to read local artifact', diff --git a/packages/remote/src/upload-client.ts b/packages/remote/src/upload-client.ts index a684db0d5a..a1220a0db7 100644 --- a/packages/remote/src/upload-client.ts +++ b/packages/remote/src/upload-client.ts @@ -8,6 +8,7 @@ import { import { prepareUploadArtifact, type PreparedUploadArtifact } from './upload-client-artifact.ts'; import { isRetryableUploadStreamError, streamFileToHttpRequest } from './upload-stream.ts'; import type { UploadProgressSink } from './upload-progress.ts'; +import { withUploadPreparationDeadline } from './upload-preparation-deadline.ts'; const UPLOAD_PREFLIGHT_TIMEOUT_MS = 30 * 1000; const ARTIFACT_HASH_ALGORITHM = 'sha256'; @@ -57,7 +58,19 @@ type UploadPreflightResult = }; export async function uploadArtifact(options: UploadArtifactOptions): Promise { - const prepared = await prepareUploadArtifact(options.localPath, options.platform, options.env); + return await withUploadPreparationDeadline( + options.signal, + async (signal) => await uploadArtifactWithinBudget({ ...options, signal }), + ); +} + +async function uploadArtifactWithinBudget(options: UploadArtifactOptions): Promise { + const prepared = await prepareUploadArtifact( + options.localPath, + options.platform, + options.env, + options.signal, + ); const normalizedBase = options.baseUrl.endsWith('/') ? options.baseUrl : `${options.baseUrl}/`; const uploadAttemptId = randomUUID(); @@ -223,7 +236,7 @@ async function uploadLegacyArtifact(options: { headers, payloadPath: artifact.payloadPath, timeoutMessage: 'Artifact upload timed out', - timeoutHint: 'The upload to the remote daemon exceeded the 5-minute timeout.', + timeoutHint: 'Inspect the upload diagnostics before retrying the artifact.', errorMessage: 'Failed to upload artifact to remote daemon', errorHint: 'Verify the remote daemon is reachable and supports artifact uploads.', signal: options.signal, diff --git a/packages/remote/src/upload-preparation-deadline.ts b/packages/remote/src/upload-preparation-deadline.ts new file mode 100644 index 0000000000..6e4fbc6c50 --- /dev/null +++ b/packages/remote/src/upload-preparation-deadline.ts @@ -0,0 +1,34 @@ +import { AppError } from '@agent-device/kernel/errors'; + +const UPLOAD_PREPARATION_TIMEOUT_MS = 600_000; + +export async function withUploadPreparationDeadline( + callerSignal: AbortSignal | undefined, + prepare: (signal: AbortSignal) => Promise, + timeout?: Readonly<{ message: string; hint?: string }>, +): Promise { + const deadline = new AbortController(); + const signal = callerSignal ? AbortSignal.any([callerSignal, deadline.signal]) : deadline.signal; + signal.throwIfAborted(); + const timer = setTimeout(() => { + deadline.abort( + new AppError('COMMAND_FAILED', timeout?.message ?? 'Artifact preparation deadline exceeded', { + reason: 'artifact_preparation_deadline_exceeded', + transferPhase: 'preparation', + timeoutMs: UPLOAD_PREPARATION_TIMEOUT_MS, + ...(timeout?.hint ? { hint: timeout.hint } : {}), + }), + ); + }, UPLOAD_PREPARATION_TIMEOUT_MS); + timer.unref(); + try { + const result = await prepare(signal); + signal.throwIfAborted(); + return result; + } catch (error) { + signal.throwIfAborted(); + throw error; + } finally { + clearTimeout(timer); + } +} diff --git a/packages/remote/src/upload-stream.ts b/packages/remote/src/upload-stream.ts index fdaf1fc018..798e039bc2 100644 --- a/packages/remote/src/upload-stream.ts +++ b/packages/remote/src/upload-stream.ts @@ -2,6 +2,7 @@ import fs from 'node:fs'; import type { IncomingHttpHeaders } from 'node:http'; import path from 'node:path'; import { pipeline } from 'node:stream/promises'; +import { addAbortSignal } from 'node:stream'; import { AppError } from '@agent-device/kernel/errors'; import { loadNodeHttpRequester, readNodeHttpResponseBody } from '@agent-device/host-kit/transport'; import { @@ -9,8 +10,8 @@ import { type UploadProgressSink, type UploadProgressStage, } from './upload-progress.ts'; +import { withUploadPreparationDeadline } from './upload-preparation-deadline.ts'; -const UPLOAD_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes const MAX_UPLOAD_REDIRECTS = 5; const UPLOAD_STREAM_RETRYABLE_DETAIL = 'uploadStreamRetryable'; @@ -40,12 +41,18 @@ export async function streamFileToHttpRequest(options: { signal?: AbortSignal; progress?: UploadStreamProgressOptions; }): Promise { - return await streamFileToHttpRequestAttempt({ - ...options, - url: options.url, - redirectCount: 0, - startOffset: 0, - }); + return await withUploadPreparationDeadline( + options.signal, + async (signal) => + await streamFileToHttpRequestAttempt({ + ...options, + signal, + url: options.url, + redirectCount: 0, + startOffset: 0, + }), + { message: options.timeoutMessage, hint: options.timeoutHint }, + ); } export function isRetryableUploadStreamError(error: unknown): boolean { @@ -67,6 +74,7 @@ async function streamFileToHttpRequestAttempt(options: { startOffset: number; progress?: UploadStreamProgressOptions; }): Promise { + options.signal?.throwIfAborted(); // This module sits in the CLI's eager import closure via the remote-artifact // upload client, but only a remote daemon ever uploads, so the HTTP stack // loads here rather than at import time. @@ -75,7 +83,8 @@ async function streamFileToHttpRequestAttempt(options: { const headers = buildUploadRequestHeaders(options.headers, options.startOffset, payloadSize); emitUploadAttemptStarted(options.progress, options.startOffset, payloadSize); - return await new Promise((resolve, reject) => { + let upload: Promise | undefined; + return await new Promise((resolve, reject) => { let responseReceived = false; const req = transport.request( { @@ -91,9 +100,9 @@ async function streamFileToHttpRequestAttempt(options: { }, (res) => { responseReceived = true; + if (options.signal) addAbortSignal(options.signal, res); void readNodeHttpResponseBody(res) .then((body) => { - clearTimeout(timeout); const statusCode = res.statusCode ?? 500; const location = res.headers.location; if (location && isUploadRedirectStatus(statusCode)) { @@ -166,24 +175,14 @@ async function streamFileToHttpRequestAttempt(options: { body, }); }) - .catch(reject); + .catch((error: unknown) => + reject(options.signal?.aborted ? options.signal.reason : error), + ); }, ); - const timeout = setTimeout(() => { - req.destroy(); - reject( - new AppError('COMMAND_FAILED', options.timeoutMessage, { - timeoutMs: UPLOAD_TIMEOUT_MS, - ...(options.timeoutHint ? { hint: options.timeoutHint } : {}), - ...retryableUploadStreamDetail(options), - }), - ); - }, UPLOAD_TIMEOUT_MS); - req.on('error', (err) => { if (responseReceived) return; - clearTimeout(timeout); // A caller that canceled owns this outcome: the signal's reason is the answer, and wrapping it // as an ordinary transport failure would let a retry path treat "we stopped" as "it broke". if (options.signal?.aborted) { @@ -202,9 +201,7 @@ async function streamFileToHttpRequestAttempt(options: { ), ); }); - req.on('close', () => clearTimeout(timeout)); - - void pipeline( + upload = pipeline( fs.createReadStream(options.payloadPath, { start: options.startOffset }), createUploadProgressTransform({ stage: options.progress?.stage ?? 'legacy', @@ -214,12 +211,19 @@ async function streamFileToHttpRequestAttempt(options: { onProgress: options.progress?.onProgress, }), req, + { signal: options.signal }, ).catch((err: unknown) => { if (responseReceived) return; req.destroy(); + if (options.signal?.aborted) { + reject(options.signal.reason); + return; + } const error = err instanceof Error ? err : new Error(String(err)); reject(new AppError('COMMAND_FAILED', 'Failed to read local artifact', {}, error)); }); + }).finally(async () => { + await upload; }); } From bec05d5d4d560bd680ddda40dd1207347dfd0d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 14:01:11 +0100 Subject: [PATCH 02/27] fix(remote): budget each artifact-transfer phase inside the preparation aggregate A single five-minute timer covered a whole upload, so an artifact that kept moving and one that stalled after the first chunk were cut off the same way, and nothing said which part of the hop ended the attempt. Each attempt now runs one sequential watchdog: 30s to connect, 120s of silence between body bytes in either direction, and 30s for response headers once the outgoing body is flushed. Only bytes actually written or read re-arm a body phase, and arming a later phase stops counting the earlier one. All of it stays inside the 600s preparation aggregate from the previous commit, which every attempt, redirect, offset resume, fallback and finalize still shares, so the earliest deadline keeps winning and no resumed work can outlive it. A phase expiry is typed evidence carrying the phase and its effective budget. It is deliberately not retryable: restarting the sequence would buy a second attempt permission the budget already refused, so neither the ticket retry nor the legacy fallback restarts those bytes. Caller cancellation stays a separate account and still answers with the signal's own reason. The two control-plane calls that bracket an upload, preflight and finalize, carry one 30s ceiling each: their payload is a few hundred bytes of JSON the transport writes with the request, so there is no separate body phase to time. --- .../support/upload-transfer-fixtures.ts | 227 ++++++++++++++++++ .../__tests__/transfer-phase-budget.test.ts | 168 +++++++++++++ .../__tests__/upload-transfer-phases.test.ts | 139 +++++++++++ packages/remote/src/transfer-phase-budget.ts | 104 ++++++++ packages/remote/src/upload-client.ts | 32 ++- .../remote/src/upload-preparation-deadline.ts | 3 +- packages/remote/src/upload-progress.ts | 9 +- packages/remote/src/upload-stream.ts | 123 +++++++--- test/wire-compat/ledger.json | 36 +-- 9 files changed, 784 insertions(+), 57 deletions(-) create mode 100644 packages/remote/src/__tests__/support/upload-transfer-fixtures.ts create mode 100644 packages/remote/src/__tests__/transfer-phase-budget.test.ts create mode 100644 packages/remote/src/__tests__/upload-transfer-phases.test.ts create mode 100644 packages/remote/src/transfer-phase-budget.ts diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts new file mode 100644 index 0000000000..320f6d3bdc --- /dev/null +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -0,0 +1,227 @@ +// Shared support for the tests that put real budget pressure on a real artifact upload (#4441). +// +// Every helper here exists so a test can spend minutes of transfer budget without waiting for them: +// the daemon holds the request body open chunk by chunk and the test drives the clock. The socket, +// the framing and the client's own stream stay real, which is the point — a mocked transport cannot +// prove that a transfer which keeps moving is never cut short, nor that one which stops is. + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import http from 'node:http'; +import path from 'node:path'; +import { vi } from 'vitest'; +import { closeLoopbackServer, listenOnLoopback, mkdtempForTestSync } from '@agent-device/test-kit'; +import { uploadArtifact } from '../../upload-client.ts'; + +export const PREFLIGHT_REQUEST = 'POST /upload/preflight'; +export const LEGACY_UPLOAD_REQUEST = 'POST /upload'; + +/** Budget step used when a test just needs time to pass: fine enough that no phase boundary is + * stepped over, cheap enough that six minutes of budget stay a few hundred loop turns. */ +const BUDGET_STEP_MS = 5_000; + +export type UploadOutcome = { value: string | undefined; error: unknown }; + +/** An upload whose completion can be observed without awaiting it, which a controlled clock needs. */ +export type SettledUpload = Promise & { settled: boolean }; + +/** Starts the clock every transfer budget in these tests is measured against. */ +export function useTransferClock(): void { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); +} + +export function startUpload(options: { + baseUrl: string; + artifactPath: string; + signal?: AbortSignal; +}): SettledUpload { + let settled = false; + const upload = uploadArtifact({ + env: process.env, + localPath: options.artifactPath, + baseUrl: options.baseUrl, + token: 'daemon-token', + ...(options.signal ? { signal: options.signal } : {}), + }).then( + (value) => ({ value, error: undefined as unknown }), + (error: unknown) => ({ value: undefined as string | undefined, error }), + ) as SettledUpload; + void upload.then(() => { + settled = true; + }); + Object.defineProperty(upload, 'settled', { get: () => settled }); + return upload; +} + +/** + * Advances the controlled clock while letting real socket work run between the steps, until `ready` + * holds or the transfer is over. + * + * Body progress re-arms an inactivity budget, so a stalled transfer is cut off only once the steps + * stop being interrupted by bytes — exactly the condition under test. A settled transfer ends the + * wait: a budget that fired is an answer, not a reason to keep ticking. + */ +async function spendUntil(options: { + ready: () => boolean; + maxMs: number; + settled?: () => boolean; +}): Promise { + const { ready, maxMs, settled } = options; + for (let elapsed = 0; elapsed < maxMs; elapsed += BUDGET_STEP_MS) { + await vi.advanceTimersByTimeAsync(BUDGET_STEP_MS); + await pumpRealWork(); + if (ready()) return true; + if (settled?.()) return ready(); + } + return ready(); +} + +/** Spends `ms` of budget in `BUDGET_STEP_MS` steps without letting a single byte through. */ +export async function spendBudget(ms: number, upload: SettledUpload): Promise { + await spendUntil({ ready: () => false, maxMs: ms, settled: () => upload.settled }); +} + +/** + * Spends `windows` windows of `windowMs` each while the body keeps moving. + * + * Each window hands back what the daemon had parked and is then charged to the transfer, so the only + * way it can be paid for is by bytes arriving: a window in which `receivedBytes` did not grow is a + * window the transfer did not earn, and the result says so. Progress is judged only once bytes have + * started moving; getting there costs the socket buffers, not the transfer. + */ +export async function spendProgressWindows( + daemon: ParkedUploadDaemon, + upload: SettledUpload, + options: { windows: number; windowMs: number }, +): Promise<{ progressedInEveryWindow: boolean; budgetMs: number }> { + await spendUntil({ ready: () => daemon.receivedBytes > 0, maxMs: 60_000 }); + let progressedInEveryWindow = true; + let spent = 0; + for (let window = 0; window < options.windows; window += 1) { + const bytesBeforeWindow = daemon.receivedBytes; + daemon.releaseAllParked(); + await vi.advanceTimersByTimeAsync(options.windowMs); + spent += options.windowMs; + await pumpRealWork(); + if (upload.settled) break; + if (daemon.receivedBytes <= bytesBeforeWindow) progressedInEveryWindow = false; + } + return { progressedInEveryWindow, budgetMs: spent }; +} + +/** + * Hands over the last few timer ticks a settled transfer needs to tear its leg down. + * + * A phase budget that expires aborts the request, and that abort reaches the client through the + * socket's own close path, which needs turns of both the controlled clock and the real loop before + * the upload promise rejects. + */ +export async function settleAfterBudget(upload: SettledUpload): Promise { + for (let round = 0; round < 200 && !upload.settled; round += 1) { + await vi.advanceTimersByTimeAsync(100); + await pumpRealWork(5); + } +} + +/** + * Lets real socket work finish without spending budget. + * + * A destroyed connection reports its close on a later turn of the real event loop, so a test that + * asserts a leg was torn down has to let that turn run. + */ +export async function pumpRealWork(rounds = 40): Promise { + for (let round = 0; round < rounds; round += 1) { + await new Promise((resolve) => setImmediate(resolve)); + } +} + +/** Writes an artifact of exactly `bytes`, so a test can size a body against a release cadence. */ +export function createArtifactFile(bytes: number, fileName = 'app.apk'): string { + const dir = mkdtempForTestSync('agent-device-upload-phases-'); + const artifactPath = path.join(dir, fileName); + fs.writeFileSync(artifactPath, Buffer.alloc(bytes, 'x')); + return artifactPath; +} + +export type ParkedUploadDaemon = { + readonly baseUrl: string; + readonly requests: readonly string[]; + /** Bytes the daemon has accepted for the upload leg. */ + readonly receivedBytes: number; + /** Lets every chunk the daemon has parked so far through. */ + releaseAllParked(): void; + close(): Promise; +}; + +/** + * A loopback daemon that accepts an artifact upload and parks its body chunk by chunk, never + * answering the leg. + * + * The body is parked so a transfer can be held open long enough to spend a budget while putting real + * backpressure on the writing side. Preflight is refused, so the upload runs on the legacy POST + * route: the one path that pipes a file at the daemon and keeps going while the daemon drains it. + */ +export async function startParkedUploadDaemon(): Promise { + const requests: string[] = []; + const parked: http.IncomingMessage[] = []; + let receivedBytes = 0; + + const server = http.createServer((req, res) => { + const key = `${req.method} ${req.url}`; + requests.push(key); + if (key === PREFLIGHT_REQUEST) { + req.resume(); + req.on('end', () => { + res.statusCode = 404; + res.end('not found'); + }); + return; + } + if (key === LEGACY_UPLOAD_REQUEST) { + req.on('data', (chunk: Buffer) => { + receivedBytes += chunk.length; + req.pause(); + parked.push(req); + }); + return; + } + req.resume(); + req.on('end', () => { + res.statusCode = 500; + res.end('only the upload leg the test set up is expected'); + }); + }); + // The clock under test is controlled, so the daemon's own idle and request ceilings must not join + // the race: they would fire on the fake clock and destroy a leg the client is still progressing. + server.requestTimeout = 0; + server.headersTimeout = 0; + server.keepAliveTimeout = 0; + server.unref(); + const port = await listenOnLoopback(server); + return { + baseUrl: `http://127.0.0.1:${port}`, + requests, + get receivedBytes() { + return receivedBytes; + }, + releaseAllParked() { + for (const entry of parked.splice(0)) entry.resume(); + }, + close: () => closeLoopbackServer(server), + }; +} + +export function assertDetailedError( + error: unknown, + details: Record, +): asserts error is { details?: Record } { + assert.ok(error && typeof error === 'object', `expected an error, got ${String(error)}`); + const actual = JSON.stringify((error as { details?: unknown }).details ?? {}); + for (const [key, value] of Object.entries(details)) { + assert.equal( + (error as { details?: Record }).details?.[key], + value, + `${key} of ${actual}`, + ); + } +} diff --git a/packages/remote/src/__tests__/transfer-phase-budget.test.ts b/packages/remote/src/__tests__/transfer-phase-budget.test.ts new file mode 100644 index 0000000000..1dbbbb3ed6 --- /dev/null +++ b/packages/remote/src/__tests__/transfer-phase-budget.test.ts @@ -0,0 +1,168 @@ +// The phase budget one artifact-transfer attempt runs under (#4441). +// +// Three ceilings govern one attempt — connecting, moving body bytes, waiting for headers — and each +// has to end a stalled attempt on its own while none of them touches the aggregate budget the +// attempt was admitted under. Those are behaviour claims about a clock, so the clock here is +// controlled rather than waited out. The loopback proofs in `upload-transfer-phases.test.ts` put the +// same watchdog in front of real sockets. + +import assert from 'node:assert/strict'; +import { afterEach, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { + createTransferPhaseWatchdog, + isTransferPhaseTimeoutError, + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + TRANSFER_CONNECT_TIMEOUT_MS, + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, +} from '../transfer-phase-budget.ts'; + +afterEach(() => { + vi.useRealTimers(); +}); + +test('the three transfer ceilings are the adopted policy values', () => { + assert.equal(TRANSFER_CONNECT_TIMEOUT_MS, 30_000); + assert.equal(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, 30_000); + assert.equal(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, 120_000); +}); + +test('a body phase restarts its ceiling on every byte that moves', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const watchdog = createTransferPhaseWatchdog(); + try { + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + // Three full ceilings' worth of wall clock, each broken by real progress: 300 seconds of a slow + // artifact is admitted work, and nothing here may end it. + for (const _ of [0, 1, 2]) { + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - 20_000); + watchdog.touchBody(); + } + assert.equal(watchdog.signal.aborted, false); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 1); + assert.equal(watchdog.signal.aborted, true); + assertPhaseDetails( + watchdog.signal.reason, + 'response-body', + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + ); + } finally { + watchdog.dispose(); + } +}); + +test('a phase that carries no bytes is not extended by progress reported in another phase', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const watchdog = createTransferPhaseWatchdog(); + try { + watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 10_000); + // Bytes moving elsewhere — an unrelated header, a keepalive, a timer callback — are not the + // response-header wait making progress. + watchdog.touchBody(); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + assertPhaseDetails( + watchdog.signal.reason, + 'response-headers', + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + ); + } finally { + watchdog.dispose(); + } +}); + +test('arming the next phase stops counting the previous one', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const watchdog = createTransferPhaseWatchdog(); + try { + watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS - 1_000); + watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + // 58 seconds of total attempt time: past the connect ceiling, which the completed hop may spend, + // and inside the header ceiling the attempt is actually waiting on. + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1_000); + assert.equal(watchdog.signal.aborted, false); + await vi.advanceTimersByTimeAsync(1_000); + assertPhaseDetails( + watchdog.signal.reason, + 'response-headers', + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + ); + } finally { + watchdog.dispose(); + } +}); + +test('an enclosing deadline wins and is never extended by a phase timer', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const caller = new AbortController(); + const reason = new Error('preparation aggregate expired'); + setTimeout(() => caller.abort(reason), 5_000); + const watchdog = createTransferPhaseWatchdog(caller.signal); + try { + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(5_001); + // The caller's reason is the answer, not a phase's own evidence, so a consumer can still tell + // "the work this upload served is gone" from "this transfer stalled". + assert.equal(watchdog.signal.aborted, true); + assert.equal(watchdog.signal.reason, reason); + assert.equal(isTransferPhaseTimeoutError(watchdog.signal.reason), false); + } finally { + watchdog.dispose(); + } +}); + +test('a phase expiry is its own typed evidence and never caller cancellation', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const caller = new AbortController(); + const watchdog = createTransferPhaseWatchdog(caller.signal); + try { + watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS + 1); + assertPhaseDetails(watchdog.signal.reason, 'connect', TRANSFER_CONNECT_TIMEOUT_MS); + // Distinctness the retry policy depends on: nobody canceled, so the caller's own signal stays + // live and an enclosing aggregate is free to reach a different outcome. + assert.equal(caller.signal.aborted, false); + } finally { + watchdog.dispose(); + } +}); + +test('a disposed watchdog stops watching', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const watchdog = createTransferPhaseWatchdog(); + watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + watchdog.dispose(); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS * 3); + assert.equal(watchdog.signal.aborted, false); +}); + +test('a phase is never armed after the attempt is already over', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const watchdog = createTransferPhaseWatchdog(); + try { + watchdog.signal.throwIfAborted(); + const caller = new AbortController(); + caller.abort(new Error('gone')); + const late = createTransferPhaseWatchdog(caller.signal); + try { + late.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS * 2); + assert.equal(late.signal.reason.message, 'gone'); + } finally { + late.dispose(); + } + } finally { + watchdog.dispose(); + } +}); + +function assertPhaseDetails(reason: unknown, phase: string, timeoutMs: number): void { + assert.ok(reason instanceof AppError, `phase expiry must be an AppError, got ${String(reason)}`); + assert.equal(isTransferPhaseTimeoutError(reason), true); + assert.deepEqual(reason.details, { + reason: 'transfer_phase_timeout', + transferPhase: phase, + timeoutMs, + }); +} diff --git a/packages/remote/src/__tests__/upload-transfer-phases.test.ts b/packages/remote/src/__tests__/upload-transfer-phases.test.ts new file mode 100644 index 0000000000..b546b31196 --- /dev/null +++ b/packages/remote/src/__tests__/upload-transfer-phases.test.ts @@ -0,0 +1,139 @@ +// Phase budgets applied to a real artifact upload over loopback (#4441). +// +// The budgets in `transfer-phase-budget.ts` are minutes long, so every case here drives a controlled +// clock and parks the request body chunk by chunk. That combination is the point: a transfer that +// keeps moving must survive budget after budget, and one that stops moving must be ended by the ONE +// budget that ran out. A mocked transport could show neither, because the backpressure that makes +// "still progressing" mean something belongs to the real socket. + +import { afterEach, assert, test, vi } from 'vitest'; +import { + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + TRANSFER_CONNECT_TIMEOUT_MS, + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, +} from '../transfer-phase-budget.ts'; +import { + LEGACY_UPLOAD_REQUEST, + PREFLIGHT_REQUEST, + assertDetailedError, + createArtifactFile, + settleAfterBudget, + spendBudget, + spendProgressWindows, + startParkedUploadDaemon, + startUpload, + useTransferClock, +} from './support/upload-transfer-fixtures.ts'; + +const MEBIBYTE = 1024 * 1024; + +afterEach(() => { + vi.useRealTimers(); +}); + +test('a body that keeps moving outlives the body-inactivity ceiling', async () => { + useTransferClock(); + const daemon = await startParkedUploadDaemon(); + const upload = startUpload({ + baseUrl: daemon.baseUrl, + // Sixteen megabytes is far past what loopback buffers absorb, so the windows below are paid for + // by bytes the peer really takes rather than by a socket that swallowed the whole file. + artifactPath: createArtifactFile(16 * MEBIBYTE), + }); + try { + const progress = await spendProgressWindows(daemon, upload, { + windows: 400, + windowMs: 1_000, + }); + + assert.equal(progress.progressedInEveryWindow, true); + assert.ok( + progress.budgetMs > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + `the transfer stayed alive past the body-inactivity ceiling, at ${progress.budgetMs}ms of budget`, + ); + assert.deepEqual(daemon.requests, [PREFLIGHT_REQUEST, LEGACY_UPLOAD_REQUEST]); + } finally { + await daemon.close(); + } +}); + +test('a body the peer stops taking expires the request-body phase on its own budget', async () => { + useTransferClock(); + const daemon = await startParkedUploadDaemon(); + const upload = startUpload({ + baseUrl: daemon.baseUrl, + artifactPath: createArtifactFile(2 * MEBIBYTE), + }); + try { + // Bytes are in flight and then the peer stops reading them, without answering or disconnecting. + // Nothing but the inactivity budget may end this. + await spendBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, upload); + await settleAfterBudget(upload); + const result = await upload; + + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'request-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); + assert.deepEqual(daemon.requests, [PREFLIGHT_REQUEST, LEGACY_UPLOAD_REQUEST]); + assert.ok(daemon.receivedBytes > 0, 'the attempt had a body in flight to stall'); + } finally { + await daemon.close(); + } +}); + +test('a body admitted past four connect ceilings still waits only its own header budget', async () => { + useTransferClock(); + const daemon = await startParkedUploadDaemon(); + const upload = startUpload({ + baseUrl: daemon.baseUrl, + artifactPath: createArtifactFile(8 * 1024), + }); + try { + // The first byte is written before any budget is spent, so the connect ceiling has already given + // way to the body phase: the body can cost more than four connect ceilings, and the wait that + // then expires is the header wait the withheld answer deserves. + const progress = await spendProgressWindows(daemon, upload, { + windows: 5, + windowMs: TRANSFER_CONNECT_TIMEOUT_MS, + }); + await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, upload); + await settleAfterBudget(upload); + const result = await upload; + + assert.equal(progress.progressedInEveryWindow, true); + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + }); + } finally { + await daemon.close(); + } +}); + +test('the caller ending the upload is reported as cancellation, not as an expired phase', async () => { + useTransferClock(); + const daemon = await startParkedUploadDaemon(); + const control = new AbortController(); + const upload = startUpload({ + baseUrl: daemon.baseUrl, + artifactPath: createArtifactFile(8 * MEBIBYTE), + signal: control.signal, + }); + try { + // Well inside every phase budget: only the caller can be the reason this stops. + await spendBudget(30_000, upload); + control.abort(); + const result = await upload; + + assert.equal(result.value, undefined); + assert.ok(result.error instanceof DOMException, `got ${String(result.error)}`); + assert.equal((result.error as DOMException).name, 'AbortError'); + } finally { + await daemon.close(); + } +}); diff --git a/packages/remote/src/transfer-phase-budget.ts b/packages/remote/src/transfer-phase-budget.ts new file mode 100644 index 0000000000..cb57e4a4e4 --- /dev/null +++ b/packages/remote/src/transfer-phase-budget.ts @@ -0,0 +1,104 @@ +import { AppError } from '@agent-device/kernel/errors'; + +/** + * Ceiling for establishing the connection one artifact-transfer attempt runs on. + * + * Kept beside its domain rather than on the socket: the same number governs every hop of an upload, + * every offset resume and a standalone artifact download, so "how long may one hop take to connect" + * has exactly one answer. + */ +export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; + +/** + * Ceiling for waiting on response headers once this side has written everything the server is + * waiting for. For an upload that is the moment its outgoing request body is fully flushed, never + * the moment the request was created; for a bodyless request it is connection establishment. + */ +export const TRANSFER_RESPONSE_HEADER_TIMEOUT_MS = 30_000; + +/** + * Ceiling for silence between body bytes in either direction. Bytes actually written or read reset + * it, so an artifact that keeps moving keeps its budget while a connection that stopped carrying + * bytes does not. Progress never touches the aggregate budget the attempt runs under. + */ +export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; + +export type TransferPhase = 'connect' | 'request-body' | 'response-headers' | 'response-body'; + +const BODY_PHASES: ReadonlySet = new Set(['request-body', 'response-body']); + +/** The `details.reason` every phase expiry carries, so no consumer has to match error text. */ +export const TRANSFER_PHASE_TIMEOUT_REASON = 'transfer_phase_timeout'; + +/** + * Watches the sequential phases of ONE transfer attempt. + * + * One attempt gets one watchdog: arming a later phase stops counting the earlier one, so the ceiling + * that applies is always the phase the transfer is actually in. Progress re-arms only the two body + * phases. Nothing here extends an enclosing aggregate budget — the caller's signal is combined in, + * never replaced, so the earliest deadline still wins and every attempt, redirect and resume keeps + * sharing the original aggregate. + */ +export type TransferPhaseWatchdog = { + /** The signal the attempt runs on: the caller's cancellation plus every phase deadline. */ + signal: AbortSignal; + /** Starts watching `phase`. The phase being watched before stops counting. */ + arm(phase: TransferPhase, timeoutMs: number): void; + /** Records body bytes moving, so only the silence since the last byte counts against a body phase. */ + touchBody(): void; + /** Stops the armed timer. Call it once the attempt is settled. */ + dispose(): void; +}; + +export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): TransferPhaseWatchdog { + const phase = new AbortController(); + const signal = callerSignal ? AbortSignal.any([callerSignal, phase.signal]) : phase.signal; + let armed: { phase: TransferPhase; timeoutMs: number } | undefined; + let timer: ReturnType | undefined; + const rearm = () => { + if (timer !== undefined) clearTimeout(timer); + const current = armed; + if (!current) return; + timer = setTimeout(() => { + timer = undefined; + phase.abort(createTransferPhaseTimeoutError(current.phase, current.timeoutMs)); + }, current.timeoutMs); + timer.unref(); + }; + return { + signal, + arm(nextPhase, timeoutMs) { + if (signal.aborted) return; + armed = { phase: nextPhase, timeoutMs }; + rearm(); + }, + touchBody() { + // Unrelated headers, keepalive traffic and timer callbacks are not body progress, and + // progress in one phase buys no time in another: only a body phase already armed re-arms. + if (!armed || !BODY_PHASES.has(armed.phase)) return; + rearm(); + }, + dispose() { + armed = undefined; + if (timer !== undefined) clearTimeout(timer); + timer = undefined; + }, + }; +} + +/** + * Typed evidence that a transfer phase, rather than the caller or the whole preparation aggregate, + * ended the attempt. The expired phase and its effective budget travel in `details`. + */ +export function createTransferPhaseTimeoutError(phase: TransferPhase, timeoutMs: number): AppError { + return new AppError('COMMAND_FAILED', `Artifact transfer ${phase} timed out`, { + reason: TRANSFER_PHASE_TIMEOUT_REASON, + transferPhase: phase, + timeoutMs, + }); +} + +/** Whether `error` is a transfer phase's own expiry, which no fallback or retry may restart. */ +export function isTransferPhaseTimeoutError(error: unknown): boolean { + return error instanceof AppError && error.details?.reason === TRANSFER_PHASE_TIMEOUT_REASON; +} diff --git a/packages/remote/src/upload-client.ts b/packages/remote/src/upload-client.ts index a1220a0db7..13400e3521 100644 --- a/packages/remote/src/upload-client.ts +++ b/packages/remote/src/upload-client.ts @@ -9,8 +9,15 @@ import { prepareUploadArtifact, type PreparedUploadArtifact } from './upload-cli import { isRetryableUploadStreamError, streamFileToHttpRequest } from './upload-stream.ts'; import type { UploadProgressSink } from './upload-progress.ts'; import { withUploadPreparationDeadline } from './upload-preparation-deadline.ts'; +import { isTransferPhaseTimeoutError } from './transfer-phase-budget.ts'; + +/** + * Ceiling for the two small control-plane calls that bracket an upload (`preflight`, `finalize`). + * Their payload is a few hundred bytes of JSON, so one window covers reaching the daemon and getting + * the answer; the artifact bytes themselves are budgeted per phase in `transfer-phase-budget.ts`. + */ +const UPLOAD_CONTROL_PLANE_TIMEOUT_MS = 30_000; -const UPLOAD_PREFLIGHT_TIMEOUT_MS = 30 * 1000; const ARTIFACT_HASH_ALGORITHM = 'sha256'; type UploadArtifactOptions = { @@ -157,6 +164,10 @@ async function tryDirectUploadWithResume(options: { // A canceled upload resumes for no reason: the thing it was uploaded for is already gone, and // re-preflighting would ask the daemon for a fresh ticket to send bytes nobody is waiting on. if (options.signal?.aborted) throw error; + // An expired phase is this client's own deadline, not a transport break. Restarting the + // sequence would only buy a second attempt permission it never had, so the typed evidence for + // the phase that ran out is what the caller sees. + if (isTransferPhaseTimeoutError(error)) throw error; if (!shouldRetryDirectUpload(error)) return undefined; return await retryDirectUpload(options, uploadOnce); } @@ -197,7 +208,10 @@ async function retryDirectUpload( if (retryPreflight?.kind === 'direct-upload') { try { return await uploadOnce(retryPreflight); - } catch { + } catch (error) { + // As above: an expired phase is this client's own deadline, so it must not buy the legacy + // fallback a third attempt at the same bytes. + if (isTransferPhaseTimeoutError(error)) throw error; return undefined; } } @@ -283,7 +297,7 @@ async function requestUploadPreflight(options: { // fetch keeps on a cross-origin hop, so any redirect fails the preflight instead. redirect: 'error', headers, - signal: combineUploadSignals(options.signal, UPLOAD_PREFLIGHT_TIMEOUT_MS), + signal: combineUploadSignals(options.signal, UPLOAD_CONTROL_PLANE_TIMEOUT_MS), body: JSON.stringify({ uploadAttemptId: options.uploadAttemptId, sha256: options.artifact.sha256, @@ -308,8 +322,12 @@ async function requestUploadPreflight(options: { } /** - * The signal one upload request runs under: the caller's cancellation and the request's own - * timeout, whichever fires first. + * The signal one small control-plane request runs under: the caller's cancellation and this + * request's own ceiling, whichever fires first. + * + * `preflight` and `finalize` carry a few hundred bytes of JSON that the transport writes with the + * request, so their ceiling is the same 30 seconds the artifact transfer spends getting connected and + * reading headers, and there is no separate body phase to time. */ function combineUploadSignals(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal { const timeout = AbortSignal.timeout(timeoutMs); @@ -418,9 +436,9 @@ async function finalizeDirectUpload(options: { // As with preflight: the route never redirects, and a redirect must not carry the token. redirect: 'error', headers, - signal: combineUploadSignals(options.signal, UPLOAD_PREFLIGHT_TIMEOUT_MS), + signal: combineUploadSignals(options.signal, UPLOAD_CONTROL_PLANE_TIMEOUT_MS), body: JSON.stringify({ uploadId: options.uploadId }), - }).catch((error) => { + }).catch((error: unknown) => { // As with preflight: a canceled finalize is not a transport failure to report, it is the caller // saying this upload is no longer worth finishing. if (options.signal?.aborted) throw error; diff --git a/packages/remote/src/upload-preparation-deadline.ts b/packages/remote/src/upload-preparation-deadline.ts index 6e4fbc6c50..d40c67c1a9 100644 --- a/packages/remote/src/upload-preparation-deadline.ts +++ b/packages/remote/src/upload-preparation-deadline.ts @@ -1,6 +1,7 @@ import { AppError } from '@agent-device/kernel/errors'; -const UPLOAD_PREPARATION_TIMEOUT_MS = 600_000; +/** The one artifact-preparation aggregate: six minutes from admission to the last finalize. */ +export const UPLOAD_PREPARATION_TIMEOUT_MS = 600_000; export async function withUploadPreparationDeadline( callerSignal: AbortSignal | undefined, diff --git a/packages/remote/src/upload-progress.ts b/packages/remote/src/upload-progress.ts index 8f537d5263..e6691bffa2 100644 --- a/packages/remote/src/upload-progress.ts +++ b/packages/remote/src/upload-progress.ts @@ -36,13 +36,20 @@ export function createUploadProgressTransform(options: { startOffset: number; totalBytes: number; onProgress?: UploadProgressSink; + /** + * Called for every chunk this side has actually handed on, which is the only thing that counts as + * transfer progress. Reporting to `onProgress` is throttled; this is not, so a trickle too small + * to be worth a UI line still proves the connection carries bytes. + */ + onBytes?: (byteLength: number) => void; }): Transform { - const { stage, fileName, startOffset, totalBytes, onProgress } = options; + const { stage, fileName, startOffset, totalBytes, onProgress, onBytes } = options; let transferredBytes = startOffset; const stepBytes = uploadProgressStepBytes(totalBytes); let nextReportAt = Math.min(totalBytes, startOffset + stepBytes); return new Transform({ transform(chunk: Buffer, _encoding, callback) { + onBytes?.(chunk.byteLength); transferredBytes = Math.min(totalBytes, transferredBytes + chunk.byteLength); if (transferredBytes >= nextReportAt) { onProgress?.({ diff --git a/packages/remote/src/upload-stream.ts b/packages/remote/src/upload-stream.ts index 798e039bc2..76ef6733bb 100644 --- a/packages/remote/src/upload-stream.ts +++ b/packages/remote/src/upload-stream.ts @@ -1,16 +1,22 @@ import fs from 'node:fs'; -import type { IncomingHttpHeaders } from 'node:http'; +import type { IncomingHttpHeaders, IncomingMessage } from 'node:http'; import path from 'node:path'; import { pipeline } from 'node:stream/promises'; import { addAbortSignal } from 'node:stream'; import { AppError } from '@agent-device/kernel/errors'; -import { loadNodeHttpRequester, readNodeHttpResponseBody } from '@agent-device/host-kit/transport'; +import { loadNodeHttpRequester } from '@agent-device/host-kit/transport'; import { createUploadProgressTransform, type UploadProgressSink, type UploadProgressStage, } from './upload-progress.ts'; import { withUploadPreparationDeadline } from './upload-preparation-deadline.ts'; +import { + createTransferPhaseWatchdog, + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + TRANSFER_CONNECT_TIMEOUT_MS, + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, +} from './transfer-phase-budget.ts'; const MAX_UPLOAD_REDIRECTS = 5; const UPLOAD_STREAM_RETRYABLE_DETAIL = 'uploadStreamRetryable'; @@ -83,9 +89,32 @@ async function streamFileToHttpRequestAttempt(options: { const headers = buildUploadRequestHeaders(options.headers, options.startOffset, payloadSize); emitUploadAttemptStarted(options.progress, options.startOffset, payloadSize); + // One watchdog for this attempt: connecting, then writing the body, then waiting for headers, then + // reading the response. A redirect or a resume is a NEW attempt and starts fresh phase timers, but + // it inherits `options.signal`, so the preparation aggregate the caller opened still expires it. + const watchdog = createTransferPhaseWatchdog(options.signal); + watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + let bodyBytesStarted = false; + const noteBodyBytes = () => { + // Nothing can reach the socket before the connection exists, so the first uploaded byte is also + // the moment the connect ceiling stops applying and the body-inactivity budget takes over. + if (!bodyBytesStarted) { + bodyBytesStarted = true; + watchdog.arm('request-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + } + watchdog.touchBody(); + }; let upload: Promise | undefined; return await new Promise((resolve, reject) => { let responseReceived = false; + const settle = (result: UploadStreamResponse) => { + watchdog.dispose(); + resolve(result); + }; + const fail = (error: unknown) => { + watchdog.dispose(); + reject(error); + }; const req = transport.request( { protocol: options.url.protocol, @@ -96,18 +125,23 @@ async function streamFileToHttpRequestAttempt(options: { headers, // Aborting destroys the request mid-stream, which is the only way to stop bytes that are // already piped at a device this client no longer holds the lease on. - ...(options.signal ? { signal: options.signal } : {}), + // This is the attempt's phase budget combined with the caller's signal, so an expired phase + // destroys the socket exactly like a cancellation does. + signal: watchdog.signal, }, (res) => { responseReceived = true; + // The response has started; from here the budget that matters is silence between body bytes. + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); if (options.signal) addAbortSignal(options.signal, res); - void readNodeHttpResponseBody(res) + addAbortSignal(watchdog.signal, res); + void readUploadResponseBody(res, () => watchdog.touchBody()) .then((body) => { const statusCode = res.statusCode ?? 500; const location = res.headers.location; if (location && isUploadRedirectStatus(statusCode)) { if (options.redirectCount >= MAX_UPLOAD_REDIRECTS) { - reject( + fail( new AppError('COMMAND_FAILED', 'Artifact upload exceeded redirect limit', { maxRedirects: MAX_UPLOAD_REDIRECTS, url: options.url.toString(), @@ -119,7 +153,7 @@ async function streamFileToHttpRequestAttempt(options: { // The request headers carry the daemon token or a signed ticket, and the body is the // artifact: neither may follow a hop to an origin the caller did not choose. if (redirectedUrl.origin !== options.url.origin) { - reject( + fail( new AppError( 'COMMAND_FAILED', 'Artifact upload refused a cross-origin redirect', @@ -136,7 +170,7 @@ async function streamFileToHttpRequestAttempt(options: { ...options, url: redirectedUrl, redirectCount: options.redirectCount + 1, - }).then(resolve, reject); + }).then(settle, fail); return; } @@ -145,7 +179,7 @@ async function streamFileToHttpRequestAttempt(options: { : undefined; if (resumeOffset !== undefined) { if (resumeOffset >= payloadSize) { - resolve({ + settle({ statusCode: 200, statusMessage: 'Upload already complete', body: '', @@ -153,7 +187,7 @@ async function streamFileToHttpRequestAttempt(options: { return; } if (resumeOffset <= options.startOffset) { - reject( + fail( new AppError('COMMAND_FAILED', 'Artifact upload resume did not advance', { offset: resumeOffset, previousOffset: options.startOffset, @@ -165,18 +199,18 @@ async function streamFileToHttpRequestAttempt(options: { void streamFileToHttpRequestAttempt({ ...options, startOffset: resumeOffset, - }).then(resolve, reject); + }).then(settle, fail); return; } - resolve({ + settle({ statusCode, statusMessage: res.statusMessage, body, }); }) .catch((error: unknown) => - reject(options.signal?.aborted ? options.signal.reason : error), + fail(watchdog.signal.aborted ? watchdog.signal.reason : error), ); }, ); @@ -185,11 +219,13 @@ async function streamFileToHttpRequestAttempt(options: { if (responseReceived) return; // A caller that canceled owns this outcome: the signal's reason is the answer, and wrapping it // as an ordinary transport failure would let a retry path treat "we stopped" as "it broke". - if (options.signal?.aborted) { - reject(options.signal.reason); + // An expired phase answers the same way — it is this client's own deadline, not a transport + // failure a retry or a legacy fallback could fix. + if (watchdog.signal.aborted) { + fail(watchdog.signal.reason); return; } - reject( + fail( new AppError( 'COMMAND_FAILED', options.errorMessage, @@ -201,28 +237,35 @@ async function streamFileToHttpRequestAttempt(options: { ), ); }); - upload = pipeline( - fs.createReadStream(options.payloadPath, { start: options.startOffset }), - createUploadProgressTransform({ - stage: options.progress?.stage ?? 'legacy', - fileName: options.progress?.fileName ?? path.basename(options.payloadPath), - startOffset: options.startOffset, - totalBytes: payloadSize, - onProgress: options.progress?.onProgress, - }), - req, - { signal: options.signal }, - ).catch((err: unknown) => { + upload = (async () => { + await pipeline( + fs.createReadStream(options.payloadPath, { start: options.startOffset }), + createUploadProgressTransform({ + stage: options.progress?.stage ?? 'legacy', + fileName: options.progress?.fileName ?? path.basename(options.payloadPath), + startOffset: options.startOffset, + totalBytes: payloadSize, + onProgress: options.progress?.onProgress, + onBytes: noteBodyBytes, + }), + req, + { signal: watchdog.signal }, + ); + // Everything the server can be waiting for is now written, so the header wait starts here and + // a body still draining never spends its 30-second budget. + if (!responseReceived) watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + })().catch((err: unknown) => { if (responseReceived) return; req.destroy(); - if (options.signal?.aborted) { - reject(options.signal.reason); + if (watchdog.signal.aborted) { + fail(watchdog.signal.reason); return; } const error = err instanceof Error ? err : new Error(String(err)); - reject(new AppError('COMMAND_FAILED', 'Failed to read local artifact', {}, error)); + fail(new AppError('COMMAND_FAILED', 'Failed to read local artifact', {}, error)); }); }).finally(async () => { + watchdog.dispose(); await upload; }); } @@ -231,6 +274,26 @@ function retryableUploadStreamDetail(options: { retryable?: boolean }): Record void): Promise { + return new Promise((resolve, reject) => { + let body = ''; + res.setEncoding('utf8'); + res.on('data', (chunk: string) => { + onBytes(); + body += chunk; + }); + res.on('end', () => resolve(body)); + res.on('error', reject); + }); +} + function emitUploadAttemptStarted( progress: UploadStreamProgressOptions | undefined, startOffset: number, diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index e464f3f929..aee46ec5f4 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -164,14 +164,14 @@ "packages/remote/src/upload-client.ts#UploadPreflightResponse": "sha256:196a1d9ef41a1aec192589fa7ccb30cf71d971a005349d5d73eed42e6ab21327", "packages/remote/src/upload-client.ts#UploadPreflightResult": "sha256:bd8fc6d617c0630325455fd64d0fe319daaced198ee6745e4673a8ace3df460d", "packages/remote/src/upload-client.ts#UploadResponse": "sha256:499e01466b5e0e03413f8d3f1986f5af6bc120704c50a3747d84de422b3f0314", - "packages/remote/src/upload-client.ts#finalizeDirectUpload": "sha256:8cd172a169aa5e54695d9185fa259d93c71b82938455398b83e7e4e214578df2", + "packages/remote/src/upload-client.ts#finalizeDirectUpload": "sha256:e8966f79d0518aa8d592113ff290efbde16120fd00ce8bb8648f243c38d0d173", "packages/remote/src/upload-client.ts#isStringRecord": "sha256:b6d6b571b90baf0976d9c347c4a45a4363a4df5ad184ea1087cb3b9febd4b624", "packages/remote/src/upload-client.ts#parseUploadPreflightResult": "sha256:7ce3f43b45c03df8b7f2690432beedc7a537c1bba540961716862f8a632d5623", - "packages/remote/src/upload-client.ts#requestUploadPreflight": "sha256:ed644d7e59c4703e3b7a05dda769b3f0554dc481598ba01906a131c543d68ffc", + "packages/remote/src/upload-client.ts#requestUploadPreflight": "sha256:e5f67a9a7799913f04cc9386291296a3d9fe13658776d2f4812a582c2e9befd5", "packages/remote/src/upload-client.ts#shouldRetryDirectUpload": "sha256:29059f076ff7f2b9540ad15408be48792b2d63a601ef2b9fb2d54f5576d49bbd", - "packages/remote/src/upload-client.ts#tryDirectUploadWithResume": "sha256:db1042b6b8f6969add9480f30a8649d83eecf60af38e80f00a4e789fcf925f97", + "packages/remote/src/upload-client.ts#tryDirectUploadWithResume": "sha256:0f55d5c81abc9010ecdbb852da81793515abab7c4d54e9f7c8b7f502e79a54a6", "packages/remote/src/upload-client.ts#uploadDirectArtifact": "sha256:3edb622eabee720085a544a4d4e6fdd74c19ec25d3e14e7f9f06ea10d550c1c7", - "packages/remote/src/upload-client.ts#uploadLegacyArtifact": "sha256:3acabacbdea5b7da7f3d8e6e87496728eb3b1df122c490751a9edabc1e9c9adc", + "packages/remote/src/upload-client.ts#uploadLegacyArtifact": "sha256:887c6a1b18dd16a51dbad89f3b8ac99727ed8fae87759c5d806f22f326a15da1", "packages/remote/src/upload-stream.ts#MAX_UPLOAD_REDIRECTS": "sha256:12ff46dfe33a1dce95bdb79c90c17a3ba4f3919e7dc4b3a1b04590fff430f79f", "packages/remote/src/upload-stream.ts#UploadStreamResponse": "sha256:724e284c2a6bd7b4b3a49b1d8c61c8923d6cc24777c3d76224a69d190e56adce", "packages/remote/src/upload-stream.ts#buildUploadRequestHeaders": "sha256:827576b13a9f3e6cea445119a6a29b2ac5eaa23b66436cf810b2d09765c82a4c", @@ -180,8 +180,8 @@ "packages/remote/src/upload-stream.ts#isUploadResumeStatus": "sha256:9f730ccd1c3401c326f8df67d1fd8243fef757357a38a1f53736df2b90bd8fdc", "packages/remote/src/upload-stream.ts#parseNonNegativeIntegerHeader": "sha256:5cbe9c6994459d673d988a51407e65556e28175b187ce0a9d27cbb6331962834", "packages/remote/src/upload-stream.ts#parseUploadResumeOffset": "sha256:297184ea22a6dc1cda9aee17034122ae0ea2f43d0355647f6fcf02f3f4b5ca0e", - "packages/remote/src/upload-stream.ts#streamFileToHttpRequest": "sha256:ce07ea33a275e06e4cceaf0c75a079bd0c7938f36df6817407dbb1cc8c4ff900", - "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:b49e17dab9a638f3e547af01717aad52264e20334d62489608bf2c7f9a238838", + "packages/remote/src/upload-stream.ts#streamFileToHttpRequest": "sha256:3ac8f379c4b508b0c28c4f41f920b40f8cc0406afcd5b2290326c4cfb9168adb", + "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:e4a67c0061c8d0999c475a4a2e8876b6c407da08e3f9b94018a772a6acc72283", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_HEADER": "sha256:d2bb3f4846470188a0fc1724271da0f0bcbdf2337bd6bf331508142cadcb65d8", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_UPSTREAM_INSTANCE_HEADER": "sha256:4564c2ce437d30d7a823e3326f4375c0750ba8d42bec22b4d02e8eb7f7db2973", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_MISMATCH_HEADER": "sha256:a197edaf7f9fa94a6aab269856aa852da3204cc586f5fd23198689eb11908b61", @@ -330,8 +330,8 @@ }, { "declaration": "packages/remote/src/upload-stream.ts#streamFileToHttpRequest", - "digest": "sha256:ce07ea33a275e06e4cceaf0c75a079bd0c7938f36df6817407dbb1cc8c4ff900", - "rationale": "#2946 widens the options with an optional abort signal handed to node:http.request. The request line, headers built by buildUploadRequestHeaders, and piped payload are unchanged, so an older daemon's 200/308 handling is unaffected." + "digest": "sha256:3ac8f379c4b508b0c28c4f41f920b40f8cc0406afcd5b2290326c4cfb9168adb", + "rationale": "#4441 replaces the one 5-minute timer over a whole upload with the preparation aggregate that bounds it, and hands that aggregate's signal to the attempt. The request a peer parses is unchanged: same route, method, the headers buildUploadRequestHeaders produces, the piped payload and the 200/308 handling. What moved is when this client stops sending, and the ceiling is now shorter rather than longer, so a protocol-2 daemon or proxy answers the same requests and simply sees a stalled leg torn down sooner." }, { "declaration": "packages/contracts/src/daemon-http.ts#DaemonHealthPayload", @@ -440,13 +440,13 @@ }, { "declaration": "packages/remote/src/upload-client.ts#tryDirectUploadWithResume", - "digest": "sha256:db1042b6b8f6969add9480f30a8649d83eecf60af38e80f00a4e789fcf925f97", - "rationale": "Clients name a held agent-device lease in an optional x-agent-device-lease-id request header on upload, artifact download and request diagnostics HTTP calls, so a gateway can route them. Released daemons ignore unknown request headers and the proxy forwards only its allowlist, so routes, bodies and responses parse as before; with no lease held the request is unchanged (ADR 0006)." + "digest": "sha256:0f55d5c81abc9010ecdbb852da81793515abab7c4d54e9f7c8b7f502e79a54a6", + "rationale": "#4441 ends the direct-upload sequence when a transfer phase's own budget expires, instead of re-preflighting for a fresh ticket or falling back, because a second attempt would be permission that budget already refused. Preflight, ticket PUT and finalize requests are unchanged in route, method, headers and body, so a protocol-2 daemon parses everything it parsed before; it just receives no further legs once the caller's aggregate has expired." }, { "declaration": "packages/remote/src/upload-client.ts#uploadLegacyArtifact", - "digest": "sha256:3acabacbdea5b7da7f3d8e6e87496728eb3b1df122c490751a9edabc1e9c9adc", - "rationale": "Clients name a held agent-device lease in an optional x-agent-device-lease-id request header on upload, artifact download and request diagnostics HTTP calls, so a gateway can route them. Released daemons ignore unknown request headers and the proxy forwards only its allowlist, so routes, bodies and responses parse as before; with no lease held the request is unchanged (ADR 0006)." + "digest": "sha256:887c6a1b18dd16a51dbad89f3b8ac99727ed8fae87759c5d806f22f326a15da1", + "rationale": "#4441 drops the hint that named the removed 5-minute timeout, and an expired transfer phase is no longer treated as a transport break, so this route is not re-entered by a fallback after the client's own budget ended the attempt. The POST /upload request itself is unchanged: same method, headers, chunked body and response parsing. A protocol-2 daemon sees the identical request in every case that completes, and no further request in a stalled one." }, { "declaration": "packages/remote/src/artifact-download.ts#downloadRemoteArtifactFromUrl", @@ -495,8 +495,8 @@ }, { "declaration": "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt", - "digest": "sha256:b49e17dab9a638f3e547af01717aad52264e20334d62489608bf2c7f9a238838", - "rationale": "The upload client refuses a redirect to another origin, including a 308 that carries a Location, instead of re-sending the daemon token or ticket headers and the artifact there. Same-origin redirects, the Location-less 308 resume contract, and the redirect limit are unchanged. A released daemon or proxy answers its upload routes on its own origin, so it parses the same requests. A direct-upload ticket that redirects cross-origin fails as a non-retryable direct upload, and uploadArtifact then takes the legacy /upload route on the daemon." + "digest": "sha256:e4a67c0061c8d0999c475a4a2e8876b6c407da08e3f9b94018a772a6acc72283", + "rationale": "#4441 runs each attempt under a sequential phase watchdog (connect, body inactivity, response headers) and fails with the expired phase's own typed evidence instead of a transport error, and reads the response through a reader that reports bytes rather than the shared host-kit one. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own send deadline moved, so a protocol-2 peer parses exactly what it parsed before." }, { "declaration": "packages/remote/src/daemon-artifacts.ts#resolveMaterializedArtifactPath", @@ -510,13 +510,13 @@ }, { "declaration": "packages/remote/src/upload-client.ts#requestUploadPreflight", - "digest": "sha256:ed644d7e59c4703e3b7a05dda769b3f0554dc481598ba01906a131c543d68ffc", - "rationale": "The request now uses fetch with redirect: 'error', so no redirect is followed: fetch keeps x-agent-device-token on a cross-origin hop. The request body, headers, and route are unchanged. A released daemon or proxy never redirects this route, so it parses the same requests. A redirected preflight falls back to the legacy /upload route, as any failed preflight does." + "digest": "sha256:e5f67a9a7799913f04cc9386291296a3d9fe13658776d2f4812a582c2e9befd5", + "rationale": "#4441 renames the constant naming this call's ceiling from UPLOAD_PREFLIGHT_TIMEOUT_MS to UPLOAD_CONTROL_PLANE_TIMEOUT_MS because it governs finalize as well. The value is the same 30 seconds the released client used, and the request body, headers, route and redirect refusal are byte-for-byte unchanged, so a released daemon or proxy parses the identical request." }, { "declaration": "packages/remote/src/upload-client.ts#finalizeDirectUpload", - "digest": "sha256:8cd172a169aa5e54695d9185fa259d93c71b82938455398b83e7e4e214578df2", - "rationale": "The request now uses fetch with redirect: 'error', so no redirect is followed: fetch keeps x-agent-device-token on a cross-origin hop. The request body, headers, and route are unchanged. A released daemon or proxy never redirects this route, so it parses the same requests. A redirected finalize fails with COMMAND_FAILED, and uploadArtifact then takes the legacy /upload route." + "digest": "sha256:e8966f79d0518aa8d592113ff290efbde16120fd00ce8bb8648f243c38d0d173", + "rationale": "#4441 renames the constant naming this call's 30-second ceiling and annotates a caught error; neither reaches the wire. The request body, headers, route and redirect refusal are unchanged, and a released daemon or proxy never redirects this route, so it parses the identical request." }, { "declaration": "packages/daemon/src/server/http/http-rpc-methods.ts#toInstallFromSourceDaemonRequest", From f1e7532d8e8d561b7f8e8e3fc741f39f713bf418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 16:31:37 +0100 Subject: [PATCH 03/27] fix(remote): end each upload attempt's owned work and time its real phases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two corrections to the phase budgeting slice, both found in review. An early response resolved, redirected or resumed while the attempt's own file → transform → request pipeline was still writing into the old hop. The answer was read while the owned file handle stayed open and the queued body kept flowing at a peer this client had already left behind. The attempt now reads the accepted answer to its end, destroys its own request only when that read completed the exchange's verdict, and joins the pipeline before resolving, starting a redirect, or starting an offset resume. The enclosing preparation aggregate still bounds what lingered, but no attempt now hands its body to the next hop. The connect ceiling ended when the request accepted its first write and body progress counted chunks read ahead of the socket. A TLS origin that never answers the handshake, or a peer that stops taking bytes after the headers, was timed against the wrong budget, and bytes sitting in the write queue re-armed a body phase that had not moved. The ceiling now ends on the events that carry the protocol: socket `connect` for plain HTTP, `secureConnect` for TLS, immediately for a pooled keep-alive socket. Body progress is the transport's own `drain` and nothing else; the progress transform's byte hook, which fired before the chunk ever reached the request, is gone. Wire bytes are unchanged, so the ledger's acknowledged digest for the attempt moves with the rationale that says so. The progress-window fixture measured budget against wall-clock scheduling, so a window could be charged before the parked peer really took a byte and the "keeps moving" case flaked. Each release now lets one quota of bytes through and the window waits with real event-loop turns until those bytes arrive before it charges the budget, which removes the race and costs the whole file a third of a second instead of two. --- .../support/upload-transfer-fixtures.ts | 526 +++++++++++++++++- .../upload-transfer-lifecycle.test.ts | 451 +++++++++++++++ .../__tests__/upload-transfer-phases.test.ts | 13 +- packages/remote/src/upload-progress.ts | 9 +- packages/remote/src/upload-stream.ts | 267 +++++---- test/wire-compat/ledger.json | 6 +- 6 files changed, 1133 insertions(+), 139 deletions(-) create mode 100644 packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts index 320f6d3bdc..9352b7aaa9 100644 --- a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -3,15 +3,20 @@ // Every helper here exists so a test can spend minutes of transfer budget without waiting for them: // the daemon holds the request body open chunk by chunk and the test drives the clock. The socket, // the framing and the client's own stream stay real, which is the point — a mocked transport cannot -// prove that a transfer which keeps moving is never cut short, nor that one which stops is. +// prove that a transfer which keeps moving is never cut short, nor that one which stops is, nor +// where an owned pipeline was still writing when an attempt ended. import assert from 'node:assert/strict'; import fs from 'node:fs'; import http from 'node:http'; +import https from 'node:https'; +import net from 'node:net'; import path from 'node:path'; +import tls from 'node:tls'; import { vi } from 'vitest'; import { closeLoopbackServer, listenOnLoopback, mkdtempForTestSync } from '@agent-device/test-kit'; import { uploadArtifact } from '../../upload-client.ts'; +import { type UploadStreamResponse, streamFileToHttpRequest } from '../../upload-stream.ts'; export const PREFLIGHT_REQUEST = 'POST /upload/preflight'; export const LEGACY_UPLOAD_REQUEST = 'POST /upload'; @@ -20,6 +25,11 @@ export const LEGACY_UPLOAD_REQUEST = 'POST /upload'; * stepped over, cheap enough that six minutes of budget stay a few hundred loop turns. */ const BUDGET_STEP_MS = 5_000; +/** Ceiling on what the parked daemon takes between two progress windows. Small enough that an + * artifact sized in megabytes cannot be drained by the windows a test runs, and large enough that + * one release always moves real bytes over the socket. */ +const RELEASE_QUOTA_BYTES = 256 * 1024; + export type UploadOutcome = { value: string | undefined; error: unknown }; /** An upload whose completion can be observed without awaiting it, which a controlled clock needs. */ @@ -53,6 +63,47 @@ export function startUpload(options: { return upload; } +/** The outcome of one `startUploadAttempt`, observable without awaiting it. */ +export type UploadAttemptOutcome = { + value: UploadStreamResponse | undefined; + error: unknown; +}; +export type SettledUploadAttempt = Promise & { settled: boolean }; + +/** + * One upload attempt straight at `url`, with its outcome observed rather than awaited. + * + * The attempt-level lifecycle — an early answer, the join that has to follow it, the phase that + * expires — belongs to `streamFileToHttpRequest`, so the tests that prove it drive that seam + * directly instead of routing a whole upload client around it. + */ +export function startUploadAttempt(options: { + url: string; + artifactPath: string; + method?: 'POST' | 'PUT'; + headers?: Record; + signal?: AbortSignal; +}): SettledUploadAttempt { + let settled = false; + const attempt = streamFileToHttpRequest({ + url: new URL(options.url), + method: options.method ?? 'POST', + headers: options.headers ?? { authorization: 'Bearer daemon-token' }, + payloadPath: options.artifactPath, + timeoutMessage: 'Artifact upload timed out', + errorMessage: 'Failed to upload artifact to remote daemon', + ...(options.signal ? { signal: options.signal } : {}), + }).then( + (value) => ({ value, error: undefined as unknown }), + (error: unknown) => ({ value: undefined as UploadStreamResponse | undefined, error }), + ) as SettledUploadAttempt; + void attempt.then(() => { + settled = true; + }); + Object.defineProperty(attempt, 'settled', { get: () => settled }); + return attempt; +} + /** * Advances the controlled clock while letting real socket work run between the steps, until `ready` * holds or the transfer is over. @@ -81,13 +132,20 @@ export async function spendBudget(ms: number, upload: SettledUpload): Promise false, maxMs: ms, settled: () => upload.settled }); } +/** Spends `ms` of budget in `BUDGET_STEP_MS` steps against a single attempt. */ +export async function spendAttemptBudget(ms: number, attempt: SettledUploadAttempt): Promise { + await spendUntil({ ready: () => false, maxMs: ms, settled: () => attempt.settled }); +} + /** * Spends `windows` windows of `windowMs` each while the body keeps moving. * - * Each window hands back what the daemon had parked and is then charged to the transfer, so the only - * way it can be paid for is by bytes arriving: a window in which `receivedBytes` did not grow is a - * window the transfer did not earn, and the result says so. Progress is judged only once bytes have - * started moving; getting there costs the socket buffers, not the transfer. + * Each window releases one quota of the parked body, waits with real event-loop turns until the peer + * really took more bytes, and only then charges `windowMs` to the transfer. Waiting before charging + * is what makes a window mean something: bytes cross the socket only while the real loop runs, so a + * window charged before they arrive measures the host's scheduling, not the transfer. Once the peer + * has taken the whole body the transfer has nothing left to move and only a response is owed, so + * later windows add budget without a growth requirement. */ export async function spendProgressWindows( daemon: ParkedUploadDaemon, @@ -98,17 +156,30 @@ export async function spendProgressWindows( let progressedInEveryWindow = true; let spent = 0; for (let window = 0; window < options.windows; window += 1) { + const bodyInFlight = !daemon.bodyEnded; const bytesBeforeWindow = daemon.receivedBytes; daemon.releaseAllParked(); + // With the body fully taken there is nothing left to move and only a response is owed, so the + // window is charged without a growth requirement. + const grew = bodyInFlight ? await awaitBytesGrowth(daemon, bytesBeforeWindow) : true; await vi.advanceTimersByTimeAsync(options.windowMs); spent += options.windowMs; - await pumpRealWork(); if (upload.settled) break; - if (daemon.receivedBytes <= bytesBeforeWindow) progressedInEveryWindow = false; + if (!grew) progressedInEveryWindow = false; } return { progressedInEveryWindow, budgetMs: spent }; } +/** Lets the real loop run until the peer has taken at least one more byte of the parked body. */ +async function awaitBytesGrowth(daemon: ParkedUploadDaemon, bytesBefore: number): Promise { + for (let round = 0; round < 400; round += 1) { + if (daemon.receivedBytes > bytesBefore) return true; + daemon.releaseAllParked(); + await pumpRealWork(15); + } + return daemon.receivedBytes > bytesBefore; +} + /** * Hands over the last few timer ticks a settled transfer needs to tear its leg down. * @@ -123,6 +194,26 @@ export async function settleAfterBudget(upload: SettledUpload): Promise { } } +/** + * Hands over the last few timer ticks a settled attempt needs to tear its leg down, and reports the + * real-time ticks it cost. + * + * A join that waits on a peer is the bug these tests look for, so a test that has to settle an + * attempt gets a number it can bound rather than a pass that says nothing about cost. + */ +export async function settleAttemptAfterBudget( + attempt: SettledUploadAttempt, + options: { maxRounds?: number } = {}, +): Promise<{ rounds: number; settled: boolean }> { + const maxRounds = options.maxRounds ?? 200; + let rounds = 0; + for (; rounds < maxRounds && !attempt.settled; rounds += 1) { + await vi.advanceTimersByTimeAsync(100); + await pumpRealWork(5); + } + return { rounds, settled: attempt.settled }; +} + /** * Lets real socket work finish without spending budget. * @@ -143,28 +234,23 @@ export function createArtifactFile(bytes: number, fileName = 'app.apk'): string return artifactPath; } -export type ParkedUploadDaemon = { - readonly baseUrl: string; - readonly requests: readonly string[]; - /** Bytes the daemon has accepted for the upload leg. */ - readonly receivedBytes: number; - /** Lets every chunk the daemon has parked so far through. */ - releaseAllParked(): void; - close(): Promise; -}; - /** - * A loopback daemon that accepts an artifact upload and parks its body chunk by chunk, never + * A loopback daemon that accepts an artifact upload and parks its body a quota at a time, never * answering the leg. * * The body is parked so a transfer can be held open long enough to spend a budget while putting real - * backpressure on the writing side. Preflight is refused, so the upload runs on the legacy POST - * route: the one path that pipes a file at the daemon and keeps going while the daemon drains it. + * backpressure on the writing side. Each release lets one quota of bytes through and parks the leg + * again, so the test decides how much the peer takes between two budget windows. Preflight is + * refused, so the upload runs on the legacy POST route: the one path that pipes a file at the daemon + * and keeps going while the daemon drains it. */ export async function startParkedUploadDaemon(): Promise { const requests: string[] = []; const parked: http.IncomingMessage[] = []; let receivedBytes = 0; + let bytesTakenSinceRelease = 0; + let parkedAtQuota: http.IncomingMessage | undefined; + let bodyEnded = false; const server = http.createServer((req, res) => { const key = `${req.method} ${req.url}`; @@ -180,8 +266,16 @@ export async function startParkedUploadDaemon(): Promise { if (key === LEGACY_UPLOAD_REQUEST) { req.on('data', (chunk: Buffer) => { receivedBytes += chunk.length; + // Each release lets one quota of bytes through and parks the stream again. The client is + // then backpressured for the rest of the window, so an artifact sized in megabytes is still + // mid-flight when the next window is charged. + bytesTakenSinceRelease += chunk.length; req.pause(); - parked.push(req); + if (bytesTakenSinceRelease < RELEASE_QUOTA_BYTES) parked.push(req); + else parkedAtQuota = req; + }); + req.on('end', () => { + bodyEnded = true; }); return; } @@ -204,6 +298,256 @@ export async function startParkedUploadDaemon(): Promise { get receivedBytes() { return receivedBytes; }, + get bodyEnded() { + return bodyEnded; + }, + releaseAllParked() { + bytesTakenSinceRelease = 0; + const resumable = parked.splice(0); + if (parkedAtQuota) { + resumable.push(parkedAtQuota); + parkedAtQuota = undefined; + } + for (const entry of resumable) entry.resume(); + }, + close: () => closeLoopbackServer(server), + }; +} + +export type ParkedUploadDaemon = { + readonly baseUrl: string; + readonly requests: readonly string[]; + /** Bytes the daemon has accepted for the upload leg. */ + readonly receivedBytes: number; + /** True once the daemon has taken the whole upload body and only a response is owed. */ + readonly bodyEnded: boolean; + /** Lets the parked body take up to one more quota of bytes. */ + releaseAllParked(): void; + close(): Promise; +}; + +/** + * An origin that answers the upload leg and then stops reading its body. + * + * The answer can be an early 4xx, redirect or offset resume, which is the peer that replies while + * the client still has bytes queued. Either way, taking one chunk and pausing is what keeps the + * client's write pipeline in real backpressure when the answer lands, so a test can see whether the + * owned file and request were closed and joined, or left writing at the old hop. + */ +export async function startAnswerUploadDaemon(options: { + respond(req: http.IncomingMessage): { + status: number; + headers?: Record; + body: string; + }; +}): Promise<{ + readonly baseUrl: string; + readonly port: number; + readonly requests: readonly string[]; + /** One entry per upload leg, in arrival order. */ + readonly legs: readonly UploadLeg[]; + close(): Promise; +}> { + const requests: string[] = []; + const legs: UploadLeg[] = []; + const server = http.createServer((req, res) => { + const key = `${req.method} ${req.url}`; + requests.push(key); + // A new leg is where an overlap would become visible: every earlier leg is put back to reading, + // so bytes its attempt is still pushing arrive and are counted rather than hiding in a paused + // stream. A client that joined its previous attempt has nothing left to push, so its leg can + // only report the bytes it took before the handoff. + for (const earlier of legs) earlier.resumeFromHandoff(); + const leg = createUploadLeg(req); + legs.push(leg); + const answer = options.respond(req); + res.statusCode = answer.status; + for (const [name, value] of Object.entries(answer.headers ?? {})) { + res.setHeader(name, value); + } + res.end(answer.body); + // One chunk, then the peer stops: the client's remaining body stays queued behind this pause, + // which is the backpressure the attempt has to be destroyed out of. + req.pause(); + }); + server.requestTimeout = 0; + server.headersTimeout = 0; + server.keepAliveTimeout = 0; + server.unref(); + const port = await listenOnLoopback(server); + return { + baseUrl: `http://127.0.0.1:${port}`, + port, + requests, + legs, + close: () => closeLoopbackServer(server), + }; +} + +export type UploadLeg = { + readonly url: string; + /** Bytes of this leg's body the origin has taken. */ + readonly bytes: number; + /** Bytes taken at the moment the NEXT leg arrived; `undefined` while this is the newest leg. */ + readonly bytesAtNextLeg: number | undefined; + /** True once the origin saw this leg's socket end. */ + readonly closed: boolean; + /** Starts counting this leg's bytes again, as the handoff check does. */ + resumeFromHandoff(): void; +}; + +/** Internal counting state; published through the read-only `UploadLeg` view. */ +type CountedUploadLeg = { + -readonly [K in keyof UploadLeg]: UploadLeg[K]; +}; + +function createUploadLeg(req: http.IncomingMessage): UploadLeg { + const leg: CountedUploadLeg = { + url: req.url ?? '', + bytes: 0, + bytesAtNextLeg: undefined, + closed: false, + resumeFromHandoff() { + if (leg.bytesAtNextLeg !== undefined) { + // Already past the handoff: keep counting whatever this attempt is still writing. + req.resume(); + return; + } + leg.bytesAtNextLeg = leg.bytes; + req.resume(); + }, + }; + req.on('data', (chunk: Buffer) => { + leg.bytes += chunk.length; + }); + req.on('error', () => {}); + req.on('aborted', () => { + leg.closed = true; + }); + req.socket.on('close', () => { + leg.closed = true; + }); + return leg; +} + +/** + * An HTTP origin that accepts the TCP connection and then reads nothing and never speaks. + * + * The connection IS carried, so the connect phase is over while the artifact never leaves the + * client: the budget that has to expire is body inactivity. `bytesTaken` stays zero because the + * origin pauses its sockets before the stream layer ever consumes a byte. + */ +export async function startConnectedStalledHttpOrigin(): Promise<{ + readonly port: number; + readonly bytesTaken: number; + close(): Promise; +}> { + let bytesTaken = 0; + const server = net.createServer((socket) => { + socket.pause(); + socket.on('data', (chunk: Buffer) => { + bytesTaken += chunk.length; + }); + socket.on('error', () => {}); + }); + server.unref(); + const port = await listenOnLoopback(server); + return { + port, + get bytesTaken() { + return bytesTaken; + }, + close: () => closeLoopbackServer(server), + }; +} + +/** + * An HTTP origin that answers with one response-body chunk and then goes silent mid-body. + * + * The answer is streaming, so the budget that applies is response-body inactivity; the bytes the + * client already consumed must survive to be reported, not be cut short with the socket. + */ +export async function startHalfAnsweredOrigin(options: { firstChunk: string }): Promise<{ + readonly baseUrl: string; + close(): Promise; +}> { + const server = http.createServer((_req, res) => { + _req.resume(); + res.statusCode = 200; + res.setHeader('content-type', 'text/plain'); + res.write(options.firstChunk); + }); + server.requestTimeout = 0; + server.headersTimeout = 0; + server.keepAliveTimeout = 0; + server.unref(); + const port = await listenOnLoopback(server); + return { baseUrl: `http://127.0.0.1:${port}`, close: () => closeLoopbackServer(server) }; +} + +/** + * A loopback daemon that reads the upload body as it arrives — one parked chunk per release, so the + * client writes only as fast as the test lets it — and answers once the body has ended. + * + * This is the peer a real daemon is: it takes every byte and eventually replies. Tests use it to + * prove that an upload which keeps progressing finishes inside the phase budgets rather than being + * cut short by one. + */ +export async function startSlowReadingUploadDaemon(options: { + status: number; + body: string; +}): Promise<{ + readonly baseUrl: string; + readonly receivedBytes: number; + readonly bodyEnded: boolean; + readonly connections: number; + releaseAllParked(): void; + close(): Promise; +}> { + const parked: http.IncomingMessage[] = []; + let receivedBytes = 0; + let bodyEnded = false; + let connections = 0; + const server = http.createServer((req, res) => { + const key = `${req.method} ${req.url}`; + if (key === PREFLIGHT_REQUEST) { + req.resume(); + req.on('end', () => { + res.statusCode = 404; + res.end('not found'); + }); + return; + } + req.on('data', (chunk: Buffer) => { + receivedBytes += chunk.length; + req.pause(); + parked.push(req); + }); + req.on('end', () => { + bodyEnded = true; + res.statusCode = options.status; + res.end(options.body); + }); + }); + server.on('connection', () => { + connections += 1; + }); + server.requestTimeout = 0; + server.headersTimeout = 0; + server.keepAliveTimeout = 0; + server.unref(); + const port = await listenOnLoopback(server); + return { + baseUrl: `http://127.0.0.1:${port}`, + get receivedBytes() { + return receivedBytes; + }, + get bodyEnded() { + return bodyEnded; + }, + get connections() { + return connections; + }, releaseAllParked() { for (const entry of parked.splice(0)) entry.resume(); }, @@ -211,6 +555,146 @@ export async function startParkedUploadDaemon(): Promise { }; } +/** + * An origin that accepts the TCP connection and never answers the TLS handshake. + * + * A plain socket sink is what keeps the handshake unfinished: the client's socket reports itself + * encrypted and still connecting, `secureConnect` never fires, and the artifact piles up in the + * write queue as `write()` keeps accepting it. That is the shape where bytes are already buffered at + * the transport while the connection is still not carried, so the connect ceiling, not a body + * budget, is what has to expire. + */ +export async function startUnfinishedHandshakeTlsOrigin(): Promise<{ + readonly port: number; + close(): Promise; +}> { + const server = net.createServer((socket) => { + socket.pause(); + socket.on('error', () => {}); + }); + server.unref(); + const port = await listenOnLoopback(server); + return { port, close: () => closeLoopbackServer(server) }; +} + +/** + * A TLS origin that completes the handshake and then reads nothing and answers nothing. + * + * The connection IS carried here, so the connect phase is over while the artifact never leaves the + * client's write queue: the budget that has to expire is body inactivity, reported with the body + * ceiling rather than the connect one. + */ +export async function startConnectedStalledTlsOrigin(): Promise<{ + readonly port: number; + close(): Promise; +}> { + const server = tls.createServer( + { key: readRepoFixture('localhost-key.pem'), cert: readRepoFixture('localhost-cert.pem') }, + (socket) => { + socket.on('data', () => socket.pause()); + socket.on('error', () => {}); + }, + ); + server.unref(); + const port = await listenOnLoopback(server); + return { port, close: () => closeLoopbackServer(server) }; +} + +/** + * Trusts the repository's self-signed localhost certificate until restored. + * + * The subject builds TLS requests with the `node:https` global agent, so this is the agent's own + * trust store: a test-local decision scoped to one `try/finally`, never a production option and + * never the `NODE_TLS_REJECT_UNAUTHORIZED` switch, which a test could not put back. + */ +export function trustSelfSignedLocalhostCertificate(): { restore(): void } { + const savedCa = https.globalAgent.options.ca; + https.globalAgent.options.ca = readRepoFixture('localhost-cert.pem'); + return { + restore() { + https.globalAgent.options.ca = savedCa; + }, + }; +} + +/** + * A loopback daemon whose upload legs behave per a caller-supplied script. + * + * Leg one is where an upload completes and its socket returns to the pool; leg two is where the + * next attempt runs on that reused connection. The script per leg says whether the peer reads the + * whole body (`readAllBody`) and whether it answers at all, so a test can put the second attempt + * into the header wait and prove the reused socket — no new connection, no connect phase — is what + * the budget then times. + */ +export async function startUploadLegsDaemon( + behaviors: readonly { + readAllBody: boolean; + respond?: () => { status: number; headers?: Record; body: string }; + }[], +): Promise<{ + readonly baseUrl: string; + readonly requests: readonly string[]; + readonly connections: number; + close(): Promise; +}> { + const requests: string[] = []; + let legs = 0; + let connections = 0; + const server = http.createServer((req, res) => { + requests.push(`${req.method} ${req.url}`); + const behavior = behaviors[legs]; + if (!behavior) throw new Error(`unexpected upload leg ${legs + 1}`); + legs += 1; + const answer = () => { + if (!behavior.respond) return; + const reply = behavior.respond(); + res.statusCode = reply.status; + for (const [name, value] of Object.entries(reply.headers ?? {})) { + res.setHeader(name, value); + } + res.end(reply.body); + }; + if (behavior.readAllBody) { + req.resume(); + req.on('end', answer); + return; + } + let tookOneChunk = false; + req.on('data', () => { + if (tookOneChunk) return; + tookOneChunk = true; + req.pause(); + }); + answer(); + }); + server.on('connection', () => { + connections += 1; + }); + server.requestTimeout = 0; + server.headersTimeout = 0; + server.keepAliveTimeout = 0; + server.unref(); + const port = await listenOnLoopback(server); + return { + baseUrl: `http://127.0.0.1:${port}`, + requests, + get connections() { + return connections; + }, + close: () => closeLoopbackServer(server), + }; +} + +function readRepoFixture(fileName: string): Buffer { + let dir = process.cwd(); + for (let depth = 0; depth < 8; depth += 1) { + const candidate = path.join(dir, 'test/integration/provider-scenarios/fixtures', fileName); + if (fs.existsSync(candidate)) return fs.readFileSync(candidate); + dir = path.dirname(dir); + } + throw new Error(`missing ${fileName} under test/integration/provider-scenarios/fixtures`); +} + export function assertDetailedError( error: unknown, details: Record, diff --git a/packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts b/packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts new file mode 100644 index 0000000000..19a088f698 --- /dev/null +++ b/packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts @@ -0,0 +1,451 @@ +// The transport lifecycle of one upload attempt while the peer misbehaves (#4441 review). +// +// A response can arrive before the outgoing body has finished: an early 413, a redirect or an offset +// resume from a peer that then stops reading. The accepted answer is a real outcome, so it must be +// read to its end; the owned file and request still pushing at the dead hop must be destroyed and +// JOINED before the attempt resolves or the next hop is dialed — no old bytes may overlap a new +// attempt. The budget story has to say what the socket says: the connect ceiling ends when the +// transport is actually carried (TCP `connect`, TLS `secureConnect`, or a pooled socket that is +// already connected), and buffered bytes never count as transport progress. +// +// Every case uses the attempt seam, a controlled clock, and real loopback sockets: the overlap being +// refused is only observable on a real socket, and the budgets are minutes long. + +import { afterEach, assert, test, vi } from 'vitest'; +import { + isTransferPhaseTimeoutError, + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + TRANSFER_CONNECT_TIMEOUT_MS, + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, +} from '../transfer-phase-budget.ts'; +import { + assertDetailedError, + createArtifactFile, + pumpRealWork, + settleAttemptAfterBudget, + spendAttemptBudget, + startAnswerUploadDaemon, + startConnectedStalledHttpOrigin, + startConnectedStalledTlsOrigin, + startHalfAnsweredOrigin, + startSlowReadingUploadDaemon, + startUnfinishedHandshakeTlsOrigin, + startUploadAttempt, + startUploadLegsDaemon, + trustSelfSignedLocalhostCertificate, + useTransferClock, +} from './support/upload-transfer-fixtures.ts'; + +const MEBIBYTE = 1024 * 1024; + +afterEach(() => { + vi.useRealTimers(); +}); + +test('an early 413 from a peer that stops reading is consumed and its owned work is ended', async () => { + useTransferClock(); + const daemon = await startAnswerUploadDaemon({ + respond: () => ({ status: 413, body: 'payload too large' }), + }); + const artifactBytes = 4 * MEBIBYTE; + // Well past what loopback buffers absorb: megabytes of owned file remain queued behind the + // peer's pause when the answer lands, which is the owned work this test expects to be ended. + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(artifactBytes), + }); + try { + const settled = await settleAttemptAfterBudget(attempt); + assert.equal(settled.settled, true, 'the attempt ended on the early response, not on a budget'); + // The join in the attempt's own teardown is what this bound proves: a pipeline still blocked on + // the parked peer would leave the attempt unresolved for its whole phase budget. + assert.ok(settled.rounds < 20, `settled in ${settled.rounds} real-loop rounds`); + const result = await attempt; + + // The accepted early response is a real outcome, not a pipeline abortion. + assert.equal(result.error, undefined, `got ${String(result.error)}`); + assert.equal(result.value?.statusCode, 413); + assert.equal(result.value?.body, 'payload too large'); + // The owned file and request were ended, not left writing: the peer took a fraction of the + // artifact and the rest never arrived. + assert.equal(daemon.legs.length, 1); + assert.ok( + (daemon.legs[0]?.bytes ?? 0) < artifactBytes, + `the peer took ${daemon.legs[0]?.bytes} of ${artifactBytes} bytes`, + ); + } finally { + await daemon.close(); + } +}); + +test('a resume response ends the stalled hop before the next attempt is dialed', async () => { + useTransferClock(); + const resumeOffset = MEBIBYTE; + const artifactBytes = 4 * MEBIBYTE; + const daemon = await startAnswerUploadDaemon({ + respond: () => ({ + status: 308, + headers: { 'x-upload-offset': String(resumeOffset) }, + body: '', + }), + }); + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(artifactBytes), + }); + try { + // Leg one is answered 308 while its body is still queued; leg two answers the same offset + // again, which is a typed terminal failure rather than a third hop. + const settled = await settleAttemptAfterBudget(attempt); + assert.equal(settled.settled, true); + const result = await attempt; + + assertDetailedError(result.error, { offset: resumeOffset, previousOffset: resumeOffset }); + const uploadLegs = daemon.requests.filter((entry) => entry === 'POST /upload'); + assert.equal(uploadLegs.length, 2); + // Bytes really were still queued when the resume was answered: the peer had taken only part of + // the artifact when the next leg was dialed. + const firstLeg = daemon.legs[0]; + assert.ok(firstLeg); + assert.ok( + (firstLeg.bytesAtNextLeg ?? artifactBytes) < artifactBytes, + `the first leg still had queued bytes at the handoff, at ${firstLeg.bytesAtNextLeg}`, + ); + // And that leg never received its queued remainder: the owned file and request were ended and + // joined before the next attempt started, so no old byte overlapped it. + assert.ok( + firstLeg.bytes < artifactBytes, + `the stalled leg kept receiving after the handoff: ${firstLeg.bytes} of ${artifactBytes}`, + ); + } finally { + await daemon.close(); + } +}); + +test('a redirect response ends the stalled hop before the next hop is dialed', async () => { + useTransferClock(); + const artifactBytes = 4 * MEBIBYTE; + const daemon = await startAnswerUploadDaemon({ + respond: (req) => + req.url === '/moved' + ? { status: 413, body: 'moved and refused' } + : { status: 307, headers: { location: '/moved' }, body: '' }, + }); + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(artifactBytes), + }); + try { + const settled = await settleAttemptAfterBudget(attempt); + assert.equal(settled.settled, true); + const result = await attempt; + + assert.equal(result.error, undefined, `got ${String(result.error)}`); + assert.equal(result.value?.statusCode, 413); + assert.equal(result.value?.body, 'moved and refused'); + const firstLeg = daemon.legs[0]; + const redirectHop = daemon.requests.indexOf('POST /moved'); + assert.ok(firstLeg && redirectHop > 0, 'the redirect hop was dialed'); + assert.ok( + (firstLeg.bytesAtNextLeg ?? artifactBytes) < artifactBytes, + `bytes were still queued when the redirect hop started, at ${firstLeg.bytesAtNextLeg}`, + ); + assert.ok( + firstLeg.bytes < artifactBytes, + `the redirected-from leg kept receiving after the hop: ${firstLeg.bytes} of ${artifactBytes}`, + ); + } finally { + await daemon.close(); + } +}); + +test('a response body that stops mid-answer expires the response-body phase after what was read', async () => { + useTransferClock(); + const origin = await startHalfAnsweredOrigin({ firstChunk: 'first chunk' }); + const attempt = startUploadAttempt({ + url: `${origin.baseUrl}/upload`, + artifactPath: createArtifactFile(64), + }); + try { + await pumpRealWork(); + await spendAttemptBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); + const settled = await settleAttemptAfterBudget(attempt); + assert.equal(settled.settled, true); + const result = await attempt; + + assert.equal(result.value, undefined); + // The stall is mid-response-body, and the budget that says so is the body one; the consumed + // first chunk made the attempt's own outgoing work already settle, not linger. + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'response-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); + } finally { + await origin.close(); + } +}); + +test('a connected peer that never takes a body byte expires body inactivity, not connect', async () => { + useTransferClock(); + const origin = await startConnectedStalledHttpOrigin(); + const attempt = startUploadAttempt({ + url: `http://127.0.0.1:${origin.port}/upload`, + artifactPath: createArtifactFile(4 * MEBIBYTE), + }); + try { + await spendAttemptBudget(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, attempt); + // The connection was carried, so the connect ceiling is over: a carried connection is never + // ended by the budget that governs dialing it. + assert.equal(attempt.settled, false); + await spendAttemptBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); + const settled = await settleAttemptAfterBudget(attempt); + assert.equal(settled.settled, true); + const result = await attempt; + + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'request-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); + } finally { + await origin.close(); + } +}); + +test('a TLS handshake that never completes expires connect even with the whole body buffered', async () => { + useTransferClock(); + // The transport accepts the socket and never completes the handshake, so `write()` accepts the + // artifact into the queue while nothing the peer answers arrives. A read-ahead progress signal + // would swap the connect ceiling for a body budget here; buffered bytes are not transport + // progress, so the connect ceiling must still expire. + const origin = await startUnfinishedHandshakeTlsOrigin(); + const attempt = startUploadAttempt({ + url: `https://127.0.0.1:${origin.port}/upload`, + artifactPath: createArtifactFile(4 * MEBIBYTE), + }); + try { + await spendAttemptBudget(TRANSFER_CONNECT_TIMEOUT_MS + 5_000, attempt); + const settled = await settleAttemptAfterBudget(attempt); + assert.equal(settled.settled, true); + const result = await attempt; + + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'connect', + timeoutMs: TRANSFER_CONNECT_TIMEOUT_MS, + }); + } finally { + await origin.close(); + } +}); + +test('a completed TLS handshake with a stalled body expires body inactivity, not connect', async () => { + useTransferClock(); + // `secureConnect` is the establishment event for TLS: once it fires, the connect phase is over + // and the stalled artifact is body inactivity, charged with the body budget. The origin's socket + // handler below answers the handshake and then stops reading, so a trusted peer connection has to + // be established for this to say anything. + const trusted = trustSelfSignedLocalhostCertificate(); + const origin = await startConnectedStalledTlsOrigin(); + try { + // The control leg first: the same trust decision, a peer that reads nothing but completes the + // handshake, and the outcome is a body-phase expiry rather than a handshake failure. + const attempt = startUploadAttempt({ + url: `https://127.0.0.1:${origin.port}/upload`, + artifactPath: createArtifactFile(4 * MEBIBYTE), + }); + await spendAttemptBudget(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, attempt); + assert.equal(attempt.settled, false, 'secureConnect ended the connect phase'); + await spendAttemptBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); + const settled = await settleAttemptAfterBudget(attempt); + assert.equal(settled.settled, true); + const result = await attempt; + + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'request-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); + } finally { + trusted.restore(); + await origin.close(); + } +}); + +test('a zero-length artifact on a peer that never answers waits on its header budget', async () => { + useTransferClock(); + const origin = await startConnectedStalledHttpOrigin(); + const attempt = startUploadAttempt({ + url: `http://127.0.0.1:${origin.port}/upload`, + artifactPath: createArtifactFile(0), + }); + try { + // The body is nothing, so the flush arrives with the connection and the wait that expires is the + // one for the answer the peer owes. That ceiling (30s) is the shorter of the two, so an expiry + // recorded against it — and not against the 120s body budget — is the header wait on its own. + await spendAttemptBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 5_000, attempt); + assert.equal(attempt.settled, false); + await spendAttemptBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, attempt); + const settled = await settleAttemptAfterBudget(attempt); + assert.equal(settled.settled, true); + const result = await attempt; + + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + }); + assert.equal(origin.bytesTaken, 0); + } finally { + await origin.close(); + } +}); + +test('an upload the peer keeps taking finishes while every phase ceiling is unexpired', async () => { + useTransferClock(); + const daemon = await startSlowReadingUploadDaemon({ + status: 200, + body: JSON.stringify({ ok: true, uploadId: 'joined-upload' }), + }); + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(MEBIBYTE), + }); + try { + // Real backpressure the whole way: the peer admits roughly one chunk between releases, so the + // upload can only finish by earning window after window of body progress. Every window is + // charged, and the attempt must be over before the two ceilings a progressing upload may never + // exceed (connect, then the header wait its flush starts) have run out. + let spent = 0; + while ( + !attempt.settled && + spent < TRANSFER_CONNECT_TIMEOUT_MS + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + ) { + daemon.releaseAllParked(); + await vi.advanceTimersByTimeAsync(1_000); + spent += 1_000; + await pumpRealWork(); + } + const result = await attempt; + + assert.equal(result.error, undefined, `upload failed: ${String(result.error)}`); + assert.equal(result.value?.statusCode, 200); + assert.ok( + spent < TRANSFER_CONNECT_TIMEOUT_MS + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + `finished in ${spent}ms of budget, inside both ceilings a progressing upload must not exceed`, + ); + // A completed upload was never destroyed mid-leg: the server saw the body arrive to its end and + // answered on the same leg the client wrote. + assert.equal(daemon.bodyEnded, true); + assert.equal(daemon.receivedBytes, MEBIBYTE); + } finally { + await daemon.close(); + } +}, 30_000); + +test('a caller cancel during a stalled connect stays caller cancellation, distinct from the connect expiry', async () => { + useTransferClock(); + const control = new AbortController(); + const origin = await startUnfinishedHandshakeTlsOrigin(); + const canceled = startUploadAttempt({ + url: `https://127.0.0.1:${origin.port}/upload`, + artifactPath: createArtifactFile(64 * 1024), + signal: control.signal, + }); + try { + // Well inside the connect ceiling: only the caller can be the reason this stops. + await spendAttemptBudget(5_000, canceled); + control.abort(); + const result = await canceled; + + assert.equal(result.value, undefined); + assert.ok(result.error instanceof DOMException, `got ${String(result.error)}`); + assert.equal((result.error as DOMException).name, 'AbortError'); + assert.equal(control.signal.aborted, true); + } finally { + await origin.close(); + } +}); + +test('an enclosing aggregate deadline ends a redirect hop with its own reason, not a phase timeout', async () => { + useTransferClock(); + const aggregate = new AbortController(); + // Leg one redirects while it stops reading; leg two is the hop that never answers, so the attempt + // is parked in a phase when the deadline it was admitted under runs out. + const daemon = await startUploadLegsDaemon([ + { + readAllBody: false, + respond: () => ({ status: 307, headers: { location: '/moved' }, body: '' }), + }, + { readAllBody: true }, + ]); + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(64 * 1024), + signal: aggregate.signal, + }); + try { + await pumpRealWork(); + // No phase budget is charged here, so nothing but the caller's deadline can end this. + const reason = new Error('preparation aggregate expired'); + aggregate.abort(reason); + const settled = await settleAttemptAfterBudget(attempt); + assert.equal(settled.settled, true); + const result = await attempt; + + assert.equal(result.value, undefined); + assert.equal(result.error, reason); + assert.equal(isTransferPhaseTimeoutError(result.error), false); + } finally { + await daemon.close(); + } +}); + +test('an attempt on a reused connection is already past its connect phase', async () => { + useTransferClock(); + // Leg one completes and its socket goes back to the pool; leg two runs on that socket, which is + // already connected, and is never answered. A pooled socket never emits `connect`, so an attempt + // that only ever transitioned on that event would still be timing the connection here. The phase + // that expires is the header wait the unanswered flush deserves, on one connection. + const daemon = await startUploadLegsDaemon([ + { + readAllBody: true, + respond: () => ({ status: 200, body: JSON.stringify({ ok: true, uploadId: 'pooled' }) }), + }, + { readAllBody: true }, + ]); + const first = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(64 * 1024), + }); + const firstSettled = await settleAttemptAfterBudget(first); + assert.equal(firstSettled.settled, true); + const firstResult = await first; + assert.equal(firstResult.error, undefined, `leg one failed: ${String(firstResult.error)}`); + + const second = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(64 * 1024), + }); + try { + await spendAttemptBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, second); + const settled = await settleAttemptAfterBudget(second); + assert.equal(settled.settled, true); + const result = await second; + + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + }); + assert.equal(daemon.connections, 1, 'the second attempt reused the pooled socket'); + } finally { + await daemon.close(); + } +}); diff --git a/packages/remote/src/__tests__/upload-transfer-phases.test.ts b/packages/remote/src/__tests__/upload-transfer-phases.test.ts index b546b31196..e44f87c311 100644 --- a/packages/remote/src/__tests__/upload-transfer-phases.test.ts +++ b/packages/remote/src/__tests__/upload-transfer-phases.test.ts @@ -1,7 +1,7 @@ // Phase budgets applied to a real artifact upload over loopback (#4441). // // The budgets in `transfer-phase-budget.ts` are minutes long, so every case here drives a controlled -// clock and parks the request body chunk by chunk. That combination is the point: a transfer that +// clock and parks the request body a quota at a time. That combination is the point: a transfer that // keeps moving must survive budget after budget, and one that stops moving must be ended by the ONE // budget that ran out. A mocked transport could show neither, because the backpressure that makes // "still progressing" mean something belongs to the real socket. @@ -36,14 +36,17 @@ test('a body that keeps moving outlives the body-inactivity ceiling', async () = const daemon = await startParkedUploadDaemon(); const upload = startUpload({ baseUrl: daemon.baseUrl, - // Sixteen megabytes is far past what loopback buffers absorb, so the windows below are paid for - // by bytes the peer really takes rather than by a socket that swallowed the whole file. + // Sixteen megabytes is far past what a handful of windows can drain, so every window below is + // paid for by bytes the peer really takes rather than by a socket that swallowed the file. artifactPath: createArtifactFile(16 * MEBIBYTE), }); try { + // Eight ceilings' worth of budget, one release-and-wait per window. Each window is paid for only + // after the peer really took bytes, so this stays cheap in real time while still spending twice + // the inactivity ceiling the transfer has to outlive. const progress = await spendProgressWindows(daemon, upload, { - windows: 400, - windowMs: 1_000, + windows: 8, + windowMs: TRANSFER_CONNECT_TIMEOUT_MS, }); assert.equal(progress.progressedInEveryWindow, true); diff --git a/packages/remote/src/upload-progress.ts b/packages/remote/src/upload-progress.ts index e6691bffa2..8f537d5263 100644 --- a/packages/remote/src/upload-progress.ts +++ b/packages/remote/src/upload-progress.ts @@ -36,20 +36,13 @@ export function createUploadProgressTransform(options: { startOffset: number; totalBytes: number; onProgress?: UploadProgressSink; - /** - * Called for every chunk this side has actually handed on, which is the only thing that counts as - * transfer progress. Reporting to `onProgress` is throttled; this is not, so a trickle too small - * to be worth a UI line still proves the connection carries bytes. - */ - onBytes?: (byteLength: number) => void; }): Transform { - const { stage, fileName, startOffset, totalBytes, onProgress, onBytes } = options; + const { stage, fileName, startOffset, totalBytes, onProgress } = options; let transferredBytes = startOffset; const stepBytes = uploadProgressStepBytes(totalBytes); let nextReportAt = Math.min(totalBytes, startOffset + stepBytes); return new Transform({ transform(chunk: Buffer, _encoding, callback) { - onBytes?.(chunk.byteLength); transferredBytes = Math.min(totalBytes, transferredBytes + chunk.byteLength); if (transferredBytes >= nextReportAt) { onProgress?.({ diff --git a/packages/remote/src/upload-stream.ts b/packages/remote/src/upload-stream.ts index 76ef6733bb..644bb238ce 100644 --- a/packages/remote/src/upload-stream.ts +++ b/packages/remote/src/upload-stream.ts @@ -94,27 +94,57 @@ async function streamFileToHttpRequestAttempt(options: { // it inherits `options.signal`, so the preparation aggregate the caller opened still expires it. const watchdog = createTransferPhaseWatchdog(options.signal); watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); - let bodyBytesStarted = false; - const noteBodyBytes = () => { - // Nothing can reach the socket before the connection exists, so the first uploaded byte is also - // the moment the connect ceiling stops applying and the body-inactivity budget takes over. - if (!bodyBytesStarted) { - bodyBytesStarted = true; - watchdog.arm('request-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); - } - watchdog.touchBody(); - }; + + // The attempt's owned outgoing pipeline: artifact file → progress transform → request. Every exit + // from this attempt joins it, so no owned byte is still moving when the attempt resolves, fails, + // or hands the body to a redirect/resume attempt. let upload: Promise | undefined; + let uploadSettled = false; + let responseReceived = false; return await new Promise((resolve, reject) => { - let responseReceived = false; + let outgoingEnded = false; + let transportEstablished = false; + let bodyFlushed = false; const settle = (result: UploadStreamResponse) => { watchdog.dispose(); resolve(result); }; const fail = (error: unknown) => { watchdog.dispose(); + // Whatever is still queued on this attempt's own outgoing pipeline has to settle before the + // join in `finally`, so a failure that ends the attempt destroys the request too. Once the + // response has been consumed this cannot cut an accepted answer short; before it, the outcome + // is already decided by this failure. + req.destroy(); reject(error); }; + // The phase this attempt is actually in, from the two events that can end the connect ceiling: + // the transport carrying the protocol, and every body byte reaching the socket. A pooled socket + // is already connected; a fresh one is not, and a TLS socket is not until its handshake ends. + // Until then the connect ceiling keeps counting even when the whole body already sits in the + // write buffer — buffered bytes are not transport progress. Once connected, a body the peer + // stopped taking is timed as body inactivity, never as a connect timeout; a body that flushed + // before the connection existed (a zero-length artifact) is already owed a response, so the + // header wait is what it waits on. + const advanceOutgoingPhase = () => { + if (responseReceived || watchdog.signal.aborted) return; + if (!transportEstablished) return; + if (bodyFlushed) { + watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + } else { + watchdog.arm('request-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + } + }; + // Body progress is what the socket accepted, which `drain` reports when a previously full write + // queue empties. Chunks merely read ahead of the request, or throttled UI progress events, say + // nothing about bytes leaving this machine, so they are not progress against any budget. + const noteWriteProgress = () => { + // A request-side drain says nothing about the response; once an answer is streaming, only + // response bytes may re-arm the inactivity budget. + if (responseReceived) return; + watchdog.touchBody(); + }; + const req = transport.request( { protocol: options.url.protocol, @@ -133,88 +163,33 @@ async function streamFileToHttpRequestAttempt(options: { responseReceived = true; // The response has started; from here the budget that matters is silence between body bytes. watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); - if (options.signal) addAbortSignal(options.signal, res); addAbortSignal(watchdog.signal, res); - void readUploadResponseBody(res, () => watchdog.touchBody()) - .then((body) => { - const statusCode = res.statusCode ?? 500; - const location = res.headers.location; - if (location && isUploadRedirectStatus(statusCode)) { - if (options.redirectCount >= MAX_UPLOAD_REDIRECTS) { - fail( - new AppError('COMMAND_FAILED', 'Artifact upload exceeded redirect limit', { - maxRedirects: MAX_UPLOAD_REDIRECTS, - url: options.url.toString(), - }), - ); - return; - } - const redirectedUrl = new URL(location, options.url); - // The request headers carry the daemon token or a signed ticket, and the body is the - // artifact: neither may follow a hop to an origin the caller did not choose. - if (redirectedUrl.origin !== options.url.origin) { - fail( - new AppError( - 'COMMAND_FAILED', - 'Artifact upload refused a cross-origin redirect', - { - reason: 'cross-origin-redirect', - url: options.url.toString(), - redirectOrigin: redirectedUrl.origin, - }, - ), - ); - return; - } - void streamFileToHttpRequestAttempt({ - ...options, - url: redirectedUrl, - redirectCount: options.redirectCount + 1, - }).then(settle, fail); - return; - } - - const resumeOffset = isUploadResumeStatus(statusCode) - ? parseUploadResumeOffset(res.headers, payloadSize) - : undefined; - if (resumeOffset !== undefined) { - if (resumeOffset >= payloadSize) { - settle({ - statusCode: 200, - statusMessage: 'Upload already complete', - body: '', - }); - return; - } - if (resumeOffset <= options.startOffset) { - fail( - new AppError('COMMAND_FAILED', 'Artifact upload resume did not advance', { - offset: resumeOffset, - previousOffset: options.startOffset, - url: options.url.toString(), - }), - ); - return; - } - void streamFileToHttpRequestAttempt({ - ...options, - startOffset: resumeOffset, - }).then(settle, fail); - return; - } - - settle({ - statusCode, - statusMessage: res.statusMessage, - body, - }); - }) + void handleUploadResponse(res) + .then((result) => settle(result)) .catch((error: unknown) => fail(watchdog.signal.aborted ? watchdog.signal.reason : error), ); }, ); + req.on('socket', (socket) => { + if (socket.connecting) { + // A fresh socket is not a connection yet: plain HTTP is carried when it connects, TLS only + // when the handshake ends. Until then the connect ceiling keeps counting, whatever the + // request's write buffer already holds. + const secured = (socket as { encrypted?: boolean }).encrypted === true; + socket.once(secured ? 'secureConnect' : 'connect', () => { + transportEstablished = true; + advanceOutgoingPhase(); + }); + } else { + // A pooled socket — keep-alive reuse — is already carrying the protocol. + transportEstablished = true; + advanceOutgoingPhase(); + } + }); + req.on('drain', noteWriteProgress); + req.on('error', (err) => { if (responseReceived) return; // A caller that canceled owns this outcome: the signal's reason is the answer, and wrapping it @@ -237,33 +212,121 @@ async function streamFileToHttpRequestAttempt(options: { ), ); }); + + // The owned outgoing work: the artifact file and the progress transform feeding the request. + // `endOutgoingWork` stops it once the response has been consumed, and `upload` is the join that + // proves the owned file handle and request are closed before this attempt reports its outcome. + const fileStream = fs.createReadStream(options.payloadPath, { start: options.startOffset }); + const progressTransform = createUploadProgressTransform({ + stage: options.progress?.stage ?? 'legacy', + fileName: options.progress?.fileName ?? path.basename(options.payloadPath), + startOffset: options.startOffset, + totalBytes: payloadSize, + onProgress: options.progress?.onProgress, + }); upload = (async () => { - await pipeline( - fs.createReadStream(options.payloadPath, { start: options.startOffset }), - createUploadProgressTransform({ - stage: options.progress?.stage ?? 'legacy', - fileName: options.progress?.fileName ?? path.basename(options.payloadPath), - startOffset: options.startOffset, - totalBytes: payloadSize, - onProgress: options.progress?.onProgress, - onBytes: noteBodyBytes, - }), - req, - { signal: watchdog.signal }, - ); - // Everything the server can be waiting for is now written, so the header wait starts here and - // a body still draining never spends its 30-second budget. - if (!responseReceived) watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + await pipeline(fileStream, progressTransform, req, { signal: watchdog.signal }); + uploadSettled = true; + // `pipeline` settles on the request's `finish`: every body byte has been flushed into the + // transport. The server may still be reading them — normal HTTP buffering — so this is this + // side's flush, not the peer's consumption; it is still the point where the server owes an + // answer. If the TLS handshake has not completed under that flush, the connect ceiling is + // still the live budget and the header wait is armed when it does. + bodyFlushed = true; + advanceOutgoingPhase(); })().catch((err: unknown) => { + // After the response has been consumed to its end, this pipeline can only have been ended by + // the teardown that consumed it: the attempt destroys its own request at that point. The + // accepted answer is the outcome, and a rejection from our own destroy must not overwrite the + // server's verdict — a 413 that the client read completely is a 413, not a transport break. if (responseReceived) return; req.destroy(); if (watchdog.signal.aborted) { fail(watchdog.signal.reason); return; } + // Before any response, a rejection here is the artifact failing to reach the socket: the real + // read failure the caller has to see, never a swallowed transport error, because a transport + // break reports through the request's own `error` handler. const error = err instanceof Error ? err : new Error(String(err)); fail(new AppError('COMMAND_FAILED', 'Failed to read local artifact', {}, error)); }); + + const endOutgoingWork = (): Promise => { + if (!outgoingEnded) { + outgoingEnded = true; + // The response has been consumed to its end, so destroying the request can no longer cut it + // short. Destroying it is what settles a pipeline still blocked on a peer that stopped + // reading: the owned file handle and the socket close now, not whenever the peer catches + // up. A pipeline that already flushed everything has nothing left to stop, and its socket + // stays in the pool exactly as it did before. + if (!uploadSettled) req.destroy(); + } + return upload!.then( + () => undefined, + () => undefined, + ); + }; + + async function handleUploadResponse(res: IncomingMessage): Promise { + // Read the accepted response to its end first, under the response-body budget; only then may + // the outgoing work be destroyed, and it must be joined before this attempt resolves or a + // redirect/resume starts — old bytes may never overlap the next hop. + const body = await readUploadResponseBody(res, () => watchdog.touchBody()); + await endOutgoingWork(); + + const statusCode = res.statusCode ?? 500; + const location = res.headers.location; + if (location && isUploadRedirectStatus(statusCode)) { + if (options.redirectCount >= MAX_UPLOAD_REDIRECTS) { + throw new AppError('COMMAND_FAILED', 'Artifact upload exceeded redirect limit', { + maxRedirects: MAX_UPLOAD_REDIRECTS, + url: options.url.toString(), + }); + } + const redirectedUrl = new URL(location, options.url); + // The request headers carry the daemon token or a signed ticket, and the body is the + // artifact: neither may follow a hop to an origin the caller did not choose. + if (redirectedUrl.origin !== options.url.origin) { + throw new AppError('COMMAND_FAILED', 'Artifact upload refused a cross-origin redirect', { + reason: 'cross-origin-redirect', + url: options.url.toString(), + redirectOrigin: redirectedUrl.origin, + }); + } + // The next hop carries its own phase budgets; the retired attempt's timers must not reach + // into it. The shared caller/aggregate signal is inherited by the new attempt, so the one + // 600-second preparation aggregate still bounds every hop. + watchdog.dispose(); + return await streamFileToHttpRequestAttempt({ + ...options, + url: redirectedUrl, + redirectCount: options.redirectCount + 1, + }); + } + + const resumeOffset = isUploadResumeStatus(statusCode) + ? parseUploadResumeOffset(res.headers, payloadSize) + : undefined; + if (resumeOffset !== undefined) { + if (resumeOffset >= payloadSize) { + return { statusCode: 200, statusMessage: 'Upload already complete', body: '' }; + } + if (resumeOffset <= options.startOffset) { + throw new AppError('COMMAND_FAILED', 'Artifact upload resume did not advance', { + offset: resumeOffset, + previousOffset: options.startOffset, + url: options.url.toString(), + }); + } + return await streamFileToHttpRequestAttempt({ + ...options, + startOffset: resumeOffset, + }); + } + + return { statusCode, statusMessage: res.statusMessage, body }; + } }).finally(async () => { watchdog.dispose(); await upload; diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index aee46ec5f4..84501c95d4 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -181,7 +181,7 @@ "packages/remote/src/upload-stream.ts#parseNonNegativeIntegerHeader": "sha256:5cbe9c6994459d673d988a51407e65556e28175b187ce0a9d27cbb6331962834", "packages/remote/src/upload-stream.ts#parseUploadResumeOffset": "sha256:297184ea22a6dc1cda9aee17034122ae0ea2f43d0355647f6fcf02f3f4b5ca0e", "packages/remote/src/upload-stream.ts#streamFileToHttpRequest": "sha256:3ac8f379c4b508b0c28c4f41f920b40f8cc0406afcd5b2290326c4cfb9168adb", - "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:e4a67c0061c8d0999c475a4a2e8876b6c407da08e3f9b94018a772a6acc72283", + "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:15a941b2bde322b0c1bce1050e9c8a1e8a556fb48770393845010180f0fd87f0", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_HEADER": "sha256:d2bb3f4846470188a0fc1724271da0f0bcbdf2337bd6bf331508142cadcb65d8", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_UPSTREAM_INSTANCE_HEADER": "sha256:4564c2ce437d30d7a823e3326f4375c0750ba8d42bec22b4d02e8eb7f7db2973", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_MISMATCH_HEADER": "sha256:a197edaf7f9fa94a6aab269856aa852da3204cc586f5fd23198689eb11908b61", @@ -495,8 +495,8 @@ }, { "declaration": "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt", - "digest": "sha256:e4a67c0061c8d0999c475a4a2e8876b6c407da08e3f9b94018a772a6acc72283", - "rationale": "#4441 runs each attempt under a sequential phase watchdog (connect, body inactivity, response headers) and fails with the expired phase's own typed evidence instead of a transport error, and reads the response through a reader that reports bytes rather than the shared host-kit one. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own send deadline moved, so a protocol-2 peer parses exactly what it parsed before." + "digest": "sha256:15a941b2bde322b0c1bce1050e9c8a1e8a556fb48770393845010180f0fd87f0", + "rationale": "#4441 keeps each attempt under its sequential phase watchdog and corrects its transport lifecycle: the connect ceiling now ends at actual socket or TLS-handshake establishment, body progress counts only transport writes, and an early response reads the accepted answer to its end, then destroys and joins the owned file/request pipeline before resolving or starting the redirect/resume attempt. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own sequencing and deadlines moved, so a protocol-2 peer parses exactly what it parsed before." }, { "declaration": "packages/remote/src/daemon-artifacts.ts#resolveMaterializedArtifactPath", From 6089d90d905071810c1b4ab4e5c89d7aabc7258c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 12:22:30 +0100 Subject: [PATCH 04/27] fix(remote): retire completed upload hop budgets before resuming --- packages/remote/src/upload-preparation-deadline.ts | 2 +- packages/remote/src/upload-stream.ts | 5 +---- test/wire-compat/ledger.json | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/remote/src/upload-preparation-deadline.ts b/packages/remote/src/upload-preparation-deadline.ts index d40c67c1a9..84fdeaabd2 100644 --- a/packages/remote/src/upload-preparation-deadline.ts +++ b/packages/remote/src/upload-preparation-deadline.ts @@ -1,6 +1,6 @@ import { AppError } from '@agent-device/kernel/errors'; -/** The one artifact-preparation aggregate: six minutes from admission to the last finalize. */ +/** The one artifact-preparation aggregate: ten minutes from admission to the last finalize. */ export const UPLOAD_PREPARATION_TIMEOUT_MS = 600_000; export async function withUploadPreparationDeadline( diff --git a/packages/remote/src/upload-stream.ts b/packages/remote/src/upload-stream.ts index 644bb238ce..09d712b61a 100644 --- a/packages/remote/src/upload-stream.ts +++ b/packages/remote/src/upload-stream.ts @@ -274,6 +274,7 @@ async function streamFileToHttpRequestAttempt(options: { // redirect/resume starts — old bytes may never overlap the next hop. const body = await readUploadResponseBody(res, () => watchdog.touchBody()); await endOutgoingWork(); + watchdog.dispose(); const statusCode = res.statusCode ?? 500; const location = res.headers.location; @@ -294,10 +295,6 @@ async function streamFileToHttpRequestAttempt(options: { redirectOrigin: redirectedUrl.origin, }); } - // The next hop carries its own phase budgets; the retired attempt's timers must not reach - // into it. The shared caller/aggregate signal is inherited by the new attempt, so the one - // 600-second preparation aggregate still bounds every hop. - watchdog.dispose(); return await streamFileToHttpRequestAttempt({ ...options, url: redirectedUrl, diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index 84501c95d4..17ce789c34 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -331,7 +331,7 @@ { "declaration": "packages/remote/src/upload-stream.ts#streamFileToHttpRequest", "digest": "sha256:3ac8f379c4b508b0c28c4f41f920b40f8cc0406afcd5b2290326c4cfb9168adb", - "rationale": "#4441 replaces the one 5-minute timer over a whole upload with the preparation aggregate that bounds it, and hands that aggregate's signal to the attempt. The request a peer parses is unchanged: same route, method, the headers buildUploadRequestHeaders produces, the piped payload and the 200/308 handling. What moved is when this client stops sending, and the ceiling is now shorter rather than longer, so a protocol-2 daemon or proxy answers the same requests and simply sees a stalled leg torn down sooner." + "rationale": "#4441 replaces the one 5-minute timer over a whole upload with the preparation aggregate that bounds it, and hands that aggregate's signal to the attempt. The request a peer parses is unchanged: same route, method, the headers buildUploadRequestHeaders produces, the piped payload and the 200/308 handling. A progressing upload may now consume the shared ten-minute aggregate instead of the former five-minute per-attempt ceiling. Connection, header and body-inactivity phase limits stop stalled work sooner; retries and resumes never restart the aggregate. A protocol-2 daemon or proxy parses the same requests, while client-side timeout and retry timing deliberately change." }, { "declaration": "packages/contracts/src/daemon-http.ts#DaemonHealthPayload", From 9544701628ab3f4838fbabddfc0c5d679696101c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 15:17:40 +0100 Subject: [PATCH 05/27] test(remote): preserve retryable failures across resumed upload hops --- .../support/upload-transfer-fixtures.ts | 32 +++++++- .../upload-transfer-lifecycle.test.ts | 78 +++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts index 9352b7aaa9..f5f350f6aa 100644 --- a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -82,6 +82,7 @@ export function startUploadAttempt(options: { artifactPath: string; method?: 'POST' | 'PUT'; headers?: Record; + retryable?: boolean; signal?: AbortSignal; }): SettledUploadAttempt { let settled = false; @@ -92,6 +93,7 @@ export function startUploadAttempt(options: { payloadPath: options.artifactPath, timeoutMessage: 'Artifact upload timed out', errorMessage: 'Failed to upload artifact to remote daemon', + ...(options.retryable === undefined ? {} : { retryable: options.retryable }), ...(options.signal ? { signal: options.signal } : {}), }).then( (value) => ({ value, error: undefined as unknown }), @@ -244,13 +246,20 @@ export function createArtifactFile(bytes: number, fileName = 'app.apk'): string * refused, so the upload runs on the legacy POST route: the one path that pipes a file at the daemon * and keeps going while the daemon drains it. */ -export async function startParkedUploadDaemon(): Promise { +export async function startParkedUploadDaemon( + options: { + /** Answers the first upload leg and stops reading it; later legs are parked at the quota. */ + answerFirstLeg?: { status: number; headers?: Record }; + } = {}, +): Promise { const requests: string[] = []; const parked: http.IncomingMessage[] = []; let receivedBytes = 0; let bytesTakenSinceRelease = 0; let parkedAtQuota: http.IncomingMessage | undefined; let bodyEnded = false; + let uploadLegs = 0; + let newestParkedLeg: http.IncomingMessage | undefined; const server = http.createServer((req, res) => { const key = `${req.method} ${req.url}`; @@ -264,6 +273,21 @@ export async function startParkedUploadDaemon(): Promise { return; } if (key === LEGACY_UPLOAD_REQUEST) { + uploadLegs += 1; + // The client destroys a leg it has moved past, and a reset leg errors on this side too. + req.on('error', () => {}); + const answer = options.answerFirstLeg; + if (answer && uploadLegs === 1) { + res.statusCode = answer.status; + for (const [name, value] of Object.entries(answer.headers ?? {})) { + res.setHeader(name, value); + } + res.end(); + // The answered leg's body is never read, so the client stays in real backpressure on it. + req.pause(); + return; + } + newestParkedLeg = req; req.on('data', (chunk: Buffer) => { receivedBytes += chunk.length; // Each release lets one quota of bytes through and parks the stream again. The client is @@ -310,6 +334,10 @@ export async function startParkedUploadDaemon(): Promise { } for (const entry of resumable) entry.resume(); }, + resetParkedLeg() { + assert.ok(newestParkedLeg, 'no parked leg to reset'); + newestParkedLeg.socket.resetAndDestroy(); + }, close: () => closeLoopbackServer(server), }; } @@ -323,6 +351,8 @@ export type ParkedUploadDaemon = { readonly bodyEnded: boolean; /** Lets the parked body take up to one more quota of bytes. */ releaseAllParked(): void; + /** Resets the connection of the newest parked leg: a real peer reset, not a clean close. */ + resetParkedLeg(): void; close(): Promise; }; diff --git a/packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts b/packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts index 19a088f698..15c7e83c64 100644 --- a/packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts +++ b/packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts @@ -11,7 +11,9 @@ // Every case uses the attempt seam, a controlled clock, and real loopback sockets: the overlap being // refused is only observable on a real socket, and the budgets are minutes long. +import http from 'node:http'; import { afterEach, assert, test, vi } from 'vitest'; +import { isRetryableUploadStreamError } from '../upload-stream.ts'; import { isTransferPhaseTimeoutError, TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, @@ -19,6 +21,8 @@ import { TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, } from '../transfer-phase-budget.ts'; import { + LEGACY_UPLOAD_REQUEST, + type ParkedUploadDaemon, assertDetailedError, createArtifactFile, pumpRealWork, @@ -28,6 +32,7 @@ import { startConnectedStalledHttpOrigin, startConnectedStalledTlsOrigin, startHalfAnsweredOrigin, + startParkedUploadDaemon, startSlowReadingUploadDaemon, startUnfinishedHandshakeTlsOrigin, startUploadAttempt, @@ -122,6 +127,79 @@ test('a resume response ends the stalled hop before the next attempt is dialed', } }); +test('a resumed hop that makes real progress past the first hop ceiling keeps its own retryable reset', async () => { + useTransferClock(); + const artifactBytes = 32 * MEBIBYTE; + let outgoingLegs = 0; + let resumedDrains = 0; + const request = http.request; + const requestSpy = vi.spyOn(http, 'request').mockImplementation((...args) => { + const req = Reflect.apply(request, http, args) as http.ClientRequest; + outgoingLegs += 1; + if (outgoingLegs === 2) + req.on('drain', () => { + resumedDrains += 1; + }); + return req; + }); + const daemon = await startParkedUploadDaemon({ + answerFirstLeg: { status: 308, headers: { 'x-upload-offset': String(MEBIBYTE) } }, + }); + // Hop one is answered while its body is still queued, so its response-body budget starts at fake + // time zero; hop two then moves bytes the peer really takes, for longer than that budget lasts. + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(artifactBytes), + method: 'POST', + retryable: true, + }); + try { + // Gate on real bytes only: no fake time is charged until hop two has really taken some. + for (let round = 0; round < 400 && !hopTwoTookBytes(daemon); round += 1) { + await pumpRealWork(15); + } + assert.ok(hopTwoTookBytes(daemon), 'the resumed hop was dialed and took bytes'); + + for (let window = 0; window < 6; window += 1) { + const drainsBefore = resumedDrains; + const bytesBefore = daemon.receivedBytes; + for (let round = 0; round < 400; round += 1) { + daemon.releaseAllParked(); + await pumpRealWork(15); + if (resumedDrains > drainsBefore && daemon.receivedBytes > bytesBefore) break; + } + assert.ok(daemon.receivedBytes > bytesBefore, 'the peer took more resumed bytes'); + assert.ok(resumedDrains > drainsBefore, 'the resumed write queue drained'); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS); + assert.equal(attempt.settled, false); + assert.equal(daemon.bodyEnded, false); + } + assert.ok(6 * TRANSFER_CONNECT_TIMEOUT_MS > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + + daemon.resetParkedLeg(); + const settled = await settleAttemptAfterBudget(attempt); + assert.equal(settled.settled, true); + const result = await attempt; + + // The peer's reset is an ordinary transport failure of hop two. Hop one's retired budget must + // not relabel it as a phase expiry, and the retry marker must survive for the client's retry. + assert.equal(result.value, undefined); + assert.equal(isTransferPhaseTimeoutError(result.error), false); + assert.equal(isRetryableUploadStreamError(result.error), true); + assert.equal(daemon.requests.filter((entry) => entry === LEGACY_UPLOAD_REQUEST).length, 2); + } finally { + requestSpy.mockRestore(); + await daemon.close(); + } +}); + +function hopTwoTookBytes(daemon: ParkedUploadDaemon): boolean { + return ( + daemon.requests.filter((entry) => entry === LEGACY_UPLOAD_REQUEST).length === 2 && + daemon.receivedBytes > 0 + ); +} + test('a redirect response ends the stalled hop before the next hop is dialed', async () => { useTransferClock(); const artifactBytes = 4 * MEBIBYTE; From 7e03b3e49912fd64bf2d5f88b715b8b7e6820cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 15:29:38 +0100 Subject: [PATCH 06/27] test(remote): clarify resumed upload progress windows --- .../src/__tests__/support/upload-transfer-fixtures.ts | 1 - .../src/__tests__/upload-transfer-lifecycle.test.ts | 11 ++++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts index f5f350f6aa..ab5bf3f6a0 100644 --- a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -283,7 +283,6 @@ export async function startParkedUploadDaemon( res.setHeader(name, value); } res.end(); - // The answered leg's body is never read, so the client stays in real backpressure on it. req.pause(); return; } diff --git a/packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts b/packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts index 15c7e83c64..82254964d2 100644 --- a/packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts +++ b/packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts @@ -129,6 +129,8 @@ test('a resume response ends the stalled hop before the next attempt is dialed', test('a resumed hop that makes real progress past the first hop ceiling keeps its own retryable reset', async () => { useTransferClock(); + const progressWindowMs = 30_000; + assert.ok(progressWindowMs < TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); const artifactBytes = 32 * MEBIBYTE; let outgoingLegs = 0; let resumedDrains = 0; @@ -145,8 +147,7 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it const daemon = await startParkedUploadDaemon({ answerFirstLeg: { status: 308, headers: { 'x-upload-offset': String(MEBIBYTE) } }, }); - // Hop one is answered while its body is still queued, so its response-body budget starts at fake - // time zero; hop two then moves bytes the peer really takes, for longer than that budget lasts. + // Hop one answers at fake time zero. Hop two must stay active beyond the old response-body deadline. const attempt = startUploadAttempt({ url: `${daemon.baseUrl}/upload`, artifactPath: createArtifactFile(artifactBytes), @@ -170,11 +171,11 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it } assert.ok(daemon.receivedBytes > bytesBefore, 'the peer took more resumed bytes'); assert.ok(resumedDrains > drainsBefore, 'the resumed write queue drained'); - await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(progressWindowMs); assert.equal(attempt.settled, false); assert.equal(daemon.bodyEnded, false); } - assert.ok(6 * TRANSFER_CONNECT_TIMEOUT_MS > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + assert.ok(6 * progressWindowMs > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); daemon.resetParkedLeg(); const settled = await settleAttemptAfterBudget(attempt); @@ -184,7 +185,7 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it // The peer's reset is an ordinary transport failure of hop two. Hop one's retired budget must // not relabel it as a phase expiry, and the retry marker must survive for the client's retry. assert.equal(result.value, undefined); - assert.equal(isTransferPhaseTimeoutError(result.error), false); + assert.equal(isTransferPhaseTimeoutError(result.error), false, JSON.stringify(result.error)); assert.equal(isRetryableUploadStreamError(result.error), true); assert.equal(daemon.requests.filter((entry) => entry === LEGACY_UPLOAD_REQUEST).length, 2); } finally { From 5f818b3796b4429623112de8fdf37022ad14eeb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 15:37:42 +0100 Subject: [PATCH 07/27] refactor(remote): keep the transfer phase owner and preparation aggregate in upload-stream The phase watchdog and the 600-second preparation aggregate lived in two new modules that every daemon-client entry now evaluated eagerly. upload-stream.ts is already in that closure and is the lowest module all three consumers reach, so it now owns both. withUploadPreparationDeadline still starts its timer synchronously when called, and isTransferPhaseTimeoutError stays a synchronous typed check. Rename the transfer tests for the module they exercise (R84). Co-Authored-By: Claude Opus 5.5 --- ...est.ts => upload-stream-lifecycle.test.ts} | 4 +- ....ts => upload-stream-phase-budget.test.ts} | 4 +- ...s.test.ts => upload-stream-phases.test.ts} | 4 +- packages/remote/src/daemon-artifacts.ts | 2 +- packages/remote/src/transfer-phase-budget.ts | 104 ------------- packages/remote/src/upload-client.ts | 11 +- .../remote/src/upload-preparation-deadline.ts | 35 ----- packages/remote/src/upload-stream.ts | 144 +++++++++++++++++- 8 files changed, 151 insertions(+), 157 deletions(-) rename packages/remote/src/__tests__/{upload-transfer-lifecycle.test.ts => upload-stream-lifecycle.test.ts} (99%) rename packages/remote/src/__tests__/{transfer-phase-budget.test.ts => upload-stream-phase-budget.test.ts} (98%) rename packages/remote/src/__tests__/{upload-transfer-phases.test.ts => upload-stream-phases.test.ts} (97%) delete mode 100644 packages/remote/src/transfer-phase-budget.ts delete mode 100644 packages/remote/src/upload-preparation-deadline.ts diff --git a/packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts similarity index 99% rename from packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts rename to packages/remote/src/__tests__/upload-stream-lifecycle.test.ts index 82254964d2..417408421d 100644 --- a/packages/remote/src/__tests__/upload-transfer-lifecycle.test.ts +++ b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts @@ -13,13 +13,13 @@ import http from 'node:http'; import { afterEach, assert, test, vi } from 'vitest'; -import { isRetryableUploadStreamError } from '../upload-stream.ts'; import { + isRetryableUploadStreamError, isTransferPhaseTimeoutError, TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, TRANSFER_CONNECT_TIMEOUT_MS, TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, -} from '../transfer-phase-budget.ts'; +} from '../upload-stream.ts'; import { LEGACY_UPLOAD_REQUEST, type ParkedUploadDaemon, diff --git a/packages/remote/src/__tests__/transfer-phase-budget.test.ts b/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts similarity index 98% rename from packages/remote/src/__tests__/transfer-phase-budget.test.ts rename to packages/remote/src/__tests__/upload-stream-phase-budget.test.ts index 1dbbbb3ed6..5d16846927 100644 --- a/packages/remote/src/__tests__/transfer-phase-budget.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts @@ -3,7 +3,7 @@ // Three ceilings govern one attempt — connecting, moving body bytes, waiting for headers — and each // has to end a stalled attempt on its own while none of them touches the aggregate budget the // attempt was admitted under. Those are behaviour claims about a clock, so the clock here is -// controlled rather than waited out. The loopback proofs in `upload-transfer-phases.test.ts` put the +// controlled rather than waited out. The loopback proofs in `upload-stream-phases.test.ts` put the // same watchdog in front of real sockets. import assert from 'node:assert/strict'; @@ -15,7 +15,7 @@ import { TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, TRANSFER_CONNECT_TIMEOUT_MS, TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, -} from '../transfer-phase-budget.ts'; +} from '../upload-stream.ts'; afterEach(() => { vi.useRealTimers(); diff --git a/packages/remote/src/__tests__/upload-transfer-phases.test.ts b/packages/remote/src/__tests__/upload-stream-phases.test.ts similarity index 97% rename from packages/remote/src/__tests__/upload-transfer-phases.test.ts rename to packages/remote/src/__tests__/upload-stream-phases.test.ts index e44f87c311..160635e8e5 100644 --- a/packages/remote/src/__tests__/upload-transfer-phases.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phases.test.ts @@ -1,6 +1,6 @@ // Phase budgets applied to a real artifact upload over loopback (#4441). // -// The budgets in `transfer-phase-budget.ts` are minutes long, so every case here drives a controlled +// The budgets in `upload-stream.ts` are minutes long, so every case here drives a controlled // clock and parks the request body a quota at a time. That combination is the point: a transfer that // keeps moving must survive budget after budget, and one that stops moving must be ended by the ONE // budget that ran out. A mocked transport could show neither, because the backpressure that makes @@ -11,7 +11,7 @@ import { TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, TRANSFER_CONNECT_TIMEOUT_MS, TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, -} from '../transfer-phase-budget.ts'; +} from '../upload-stream.ts'; import { LEGACY_UPLOAD_REQUEST, PREFLIGHT_REQUEST, diff --git a/packages/remote/src/daemon-artifacts.ts b/packages/remote/src/daemon-artifacts.ts index b57d4db170..226980ad16 100644 --- a/packages/remote/src/daemon-artifacts.ts +++ b/packages/remote/src/daemon-artifacts.ts @@ -15,7 +15,7 @@ import { } from '@agent-device/capture-kit/recording-output-path'; import { parsePerfRuntimeRequest } from '@agent-device/capture-kit/perf-runtime-plan'; import { uploadArtifact } from './upload-client.ts'; -import { withUploadPreparationDeadline } from './upload-preparation-deadline.ts'; +import { withUploadPreparationDeadline } from './upload-stream.ts'; import { createStderrUploadProgressReporter, type UploadProgressSink } from './upload-progress.ts'; // Mirrors `DEFAULT_TEST_ARTIFACTS_ROOT` in `packages/replay-test/src/internal/session-test-artifacts.ts` diff --git a/packages/remote/src/transfer-phase-budget.ts b/packages/remote/src/transfer-phase-budget.ts deleted file mode 100644 index cb57e4a4e4..0000000000 --- a/packages/remote/src/transfer-phase-budget.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { AppError } from '@agent-device/kernel/errors'; - -/** - * Ceiling for establishing the connection one artifact-transfer attempt runs on. - * - * Kept beside its domain rather than on the socket: the same number governs every hop of an upload, - * every offset resume and a standalone artifact download, so "how long may one hop take to connect" - * has exactly one answer. - */ -export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; - -/** - * Ceiling for waiting on response headers once this side has written everything the server is - * waiting for. For an upload that is the moment its outgoing request body is fully flushed, never - * the moment the request was created; for a bodyless request it is connection establishment. - */ -export const TRANSFER_RESPONSE_HEADER_TIMEOUT_MS = 30_000; - -/** - * Ceiling for silence between body bytes in either direction. Bytes actually written or read reset - * it, so an artifact that keeps moving keeps its budget while a connection that stopped carrying - * bytes does not. Progress never touches the aggregate budget the attempt runs under. - */ -export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; - -export type TransferPhase = 'connect' | 'request-body' | 'response-headers' | 'response-body'; - -const BODY_PHASES: ReadonlySet = new Set(['request-body', 'response-body']); - -/** The `details.reason` every phase expiry carries, so no consumer has to match error text. */ -export const TRANSFER_PHASE_TIMEOUT_REASON = 'transfer_phase_timeout'; - -/** - * Watches the sequential phases of ONE transfer attempt. - * - * One attempt gets one watchdog: arming a later phase stops counting the earlier one, so the ceiling - * that applies is always the phase the transfer is actually in. Progress re-arms only the two body - * phases. Nothing here extends an enclosing aggregate budget — the caller's signal is combined in, - * never replaced, so the earliest deadline still wins and every attempt, redirect and resume keeps - * sharing the original aggregate. - */ -export type TransferPhaseWatchdog = { - /** The signal the attempt runs on: the caller's cancellation plus every phase deadline. */ - signal: AbortSignal; - /** Starts watching `phase`. The phase being watched before stops counting. */ - arm(phase: TransferPhase, timeoutMs: number): void; - /** Records body bytes moving, so only the silence since the last byte counts against a body phase. */ - touchBody(): void; - /** Stops the armed timer. Call it once the attempt is settled. */ - dispose(): void; -}; - -export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): TransferPhaseWatchdog { - const phase = new AbortController(); - const signal = callerSignal ? AbortSignal.any([callerSignal, phase.signal]) : phase.signal; - let armed: { phase: TransferPhase; timeoutMs: number } | undefined; - let timer: ReturnType | undefined; - const rearm = () => { - if (timer !== undefined) clearTimeout(timer); - const current = armed; - if (!current) return; - timer = setTimeout(() => { - timer = undefined; - phase.abort(createTransferPhaseTimeoutError(current.phase, current.timeoutMs)); - }, current.timeoutMs); - timer.unref(); - }; - return { - signal, - arm(nextPhase, timeoutMs) { - if (signal.aborted) return; - armed = { phase: nextPhase, timeoutMs }; - rearm(); - }, - touchBody() { - // Unrelated headers, keepalive traffic and timer callbacks are not body progress, and - // progress in one phase buys no time in another: only a body phase already armed re-arms. - if (!armed || !BODY_PHASES.has(armed.phase)) return; - rearm(); - }, - dispose() { - armed = undefined; - if (timer !== undefined) clearTimeout(timer); - timer = undefined; - }, - }; -} - -/** - * Typed evidence that a transfer phase, rather than the caller or the whole preparation aggregate, - * ended the attempt. The expired phase and its effective budget travel in `details`. - */ -export function createTransferPhaseTimeoutError(phase: TransferPhase, timeoutMs: number): AppError { - return new AppError('COMMAND_FAILED', `Artifact transfer ${phase} timed out`, { - reason: TRANSFER_PHASE_TIMEOUT_REASON, - transferPhase: phase, - timeoutMs, - }); -} - -/** Whether `error` is a transfer phase's own expiry, which no fallback or retry may restart. */ -export function isTransferPhaseTimeoutError(error: unknown): boolean { - return error instanceof AppError && error.details?.reason === TRANSFER_PHASE_TIMEOUT_REASON; -} diff --git a/packages/remote/src/upload-client.ts b/packages/remote/src/upload-client.ts index 13400e3521..3a7a65f02f 100644 --- a/packages/remote/src/upload-client.ts +++ b/packages/remote/src/upload-client.ts @@ -6,15 +6,18 @@ import { buildDaemonHttpLeaseHeaders, } from '@agent-device/contracts/daemon-http'; import { prepareUploadArtifact, type PreparedUploadArtifact } from './upload-client-artifact.ts'; -import { isRetryableUploadStreamError, streamFileToHttpRequest } from './upload-stream.ts'; +import { + isRetryableUploadStreamError, + isTransferPhaseTimeoutError, + streamFileToHttpRequest, + withUploadPreparationDeadline, +} from './upload-stream.ts'; import type { UploadProgressSink } from './upload-progress.ts'; -import { withUploadPreparationDeadline } from './upload-preparation-deadline.ts'; -import { isTransferPhaseTimeoutError } from './transfer-phase-budget.ts'; /** * Ceiling for the two small control-plane calls that bracket an upload (`preflight`, `finalize`). * Their payload is a few hundred bytes of JSON, so one window covers reaching the daemon and getting - * the answer; the artifact bytes themselves are budgeted per phase in `transfer-phase-budget.ts`. + * the answer; the artifact bytes themselves are budgeted per phase in `upload-stream.ts`. */ const UPLOAD_CONTROL_PLANE_TIMEOUT_MS = 30_000; diff --git a/packages/remote/src/upload-preparation-deadline.ts b/packages/remote/src/upload-preparation-deadline.ts deleted file mode 100644 index 84fdeaabd2..0000000000 --- a/packages/remote/src/upload-preparation-deadline.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { AppError } from '@agent-device/kernel/errors'; - -/** The one artifact-preparation aggregate: ten minutes from admission to the last finalize. */ -export const UPLOAD_PREPARATION_TIMEOUT_MS = 600_000; - -export async function withUploadPreparationDeadline( - callerSignal: AbortSignal | undefined, - prepare: (signal: AbortSignal) => Promise, - timeout?: Readonly<{ message: string; hint?: string }>, -): Promise { - const deadline = new AbortController(); - const signal = callerSignal ? AbortSignal.any([callerSignal, deadline.signal]) : deadline.signal; - signal.throwIfAborted(); - const timer = setTimeout(() => { - deadline.abort( - new AppError('COMMAND_FAILED', timeout?.message ?? 'Artifact preparation deadline exceeded', { - reason: 'artifact_preparation_deadline_exceeded', - transferPhase: 'preparation', - timeoutMs: UPLOAD_PREPARATION_TIMEOUT_MS, - ...(timeout?.hint ? { hint: timeout.hint } : {}), - }), - ); - }, UPLOAD_PREPARATION_TIMEOUT_MS); - timer.unref(); - try { - const result = await prepare(signal); - signal.throwIfAborted(); - return result; - } catch (error) { - signal.throwIfAborted(); - throw error; - } finally { - clearTimeout(timer); - } -} diff --git a/packages/remote/src/upload-stream.ts b/packages/remote/src/upload-stream.ts index 09d712b61a..fe9c5bc93c 100644 --- a/packages/remote/src/upload-stream.ts +++ b/packages/remote/src/upload-stream.ts @@ -10,13 +10,143 @@ import { type UploadProgressSink, type UploadProgressStage, } from './upload-progress.ts'; -import { withUploadPreparationDeadline } from './upload-preparation-deadline.ts'; -import { - createTransferPhaseWatchdog, - TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, - TRANSFER_CONNECT_TIMEOUT_MS, - TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, -} from './transfer-phase-budget.ts'; + +/** + * Ceiling for establishing the connection one artifact-transfer attempt runs on. + * + * Kept beside its domain rather than on the socket: the same number governs every hop of an upload, + * every offset resume and a standalone artifact download, so "how long may one hop take to connect" + * has exactly one answer. + */ +export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; + +/** + * Ceiling for waiting on response headers once this side has written everything the server is + * waiting for. For an upload that is the moment its outgoing request body is fully flushed, never + * the moment the request was created; for a bodyless request it is connection establishment. + */ +export const TRANSFER_RESPONSE_HEADER_TIMEOUT_MS = 30_000; + +/** + * Ceiling for silence between body bytes in either direction. Bytes actually written or read reset + * it, so an artifact that keeps moving keeps its budget while a connection that stopped carrying + * bytes does not. Progress never touches the aggregate budget the attempt runs under. + */ +export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; + +export type TransferPhase = 'connect' | 'request-body' | 'response-headers' | 'response-body'; + +const BODY_PHASES: ReadonlySet = new Set(['request-body', 'response-body']); + +/** The `details.reason` every phase expiry carries, so no consumer has to match error text. */ +export const TRANSFER_PHASE_TIMEOUT_REASON = 'transfer_phase_timeout'; + +/** + * Watches the sequential phases of ONE transfer attempt. + * + * One attempt gets one watchdog: arming a later phase stops counting the earlier one, so the ceiling + * that applies is always the phase the transfer is actually in. Progress re-arms only the two body + * phases. Nothing here extends an enclosing aggregate budget — the caller's signal is combined in, + * never replaced, so the earliest deadline still wins and every attempt, redirect and resume keeps + * sharing the original aggregate. + */ +export type TransferPhaseWatchdog = { + /** The signal the attempt runs on: the caller's cancellation plus every phase deadline. */ + signal: AbortSignal; + /** Starts watching `phase`. The phase being watched before stops counting. */ + arm(phase: TransferPhase, timeoutMs: number): void; + /** Records body bytes moving, so only the silence since the last byte counts against a body phase. */ + touchBody(): void; + /** Stops the armed timer. Call it once the attempt is settled. */ + dispose(): void; +}; + +export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): TransferPhaseWatchdog { + const phase = new AbortController(); + const signal = callerSignal ? AbortSignal.any([callerSignal, phase.signal]) : phase.signal; + let armed: { phase: TransferPhase; timeoutMs: number } | undefined; + let timer: ReturnType | undefined; + const rearm = () => { + if (timer !== undefined) clearTimeout(timer); + const current = armed; + if (!current) return; + timer = setTimeout(() => { + timer = undefined; + phase.abort(createTransferPhaseTimeoutError(current.phase, current.timeoutMs)); + }, current.timeoutMs); + timer.unref(); + }; + return { + signal, + arm(nextPhase, timeoutMs) { + if (signal.aborted) return; + armed = { phase: nextPhase, timeoutMs }; + rearm(); + }, + touchBody() { + // Unrelated headers, keepalive traffic and timer callbacks are not body progress, and + // progress in one phase buys no time in another: only a body phase already armed re-arms. + if (!armed || !BODY_PHASES.has(armed.phase)) return; + rearm(); + }, + dispose() { + armed = undefined; + if (timer !== undefined) clearTimeout(timer); + timer = undefined; + }, + }; +} + +/** + * Typed evidence that a transfer phase, rather than the caller or the whole preparation aggregate, + * ended the attempt. The expired phase and its effective budget travel in `details`. + */ +export function createTransferPhaseTimeoutError(phase: TransferPhase, timeoutMs: number): AppError { + return new AppError('COMMAND_FAILED', `Artifact transfer ${phase} timed out`, { + reason: TRANSFER_PHASE_TIMEOUT_REASON, + transferPhase: phase, + timeoutMs, + }); +} + +/** Whether `error` is a transfer phase's own expiry, which no fallback or retry may restart. */ +export function isTransferPhaseTimeoutError(error: unknown): boolean { + return error instanceof AppError && error.details?.reason === TRANSFER_PHASE_TIMEOUT_REASON; +} + +/** The one artifact-preparation aggregate: ten minutes from admission to the last finalize. */ +const UPLOAD_PREPARATION_TIMEOUT_MS = 600_000; + +export async function withUploadPreparationDeadline( + callerSignal: AbortSignal | undefined, + prepare: (signal: AbortSignal) => Promise, + timeout?: Readonly<{ message: string; hint?: string }>, +): Promise { + const deadline = new AbortController(); + const signal = callerSignal ? AbortSignal.any([callerSignal, deadline.signal]) : deadline.signal; + signal.throwIfAborted(); + const timer = setTimeout(() => { + deadline.abort( + new AppError('COMMAND_FAILED', timeout?.message ?? 'Artifact preparation deadline exceeded', { + reason: 'artifact_preparation_deadline_exceeded', + transferPhase: 'preparation', + timeoutMs: UPLOAD_PREPARATION_TIMEOUT_MS, + ...(timeout?.hint ? { hint: timeout.hint } : {}), + }), + ); + }, UPLOAD_PREPARATION_TIMEOUT_MS); + timer.unref(); + try { + const result = await prepare(signal); + signal.throwIfAborted(); + return result; + } catch (error) { + signal.throwIfAborted(); + throw error; + } finally { + clearTimeout(timer); + } +} const MAX_UPLOAD_REDIRECTS = 5; const UPLOAD_STREAM_RETRYABLE_DETAIL = 'uploadStreamRetryable'; From 8a62f24668ff8a43dcd2658d8adf35c3b462b1af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 15:37:42 +0100 Subject: [PATCH 08/27] chore(gates): acknowledge the retired hop watchdog in the upload attempt wire digest 6089d90 moved the attempt's watchdog.dispose() ahead of the redirect/resume decision without refreshing the ledger. The digest pasted here is the one wire-compat.test.ts prints for the current source; the rationale now names the retired watchdog. Requests on the wire are unchanged. Co-Authored-By: Claude Opus 5.5 --- test/wire-compat/ledger.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index 17ce789c34..13b9ef2eef 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -181,7 +181,7 @@ "packages/remote/src/upload-stream.ts#parseNonNegativeIntegerHeader": "sha256:5cbe9c6994459d673d988a51407e65556e28175b187ce0a9d27cbb6331962834", "packages/remote/src/upload-stream.ts#parseUploadResumeOffset": "sha256:297184ea22a6dc1cda9aee17034122ae0ea2f43d0355647f6fcf02f3f4b5ca0e", "packages/remote/src/upload-stream.ts#streamFileToHttpRequest": "sha256:3ac8f379c4b508b0c28c4f41f920b40f8cc0406afcd5b2290326c4cfb9168adb", - "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:15a941b2bde322b0c1bce1050e9c8a1e8a556fb48770393845010180f0fd87f0", + "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:5ba978e2740430e6ed03e06a1e4e056115fa393a3e32658739b97a444aa939fa", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_HEADER": "sha256:d2bb3f4846470188a0fc1724271da0f0bcbdf2337bd6bf331508142cadcb65d8", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_UPSTREAM_INSTANCE_HEADER": "sha256:4564c2ce437d30d7a823e3326f4375c0750ba8d42bec22b4d02e8eb7f7db2973", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_MISMATCH_HEADER": "sha256:a197edaf7f9fa94a6aab269856aa852da3204cc586f5fd23198689eb11908b61", @@ -495,8 +495,8 @@ }, { "declaration": "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt", - "digest": "sha256:15a941b2bde322b0c1bce1050e9c8a1e8a556fb48770393845010180f0fd87f0", - "rationale": "#4441 keeps each attempt under its sequential phase watchdog and corrects its transport lifecycle: the connect ceiling now ends at actual socket or TLS-handshake establishment, body progress counts only transport writes, and an early response reads the accepted answer to its end, then destroys and joins the owned file/request pipeline before resolving or starting the redirect/resume attempt. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own sequencing and deadlines moved, so a protocol-2 peer parses exactly what it parsed before." + "digest": "sha256:5ba978e2740430e6ed03e06a1e4e056115fa393a3e32658739b97a444aa939fa", + "rationale": "#4441 keeps each attempt under its sequential phase watchdog and corrects its transport lifecycle: the connect ceiling now ends at actual socket or TLS-handshake establishment, body progress counts only transport writes, and an early response reads the accepted answer to its end, then destroys and joins the owned file/request pipeline and retires this attempt's phase watchdog before resolving or starting the redirect/resume attempt, so a completed hop's timers never reach the next hop. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own sequencing and deadlines moved, so a protocol-2 peer parses exactly what it parsed before." }, { "declaration": "packages/remote/src/daemon-artifacts.ts#resolveMaterializedArtifactPath", From 09e1e760b9c688f02c935882308c08b5fed77ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 15:39:24 +0100 Subject: [PATCH 09/27] refactor(remote): keep the phase-expiry reason and error constructor private to upload-stream No module outside upload-stream reads them; consumers classify through isTransferPhaseTimeoutError. Co-Authored-By: Claude Opus 5.5 --- packages/remote/src/upload-stream.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/remote/src/upload-stream.ts b/packages/remote/src/upload-stream.ts index fe9c5bc93c..e9cc6e58eb 100644 --- a/packages/remote/src/upload-stream.ts +++ b/packages/remote/src/upload-stream.ts @@ -39,7 +39,7 @@ export type TransferPhase = 'connect' | 'request-body' | 'response-headers' | 'r const BODY_PHASES: ReadonlySet = new Set(['request-body', 'response-body']); /** The `details.reason` every phase expiry carries, so no consumer has to match error text. */ -export const TRANSFER_PHASE_TIMEOUT_REASON = 'transfer_phase_timeout'; +const TRANSFER_PHASE_TIMEOUT_REASON = 'transfer_phase_timeout'; /** * Watches the sequential phases of ONE transfer attempt. @@ -101,7 +101,7 @@ export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): Transfe * Typed evidence that a transfer phase, rather than the caller or the whole preparation aggregate, * ended the attempt. The expired phase and its effective budget travel in `details`. */ -export function createTransferPhaseTimeoutError(phase: TransferPhase, timeoutMs: number): AppError { +function createTransferPhaseTimeoutError(phase: TransferPhase, timeoutMs: number): AppError { return new AppError('COMMAND_FAILED', `Artifact transfer ${phase} timed out`, { reason: TRANSFER_PHASE_TIMEOUT_REASON, transferPhase: phase, From fae463fc2107b30748b5eb2d63224245ba16a35b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 17:02:14 +0100 Subject: [PATCH 10/27] fix(remote): reuse shared response reader and tighten review guards --- .../host-kit/src/internal/transport.test.ts | 7 ++ packages/host-kit/src/internal/transport.ts | 8 +- .../support/upload-transfer-fixtures.ts | 11 ++- .../__tests__/upload-stream-lifecycle.test.ts | 80 +++++++++---------- .../upload-stream-phase-budget.test.ts | 1 + .../__tests__/upload-stream-phases.test.ts | 8 +- packages/remote/src/upload-client.ts | 1 + packages/remote/src/upload-stream.ts | 26 +----- 8 files changed, 71 insertions(+), 71 deletions(-) diff --git a/packages/host-kit/src/internal/transport.test.ts b/packages/host-kit/src/internal/transport.test.ts index 3ae23547d4..0cc966a8a8 100644 --- a/packages/host-kit/src/internal/transport.test.ts +++ b/packages/host-kit/src/internal/transport.test.ts @@ -44,6 +44,13 @@ test('loadNodeHttpRequester selects the protocol module and keeps request mutabl test('readNodeHttpResponseBody decodes the complete response stream', async () => { const response = Readable.from(['hello', Buffer.from(' world')]) as unknown as IncomingMessage; assert.equal(await readNodeHttpResponseBody(response), 'hello world'); + const observed: string[] = []; + const progressing = Readable.from(['hello', Buffer.from(' world')]) as unknown as IncomingMessage; + assert.equal( + await readNodeHttpResponseBody(progressing, (chunk) => observed.push(chunk)), + 'hello world', + ); + assert.deepEqual(observed, ['hello', ' world']); }); test('readNodeHttpRequestBody returns bytes and preserves the caller error message at the limit', async () => { diff --git a/packages/host-kit/src/internal/transport.ts b/packages/host-kit/src/internal/transport.ts index 13fa052b69..90e68c6cc6 100644 --- a/packages/host-kit/src/internal/transport.ts +++ b/packages/host-kit/src/internal/transport.ts @@ -42,11 +42,15 @@ export async function loadNodeHttpRequester(protocol: string): Promise { +export function readNodeHttpResponseBody( + res: IncomingMessage, + onChunk?: (chunk: string) => void, +): Promise { return new Promise((resolve, reject) => { let body = ''; res.setEncoding('utf8'); - res.on('data', (chunk) => { + res.on('data', (chunk: string) => { + onChunk?.(chunk); body += chunk; }); res.on('end', () => resolve(body)); diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts index ab5bf3f6a0..e06c5d6d21 100644 --- a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -18,6 +18,11 @@ import { closeLoopbackServer, listenOnLoopback, mkdtempForTestSync } from '@agen import { uploadArtifact } from '../../upload-client.ts'; import { type UploadStreamResponse, streamFileToHttpRequest } from '../../upload-stream.ts'; +// Adopted phase budgets are independent expectations for the loopback tests. +export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; +export const TRANSFER_RESPONSE_HEADER_TIMEOUT_MS = 30_000; +export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; + export const PREFLIGHT_REQUEST = 'POST /upload/preflight'; export const LEGACY_UPLOAD_REQUEST = 'POST /upload'; @@ -672,7 +677,11 @@ export async function startUploadLegsDaemon( const server = http.createServer((req, res) => { requests.push(`${req.method} ${req.url}`); const behavior = behaviors[legs]; - if (!behavior) throw new Error(`unexpected upload leg ${legs + 1}`); + if (!behavior) { + res.statusCode = 500; + res.end(`unexpected upload leg ${legs + 1}`); + return; + } legs += 1; const answer = () => { if (!behavior.respond) return; diff --git a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts index 417408421d..19201cbff9 100644 --- a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts +++ b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts @@ -9,19 +9,16 @@ // already connected), and buffered bytes never count as transport progress. // // Every case uses the attempt seam, a controlled clock, and real loopback sockets: the overlap being -// refused is only observable on a real socket, and the budgets are minutes long. +// refused is only observable on a real socket, and phase budgets range from 30 to 120 seconds. import http from 'node:http'; import { afterEach, assert, test, vi } from 'vitest'; +import { isRetryableUploadStreamError, isTransferPhaseTimeoutError } from '../upload-stream.ts'; import { - isRetryableUploadStreamError, - isTransferPhaseTimeoutError, + LEGACY_UPLOAD_REQUEST, TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, TRANSFER_CONNECT_TIMEOUT_MS, TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, -} from '../upload-stream.ts'; -import { - LEGACY_UPLOAD_REQUEST, type ParkedUploadDaemon, assertDetailedError, createArtifactFile, @@ -328,30 +325,33 @@ test('a completed TLS handshake with a stalled body expires body inactivity, not // handler below answers the handshake and then stops reading, so a trusted peer connection has to // be established for this to say anything. const trusted = trustSelfSignedLocalhostCertificate(); - const origin = await startConnectedStalledTlsOrigin(); try { - // The control leg first: the same trust decision, a peer that reads nothing but completes the - // handshake, and the outcome is a body-phase expiry rather than a handshake failure. - const attempt = startUploadAttempt({ - url: `https://127.0.0.1:${origin.port}/upload`, - artifactPath: createArtifactFile(4 * MEBIBYTE), - }); - await spendAttemptBudget(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, attempt); - assert.equal(attempt.settled, false, 'secureConnect ended the connect phase'); - await spendAttemptBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); - const settled = await settleAttemptAfterBudget(attempt); - assert.equal(settled.settled, true); - const result = await attempt; - - assert.equal(result.value, undefined); - assertDetailedError(result.error, { - reason: 'transfer_phase_timeout', - transferPhase: 'request-body', - timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, - }); + const origin = await startConnectedStalledTlsOrigin(); + try { + // The control leg first: the same trust decision, a peer that reads nothing but completes the + // handshake, and the outcome is a body-phase expiry rather than a handshake failure. + const attempt = startUploadAttempt({ + url: `https://127.0.0.1:${origin.port}/upload`, + artifactPath: createArtifactFile(4 * MEBIBYTE), + }); + await spendAttemptBudget(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, attempt); + assert.equal(attempt.settled, false, 'secureConnect ended the connect phase'); + await spendAttemptBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); + const settled = await settleAttemptAfterBudget(attempt); + assert.equal(settled.settled, true); + const result = await attempt; + + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'request-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); + } finally { + await origin.close(); + } } finally { trusted.restore(); - await origin.close(); } }); @@ -498,20 +498,20 @@ test('an attempt on a reused connection is already past its connect phase', asyn }, { readAllBody: true }, ]); - const first = startUploadAttempt({ - url: `${daemon.baseUrl}/upload`, - artifactPath: createArtifactFile(64 * 1024), - }); - const firstSettled = await settleAttemptAfterBudget(first); - assert.equal(firstSettled.settled, true); - const firstResult = await first; - assert.equal(firstResult.error, undefined, `leg one failed: ${String(firstResult.error)}`); - - const second = startUploadAttempt({ - url: `${daemon.baseUrl}/upload`, - artifactPath: createArtifactFile(64 * 1024), - }); try { + const first = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(64 * 1024), + }); + const firstSettled = await settleAttemptAfterBudget(first); + assert.equal(firstSettled.settled, true); + const firstResult = await first; + assert.equal(firstResult.error, undefined, `leg one failed: ${String(firstResult.error)}`); + + const second = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(64 * 1024), + }); await spendAttemptBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, second); const settled = await settleAttemptAfterBudget(second); assert.equal(settled.settled, true); diff --git a/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts b/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts index 5d16846927..bd11c67683 100644 --- a/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts @@ -147,6 +147,7 @@ test('a phase is never armed after the attempt is already over', async () => { const late = createTransferPhaseWatchdog(caller.signal); try { late.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); + assert.equal(vi.getTimerCount(), 0); await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS * 2); assert.equal(late.signal.reason.message, 'gone'); } finally { diff --git a/packages/remote/src/__tests__/upload-stream-phases.test.ts b/packages/remote/src/__tests__/upload-stream-phases.test.ts index 160635e8e5..910b5d01e1 100644 --- a/packages/remote/src/__tests__/upload-stream-phases.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phases.test.ts @@ -1,19 +1,17 @@ // Phase budgets applied to a real artifact upload over loopback (#4441). // -// The budgets in `upload-stream.ts` are minutes long, so every case here drives a controlled -// clock and parks the request body a quota at a time. That combination is the point: a transfer that +// The 30–120-second phase budgets use a controlled clock here. Each case parks the request +// body a quota at a time. That combination is the point: a transfer that // keeps moving must survive budget after budget, and one that stops moving must be ended by the ONE // budget that ran out. A mocked transport could show neither, because the backpressure that makes // "still progressing" mean something belongs to the real socket. import { afterEach, assert, test, vi } from 'vitest'; import { + LEGACY_UPLOAD_REQUEST, TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, TRANSFER_CONNECT_TIMEOUT_MS, TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, -} from '../upload-stream.ts'; -import { - LEGACY_UPLOAD_REQUEST, PREFLIGHT_REQUEST, assertDetailedError, createArtifactFile, diff --git a/packages/remote/src/upload-client.ts b/packages/remote/src/upload-client.ts index 3a7a65f02f..40919f6512 100644 --- a/packages/remote/src/upload-client.ts +++ b/packages/remote/src/upload-client.ts @@ -212,6 +212,7 @@ async function retryDirectUpload( try { return await uploadOnce(retryPreflight); } catch (error) { + options.signal?.throwIfAborted(); // As above: an expired phase is this client's own deadline, so it must not buy the legacy // fallback a third attempt at the same bytes. if (isTransferPhaseTimeoutError(error)) throw error; diff --git a/packages/remote/src/upload-stream.ts b/packages/remote/src/upload-stream.ts index e9cc6e58eb..d36842e2a1 100644 --- a/packages/remote/src/upload-stream.ts +++ b/packages/remote/src/upload-stream.ts @@ -4,7 +4,7 @@ import path from 'node:path'; import { pipeline } from 'node:stream/promises'; import { addAbortSignal } from 'node:stream'; import { AppError } from '@agent-device/kernel/errors'; -import { loadNodeHttpRequester } from '@agent-device/host-kit/transport'; +import { loadNodeHttpRequester, readNodeHttpResponseBody } from '@agent-device/host-kit/transport'; import { createUploadProgressTransform, type UploadProgressSink, @@ -15,7 +15,7 @@ import { * Ceiling for establishing the connection one artifact-transfer attempt runs on. * * Kept beside its domain rather than on the socket: the same number governs every hop of an upload, - * every offset resume and a standalone artifact download, so "how long may one hop take to connect" + * and every offset resume, so "how long may one hop take to connect" * has exactly one answer. */ export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; @@ -402,7 +402,7 @@ async function streamFileToHttpRequestAttempt(options: { // Read the accepted response to its end first, under the response-body budget; only then may // the outgoing work be destroyed, and it must be joined before this attempt resolves or a // redirect/resume starts — old bytes may never overlap the next hop. - const body = await readUploadResponseBody(res, () => watchdog.touchBody()); + const body = await readNodeHttpResponseBody(res, () => watchdog.touchBody()); await endOutgoingWork(); watchdog.dispose(); @@ -464,26 +464,6 @@ function retryableUploadStreamDetail(options: { retryable?: boolean }): Record void): Promise { - return new Promise((resolve, reject) => { - let body = ''; - res.setEncoding('utf8'); - res.on('data', (chunk: string) => { - onBytes(); - body += chunk; - }); - res.on('end', () => resolve(body)); - res.on('error', reject); - }); -} - function emitUploadAttemptStarted( progress: UploadStreamProgressOptions | undefined, startOffset: number, From c9f91d0a73112dd93aede8a2e1488bf340128e13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 17:25:44 +0100 Subject: [PATCH 11/27] fix(remote): count completed body writes as upload progress Request-body progress came only from `drain`, which fires after a write returned false. A slow producer whose writes stay under the socket's high-water mark never sees a drain, so a progressing upload expired its 120-second body budget. The attempt now wraps the request's `write` and touches the body phase when a non-empty write completes without error: its bytes reached the local kernel. Arguments, `this`, the backpressure result and the caller's callback pass through unchanged, and completions after the response, an abort, a destroy or the attempt's end do not count. streamFileToHttpRequest no longer opens a second copy of the preparation aggregate: its only callers run inside uploadArtifact's, so it requires that signal and drops the timeout message and hint the outer deadline always preempted. uploadArtifact and prepareRemoteRequestArtifacts keep their own 600-second admission boundaries. Tests: a real loopback slow producer with 1 KiB writes survives six 30-second windows with zero drains and then expires one ceiling after its last completed write; the header-wait case now proves the body is still in flight past four connect ceilings before only the header wait expires; a cancel during the retried direct upload announces no fallback. Loopback progress windows gate on completed writes. Co-Authored-By: Claude Opus 5.5 --- .../support/upload-transfer-fixtures.ts | 79 ++++++++++++++-- .../upload-client-cancellation.test.ts | 53 +++++++++++ .../__tests__/upload-stream-lifecycle.test.ts | 24 ++--- .../__tests__/upload-stream-phases.test.ts | 81 ++++++++++++++-- .../src/__tests__/upload-stream.test.ts | 2 +- packages/remote/src/upload-client.ts | 14 ++- packages/remote/src/upload-stream.ts | 93 ++++++++++++------- test/wire-compat/ledger.json | 30 +++--- 8 files changed, 285 insertions(+), 91 deletions(-) diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts index e06c5d6d21..2384e5e560 100644 --- a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -96,10 +96,9 @@ export function startUploadAttempt(options: { method: options.method ?? 'POST', headers: options.headers ?? { authorization: 'Bearer daemon-token' }, payloadPath: options.artifactPath, - timeoutMessage: 'Artifact upload timed out', errorMessage: 'Failed to upload artifact to remote daemon', ...(options.retryable === undefined ? {} : { retryable: options.retryable }), - ...(options.signal ? { signal: options.signal } : {}), + signal: options.signal ?? new AbortController().signal, }).then( (value) => ({ value, error: undefined as unknown }), (error: unknown) => ({ value: undefined as UploadStreamResponse | undefined, error }), @@ -157,18 +156,18 @@ export async function spendAttemptBudget(ms: number, attempt: SettledUploadAttem export async function spendProgressWindows( daemon: ParkedUploadDaemon, upload: SettledUpload, - options: { windows: number; windowMs: number }, + options: { windows: number; windowMs: number; writes: OutgoingWrites }, ): Promise<{ progressedInEveryWindow: boolean; budgetMs: number }> { await spendUntil({ ready: () => daemon.receivedBytes > 0, maxMs: 60_000 }); let progressedInEveryWindow = true; let spent = 0; for (let window = 0; window < options.windows; window += 1) { const bodyInFlight = !daemon.bodyEnded; - const bytesBeforeWindow = daemon.receivedBytes; + const before = { bytes: daemon.receivedBytes, completed: options.writes.completed }; daemon.releaseAllParked(); // With the body fully taken there is nothing left to move and only a response is owed, so the // window is charged without a growth requirement. - const grew = bodyInFlight ? await awaitBytesGrowth(daemon, bytesBeforeWindow) : true; + const grew = bodyInFlight ? await awaitBodyProgress(daemon, options.writes, before) : true; await vi.advanceTimersByTimeAsync(options.windowMs); spent += options.windowMs; if (upload.settled) break; @@ -177,14 +176,66 @@ export async function spendProgressWindows( return { progressedInEveryWindow, budgetMs: spent }; } -/** Lets the real loop run until the peer has taken at least one more byte of the parked body. */ -async function awaitBytesGrowth(daemon: ParkedUploadDaemon, bytesBefore: number): Promise { +/** + * Lets the real loop run until the client completed at least one more body write and the peer took + * at least one more byte. A completed write is the client's body progress; peer growth shows those + * bytes really crossed the socket. + */ +export async function awaitBodyProgress( + daemon: { readonly receivedBytes: number; releaseAllParked(): void }, + writes: OutgoingWrites, + before: { bytes: number; completed: number }, +): Promise { + const progressed = () => + writes.completed > before.completed && daemon.receivedBytes > before.bytes; for (let round = 0; round < 400; round += 1) { - if (daemon.receivedBytes > bytesBefore) return true; + if (progressed()) return true; daemon.releaseAllParked(); await pumpRealWork(15); } - return daemon.receivedBytes > bytesBefore; + return progressed(); +} + +/** The body writes of every outgoing `node:http` request, totalled over the legs of one upload. */ +export type OutgoingWrites = { + /** `write` calls that returned `false`. */ + readonly falseReturns: number; + /** Writes whose callback reported success: bytes handed to the local kernel. */ + readonly completed: number; + readonly drains: number; + /** True once the newest leg emitted `finish`: its whole body reached the transport. */ + readonly finished: boolean; +}; + +/** + * Records the body writes of every outgoing `node:http` request; `vi.restoreAllMocks` ends it. + * + * The wrapper goes on each request as it is created, before the subject observes it, so the + * subject's own write observation calls through this one and both see the same completions. + */ +export function recordOutgoingWrites(): OutgoingWrites { + const log = { falseReturns: 0, completed: 0, drains: 0, finished: false }; + const request = http.request; + vi.spyOn(http, 'request').mockImplementation((...args) => { + const req = Reflect.apply(request, http, args) as http.ClientRequest; + log.finished = false; + const write = req.write; + req.write = function (this: http.ClientRequest, ...writeArgs: unknown[]): boolean { + const callbackIndex = typeof writeArgs[1] === 'function' ? 1 : 2; + const callback = writeArgs[callbackIndex]; + writeArgs[callbackIndex] = (error?: Error | null) => { + if (!error) log.completed += 1; + if (typeof callback === 'function') callback(error); + }; + const accepted: boolean = Reflect.apply(write, this, writeArgs); + if (!accepted) log.falseReturns += 1; + return accepted; + }; + req.on('drain', () => (log.drains += 1)); + req.on('finish', () => (log.finished = true)); + return req; + }); + return log; } /** @@ -265,6 +316,7 @@ export async function startParkedUploadDaemon( let bodyEnded = false; let uploadLegs = 0; let newestParkedLeg: http.IncomingMessage | undefined; + let readingToEnd = false; const server = http.createServer((req, res) => { const key = `${req.method} ${req.url}`; @@ -298,6 +350,7 @@ export async function startParkedUploadDaemon( // then backpressured for the rest of the window, so an artifact sized in megabytes is still // mid-flight when the next window is charged. bytesTakenSinceRelease += chunk.length; + if (readingToEnd) return; req.pause(); if (bytesTakenSinceRelease < RELEASE_QUOTA_BYTES) parked.push(req); else parkedAtQuota = req; @@ -338,6 +391,12 @@ export async function startParkedUploadDaemon( } for (const entry of resumable) entry.resume(); }, + readToEnd() { + readingToEnd = true; + for (const entry of parked.splice(0)) entry.resume(); + parkedAtQuota?.resume(); + parkedAtQuota = undefined; + }, resetParkedLeg() { assert.ok(newestParkedLeg, 'no parked leg to reset'); newestParkedLeg.socket.resetAndDestroy(); @@ -355,6 +414,8 @@ export type ParkedUploadDaemon = { readonly bodyEnded: boolean; /** Lets the parked body take up to one more quota of bytes. */ releaseAllParked(): void; + /** Stops parking: the daemon reads every remaining body byte as it arrives, still never answering. */ + readToEnd(): void; /** Resets the connection of the newest parked leg: a real peer reset, not a clean close. */ resetParkedLeg(): void; close(): Promise; diff --git a/packages/remote/src/__tests__/upload-client-cancellation.test.ts b/packages/remote/src/__tests__/upload-client-cancellation.test.ts index c5ebb6f4af..74795076f5 100644 --- a/packages/remote/src/__tests__/upload-client-cancellation.test.ts +++ b/packages/remote/src/__tests__/upload-client-cancellation.test.ts @@ -145,6 +145,59 @@ test('an aborted signal ends a legacy upload mid-stream instead of finishing the } }); +test('a cancel during the retried direct upload ends the upload without announcing a fallback', async () => { + const artifactPath = createTempFile('app.apk', 'retried-direct-payload'); + const control = new AbortController(); + const reason = new Error('request owner stopped'); + const requests: string[] = []; + const events: string[] = []; + let ticketLegs = 0; + const server = await startServer(async (req, res) => { + requests.push(`${req.method} ${req.url}`); + if (req.url === '/upload/preflight') { + await readRequestBody(req); + sendJson(res, { + ok: true, + uploadId: 'retried-ticket', + upload: { url: `${server.baseUrl}/ticket`, headers: {} }, + }); + return; + } + if (req.url === '/ticket') { + ticketLegs += 1; + // Leg one breaks as a retryable transport failure; the caller cancels during leg two. + if (ticketLegs === 1) req.destroy(); + else control.abort(reason); + return; + } + res.statusCode = 404; + res.end('not found'); + }); + + try { + await assert.rejects( + uploadArtifact({ + env: process.env, + localPath: artifactPath, + baseUrl: server.baseUrl, + token: TEST_TOKEN, + signal: control.signal, + onProgress: (event) => events.push(event.type), + }), + (error: unknown) => error === reason, + ); + assert.equal(events.includes('fallback'), false, events.join(',')); + assert.deepEqual(requests, [ + 'POST /upload/preflight', + 'PUT /ticket', + 'POST /upload/preflight', + 'PUT /ticket', + ]); + } finally { + await server.close(); + } +}); + test('an aborted signal before preflight refuses to ask the daemon for a ticket', async () => { const artifactPath = createTempFile('app.apk', 'payload'); const control = new AbortController(); diff --git a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts index 19201cbff9..8a0f8da676 100644 --- a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts +++ b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts @@ -11,7 +11,6 @@ // Every case uses the attempt seam, a controlled clock, and real loopback sockets: the overlap being // refused is only observable on a real socket, and phase budgets range from 30 to 120 seconds. -import http from 'node:http'; import { afterEach, assert, test, vi } from 'vitest'; import { isRetryableUploadStreamError, isTransferPhaseTimeoutError } from '../upload-stream.ts'; import { @@ -23,6 +22,7 @@ import { assertDetailedError, createArtifactFile, pumpRealWork, + recordOutgoingWrites, settleAttemptAfterBudget, spendAttemptBudget, startAnswerUploadDaemon, @@ -41,6 +41,7 @@ import { const MEBIBYTE = 1024 * 1024; afterEach(() => { + vi.restoreAllMocks(); vi.useRealTimers(); }); @@ -129,18 +130,7 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it const progressWindowMs = 30_000; assert.ok(progressWindowMs < TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); const artifactBytes = 32 * MEBIBYTE; - let outgoingLegs = 0; - let resumedDrains = 0; - const request = http.request; - const requestSpy = vi.spyOn(http, 'request').mockImplementation((...args) => { - const req = Reflect.apply(request, http, args) as http.ClientRequest; - outgoingLegs += 1; - if (outgoingLegs === 2) - req.on('drain', () => { - resumedDrains += 1; - }); - return req; - }); + const writes = recordOutgoingWrites(); const daemon = await startParkedUploadDaemon({ answerFirstLeg: { status: 308, headers: { 'x-upload-offset': String(MEBIBYTE) } }, }); @@ -159,15 +149,16 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it assert.ok(hopTwoTookBytes(daemon), 'the resumed hop was dialed and took bytes'); for (let window = 0; window < 6; window += 1) { - const drainsBefore = resumedDrains; + const writesBefore = writes.completed; const bytesBefore = daemon.receivedBytes; for (let round = 0; round < 400; round += 1) { daemon.releaseAllParked(); await pumpRealWork(15); - if (resumedDrains > drainsBefore && daemon.receivedBytes > bytesBefore) break; + if (writes.completed > writesBefore && daemon.receivedBytes > bytesBefore) break; } assert.ok(daemon.receivedBytes > bytesBefore, 'the peer took more resumed bytes'); - assert.ok(resumedDrains > drainsBefore, 'the resumed write queue drained'); + // Hop one was joined before hop two was dialed, so every new completion is hop two's. + assert.ok(writes.completed > writesBefore, 'the resumed hop completed more writes'); await vi.advanceTimersByTimeAsync(progressWindowMs); assert.equal(attempt.settled, false); assert.equal(daemon.bodyEnded, false); @@ -186,7 +177,6 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it assert.equal(isRetryableUploadStreamError(result.error), true); assert.equal(daemon.requests.filter((entry) => entry === LEGACY_UPLOAD_REQUEST).length, 2); } finally { - requestSpy.mockRestore(); await daemon.close(); } }); diff --git a/packages/remote/src/__tests__/upload-stream-phases.test.ts b/packages/remote/src/__tests__/upload-stream-phases.test.ts index 910b5d01e1..04c6fe5181 100644 --- a/packages/remote/src/__tests__/upload-stream-phases.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phases.test.ts @@ -6,6 +6,8 @@ // budget that ran out. A mocked transport could show neither, because the backpressure that makes // "still progressing" mean something belongs to the real socket. +import fs from 'node:fs'; +import { PassThrough } from 'node:stream'; import { afterEach, assert, test, vi } from 'vitest'; import { LEGACY_UPLOAD_REQUEST, @@ -14,23 +16,30 @@ import { TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, PREFLIGHT_REQUEST, assertDetailedError, + awaitBodyProgress, createArtifactFile, + pumpRealWork, + recordOutgoingWrites, settleAfterBudget, + settleAttemptAfterBudget, spendBudget, spendProgressWindows, startParkedUploadDaemon, startUpload, + startUploadAttempt, useTransferClock, } from './support/upload-transfer-fixtures.ts'; const MEBIBYTE = 1024 * 1024; afterEach(() => { + vi.restoreAllMocks(); vi.useRealTimers(); }); test('a body that keeps moving outlives the body-inactivity ceiling', async () => { useTransferClock(); + const writes = recordOutgoingWrites(); const daemon = await startParkedUploadDaemon(); const upload = startUpload({ baseUrl: daemon.baseUrl, @@ -45,6 +54,7 @@ test('a body that keeps moving outlives the body-inactivity ceiling', async () = const progress = await spendProgressWindows(daemon, upload, { windows: 8, windowMs: TRANSFER_CONNECT_TIMEOUT_MS, + writes, }); assert.equal(progress.progressedInEveryWindow, true); @@ -87,24 +97,38 @@ test('a body the peer stops taking expires the request-body phase on its own bud test('a body admitted past four connect ceilings still waits only its own header budget', async () => { useTransferClock(); + const writes = recordOutgoingWrites(); const daemon = await startParkedUploadDaemon(); const upload = startUpload({ baseUrl: daemon.baseUrl, - artifactPath: createArtifactFile(8 * 1024), + // Sixteen megabytes outlasts five release windows: the body is still in flight past four ceilings. + artifactPath: createArtifactFile(16 * MEBIBYTE), }); try { - // The first byte is written before any budget is spent, so the connect ceiling has already given - // way to the body phase: the body can cost more than four connect ceilings, and the wait that - // then expires is the header wait the withheld answer deserves. const progress = await spendProgressWindows(daemon, upload, { windows: 5, windowMs: TRANSFER_CONNECT_TIMEOUT_MS, + writes, }); - await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, upload); + assert.equal(progress.progressedInEveryWindow, true); + assert.ok(progress.budgetMs > 4 * TRANSFER_CONNECT_TIMEOUT_MS, `${progress.budgetMs}ms`); + assert.equal(writes.finished, false, 'the body was still being written'); + assert.equal(daemon.bodyEnded, false); + assert.equal(upload.settled, false); + + // The peer takes the rest with no fake time passing; `finish` starts the only wait that expires. + daemon.readToEnd(); + for (let round = 0; round < 400 && !(writes.finished && daemon.bodyEnded); round += 1) { + await pumpRealWork(15); + } + assert.equal(writes.finished && daemon.bodyEnded, true, 'the whole body reached the peer'); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1); + await pumpRealWork(); + assert.equal(upload.settled, false, 'the header wait counts from the flush'); + await vi.advanceTimersByTimeAsync(1); await settleAfterBudget(upload); const result = await upload; - assert.equal(progress.progressedInEveryWindow, true); assert.equal(result.value, undefined); assertDetailedError(result.error, { reason: 'transfer_phase_timeout', @@ -116,6 +140,51 @@ test('a body admitted past four connect ceilings still waits only its own header } }); +test('writes below the high-water mark keep the body budget on each completed write', async () => { + useTransferClock(); + // A slow producer hands the request 1 KiB at a time. Each write fits under the socket's high-water + // mark, so it returns true and no `drain` ever fires: the write's completion is the only report + // that its bytes reached the local kernel. + const source = new PassThrough(); + vi.spyOn(fs, 'createReadStream').mockReturnValueOnce(source as unknown as fs.ReadStream); + const writes = recordOutgoingWrites(); + const daemon = await startParkedUploadDaemon(); + const attempt = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(64 * 1024), + }); + try { + for (let window = 0; window < 6; window += 1) { + const before = { bytes: daemon.receivedBytes, completed: writes.completed }; + source.write(Buffer.alloc(1024, 'x')); + assert.equal(await awaitBodyProgress(daemon, writes, before), true, `window ${window}`); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS); + assert.equal(attempt.settled, false, `window ${window}`); + } + assert.ok(6 * TRANSFER_CONNECT_TIMEOUT_MS > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + assert.equal(writes.drains, 0); + assert.equal(writes.falseReturns, 0); + + // The producer stops. The last completed write was one window ago, so the body budget expires + // exactly one inactivity ceiling after it. + await vi.advanceTimersByTimeAsync( + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - TRANSFER_CONNECT_TIMEOUT_MS - 1, + ); + await pumpRealWork(); + assert.equal(attempt.settled, false); + await vi.advanceTimersByTimeAsync(1); + assert.equal((await settleAttemptAfterBudget(attempt)).settled, true); + assertDetailedError((await attempt).error, { + reason: 'transfer_phase_timeout', + transferPhase: 'request-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); + } finally { + source.destroy(); + await daemon.close(); + } +}); + test('the caller ending the upload is reported as cancellation, not as an expired phase', async () => { useTransferClock(); const daemon = await startParkedUploadDaemon(); diff --git a/packages/remote/src/__tests__/upload-stream.test.ts b/packages/remote/src/__tests__/upload-stream.test.ts index 92a4ebf1a7..6a148c483b 100644 --- a/packages/remote/src/__tests__/upload-stream.test.ts +++ b/packages/remote/src/__tests__/upload-stream.test.ts @@ -48,7 +48,7 @@ test('an upload refuses a redirect to another origin and never sends it the toke method: 'POST', headers: { authorization: 'Bearer daemon-token', 'transfer-encoding': 'chunked' }, payloadPath, - timeoutMessage: 'timed out', + signal: new AbortController().signal, errorMessage: 'failed', }), (error: unknown) => diff --git a/packages/remote/src/upload-client.ts b/packages/remote/src/upload-client.ts index 40919f6512..9c7bdb04d7 100644 --- a/packages/remote/src/upload-client.ts +++ b/packages/remote/src/upload-client.ts @@ -74,7 +74,9 @@ export async function uploadArtifact(options: UploadArtifactOptions): Promise { +async function uploadArtifactWithinBudget( + options: UploadArtifactOptions & { signal: AbortSignal }, +): Promise { const prepared = await prepareUploadArtifact( options.localPath, options.platform, @@ -146,7 +148,7 @@ async function tryDirectUploadWithResume(options: { preflight: Extract; uploadAttemptId: string; onProgress?: UploadProgressSink; - signal?: AbortSignal; + signal: AbortSignal; }): Promise { const uploadOnce = async ( preflight: Extract, @@ -233,7 +235,7 @@ async function uploadLegacyArtifact(options: { leaseId?: string; artifact: PreparedUploadArtifact; onProgress?: UploadProgressSink; - signal?: AbortSignal; + signal: AbortSignal; }): Promise { const { normalizedBase, token, leaseId, artifact } = options; const uploadUrl = new URL('upload', normalizedBase); @@ -253,8 +255,6 @@ async function uploadLegacyArtifact(options: { method: 'POST', headers, payloadPath: artifact.payloadPath, - timeoutMessage: 'Artifact upload timed out', - timeoutHint: 'Inspect the upload diagnostics before retrying the artifact.', errorMessage: 'Failed to upload artifact to remote daemon', errorHint: 'Verify the remote daemon is reachable and supports artifact uploads.', signal: options.signal, @@ -382,7 +382,7 @@ async function uploadDirectArtifact( leaseId?: string; artifact: PreparedUploadArtifact; onProgress?: UploadProgressSink; - signal?: AbortSignal; + signal: AbortSignal; }, ticket: Extract, ): Promise { @@ -398,8 +398,6 @@ async function uploadDirectArtifact( ? { ...ticket.headers, ...buildDaemonHttpLeaseHeaders(options.leaseId) } : ticket.headers, payloadPath: artifact.payloadPath, - timeoutMessage: 'Direct artifact upload timed out', - timeoutHint: 'The direct upload ticket did not accept the artifact within the timeout.', errorMessage: 'Failed to upload artifact with direct upload ticket', retryable: true, signal, diff --git a/packages/remote/src/upload-stream.ts b/packages/remote/src/upload-stream.ts index d36842e2a1..bab2666960 100644 --- a/packages/remote/src/upload-stream.ts +++ b/packages/remote/src/upload-stream.ts @@ -1,5 +1,5 @@ import fs from 'node:fs'; -import type { IncomingHttpHeaders, IncomingMessage } from 'node:http'; +import type { ClientRequest, IncomingHttpHeaders, IncomingMessage } from 'node:http'; import path from 'node:path'; import { pipeline } from 'node:stream/promises'; import { addAbortSignal } from 'node:stream'; @@ -120,18 +120,16 @@ const UPLOAD_PREPARATION_TIMEOUT_MS = 600_000; export async function withUploadPreparationDeadline( callerSignal: AbortSignal | undefined, prepare: (signal: AbortSignal) => Promise, - timeout?: Readonly<{ message: string; hint?: string }>, ): Promise { const deadline = new AbortController(); const signal = callerSignal ? AbortSignal.any([callerSignal, deadline.signal]) : deadline.signal; signal.throwIfAborted(); const timer = setTimeout(() => { deadline.abort( - new AppError('COMMAND_FAILED', timeout?.message ?? 'Artifact preparation deadline exceeded', { + new AppError('COMMAND_FAILED', 'Artifact preparation deadline exceeded', { reason: 'artifact_preparation_deadline_exceeded', transferPhase: 'preparation', timeoutMs: UPLOAD_PREPARATION_TIMEOUT_MS, - ...(timeout?.hint ? { hint: timeout.hint } : {}), }), ); }, UPLOAD_PREPARATION_TIMEOUT_MS); @@ -168,27 +166,17 @@ export async function streamFileToHttpRequest(options: { method: 'POST' | 'PUT'; headers: Record; payloadPath: string; - timeoutMessage: string; - timeoutHint?: string; errorMessage: string; errorHint?: string; retryable?: boolean; - /** Ends this request when the work the upload serves is over; see `uploadArtifact`. */ - signal?: AbortSignal; + /** + * The preparation aggregate `uploadArtifact` opened, combined with its caller's cancellation. Every + * attempt, redirect and resume runs under it; none of them opens a deadline of its own. + */ + signal: AbortSignal; progress?: UploadStreamProgressOptions; }): Promise { - return await withUploadPreparationDeadline( - options.signal, - async (signal) => - await streamFileToHttpRequestAttempt({ - ...options, - signal, - url: options.url, - redirectCount: 0, - startOffset: 0, - }), - { message: options.timeoutMessage, hint: options.timeoutHint }, - ); + return await streamFileToHttpRequestAttempt({ ...options, redirectCount: 0, startOffset: 0 }); } export function isRetryableUploadStreamError(error: unknown): boolean { @@ -200,17 +188,15 @@ async function streamFileToHttpRequestAttempt(options: { method: 'POST' | 'PUT'; headers: Record; payloadPath: string; - timeoutMessage: string; - timeoutHint?: string; errorMessage: string; errorHint?: string; retryable?: boolean; - signal?: AbortSignal; + signal: AbortSignal; redirectCount: number; startOffset: number; progress?: UploadStreamProgressOptions; }): Promise { - options.signal?.throwIfAborted(); + options.signal.throwIfAborted(); // This module sits in the CLI's eager import closure via the remote-artifact // upload client, but only a remote daemon ever uploads, so the HTTP stack // loads here rather than at import time. @@ -236,10 +222,12 @@ async function streamFileToHttpRequestAttempt(options: { let transportEstablished = false; let bodyFlushed = false; const settle = (result: UploadStreamResponse) => { + stopObservingWrites(); watchdog.dispose(); resolve(result); }; const fail = (error: unknown) => { + stopObservingWrites(); watchdog.dispose(); // Whatever is still queued on this attempt's own outgoing pipeline has to settle before the // join in `finally`, so a failure that ends the attempt destroys the request too. Once the @@ -265,16 +253,6 @@ async function streamFileToHttpRequestAttempt(options: { watchdog.arm('request-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); } }; - // Body progress is what the socket accepted, which `drain` reports when a previously full write - // queue empties. Chunks merely read ahead of the request, or throttled UI progress events, say - // nothing about bytes leaving this machine, so they are not progress against any budget. - const noteWriteProgress = () => { - // A request-side drain says nothing about the response; once an answer is streaming, only - // response bytes may re-arm the inactivity budget. - if (responseReceived) return; - watchdog.touchBody(); - }; - const req = transport.request( { protocol: options.url.protocol, @@ -318,7 +296,13 @@ async function streamFileToHttpRequestAttempt(options: { advanceOutgoingPhase(); } }); - req.on('drain', noteWriteProgress); + // Body progress is a completed body write: its bytes reached the local kernel, not the peer. + // Read-ahead chunks, queued writes and UI progress are not progress, and a completion after the + // response, an abort or a destroy belongs to an exchange with no request-body phase left. + const stopObservingWrites = observeCompletedWrites(req, () => { + if (responseReceived || req.destroyed || watchdog.signal.aborted) return; + watchdog.touchBody(); + }); req.on('error', (err) => { if (responseReceived) return; @@ -404,6 +388,7 @@ async function streamFileToHttpRequestAttempt(options: { // redirect/resume starts — old bytes may never overlap the next hop. const body = await readNodeHttpResponseBody(res, () => watchdog.touchBody()); await endOutgoingWork(); + stopObservingWrites(); watchdog.dispose(); const statusCode = res.statusCode ?? 500; @@ -460,6 +445,44 @@ async function streamFileToHttpRequestAttempt(options: { }); } +/** + * Reports each non-empty `req.write` chunk that Node completes without an error, until stopped. + * + * `pipeline` writes without a callback, and `drain` fires only after a write returned `false`. The + * wrapper passes the arguments, `this` and the backpressure result through, and runs the caller's + * callback exactly once with the same error. Stopping restores `write` only while this wrapper is + * still installed, and completions after the stop are not reported. + */ +function observeCompletedWrites(req: ClientRequest, onCompleted: () => void): () => void { + const ownWrite = Object.getOwnPropertyDescriptor(req, 'write'); + const write = req.write; + let observing = true; + const observedWrite = function (this: ClientRequest, ...args: unknown[]): boolean { + const [chunk, encoding] = args; + const callbackIndex = typeof encoding === 'function' ? 1 : 2; + const callback = args[callbackIndex]; + const hasBytes = + typeof chunk === 'string' + ? chunk.length > 0 + : chunk instanceof Uint8Array && chunk.byteLength > 0; + if (observing && hasBytes) { + args[callbackIndex] = (error?: Error | null) => { + if (!error && observing) onCompleted(); + if (typeof callback === 'function') callback(error); + }; + } + const accepted: boolean = Reflect.apply(write, this, args); + return accepted; + }; + req.write = observedWrite; + return () => { + observing = false; + if (req.write !== observedWrite) return; + if (ownWrite) Object.defineProperty(req, 'write', ownWrite); + else Reflect.deleteProperty(req, 'write'); + }; +} + function retryableUploadStreamDetail(options: { retryable?: boolean }): Record { return options.retryable === true ? { [UPLOAD_STREAM_RETRYABLE_DETAIL]: true } : {}; } diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index 13b9ef2eef..ab27ac2820 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -169,9 +169,9 @@ "packages/remote/src/upload-client.ts#parseUploadPreflightResult": "sha256:7ce3f43b45c03df8b7f2690432beedc7a537c1bba540961716862f8a632d5623", "packages/remote/src/upload-client.ts#requestUploadPreflight": "sha256:e5f67a9a7799913f04cc9386291296a3d9fe13658776d2f4812a582c2e9befd5", "packages/remote/src/upload-client.ts#shouldRetryDirectUpload": "sha256:29059f076ff7f2b9540ad15408be48792b2d63a601ef2b9fb2d54f5576d49bbd", - "packages/remote/src/upload-client.ts#tryDirectUploadWithResume": "sha256:0f55d5c81abc9010ecdbb852da81793515abab7c4d54e9f7c8b7f502e79a54a6", - "packages/remote/src/upload-client.ts#uploadDirectArtifact": "sha256:3edb622eabee720085a544a4d4e6fdd74c19ec25d3e14e7f9f06ea10d550c1c7", - "packages/remote/src/upload-client.ts#uploadLegacyArtifact": "sha256:887c6a1b18dd16a51dbad89f3b8ac99727ed8fae87759c5d806f22f326a15da1", + "packages/remote/src/upload-client.ts#tryDirectUploadWithResume": "sha256:8b1b35bcac4977f79fdc126de1fe476fd1439e2b1a65b56f53b82366421e6986", + "packages/remote/src/upload-client.ts#uploadDirectArtifact": "sha256:93a36a9d3dea88064569517f3cdc34f27b33df6bf5f0c0ecc9efc29b47f24b77", + "packages/remote/src/upload-client.ts#uploadLegacyArtifact": "sha256:1546c696134b47bfaaaffcf95deb176cccdce243818b658ae0494dafb57fafb6", "packages/remote/src/upload-stream.ts#MAX_UPLOAD_REDIRECTS": "sha256:12ff46dfe33a1dce95bdb79c90c17a3ba4f3919e7dc4b3a1b04590fff430f79f", "packages/remote/src/upload-stream.ts#UploadStreamResponse": "sha256:724e284c2a6bd7b4b3a49b1d8c61c8923d6cc24777c3d76224a69d190e56adce", "packages/remote/src/upload-stream.ts#buildUploadRequestHeaders": "sha256:827576b13a9f3e6cea445119a6a29b2ac5eaa23b66436cf810b2d09765c82a4c", @@ -180,8 +180,8 @@ "packages/remote/src/upload-stream.ts#isUploadResumeStatus": "sha256:9f730ccd1c3401c326f8df67d1fd8243fef757357a38a1f53736df2b90bd8fdc", "packages/remote/src/upload-stream.ts#parseNonNegativeIntegerHeader": "sha256:5cbe9c6994459d673d988a51407e65556e28175b187ce0a9d27cbb6331962834", "packages/remote/src/upload-stream.ts#parseUploadResumeOffset": "sha256:297184ea22a6dc1cda9aee17034122ae0ea2f43d0355647f6fcf02f3f4b5ca0e", - "packages/remote/src/upload-stream.ts#streamFileToHttpRequest": "sha256:3ac8f379c4b508b0c28c4f41f920b40f8cc0406afcd5b2290326c4cfb9168adb", - "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:5ba978e2740430e6ed03e06a1e4e056115fa393a3e32658739b97a444aa939fa", + "packages/remote/src/upload-stream.ts#streamFileToHttpRequest": "sha256:c2b5e1369c8b1c9420036e23163d21e43fa2ac8b889c6a8f028cc6095055a16f", + "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:38cf753a537423c93e48d89b1fc45eb0237180ce4f3886173660718d1fcfb79e", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_HEADER": "sha256:d2bb3f4846470188a0fc1724271da0f0bcbdf2337bd6bf331508142cadcb65d8", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_UPSTREAM_INSTANCE_HEADER": "sha256:4564c2ce437d30d7a823e3326f4375c0750ba8d42bec22b4d02e8eb7f7db2973", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_MISMATCH_HEADER": "sha256:a197edaf7f9fa94a6aab269856aa852da3204cc586f5fd23198689eb11908b61", @@ -330,8 +330,8 @@ }, { "declaration": "packages/remote/src/upload-stream.ts#streamFileToHttpRequest", - "digest": "sha256:3ac8f379c4b508b0c28c4f41f920b40f8cc0406afcd5b2290326c4cfb9168adb", - "rationale": "#4441 replaces the one 5-minute timer over a whole upload with the preparation aggregate that bounds it, and hands that aggregate's signal to the attempt. The request a peer parses is unchanged: same route, method, the headers buildUploadRequestHeaders produces, the piped payload and the 200/308 handling. A progressing upload may now consume the shared ten-minute aggregate instead of the former five-minute per-attempt ceiling. Connection, header and body-inactivity phase limits stop stalled work sooner; retries and resumes never restart the aggregate. A protocol-2 daemon or proxy parses the same requests, while client-side timeout and retry timing deliberately change." + "digest": "sha256:c2b5e1369c8b1c9420036e23163d21e43fa2ac8b889c6a8f028cc6095055a16f", + "rationale": "#4441 replaces the one 5-minute timer over a whole upload with the preparation aggregate that bounds it, and hands that aggregate's signal to the attempt. The request a peer parses is unchanged: same route, method, the headers buildUploadRequestHeaders produces, the piped payload and the 200/308 handling. A progressing upload may now consume the shared ten-minute aggregate instead of the former five-minute per-attempt ceiling. Connection, header and body-inactivity phase limits stop stalled work sooner; retries and resumes never restart the aggregate. A protocol-2 daemon or proxy parses the same requests, while client-side timeout and retry timing deliberately change. Its only callers run inside uploadArtifact's aggregate, so it no longer opens a second copy of that deadline: it requires the aggregate's signal and drops the timeout message and hint that the outer deadline always preempted." }, { "declaration": "packages/contracts/src/daemon-http.ts#DaemonHealthPayload", @@ -435,18 +435,18 @@ }, { "declaration": "packages/remote/src/upload-client.ts#uploadDirectArtifact", - "digest": "sha256:3edb622eabee720085a544a4d4e6fdd74c19ec25d3e14e7f9f06ea10d550c1c7", - "rationale": "Clients name a held agent-device lease in an optional x-agent-device-lease-id request header on upload, artifact download and request diagnostics HTTP calls, so a gateway can route them. Released daemons ignore unknown request headers and the proxy forwards only its allowlist, so routes, bodies and responses parse as before; with no lease held the request is unchanged (ADR 0006)." + "digest": "sha256:93a36a9d3dea88064569517f3cdc34f27b33df6bf5f0c0ecc9efc29b47f24b77", + "rationale": "Clients name a held agent-device lease in an optional x-agent-device-lease-id request header on upload, artifact download and request diagnostics HTTP calls, so a gateway can route them. Released daemons ignore unknown request headers and the proxy forwards only its allowlist, so routes, bodies and responses parse as before; with no lease held the request is unchanged (ADR 0006). #4441 also stops passing a timeout message and hint that no caller could see, and requires the preparation aggregate's signal. The ticket PUT keeps its route, method, headers and body." }, { "declaration": "packages/remote/src/upload-client.ts#tryDirectUploadWithResume", - "digest": "sha256:0f55d5c81abc9010ecdbb852da81793515abab7c4d54e9f7c8b7f502e79a54a6", - "rationale": "#4441 ends the direct-upload sequence when a transfer phase's own budget expires, instead of re-preflighting for a fresh ticket or falling back, because a second attempt would be permission that budget already refused. Preflight, ticket PUT and finalize requests are unchanged in route, method, headers and body, so a protocol-2 daemon parses everything it parsed before; it just receives no further legs once the caller's aggregate has expired." + "digest": "sha256:8b1b35bcac4977f79fdc126de1fe476fd1439e2b1a65b56f53b82366421e6986", + "rationale": "#4441 ends the direct-upload sequence when a transfer phase's own budget expires, instead of re-preflighting for a fresh ticket or falling back, because a second attempt would be permission that budget already refused. Preflight, ticket PUT and finalize requests are unchanged in route, method, headers and body, so a protocol-2 daemon parses everything it parsed before; it just receives no further legs once the caller's aggregate has expired. Its signal is now required, because the preparation aggregate is always the one passed down; no request changes." }, { "declaration": "packages/remote/src/upload-client.ts#uploadLegacyArtifact", - "digest": "sha256:887c6a1b18dd16a51dbad89f3b8ac99727ed8fae87759c5d806f22f326a15da1", - "rationale": "#4441 drops the hint that named the removed 5-minute timeout, and an expired transfer phase is no longer treated as a transport break, so this route is not re-entered by a fallback after the client's own budget ended the attempt. The POST /upload request itself is unchanged: same method, headers, chunked body and response parsing. A protocol-2 daemon sees the identical request in every case that completes, and no further request in a stalled one." + "digest": "sha256:1546c696134b47bfaaaffcf95deb176cccdce243818b658ae0494dafb57fafb6", + "rationale": "#4441 drops the hint that named the removed 5-minute timeout, and an expired transfer phase is no longer treated as a transport break, so this route is not re-entered by a fallback after the client's own budget ended the attempt. The POST /upload request itself is unchanged: same method, headers, chunked body and response parsing. A protocol-2 daemon sees the identical request in every case that completes, and no further request in a stalled one. The unreachable timeout message and hint are no longer passed, and the aggregate signal is required." }, { "declaration": "packages/remote/src/artifact-download.ts#downloadRemoteArtifactFromUrl", @@ -495,8 +495,8 @@ }, { "declaration": "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt", - "digest": "sha256:5ba978e2740430e6ed03e06a1e4e056115fa393a3e32658739b97a444aa939fa", - "rationale": "#4441 keeps each attempt under its sequential phase watchdog and corrects its transport lifecycle: the connect ceiling now ends at actual socket or TLS-handshake establishment, body progress counts only transport writes, and an early response reads the accepted answer to its end, then destroys and joins the owned file/request pipeline and retires this attempt's phase watchdog before resolving or starting the redirect/resume attempt, so a completed hop's timers never reach the next hop. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own sequencing and deadlines moved, so a protocol-2 peer parses exactly what it parsed before." + "digest": "sha256:38cf753a537423c93e48d89b1fc45eb0237180ce4f3886173660718d1fcfb79e", + "rationale": "#4441 keeps each attempt under its sequential phase watchdog and corrects its transport lifecycle: the connect ceiling now ends at actual socket or TLS-handshake establishment, body progress counts only transport writes, and an early response reads the accepted answer to its end, then destroys and joins the owned file/request pipeline and retires this attempt's phase watchdog before resolving or starting the redirect/resume attempt, so a completed hop's timers never reach the next hop. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own sequencing and deadlines moved, so a protocol-2 peer parses exactly what it parsed before. Request-body progress is now each non-empty request write that completes without an error, which Node reports once the bytes reach the local kernel, instead of drain, which fires only after a write returned false. The write wrapper passes every argument and the backpressure result through, so the bytes and framing on the wire do not change." }, { "declaration": "packages/remote/src/daemon-artifacts.ts#resolveMaterializedArtifactPath", From 0eb01daa2e93767f8f2659911a9ca669827883de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 18:30:26 +0100 Subject: [PATCH 12/27] fix(transport): own completed request writes in host-kit and time fixtures on owned I/O Host-kit's transport owns observeCompletedRequestWrites: it reports each non-empty write Node completes without an error, encoded-empty strings excluded, and stops after the response, an abort, a destroy or disposal. The remote upload attempt consumes it instead of a private copy, and the generic contract has direct tests beside its owner. The loopback fixtures charge fake time only after the owned I/O a phase needs was observed, tie pending waits to the test lifecycle, assert the stalled body is unfinished, and prove the early-answer join with a producer that never ends instead of a host-speed round bound. Co-Authored-By: Claude Opus 5.5 --- .../src/internal/transport-writes.test.ts | 148 +++++ packages/host-kit/src/internal/transport.ts | 53 +- packages/host-kit/src/transport.ts | 1 + .../support/upload-transfer-fixtures.ts | 628 +++++++----------- .../__tests__/upload-stream-lifecycle.test.ts | 213 ++---- .../upload-stream-phase-budget.test.ts | 9 +- .../__tests__/upload-stream-phases.test.ts | 51 +- packages/remote/src/upload-client.ts | 18 +- packages/remote/src/upload-stream.ts | 136 +--- 9 files changed, 576 insertions(+), 681 deletions(-) create mode 100644 packages/host-kit/src/internal/transport-writes.test.ts diff --git a/packages/host-kit/src/internal/transport-writes.test.ts b/packages/host-kit/src/internal/transport-writes.test.ts new file mode 100644 index 0000000000..fbbe8c3161 --- /dev/null +++ b/packages/host-kit/src/internal/transport-writes.test.ts @@ -0,0 +1,148 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import type { ClientRequest } from 'node:http'; +import { test } from 'vitest'; +import { observeCompletedRequestWrites } from '../transport.ts'; + +/** + * A request whose `write` the test completes by hand, as Node does once a chunk reaches the kernel. + * Every write, empty or not, holds its callback until `complete` runs it. + */ +class FakeRequest extends EventEmitter { + destroyed = false; + accepts = true; + readonly writes: { self: unknown; args: unknown[] }[] = []; + write(this: FakeRequest, ...args: unknown[]): boolean { + this.writes.push({ self: this, args }); + return this.accepts; + } + complete(index: number, error: Error | null = null): void { + const callback = this.writes[index]?.args.find((arg) => typeof arg === 'function'); + assert.ok(typeof callback === 'function', `write ${index} has no callback`); + callback(error); + } +} + +type Observed = { + fake: FakeRequest; + req: ClientRequest; + caller: AbortController; + completed: { count: number }; + dispose(): void; +}; + +function observe(prepare?: (fake: FakeRequest) => void): Observed { + const fake = new FakeRequest(); + prepare?.(fake); + const caller = new AbortController(); + const completed = { count: 0 }; + const req = fake as unknown as ClientRequest; + const dispose = observeCompletedRequestWrites(req, caller.signal, () => (completed.count += 1)); + return { fake, req, caller, completed, dispose }; +} + +test('only a non-empty write that completes without an error is reported', () => { + const { fake, req, completed, dispose } = observe(); + req.write(Buffer.from('queued')); + req.write('', () => {}); + fake.complete(1); + req.write(new Uint8Array(0), () => {}); + fake.complete(2); + req.write('failed', () => {}); + fake.complete(3, new Error('EPIPE')); + for (const [text, encoding] of [ + ['a', 'hex'], + ['zz', 'hex'], + ['=', 'base64'], + ['=', 'base64url'], + ] as const) { + req.write(text, encoding, () => {}); + fake.complete(fake.writes.length - 1); + } + assert.equal( + completed.count, + 0, + 'a queued, empty, failed or encoded-empty write is not progress', + ); + fake.complete(0); + assert.equal(completed.count, 1); + dispose(); +}); + +test.each([ + ['utf8', 'é'], + ['hex', 'ff'], + ['base64', 'YQ=='], + ['base64url', 'YQ'], +] as const)('a successful %s string write is reported', (encoding, text) => { + const { fake, req, completed, dispose } = observe(); + req.write(text, encoding); + fake.complete(0); + assert.equal(completed.count, 1); + dispose(); +}); + +test('the observed write keeps every argument, this, the result and one callback', () => { + const { fake, req, dispose } = observe(); + const calls: unknown[] = []; + const error = new Error('EPIPE'); + fake.accepts = false; + assert.equal( + req.write('a', 'latin1', (cause) => calls.push(cause)), + false, + ); + fake.accepts = true; + assert.equal( + req.write('b', (cause) => calls.push(cause)), + true, + ); + assert.equal(req.write('c'), true); + assert.deepEqual( + fake.writes.map(({ self, args }) => [self === fake, ...args.map((arg) => typeof arg)]), + [ + [true, 'string', 'string', 'function'], + [true, 'string', 'function'], + [true, 'string', 'undefined', 'function'], + ], + ); + assert.equal(fake.writes[0]?.args[1], 'latin1'); + fake.complete(0, error); + fake.complete(1); + fake.complete(2); + assert.deepEqual(calls, [error, null]); + dispose(); +}); + +const endings: [string, (observed: Observed) => unknown][] = [ + ['the response started', ({ fake }) => fake.emit('response')], + ['the signal aborted', ({ caller }) => caller.abort()], + ['the request was destroyed', ({ fake }) => (fake.destroyed = true)], + ['the observer was disposed', ({ dispose }) => dispose()], +]; + +test.each(endings)('a completion after %s is not reported', (_, end) => { + const observed = observe(); + const calls: unknown[] = []; + observed.req.write(Buffer.from('late'), (cause) => calls.push(cause)); + end(observed); + observed.fake.complete(0); + assert.equal(observed.completed.count, 0); + assert.deepEqual(calls, [null], 'the caller still hears its own completion'); + observed.dispose(); +}); + +test('disposing removes the listener and restores only the write it wrapped', () => { + const inherited = observe(); + inherited.dispose(); + assert.equal(inherited.fake.listenerCount('response'), 0); + assert.equal(Object.hasOwn(inherited.fake, 'write'), false, 'the inherited write is back'); + const ownWrite = () => true; + const owned = observe((fake) => Object.assign(fake, { write: ownWrite })); + owned.dispose(); + assert.equal(owned.fake.write, ownWrite, 'an own write is put back'); + const wrapped = observe(); + const foreign = () => true; + Object.assign(wrapped.fake, { write: foreign }); + wrapped.dispose(); + assert.equal(wrapped.fake.write, foreign, 'a later wrapper stays installed'); +}); diff --git a/packages/host-kit/src/internal/transport.ts b/packages/host-kit/src/internal/transport.ts index 90e68c6cc6..f80b58637d 100644 --- a/packages/host-kit/src/internal/transport.ts +++ b/packages/host-kit/src/internal/transport.ts @@ -1,5 +1,5 @@ import crypto from 'node:crypto'; -import type { IncomingMessage } from 'node:http'; +import type { ClientRequest, IncomingMessage } from 'node:http'; import { AppError } from '@agent-device/kernel/errors'; export function consumeTextLines( @@ -86,3 +86,54 @@ export function timingSafeStringEqual(a: string, b: string): boolean { const hashB = crypto.createHash('sha256').update(b).digest(); return crypto.timingSafeEqual(hashA, hashB); } + +/** + * Reports each non-empty `req.write` chunk that Node completes without an error: its bytes reached + * the local kernel, not the peer. A queued write, `drain`, and a completion after the response + * started, after `signal` aborted, after a destroy or after disposal are not reported. + * + * Attach it in the tick that creates the request, before anything writes to it. The wrapper keeps + * the arguments, `this`, the backpressure result and one call of the caller's callback with its + * error. Disposing removes its listener and restores `write` only while this wrapper is installed. + */ +export function observeCompletedRequestWrites( + req: ClientRequest, + signal: AbortSignal, + onCompleted: () => void, +): () => void { + const ownWrite = Object.getOwnPropertyDescriptor(req, 'write'); + const write = req.write; + let observing = true; + const onResponse = () => { + observing = false; + }; + const observedWrite = function (this: ClientRequest, ...args: unknown[]): boolean { + const [chunk, encoding] = args; + const callbackIndex = typeof encoding === 'function' ? 1 : 2; + const callback = args[callbackIndex]; + const encodingName = typeof encoding === 'string' ? encoding.toLowerCase() : undefined; + const hasBytes = + typeof chunk === 'string' + ? encodingName === 'hex' || encodingName === 'base64' || encodingName === 'base64url' + ? Buffer.from(chunk, encodingName).byteLength > 0 + : chunk.length > 0 + : chunk instanceof Uint8Array && chunk.byteLength > 0; + if (observing && hasBytes) { + args[callbackIndex] = (error?: Error | null) => { + if (!error && observing && !req.destroyed && !signal.aborted) onCompleted(); + if (typeof callback === 'function') callback(error); + }; + } + const accepted: boolean = Reflect.apply(write, this, args); + return accepted; + }; + req.write = observedWrite; + req.prependListener('response', onResponse); + return () => { + observing = false; + req.off('response', onResponse); + if (req.write !== observedWrite) return; + if (ownWrite) Object.defineProperty(req, 'write', ownWrite); + else Reflect.deleteProperty(req, 'write'); + }; +} diff --git a/packages/host-kit/src/transport.ts b/packages/host-kit/src/transport.ts index a2d250af38..6b55289d46 100644 --- a/packages/host-kit/src/transport.ts +++ b/packages/host-kit/src/transport.ts @@ -1,6 +1,7 @@ export { consumeTextLines, loadNodeHttpRequester, + observeCompletedRequestWrites, readNodeHttpRequestBody, readNodeHttpResponseBody, timingSafeStringEqual, diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts index 2384e5e560..063ef35b90 100644 --- a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -1,10 +1,6 @@ -// Shared support for the tests that put real budget pressure on a real artifact upload (#4441). -// -// Every helper here exists so a test can spend minutes of transfer budget without waiting for them: -// the daemon holds the request body open chunk by chunk and the test drives the clock. The socket, -// the framing and the client's own stream stay real, which is the point — a mocked transport cannot -// prove that a transfer which keeps moving is never cut short, nor that one which stops is, nor -// where an owned pipeline was still writing when an attempt ended. +// Support for the tests that put real budget pressure on a real artifact upload (#4441). The peer +// holds the body open chunk by chunk and the test drives the clock; sockets, framing and the client's +// own stream stay real, because a mocked transport cannot show what an owned pipeline was doing. import assert from 'node:assert/strict'; import fs from 'node:fs'; @@ -12,8 +8,9 @@ import http from 'node:http'; import https from 'node:https'; import net from 'node:net'; import path from 'node:path'; +import { PassThrough } from 'node:stream'; import tls from 'node:tls'; -import { vi } from 'vitest'; +import { onTestFinished, vi } from 'vitest'; import { closeLoopbackServer, listenOnLoopback, mkdtempForTestSync } from '@agent-device/test-kit'; import { uploadArtifact } from '../../upload-client.ts'; import { type UploadStreamResponse, streamFileToHttpRequest } from '../../upload-stream.ts'; @@ -26,62 +23,92 @@ export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; export const PREFLIGHT_REQUEST = 'POST /upload/preflight'; export const LEGACY_UPLOAD_REQUEST = 'POST /upload'; -/** Budget step used when a test just needs time to pass: fine enough that no phase boundary is - * stepped over, cheap enough that six minutes of budget stay a few hundred loop turns. */ +/** Fine enough that no phase boundary is stepped over, cheap enough for minutes of budget. */ const BUDGET_STEP_MS = 5_000; -/** Ceiling on what the parked daemon takes between two progress windows. Small enough that an - * artifact sized in megabytes cannot be drained by the windows a test runs, and large enough that - * one release always moves real bytes over the socket. */ +/** What the parked daemon takes per release: megabytes outlast the windows a test runs. */ const RELEASE_QUOTA_BYTES = 256 * 1024; -export type UploadOutcome = { value: string | undefined; error: unknown }; +/** Anything whose settlement a test observes without awaiting it. */ +export type Settles = { readonly settled: boolean }; -/** An upload whose completion can be observed without awaiting it, which a controlled clock needs. */ -export type SettledUpload = Promise & { settled: boolean }; +/** The pending `untilIo` waits, each with the rejection the end of its test delivers. */ +const ioWaiters = new Map<() => void, () => void>(); +let ioWaitersEndWithTest = false; + +/** Called by every owned I/O callback below, after it records what it observed. */ +function noteIo(): void { + for (const wake of ioWaiters.keys()) wake(); +} + +/** + * Resolves once `ready` holds, rechecked after each owned I/O event below. It charges no budget and + * counts no loop turns; a wait still pending when its test finishes is rejected with its stack. + */ +export async function untilIo(ready: () => boolean): Promise { + if (ready()) return; + const unmet = new Error('untilIo: the I/O this wait needs never happened before its test ended'); + if (!ioWaitersEndWithTest) { + ioWaitersEndWithTest = true; + onTestFinished(() => { + ioWaitersEndWithTest = false; + for (const release of ioWaiters.values()) release(); + ioWaiters.clear(); + }); + } + await new Promise((resolve, reject) => { + const wake = () => { + try { + if (!ready()) return; + ioWaiters.delete(wake); + resolve(); + } catch (error) { + ioWaiters.delete(wake); + reject(error); + } + }; + ioWaiters.set(wake, () => reject(unmet)); + }); +} /** Starts the clock every transfer budget in these tests is measured against. */ export function useTransferClock(): void { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); } -export function startUpload(options: { - baseUrl: string; - artifactPath: string; - signal?: AbortSignal; -}): SettledUpload { +/** An outcome observable without awaiting it, which a controlled clock needs. */ +export type ObservedOutcome = Promise<{ value: T | undefined; error: unknown }> & Settles; + +function observeOutcome(work: Promise): ObservedOutcome { let settled = false; - const upload = uploadArtifact({ - env: process.env, - localPath: options.artifactPath, - baseUrl: options.baseUrl, - token: 'daemon-token', - ...(options.signal ? { signal: options.signal } : {}), - }).then( + const outcome = work.then( (value) => ({ value, error: undefined as unknown }), - (error: unknown) => ({ value: undefined as string | undefined, error }), - ) as SettledUpload; - void upload.then(() => { + (error: unknown) => ({ value: undefined as T | undefined, error }), + ); + void outcome.then(() => { settled = true; + noteIo(); }); - Object.defineProperty(upload, 'settled', { get: () => settled }); - return upload; + return Object.defineProperty(outcome, 'settled', { get: () => settled }) as ObservedOutcome; } -/** The outcome of one `startUploadAttempt`, observable without awaiting it. */ -export type UploadAttemptOutcome = { - value: UploadStreamResponse | undefined; - error: unknown; -}; -export type SettledUploadAttempt = Promise & { settled: boolean }; +export function startUpload(options: { + baseUrl: string; + artifactPath: string; + signal?: AbortSignal; +}): ObservedOutcome { + return observeOutcome( + uploadArtifact({ + env: process.env, + localPath: options.artifactPath, + baseUrl: options.baseUrl, + token: 'daemon-token', + ...(options.signal ? { signal: options.signal } : {}), + }), + ); +} -/** - * One upload attempt straight at `url`, with its outcome observed rather than awaited. - * - * The attempt-level lifecycle — an early answer, the join that has to follow it, the phase that - * expires — belongs to `streamFileToHttpRequest`, so the tests that prove it drive that seam - * directly instead of routing a whole upload client around it. - */ +/** One attempt straight at `url`: the attempt lifecycle belongs to `streamFileToHttpRequest`. */ export function startUploadAttempt(options: { url: string; artifactPath: string; @@ -89,85 +116,50 @@ export function startUploadAttempt(options: { headers?: Record; retryable?: boolean; signal?: AbortSignal; -}): SettledUploadAttempt { - let settled = false; - const attempt = streamFileToHttpRequest({ - url: new URL(options.url), - method: options.method ?? 'POST', - headers: options.headers ?? { authorization: 'Bearer daemon-token' }, - payloadPath: options.artifactPath, - errorMessage: 'Failed to upload artifact to remote daemon', - ...(options.retryable === undefined ? {} : { retryable: options.retryable }), - signal: options.signal ?? new AbortController().signal, - }).then( - (value) => ({ value, error: undefined as unknown }), - (error: unknown) => ({ value: undefined as UploadStreamResponse | undefined, error }), - ) as SettledUploadAttempt; - void attempt.then(() => { - settled = true; - }); - Object.defineProperty(attempt, 'settled', { get: () => settled }); - return attempt; +}): ObservedOutcome { + return observeOutcome( + streamFileToHttpRequest({ + url: new URL(options.url), + method: options.method ?? 'POST', + headers: options.headers ?? { authorization: 'Bearer daemon-token' }, + payloadPath: options.artifactPath, + errorMessage: 'Failed to upload artifact to remote daemon', + ...(options.retryable === undefined ? {} : { retryable: options.retryable }), + signal: options.signal ?? new AbortController().signal, + }), + ); } /** - * Advances the controlled clock while letting real socket work run between the steps, until `ready` - * holds or the transfer is over. - * - * Body progress re-arms an inactivity budget, so a stalled transfer is cut off only once the steps - * stop being interrupted by bytes — exactly the condition under test. A settled transfer ends the - * wait: a budget that fired is an answer, not a reason to keep ticking. + * Spends `ms` of budget in `BUDGET_STEP_MS` steps, letting real socket work run between them, until + * the transfer is over. A budget that fired is an answer, not a reason to keep ticking. */ -async function spendUntil(options: { - ready: () => boolean; - maxMs: number; - settled?: () => boolean; -}): Promise { - const { ready, maxMs, settled } = options; - for (let elapsed = 0; elapsed < maxMs; elapsed += BUDGET_STEP_MS) { +export async function spendBudget(ms: number, transfer: Settles): Promise { + for (let elapsed = 0; elapsed < ms && !transfer.settled; elapsed += BUDGET_STEP_MS) { await vi.advanceTimersByTimeAsync(BUDGET_STEP_MS); await pumpRealWork(); - if (ready()) return true; - if (settled?.()) return ready(); } - return ready(); -} - -/** Spends `ms` of budget in `BUDGET_STEP_MS` steps without letting a single byte through. */ -export async function spendBudget(ms: number, upload: SettledUpload): Promise { - await spendUntil({ ready: () => false, maxMs: ms, settled: () => upload.settled }); -} - -/** Spends `ms` of budget in `BUDGET_STEP_MS` steps against a single attempt. */ -export async function spendAttemptBudget(ms: number, attempt: SettledUploadAttempt): Promise { - await spendUntil({ ready: () => false, maxMs: ms, settled: () => attempt.settled }); } /** - * Spends `windows` windows of `windowMs` each while the body keeps moving. - * - * Each window releases one quota of the parked body, waits with real event-loop turns until the peer - * really took more bytes, and only then charges `windowMs` to the transfer. Waiting before charging - * is what makes a window mean something: bytes cross the socket only while the real loop runs, so a - * window charged before they arrive measures the host's scheduling, not the transfer. Once the peer - * has taken the whole body the transfer has nothing left to move and only a response is owed, so - * later windows add budget without a growth requirement. + * Charges `windows` windows of `windowMs`, each only after one more completed write reached the + * peer, and nothing before the first. Once the peer has the whole body, no growth is required. */ export async function spendProgressWindows( daemon: ParkedUploadDaemon, - upload: SettledUpload, + upload: Settles, options: { windows: number; windowMs: number; writes: OutgoingWrites }, ): Promise<{ progressedInEveryWindow: boolean; budgetMs: number }> { - await spendUntil({ ready: () => daemon.receivedBytes > 0, maxMs: 60_000 }); + await untilIo(() => upload.settled || (options.writes.completed > 0 && daemon.receivedBytes > 0)); let progressedInEveryWindow = true; let spent = 0; for (let window = 0; window < options.windows; window += 1) { const bodyInFlight = !daemon.bodyEnded; const before = { bytes: daemon.receivedBytes, completed: options.writes.completed }; daemon.releaseAllParked(); - // With the body fully taken there is nothing left to move and only a response is owed, so the - // window is charged without a growth requirement. - const grew = bodyInFlight ? await awaitBodyProgress(daemon, options.writes, before) : true; + const grew = bodyInFlight + ? await awaitBodyProgress(daemon, options.writes, before, upload) + : true; await vi.advanceTimersByTimeAsync(options.windowMs); spent += options.windowMs; if (upload.settled) break; @@ -177,107 +169,145 @@ export async function spendProgressWindows( } /** - * Lets the real loop run until the client completed at least one more body write and the peer took - * at least one more byte. A completed write is the client's body progress; peer growth shows those - * bytes really crossed the socket. + * Waits until the client completed one more body write and the peer took one more byte, or the + * transfer settled. The parked peer is released again after each I/O event. */ export async function awaitBodyProgress( daemon: { readonly receivedBytes: number; releaseAllParked(): void }, writes: OutgoingWrites, before: { bytes: number; completed: number }, + transfer: Settles, ): Promise { const progressed = () => writes.completed > before.completed && daemon.receivedBytes > before.bytes; - for (let round = 0; round < 400; round += 1) { - if (progressed()) return true; + await untilIo(() => { + if (progressed() || transfer.settled) return true; daemon.releaseAllParked(); - await pumpRealWork(15); - } + return false; + }); return progressed(); } -/** The body writes of every outgoing `node:http` request, totalled over the legs of one upload. */ +/** The body writes of every outgoing request, totalled over the legs of one upload. */ export type OutgoingWrites = { /** `write` calls that returned `false`. */ readonly falseReturns: number; /** Writes whose callback reported success: bytes handed to the local kernel. */ readonly completed: number; readonly drains: number; + /** Controlled-clock time of the newest leg's latest completed write. */ + readonly lastCompletedAt: number | undefined; /** True once the newest leg emitted `finish`: its whole body reached the transport. */ readonly finished: boolean; }; -/** - * Records the body writes of every outgoing `node:http` request; `vi.restoreAllMocks` ends it. - * - * The wrapper goes on each request as it is created, before the subject observes it, so the - * subject's own write observation calls through this one and both see the same completions. - */ +/** Records each outgoing request's writes, wrapped before the subject observes them. */ export function recordOutgoingWrites(): OutgoingWrites { - const log = { falseReturns: 0, completed: 0, drains: 0, finished: false }; - const request = http.request; - vi.spyOn(http, 'request').mockImplementation((...args) => { - const req = Reflect.apply(request, http, args) as http.ClientRequest; + const log: { -readonly [K in keyof OutgoingWrites]: OutgoingWrites[K] } = { + falseReturns: 0, + completed: 0, + drains: 0, + finished: false, + lastCompletedAt: undefined, + }; + const observe = (req: http.ClientRequest): http.ClientRequest => { log.finished = false; + log.lastCompletedAt = undefined; const write = req.write; req.write = function (this: http.ClientRequest, ...writeArgs: unknown[]): boolean { const callbackIndex = typeof writeArgs[1] === 'function' ? 1 : 2; const callback = writeArgs[callbackIndex]; writeArgs[callbackIndex] = (error?: Error | null) => { - if (!error) log.completed += 1; + if (!error) { + log.completed += 1; + log.lastCompletedAt = vi.getMockedSystemTime()?.getTime(); + } if (typeof callback === 'function') callback(error); + noteIo(); }; const accepted: boolean = Reflect.apply(write, this, writeArgs); if (!accepted) log.falseReturns += 1; + noteIo(); return accepted; }; - req.on('drain', () => (log.drains += 1)); - req.on('finish', () => (log.finished = true)); + req.on('drain', () => { + log.drains += 1; + noteIo(); + }); + req.on('finish', () => { + log.finished = true; + noteIo(); + }); return req; - }); + }; + const httpRequest = http.request; + vi.spyOn(http, 'request').mockImplementation((...args) => + observe(Reflect.apply(httpRequest, http, args) as http.ClientRequest), + ); + const httpsRequest = https.request; + vi.spyOn(https, 'request').mockImplementation((...args) => + observe(Reflect.apply(httpsRequest, https, args) as http.ClientRequest), + ); return log; } /** - * Hands over the last few timer ticks a settled transfer needs to tear its leg down. - * - * A phase budget that expires aborts the request, and that abort reaches the client through the - * socket's own close path, which needs turns of both the controlled clock and the real loop before - * the upload promise rejects. + * Spends budget until `ms` of it has passed since the newest leg's last completed write, charging + * nothing before the first one. The kernel may still take bytes late after a stall began; each + * completion comes from the callback the subject observes, so it moves both deadlines alike. */ -export async function settleAfterBudget(upload: SettledUpload): Promise { - for (let round = 0; round < 200 && !upload.settled; round += 1) { - await vi.advanceTimersByTimeAsync(100); - await pumpRealWork(5); +export async function spendSinceLastWrite( + ms: number, + writes: OutgoingWrites, + transfer: Settles, +): Promise { + await untilIo(() => writes.lastCompletedAt !== undefined || transfer.settled); + while (!transfer.settled) { + const remaining = (writes.lastCompletedAt ?? 0) + ms - controlledNow(); + if (remaining <= 0) return; + await vi.advanceTimersByTimeAsync(Math.min(remaining, BUDGET_STEP_MS)); + await pumpRealWork(); } } -/** - * Hands over the last few timer ticks a settled attempt needs to tear its leg down, and reports the - * real-time ticks it cost. - * - * A join that waits on a peer is the bug these tests look for, so a test that has to settle an - * attempt gets a number it can bound rather than a pass that says nothing about cost. - */ -export async function settleAttemptAfterBudget( - attempt: SettledUploadAttempt, - options: { maxRounds?: number } = {}, -): Promise<{ rounds: number; settled: boolean }> { - const maxRounds = options.maxRounds ?? 200; - let rounds = 0; - for (; rounds < maxRounds && !attempt.settled; rounds += 1) { +function controlledNow(): number { + const now = vi.getMockedSystemTime(); + assert.ok(now, 'the transfer clock is installed'); + return now.getTime(); +} + +/** Gives an ended transfer the clock and loop turns its teardown needs; reports settlement. */ +export async function settleAfterBudget(transfer: Settles): Promise { + for (let round = 0; round < 200 && !transfer.settled; round += 1) { await vi.advanceTimersByTimeAsync(100); await pumpRealWork(5); } - return { rounds, settled: attempt.settled }; + return transfer.settled; } /** - * Lets real socket work finish without spending budget. - * - * A destroyed connection reports its close on a later turn of the real event loop, so a test that - * asserts a leg was torn down has to let that turn run. + * Feeds the next `fs.createReadStream` from a source that holds `bytes` and never ends, so the + * attempt's pipeline can stop only when the attempt destroys it, whatever the kernel buffers. The + * open after it reads the real file and records whether this source was destroyed by then. */ +export function holdArtifactSourceOpen(bytes = 0): { + readonly source: PassThrough; + readonly destroyedAtNextOpen: boolean | undefined; +} { + const source = new PassThrough(); + if (bytes > 0) source.write(Buffer.alloc(bytes, 'x')); + const open = fs.createReadStream; + const held = { source, destroyedAtNextOpen: undefined as boolean | undefined }; + vi.spyOn(fs, 'createReadStream') + .mockImplementationOnce(() => source as unknown as fs.ReadStream) + .mockImplementationOnce((...args) => { + held.destroyedAtNextOpen = source.destroyed; + return Reflect.apply(open, fs, args); + }); + return held; +} + +/** Lets real socket work, such as a destroyed connection's close, run without spending budget. */ export async function pumpRealWork(rounds = 40): Promise { for (let round = 0; round < rounds; round += 1) { await new Promise((resolve) => setImmediate(resolve)); @@ -292,15 +322,31 @@ export function createArtifactFile(bytes: number, fileName = 'app.apk'): string return artifactPath; } +/** Listens on loopback without server ceilings, which would fire on the controlled clock. */ +async function listenUntimed(server: net.Server): Promise { + if (server instanceof http.Server) { + server.requestTimeout = 0; + server.headersTimeout = 0; + server.keepAliveTimeout = 0; + } + server.unref(); + return await listenOnLoopback(server); +} + +/** Refuses preflight, so the upload runs on the legacy POST route that pipes the file. */ +function refusedPreflight(req: http.IncomingMessage, res: http.ServerResponse): boolean { + if (`${req.method} ${req.url}` !== PREFLIGHT_REQUEST) return false; + req.resume(); + req.on('end', () => { + res.statusCode = 404; + res.end('not found'); + }); + return true; +} + /** - * A loopback daemon that accepts an artifact upload and parks its body a quota at a time, never - * answering the leg. - * - * The body is parked so a transfer can be held open long enough to spend a budget while putting real - * backpressure on the writing side. Each release lets one quota of bytes through and parks the leg - * again, so the test decides how much the peer takes between two budget windows. Preflight is - * refused, so the upload runs on the legacy POST route: the one path that pipes a file at the daemon - * and keeps going while the daemon drains it. + * A loopback daemon that parks the upload body a quota at a time and never answers it, so a test + * decides how much the peer takes between two budget windows under real backpressure. */ export async function startParkedUploadDaemon( options: { @@ -321,14 +367,8 @@ export async function startParkedUploadDaemon( const server = http.createServer((req, res) => { const key = `${req.method} ${req.url}`; requests.push(key); - if (key === PREFLIGHT_REQUEST) { - req.resume(); - req.on('end', () => { - res.statusCode = 404; - res.end('not found'); - }); - return; - } + noteIo(); + if (refusedPreflight(req, res)) return; if (key === LEGACY_UPLOAD_REQUEST) { uploadLegs += 1; // The client destroys a leg it has moved past, and a reset leg errors on this side too. @@ -346,17 +386,17 @@ export async function startParkedUploadDaemon( newestParkedLeg = req; req.on('data', (chunk: Buffer) => { receivedBytes += chunk.length; - // Each release lets one quota of bytes through and parks the stream again. The client is - // then backpressured for the rest of the window, so an artifact sized in megabytes is still - // mid-flight when the next window is charged. bytesTakenSinceRelease += chunk.length; - if (readingToEnd) return; - req.pause(); - if (bytesTakenSinceRelease < RELEASE_QUOTA_BYTES) parked.push(req); - else parkedAtQuota = req; + if (!readingToEnd) { + req.pause(); + if (bytesTakenSinceRelease < RELEASE_QUOTA_BYTES) parked.push(req); + else parkedAtQuota = req; + } + noteIo(); }); req.on('end', () => { bodyEnded = true; + noteIo(); }); return; } @@ -366,13 +406,7 @@ export async function startParkedUploadDaemon( res.end('only the upload leg the test set up is expected'); }); }); - // The clock under test is controlled, so the daemon's own idle and request ceilings must not join - // the race: they would fire on the fake clock and destroy a leg the client is still progressing. - server.requestTimeout = 0; - server.headersTimeout = 0; - server.keepAliveTimeout = 0; - server.unref(); - const port = await listenOnLoopback(server); + const port = await listenUntimed(server); return { baseUrl: `http://127.0.0.1:${port}`, requests, @@ -422,12 +456,8 @@ export type ParkedUploadDaemon = { }; /** - * An origin that answers the upload leg and then stops reading its body. - * - * The answer can be an early 4xx, redirect or offset resume, which is the peer that replies while - * the client still has bytes queued. Either way, taking one chunk and pausing is what keeps the - * client's write pipeline in real backpressure when the answer lands, so a test can see whether the - * owned file and request were closed and joined, or left writing at the old hop. + * An origin that answers each upload leg at once — an early 4xx, a redirect or an offset resume — + * and then stops reading its body while the client still has bytes queued. */ export async function startAnswerUploadDaemon(options: { respond(req: http.IncomingMessage): { @@ -435,108 +465,32 @@ export async function startAnswerUploadDaemon(options: { headers?: Record; body: string; }; -}): Promise<{ - readonly baseUrl: string; - readonly port: number; - readonly requests: readonly string[]; - /** One entry per upload leg, in arrival order. */ - readonly legs: readonly UploadLeg[]; - close(): Promise; -}> { +}) { const requests: string[] = []; - const legs: UploadLeg[] = []; const server = http.createServer((req, res) => { - const key = `${req.method} ${req.url}`; - requests.push(key); - // A new leg is where an overlap would become visible: every earlier leg is put back to reading, - // so bytes its attempt is still pushing arrive and are counted rather than hiding in a paused - // stream. A client that joined its previous attempt has nothing left to push, so its leg can - // only report the bytes it took before the handoff. - for (const earlier of legs) earlier.resumeFromHandoff(); - const leg = createUploadLeg(req); - legs.push(leg); + requests.push(`${req.method} ${req.url}`); + req.on('error', () => {}); const answer = options.respond(req); res.statusCode = answer.status; for (const [name, value] of Object.entries(answer.headers ?? {})) { res.setHeader(name, value); } res.end(answer.body); - // One chunk, then the peer stops: the client's remaining body stays queued behind this pause, - // which is the backpressure the attempt has to be destroyed out of. req.pause(); }); - server.requestTimeout = 0; - server.headersTimeout = 0; - server.keepAliveTimeout = 0; - server.unref(); - const port = await listenOnLoopback(server); + const port = await listenUntimed(server); return { baseUrl: `http://127.0.0.1:${port}`, - port, requests, - legs, close: () => closeLoopbackServer(server), }; } -export type UploadLeg = { - readonly url: string; - /** Bytes of this leg's body the origin has taken. */ - readonly bytes: number; - /** Bytes taken at the moment the NEXT leg arrived; `undefined` while this is the newest leg. */ - readonly bytesAtNextLeg: number | undefined; - /** True once the origin saw this leg's socket end. */ - readonly closed: boolean; - /** Starts counting this leg's bytes again, as the handoff check does. */ - resumeFromHandoff(): void; -}; - -/** Internal counting state; published through the read-only `UploadLeg` view. */ -type CountedUploadLeg = { - -readonly [K in keyof UploadLeg]: UploadLeg[K]; -}; - -function createUploadLeg(req: http.IncomingMessage): UploadLeg { - const leg: CountedUploadLeg = { - url: req.url ?? '', - bytes: 0, - bytesAtNextLeg: undefined, - closed: false, - resumeFromHandoff() { - if (leg.bytesAtNextLeg !== undefined) { - // Already past the handoff: keep counting whatever this attempt is still writing. - req.resume(); - return; - } - leg.bytesAtNextLeg = leg.bytes; - req.resume(); - }, - }; - req.on('data', (chunk: Buffer) => { - leg.bytes += chunk.length; - }); - req.on('error', () => {}); - req.on('aborted', () => { - leg.closed = true; - }); - req.socket.on('close', () => { - leg.closed = true; - }); - return leg; -} - /** - * An HTTP origin that accepts the TCP connection and then reads nothing and never speaks. - * - * The connection IS carried, so the connect phase is over while the artifact never leaves the - * client: the budget that has to expire is body inactivity. `bytesTaken` stays zero because the - * origin pauses its sockets before the stream layer ever consumes a byte. + * An HTTP origin that accepts the TCP connection and then reads nothing and never speaks: the + * connection is carried, so the budget that has to expire is body inactivity. */ -export async function startConnectedStalledHttpOrigin(): Promise<{ - readonly port: number; - readonly bytesTaken: number; - close(): Promise; -}> { +export async function startConnectedStalledHttpOrigin() { let bytesTaken = 0; const server = net.createServer((socket) => { socket.pause(); @@ -545,8 +499,7 @@ export async function startConnectedStalledHttpOrigin(): Promise<{ }); socket.on('error', () => {}); }); - server.unref(); - const port = await listenOnLoopback(server); + const port = await listenUntimed(server); return { port, get bytesTaken() { @@ -556,82 +509,43 @@ export async function startConnectedStalledHttpOrigin(): Promise<{ }; } -/** - * An HTTP origin that answers with one response-body chunk and then goes silent mid-body. - * - * The answer is streaming, so the budget that applies is response-body inactivity; the bytes the - * client already consumed must survive to be reported, not be cut short with the socket. - */ -export async function startHalfAnsweredOrigin(options: { firstChunk: string }): Promise<{ - readonly baseUrl: string; - close(): Promise; -}> { - const server = http.createServer((_req, res) => { - _req.resume(); +/** An HTTP origin that answers with one response-body chunk and then goes silent mid-body. */ +export async function startHalfAnsweredOrigin(options: { firstChunk: string }) { + const server = http.createServer((req, res) => { + req.resume(); res.statusCode = 200; res.setHeader('content-type', 'text/plain'); res.write(options.firstChunk); }); - server.requestTimeout = 0; - server.headersTimeout = 0; - server.keepAliveTimeout = 0; - server.unref(); - const port = await listenOnLoopback(server); + const port = await listenUntimed(server); return { baseUrl: `http://127.0.0.1:${port}`, close: () => closeLoopbackServer(server) }; } -/** - * A loopback daemon that reads the upload body as it arrives — one parked chunk per release, so the - * client writes only as fast as the test lets it — and answers once the body has ended. - * - * This is the peer a real daemon is: it takes every byte and eventually replies. Tests use it to - * prove that an upload which keeps progressing finishes inside the phase budgets rather than being - * cut short by one. - */ -export async function startSlowReadingUploadDaemon(options: { - status: number; - body: string; -}): Promise<{ - readonly baseUrl: string; - readonly receivedBytes: number; - readonly bodyEnded: boolean; - readonly connections: number; - releaseAllParked(): void; - close(): Promise; -}> { +/** A daemon that reads one parked chunk per release and answers once the body has ended. */ +export async function startSlowReadingUploadDaemon(options: { status: number; body: string }) { const parked: http.IncomingMessage[] = []; let receivedBytes = 0; let bodyEnded = false; let connections = 0; const server = http.createServer((req, res) => { - const key = `${req.method} ${req.url}`; - if (key === PREFLIGHT_REQUEST) { - req.resume(); - req.on('end', () => { - res.statusCode = 404; - res.end('not found'); - }); - return; - } + if (refusedPreflight(req, res)) return; req.on('data', (chunk: Buffer) => { receivedBytes += chunk.length; req.pause(); parked.push(req); + noteIo(); }); req.on('end', () => { bodyEnded = true; res.statusCode = options.status; res.end(options.body); + noteIo(); }); }); server.on('connection', () => { connections += 1; }); - server.requestTimeout = 0; - server.headersTimeout = 0; - server.keepAliveTimeout = 0; - server.unref(); - const port = await listenOnLoopback(server); + const port = await listenUntimed(server); return { baseUrl: `http://127.0.0.1:${port}`, get receivedBytes() { @@ -651,38 +565,20 @@ export async function startSlowReadingUploadDaemon(options: { } /** - * An origin that accepts the TCP connection and never answers the TLS handshake. - * - * A plain socket sink is what keeps the handshake unfinished: the client's socket reports itself - * encrypted and still connecting, `secureConnect` never fires, and the artifact piles up in the - * write queue as `write()` keeps accepting it. That is the shape where bytes are already buffered at - * the transport while the connection is still not carried, so the connect ceiling, not a body - * budget, is what has to expire. + * An origin that accepts the TCP connection and never answers the TLS handshake: the client's socket + * stays encrypted and connecting while the artifact piles up in its write queue. */ -export async function startUnfinishedHandshakeTlsOrigin(): Promise<{ - readonly port: number; - close(): Promise; -}> { +export async function startUnfinishedHandshakeTlsOrigin() { const server = net.createServer((socket) => { socket.pause(); socket.on('error', () => {}); }); - server.unref(); - const port = await listenOnLoopback(server); + const port = await listenUntimed(server); return { port, close: () => closeLoopbackServer(server) }; } -/** - * A TLS origin that completes the handshake and then reads nothing and answers nothing. - * - * The connection IS carried here, so the connect phase is over while the artifact never leaves the - * client's write queue: the budget that has to expire is body inactivity, reported with the body - * ceiling rather than the connect one. - */ -export async function startConnectedStalledTlsOrigin(): Promise<{ - readonly port: number; - close(): Promise; -}> { +/** A TLS origin that completes the handshake and then reads nothing and answers nothing. */ +export async function startConnectedStalledTlsOrigin() { const server = tls.createServer( { key: readRepoFixture('localhost-key.pem'), cert: readRepoFixture('localhost-cert.pem') }, (socket) => { @@ -690,17 +586,13 @@ export async function startConnectedStalledTlsOrigin(): Promise<{ socket.on('error', () => {}); }, ); - server.unref(); - const port = await listenOnLoopback(server); + const port = await listenUntimed(server); return { port, close: () => closeLoopbackServer(server) }; } /** - * Trusts the repository's self-signed localhost certificate until restored. - * - * The subject builds TLS requests with the `node:https` global agent, so this is the agent's own - * trust store: a test-local decision scoped to one `try/finally`, never a production option and - * never the `NODE_TLS_REJECT_UNAUTHORIZED` switch, which a test could not put back. + * Trusts the repository's self-signed localhost certificate in the `node:https` global agent the + * subject uses, until restored; never `NODE_TLS_REJECT_UNAUTHORIZED`, which a test cannot put back. */ export function trustSelfSignedLocalhostCertificate(): { restore(): void } { const savedCa = https.globalAgent.options.ca; @@ -712,31 +604,19 @@ export function trustSelfSignedLocalhostCertificate(): { restore(): void } { }; } -/** - * A loopback daemon whose upload legs behave per a caller-supplied script. - * - * Leg one is where an upload completes and its socket returns to the pool; leg two is where the - * next attempt runs on that reused connection. The script per leg says whether the peer reads the - * whole body (`readAllBody`) and whether it answers at all, so a test can put the second attempt - * into the header wait and prove the reused socket — no new connection, no connect phase — is what - * the budget then times. - */ +/** A daemon whose legs follow a script: whether the peer reads the whole body, and answers. */ export async function startUploadLegsDaemon( behaviors: readonly { readAllBody: boolean; respond?: () => { status: number; headers?: Record; body: string }; }[], -): Promise<{ - readonly baseUrl: string; - readonly requests: readonly string[]; - readonly connections: number; - close(): Promise; -}> { +) { const requests: string[] = []; let legs = 0; let connections = 0; const server = http.createServer((req, res) => { requests.push(`${req.method} ${req.url}`); + noteIo(); const behavior = behaviors[legs]; if (!behavior) { res.statusCode = 500; @@ -769,11 +649,7 @@ export async function startUploadLegsDaemon( server.on('connection', () => { connections += 1; }); - server.requestTimeout = 0; - server.headersTimeout = 0; - server.keepAliveTimeout = 0; - server.unref(); - const port = await listenOnLoopback(server); + const port = await listenUntimed(server); return { baseUrl: `http://127.0.0.1:${port}`, requests, diff --git a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts index 8a0f8da676..06b20c8792 100644 --- a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts +++ b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts @@ -1,15 +1,6 @@ -// The transport lifecycle of one upload attempt while the peer misbehaves (#4441 review). -// -// A response can arrive before the outgoing body has finished: an early 413, a redirect or an offset -// resume from a peer that then stops reading. The accepted answer is a real outcome, so it must be -// read to its end; the owned file and request still pushing at the dead hop must be destroyed and -// JOINED before the attempt resolves or the next hop is dialed — no old bytes may overlap a new -// attempt. The budget story has to say what the socket says: the connect ceiling ends when the -// transport is actually carried (TCP `connect`, TLS `secureConnect`, or a pooled socket that is -// already connected), and buffered bytes never count as transport progress. -// -// Every case uses the attempt seam, a controlled clock, and real loopback sockets: the overlap being -// refused is only observable on a real socket, and phase budgets range from 30 to 120 seconds. +// One upload attempt against a misbehaving loopback peer (#4441 review), on a controlled clock. An +// early answer is read to its end, and the owned work is destroyed and joined before the attempt +// resolves or the next hop starts. The connect ceiling ends only when the transport is carried. import { afterEach, assert, test, vi } from 'vitest'; import { isRetryableUploadStreamError, isTransferPhaseTimeoutError } from '../upload-stream.ts'; @@ -20,11 +11,14 @@ import { TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, type ParkedUploadDaemon, assertDetailedError, + awaitBodyProgress, createArtifactFile, + holdArtifactSourceOpen, pumpRealWork, recordOutgoingWrites, - settleAttemptAfterBudget, - spendAttemptBudget, + settleAfterBudget, + spendBudget, + spendSinceLastWrite, startAnswerUploadDaemon, startConnectedStalledHttpOrigin, startConnectedStalledTlsOrigin, @@ -35,6 +29,7 @@ import { startUploadAttempt, startUploadLegsDaemon, trustSelfSignedLocalhostCertificate, + untilIo, useTransferClock, } from './support/upload-transfer-fixtures.ts'; @@ -50,32 +45,23 @@ test('an early 413 from a peer that stops reading is consumed and its owned work const daemon = await startAnswerUploadDaemon({ respond: () => ({ status: 413, body: 'payload too large' }), }); - const artifactBytes = 4 * MEBIBYTE; - // Well past what loopback buffers absorb: megabytes of owned file remain queued behind the - // peer's pause when the answer lands, which is the owned work this test expects to be ended. + const writes = recordOutgoingWrites(); + const { source } = holdArtifactSourceOpen(MEBIBYTE); const attempt = startUploadAttempt({ url: `${daemon.baseUrl}/upload`, - artifactPath: createArtifactFile(artifactBytes), + artifactPath: createArtifactFile(4 * MEBIBYTE), }); + const atSettle = attempt.then(() => ({ destroyed: source.destroyed, finished: writes.finished })); try { - const settled = await settleAttemptAfterBudget(attempt); - assert.equal(settled.settled, true, 'the attempt ended on the early response, not on a budget'); - // The join in the attempt's own teardown is what this bound proves: a pipeline still blocked on - // the parked peer would leave the attempt unresolved for its whole phase budget. - assert.ok(settled.rounds < 20, `settled in ${settled.rounds} real-loop rounds`); + assert.equal(await settleAfterBudget(attempt), true, 'the attempt ended on the early answer'); + // The body never finished, so only the attempt's own destroy-and-join can have ended it. + assert.deepEqual(await atSettle, { destroyed: true, finished: false }); const result = await attempt; - // The accepted early response is a real outcome, not a pipeline abortion. assert.equal(result.error, undefined, `got ${String(result.error)}`); assert.equal(result.value?.statusCode, 413); assert.equal(result.value?.body, 'payload too large'); - // The owned file and request were ended, not left writing: the peer took a fraction of the - // artifact and the rest never arrived. - assert.equal(daemon.legs.length, 1); - assert.ok( - (daemon.legs[0]?.bytes ?? 0) < artifactBytes, - `the peer took ${daemon.legs[0]?.bytes} of ${artifactBytes} bytes`, - ); + assert.deepEqual(daemon.requests, [LEGACY_UPLOAD_REQUEST]); } finally { await daemon.close(); } @@ -84,7 +70,6 @@ test('an early 413 from a peer that stops reading is consumed and its owned work test('a resume response ends the stalled hop before the next attempt is dialed', async () => { useTransferClock(); const resumeOffset = MEBIBYTE; - const artifactBytes = 4 * MEBIBYTE; const daemon = await startAnswerUploadDaemon({ respond: () => ({ status: 308, @@ -92,34 +77,19 @@ test('a resume response ends the stalled hop before the next attempt is dialed', body: '', }), }); + const held = holdArtifactSourceOpen(MEBIBYTE); const attempt = startUploadAttempt({ url: `${daemon.baseUrl}/upload`, - artifactPath: createArtifactFile(artifactBytes), + artifactPath: createArtifactFile(4 * MEBIBYTE), }); try { - // Leg one is answered 308 while its body is still queued; leg two answers the same offset - // again, which is a typed terminal failure rather than a third hop. - const settled = await settleAttemptAfterBudget(attempt); - assert.equal(settled.settled, true); + // Leg two answers the same offset again: a typed terminal failure, not a third hop. + assert.equal(await settleAfterBudget(attempt), true); const result = await attempt; assertDetailedError(result.error, { offset: resumeOffset, previousOffset: resumeOffset }); - const uploadLegs = daemon.requests.filter((entry) => entry === 'POST /upload'); - assert.equal(uploadLegs.length, 2); - // Bytes really were still queued when the resume was answered: the peer had taken only part of - // the artifact when the next leg was dialed. - const firstLeg = daemon.legs[0]; - assert.ok(firstLeg); - assert.ok( - (firstLeg.bytesAtNextLeg ?? artifactBytes) < artifactBytes, - `the first leg still had queued bytes at the handoff, at ${firstLeg.bytesAtNextLeg}`, - ); - // And that leg never received its queued remainder: the owned file and request were ended and - // joined before the next attempt started, so no old byte overlapped it. - assert.ok( - firstLeg.bytes < artifactBytes, - `the stalled leg kept receiving after the handoff: ${firstLeg.bytes} of ${artifactBytes}`, - ); + assert.equal(daemon.requests.filter((entry) => entry === LEGACY_UPLOAD_REQUEST).length, 2); + assert.equal(held.destroyedAtNextOpen, true, 'hop one was ended before hop two opened'); } finally { await daemon.close(); } @@ -134,7 +104,6 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it const daemon = await startParkedUploadDaemon({ answerFirstLeg: { status: 308, headers: { 'x-upload-offset': String(MEBIBYTE) } }, }); - // Hop one answers at fake time zero. Hop two must stay active beyond the old response-body deadline. const attempt = startUploadAttempt({ url: `${daemon.baseUrl}/upload`, artifactPath: createArtifactFile(artifactBytes), @@ -142,23 +111,15 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it retryable: true, }); try { - // Gate on real bytes only: no fake time is charged until hop two has really taken some. - for (let round = 0; round < 400 && !hopTwoTookBytes(daemon); round += 1) { - await pumpRealWork(15); - } + // No fake time is charged until hop two has really taken bytes. + await untilIo(() => hopTwoTookBytes(daemon) || attempt.settled); assert.ok(hopTwoTookBytes(daemon), 'the resumed hop was dialed and took bytes'); for (let window = 0; window < 6; window += 1) { - const writesBefore = writes.completed; - const bytesBefore = daemon.receivedBytes; - for (let round = 0; round < 400; round += 1) { - daemon.releaseAllParked(); - await pumpRealWork(15); - if (writes.completed > writesBefore && daemon.receivedBytes > bytesBefore) break; - } - assert.ok(daemon.receivedBytes > bytesBefore, 'the peer took more resumed bytes'); + const before = { bytes: daemon.receivedBytes, completed: writes.completed }; // Hop one was joined before hop two was dialed, so every new completion is hop two's. - assert.ok(writes.completed > writesBefore, 'the resumed hop completed more writes'); + const progressed = await awaitBodyProgress(daemon, writes, before, attempt); + assert.equal(progressed, true, `window ${window}`); await vi.advanceTimersByTimeAsync(progressWindowMs); assert.equal(attempt.settled, false); assert.equal(daemon.bodyEnded, false); @@ -166,12 +127,10 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it assert.ok(6 * progressWindowMs > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); daemon.resetParkedLeg(); - const settled = await settleAttemptAfterBudget(attempt); - assert.equal(settled.settled, true); + assert.equal(await settleAfterBudget(attempt), true); const result = await attempt; - // The peer's reset is an ordinary transport failure of hop two. Hop one's retired budget must - // not relabel it as a phase expiry, and the retry marker must survive for the client's retry. + // Hop one's retired budget must not relabel hop two's reset, and the retry marker survives. assert.equal(result.value, undefined); assert.equal(isTransferPhaseTimeoutError(result.error), false, JSON.stringify(result.error)); assert.equal(isRetryableUploadStreamError(result.error), true); @@ -190,36 +149,26 @@ function hopTwoTookBytes(daemon: ParkedUploadDaemon): boolean { test('a redirect response ends the stalled hop before the next hop is dialed', async () => { useTransferClock(); - const artifactBytes = 4 * MEBIBYTE; const daemon = await startAnswerUploadDaemon({ respond: (req) => req.url === '/moved' ? { status: 413, body: 'moved and refused' } : { status: 307, headers: { location: '/moved' }, body: '' }, }); + const held = holdArtifactSourceOpen(MEBIBYTE); const attempt = startUploadAttempt({ url: `${daemon.baseUrl}/upload`, - artifactPath: createArtifactFile(artifactBytes), + artifactPath: createArtifactFile(4 * MEBIBYTE), }); try { - const settled = await settleAttemptAfterBudget(attempt); - assert.equal(settled.settled, true); + assert.equal(await settleAfterBudget(attempt), true); const result = await attempt; assert.equal(result.error, undefined, `got ${String(result.error)}`); assert.equal(result.value?.statusCode, 413); assert.equal(result.value?.body, 'moved and refused'); - const firstLeg = daemon.legs[0]; - const redirectHop = daemon.requests.indexOf('POST /moved'); - assert.ok(firstLeg && redirectHop > 0, 'the redirect hop was dialed'); - assert.ok( - (firstLeg.bytesAtNextLeg ?? artifactBytes) < artifactBytes, - `bytes were still queued when the redirect hop started, at ${firstLeg.bytesAtNextLeg}`, - ); - assert.ok( - firstLeg.bytes < artifactBytes, - `the redirected-from leg kept receiving after the hop: ${firstLeg.bytes} of ${artifactBytes}`, - ); + assert.deepEqual(daemon.requests, ['POST /upload', 'POST /moved']); + assert.equal(held.destroyedAtNextOpen, true, 'the redirected-from hop ended before the next'); } finally { await daemon.close(); } @@ -234,14 +183,11 @@ test('a response body that stops mid-answer expires the response-body phase afte }); try { await pumpRealWork(); - await spendAttemptBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); - const settled = await settleAttemptAfterBudget(attempt); - assert.equal(settled.settled, true); + await spendBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); + assert.equal(await settleAfterBudget(attempt), true); const result = await attempt; assert.equal(result.value, undefined); - // The stall is mid-response-body, and the budget that says so is the body one; the consumed - // first chunk made the attempt's own outgoing work already settle, not linger. assertDetailedError(result.error, { reason: 'transfer_phase_timeout', transferPhase: 'response-body', @@ -254,19 +200,18 @@ test('a response body that stops mid-answer expires the response-body phase afte test('a connected peer that never takes a body byte expires body inactivity, not connect', async () => { useTransferClock(); + const writes = recordOutgoingWrites(); const origin = await startConnectedStalledHttpOrigin(); const attempt = startUploadAttempt({ url: `http://127.0.0.1:${origin.port}/upload`, artifactPath: createArtifactFile(4 * MEBIBYTE), }); try { - await spendAttemptBudget(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, attempt); - // The connection was carried, so the connect ceiling is over: a carried connection is never - // ended by the budget that governs dialing it. + await spendSinceLastWrite(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, writes, attempt); assert.equal(attempt.settled, false); - await spendAttemptBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); - const settled = await settleAttemptAfterBudget(attempt); - assert.equal(settled.settled, true); + await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, attempt); + assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); + assert.equal(await settleAfterBudget(attempt), true); const result = await attempt; assert.equal(result.value, undefined); @@ -282,19 +227,14 @@ test('a connected peer that never takes a body byte expires body inactivity, not test('a TLS handshake that never completes expires connect even with the whole body buffered', async () => { useTransferClock(); - // The transport accepts the socket and never completes the handshake, so `write()` accepts the - // artifact into the queue while nothing the peer answers arrives. A read-ahead progress signal - // would swap the connect ceiling for a body budget here; buffered bytes are not transport - // progress, so the connect ceiling must still expire. const origin = await startUnfinishedHandshakeTlsOrigin(); const attempt = startUploadAttempt({ url: `https://127.0.0.1:${origin.port}/upload`, artifactPath: createArtifactFile(4 * MEBIBYTE), }); try { - await spendAttemptBudget(TRANSFER_CONNECT_TIMEOUT_MS + 5_000, attempt); - const settled = await settleAttemptAfterBudget(attempt); - assert.equal(settled.settled, true); + await spendBudget(TRANSFER_CONNECT_TIMEOUT_MS + 5_000, attempt); + assert.equal(await settleAfterBudget(attempt), true); const result = await attempt; assert.equal(result.value, undefined); @@ -310,25 +250,20 @@ test('a TLS handshake that never completes expires connect even with the whole b test('a completed TLS handshake with a stalled body expires body inactivity, not connect', async () => { useTransferClock(); - // `secureConnect` is the establishment event for TLS: once it fires, the connect phase is over - // and the stalled artifact is body inactivity, charged with the body budget. The origin's socket - // handler below answers the handshake and then stops reading, so a trusted peer connection has to - // be established for this to say anything. const trusted = trustSelfSignedLocalhostCertificate(); try { const origin = await startConnectedStalledTlsOrigin(); try { - // The control leg first: the same trust decision, a peer that reads nothing but completes the - // handshake, and the outcome is a body-phase expiry rather than a handshake failure. + const writes = recordOutgoingWrites(); const attempt = startUploadAttempt({ url: `https://127.0.0.1:${origin.port}/upload`, artifactPath: createArtifactFile(4 * MEBIBYTE), }); - await spendAttemptBudget(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, attempt); + await spendSinceLastWrite(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, writes, attempt); assert.equal(attempt.settled, false, 'secureConnect ended the connect phase'); - await spendAttemptBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); - const settled = await settleAttemptAfterBudget(attempt); - assert.equal(settled.settled, true); + await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, attempt); + assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); + assert.equal(await settleAfterBudget(attempt), true); const result = await attempt; assert.equal(result.value, undefined); @@ -353,14 +288,10 @@ test('a zero-length artifact on a peer that never answers waits on its header bu artifactPath: createArtifactFile(0), }); try { - // The body is nothing, so the flush arrives with the connection and the wait that expires is the - // one for the answer the peer owes. That ceiling (30s) is the shorter of the two, so an expiry - // recorded against it — and not against the 120s body budget — is the header wait on its own. - await spendAttemptBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 5_000, attempt); + await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 5_000, attempt); assert.equal(attempt.settled, false); - await spendAttemptBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, attempt); - const settled = await settleAttemptAfterBudget(attempt); - assert.equal(settled.settled, true); + await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, attempt); + assert.equal(await settleAfterBudget(attempt), true); const result = await attempt; assert.equal(result.value, undefined); @@ -377,6 +308,7 @@ test('a zero-length artifact on a peer that never answers waits on its header bu test('an upload the peer keeps taking finishes while every phase ceiling is unexpired', async () => { useTransferClock(); + const writes = recordOutgoingWrites(); const daemon = await startSlowReadingUploadDaemon({ status: 200, body: JSON.stringify({ ok: true, uploadId: 'joined-upload' }), @@ -386,19 +318,20 @@ test('an upload the peer keeps taking finishes while every phase ceiling is unex artifactPath: createArtifactFile(MEBIBYTE), }); try { - // Real backpressure the whole way: the peer admits roughly one chunk between releases, so the - // upload can only finish by earning window after window of body progress. Every window is - // charged, and the attempt must be over before the two ceilings a progressing upload may never - // exceed (connect, then the header wait its flush starts) have run out. + // One second is charged per release only while the client still writes: after its flush the + // peer reads the kernel's copy, and charging that would measure socket buffer depth. let spent = 0; while ( !attempt.settled && spent < TRANSFER_CONNECT_TIMEOUT_MS + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS ) { + const bytesBefore = daemon.receivedBytes; daemon.releaseAllParked(); - await vi.advanceTimersByTimeAsync(1_000); - spent += 1_000; - await pumpRealWork(); + if (!writes.finished) { + await vi.advanceTimersByTimeAsync(1_000); + spent += 1_000; + } + await untilIo(() => attempt.settled || daemon.receivedBytes > bytesBefore); } const result = await attempt; @@ -408,8 +341,6 @@ test('an upload the peer keeps taking finishes while every phase ceiling is unex spent < TRANSFER_CONNECT_TIMEOUT_MS + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, `finished in ${spent}ms of budget, inside both ceilings a progressing upload must not exceed`, ); - // A completed upload was never destroyed mid-leg: the server saw the body arrive to its end and - // answered on the same leg the client wrote. assert.equal(daemon.bodyEnded, true); assert.equal(daemon.receivedBytes, MEBIBYTE); } finally { @@ -428,7 +359,7 @@ test('a caller cancel during a stalled connect stays caller cancellation, distin }); try { // Well inside the connect ceiling: only the caller can be the reason this stops. - await spendAttemptBudget(5_000, canceled); + await spendBudget(5_000, canceled); control.abort(); const result = await canceled; @@ -444,8 +375,6 @@ test('a caller cancel during a stalled connect stays caller cancellation, distin test('an enclosing aggregate deadline ends a redirect hop with its own reason, not a phase timeout', async () => { useTransferClock(); const aggregate = new AbortController(); - // Leg one redirects while it stops reading; leg two is the hop that never answers, so the attempt - // is parked in a phase when the deadline it was admitted under runs out. const daemon = await startUploadLegsDaemon([ { readAllBody: false, @@ -463,8 +392,7 @@ test('an enclosing aggregate deadline ends a redirect hop with its own reason, n // No phase budget is charged here, so nothing but the caller's deadline can end this. const reason = new Error('preparation aggregate expired'); aggregate.abort(reason); - const settled = await settleAttemptAfterBudget(attempt); - assert.equal(settled.settled, true); + assert.equal(await settleAfterBudget(attempt), true); const result = await attempt; assert.equal(result.value, undefined); @@ -477,10 +405,8 @@ test('an enclosing aggregate deadline ends a redirect hop with its own reason, n test('an attempt on a reused connection is already past its connect phase', async () => { useTransferClock(); - // Leg one completes and its socket goes back to the pool; leg two runs on that socket, which is - // already connected, and is never answered. A pooled socket never emits `connect`, so an attempt - // that only ever transitioned on that event would still be timing the connection here. The phase - // that expires is the header wait the unanswered flush deserves, on one connection. + // A pooled socket never emits `connect`; leg two reuses leg one's socket and is never answered. + const writes = recordOutgoingWrites(); const daemon = await startUploadLegsDaemon([ { readAllBody: true, @@ -493,8 +419,6 @@ test('an attempt on a reused connection is already past its connect phase', asyn url: `${daemon.baseUrl}/upload`, artifactPath: createArtifactFile(64 * 1024), }); - const firstSettled = await settleAttemptAfterBudget(first); - assert.equal(firstSettled.settled, true); const firstResult = await first; assert.equal(firstResult.error, undefined, `leg one failed: ${String(firstResult.error)}`); @@ -502,9 +426,10 @@ test('an attempt on a reused connection is already past its connect phase', asyn url: `${daemon.baseUrl}/upload`, artifactPath: createArtifactFile(64 * 1024), }); - await spendAttemptBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, second); - const settled = await settleAttemptAfterBudget(second); - assert.equal(settled.settled, true); + // The header wait starts at leg two's flush; no budget is charged before it. + await untilIo(() => (daemon.requests.length === 2 && writes.finished) || second.settled); + await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, second); + assert.equal(await settleAfterBudget(second), true); const result = await second; assert.equal(result.value, undefined); diff --git a/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts b/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts index bd11c67683..bdb65016a5 100644 --- a/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts @@ -1,10 +1,5 @@ -// The phase budget one artifact-transfer attempt runs under (#4441). -// -// Three ceilings govern one attempt — connecting, moving body bytes, waiting for headers — and each -// has to end a stalled attempt on its own while none of them touches the aggregate budget the -// attempt was admitted under. Those are behaviour claims about a clock, so the clock here is -// controlled rather than waited out. The loopback proofs in `upload-stream-phases.test.ts` put the -// same watchdog in front of real sockets. +// The phase budget of one artifact-transfer attempt (#4441), on a controlled clock. Each ceiling ends a +// stalled attempt on its own, and none of them touches the aggregate the attempt was admitted under. import assert from 'node:assert/strict'; import { afterEach, test, vi } from 'vitest'; diff --git a/packages/remote/src/__tests__/upload-stream-phases.test.ts b/packages/remote/src/__tests__/upload-stream-phases.test.ts index 04c6fe5181..36d4135b25 100644 --- a/packages/remote/src/__tests__/upload-stream-phases.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phases.test.ts @@ -1,13 +1,6 @@ -// Phase budgets applied to a real artifact upload over loopback (#4441). -// -// The 30–120-second phase budgets use a controlled clock here. Each case parks the request -// body a quota at a time. That combination is the point: a transfer that -// keeps moving must survive budget after budget, and one that stops moving must be ended by the ONE -// budget that ran out. A mocked transport could show neither, because the backpressure that makes -// "still progressing" mean something belongs to the real socket. +// Phase budgets on a real loopback upload (#4441): a transfer that keeps moving survives budget after +// budget, and one that stops is ended by the one budget that ran out. -import fs from 'node:fs'; -import { PassThrough } from 'node:stream'; import { afterEach, assert, test, vi } from 'vitest'; import { LEGACY_UPLOAD_REQUEST, @@ -18,15 +11,17 @@ import { assertDetailedError, awaitBodyProgress, createArtifactFile, + holdArtifactSourceOpen, pumpRealWork, recordOutgoingWrites, settleAfterBudget, - settleAttemptAfterBudget, spendBudget, spendProgressWindows, + spendSinceLastWrite, startParkedUploadDaemon, startUpload, startUploadAttempt, + untilIo, useTransferClock, } from './support/upload-transfer-fixtures.ts'; @@ -43,14 +38,9 @@ test('a body that keeps moving outlives the body-inactivity ceiling', async () = const daemon = await startParkedUploadDaemon(); const upload = startUpload({ baseUrl: daemon.baseUrl, - // Sixteen megabytes is far past what a handful of windows can drain, so every window below is - // paid for by bytes the peer really takes rather than by a socket that swallowed the file. artifactPath: createArtifactFile(16 * MEBIBYTE), }); try { - // Eight ceilings' worth of budget, one release-and-wait per window. Each window is paid for only - // after the peer really took bytes, so this stays cheap in real time while still spending twice - // the inactivity ceiling the transfer has to outlive. const progress = await spendProgressWindows(daemon, upload, { windows: 8, windowMs: TRANSFER_CONNECT_TIMEOUT_MS, @@ -70,16 +60,17 @@ test('a body that keeps moving outlives the body-inactivity ceiling', async () = test('a body the peer stops taking expires the request-body phase on its own budget', async () => { useTransferClock(); + const writes = recordOutgoingWrites(); const daemon = await startParkedUploadDaemon(); const upload = startUpload({ baseUrl: daemon.baseUrl, artifactPath: createArtifactFile(2 * MEBIBYTE), }); try { - // Bytes are in flight and then the peer stops reading them, without answering or disconnecting. - // Nothing but the inactivity budget may end this. - await spendBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, upload); - await settleAfterBudget(upload); + // The peer stops reading without answering or disconnecting. + await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, upload); + assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); + assert.equal(await settleAfterBudget(upload), true); const result = await upload; assert.equal(result.value, undefined); @@ -101,7 +92,6 @@ test('a body admitted past four connect ceilings still waits only its own header const daemon = await startParkedUploadDaemon(); const upload = startUpload({ baseUrl: daemon.baseUrl, - // Sixteen megabytes outlasts five release windows: the body is still in flight past four ceilings. artifactPath: createArtifactFile(16 * MEBIBYTE), }); try { @@ -118,15 +108,13 @@ test('a body admitted past four connect ceilings still waits only its own header // The peer takes the rest with no fake time passing; `finish` starts the only wait that expires. daemon.readToEnd(); - for (let round = 0; round < 400 && !(writes.finished && daemon.bodyEnded); round += 1) { - await pumpRealWork(15); - } + await untilIo(() => (writes.finished && daemon.bodyEnded) || upload.settled); assert.equal(writes.finished && daemon.bodyEnded, true, 'the whole body reached the peer'); await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1); await pumpRealWork(); assert.equal(upload.settled, false, 'the header wait counts from the flush'); await vi.advanceTimersByTimeAsync(1); - await settleAfterBudget(upload); + assert.equal(await settleAfterBudget(upload), true); const result = await upload; assert.equal(result.value, undefined); @@ -142,11 +130,8 @@ test('a body admitted past four connect ceilings still waits only its own header test('writes below the high-water mark keep the body budget on each completed write', async () => { useTransferClock(); - // A slow producer hands the request 1 KiB at a time. Each write fits under the socket's high-water - // mark, so it returns true and no `drain` ever fires: the write's completion is the only report - // that its bytes reached the local kernel. - const source = new PassThrough(); - vi.spyOn(fs, 'createReadStream').mockReturnValueOnce(source as unknown as fs.ReadStream); + // 1 KiB writes return true and never `drain`: only their completion reports kernel progress. + const { source } = holdArtifactSourceOpen(); const writes = recordOutgoingWrites(); const daemon = await startParkedUploadDaemon(); const attempt = startUploadAttempt({ @@ -157,7 +142,8 @@ test('writes below the high-water mark keep the body budget on each completed wr for (let window = 0; window < 6; window += 1) { const before = { bytes: daemon.receivedBytes, completed: writes.completed }; source.write(Buffer.alloc(1024, 'x')); - assert.equal(await awaitBodyProgress(daemon, writes, before), true, `window ${window}`); + const progressed = await awaitBodyProgress(daemon, writes, before, attempt); + assert.equal(progressed, true, `window ${window}`); await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS); assert.equal(attempt.settled, false, `window ${window}`); } @@ -165,15 +151,14 @@ test('writes below the high-water mark keep the body budget on each completed wr assert.equal(writes.drains, 0); assert.equal(writes.falseReturns, 0); - // The producer stops. The last completed write was one window ago, so the body budget expires - // exactly one inactivity ceiling after it. + // The producer stops; the budget expires one ceiling after the last completed write. await vi.advanceTimersByTimeAsync( TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - TRANSFER_CONNECT_TIMEOUT_MS - 1, ); await pumpRealWork(); assert.equal(attempt.settled, false); await vi.advanceTimersByTimeAsync(1); - assert.equal((await settleAttemptAfterBudget(attempt)).settled, true); + assert.equal(await settleAfterBudget(attempt), true); assertDetailedError((await attempt).error, { reason: 'transfer_phase_timeout', transferPhase: 'request-body', diff --git a/packages/remote/src/upload-client.ts b/packages/remote/src/upload-client.ts index 9c7bdb04d7..3cd9978aa1 100644 --- a/packages/remote/src/upload-client.ts +++ b/packages/remote/src/upload-client.ts @@ -14,11 +14,7 @@ import { } from './upload-stream.ts'; import type { UploadProgressSink } from './upload-progress.ts'; -/** - * Ceiling for the two small control-plane calls that bracket an upload (`preflight`, `finalize`). - * Their payload is a few hundred bytes of JSON, so one window covers reaching the daemon and getting - * the answer; the artifact bytes themselves are budgeted per phase in `upload-stream.ts`. - */ +/** Ceiling for the small JSON `preflight` and `finalize` calls that bracket an upload. */ const UPLOAD_CONTROL_PLANE_TIMEOUT_MS = 30_000; const ARTIFACT_HASH_ALGORITHM = 'sha256'; @@ -168,10 +164,8 @@ async function tryDirectUploadWithResume(options: { } catch (error) { // A canceled upload resumes for no reason: the thing it was uploaded for is already gone, and // re-preflighting would ask the daemon for a fresh ticket to send bytes nobody is waiting on. - if (options.signal?.aborted) throw error; - // An expired phase is this client's own deadline, not a transport break. Restarting the - // sequence would only buy a second attempt permission it never had, so the typed evidence for - // the phase that ran out is what the caller sees. + if (options.signal.aborted) throw error; + // An expired phase is this client's own deadline, never a transport break to retry. if (isTransferPhaseTimeoutError(error)) throw error; if (!shouldRetryDirectUpload(error)) return undefined; return await retryDirectUpload(options, uploadOnce); @@ -193,7 +187,7 @@ async function retryDirectUpload( artifact: PreparedUploadArtifact; uploadAttemptId: string; onProgress?: UploadProgressSink; - signal?: AbortSignal; + signal: AbortSignal; }, uploadOnce: ( preflight: Extract, @@ -214,9 +208,7 @@ async function retryDirectUpload( try { return await uploadOnce(retryPreflight); } catch (error) { - options.signal?.throwIfAborted(); - // As above: an expired phase is this client's own deadline, so it must not buy the legacy - // fallback a third attempt at the same bytes. + options.signal.throwIfAborted(); if (isTransferPhaseTimeoutError(error)) throw error; return undefined; } diff --git a/packages/remote/src/upload-stream.ts b/packages/remote/src/upload-stream.ts index bab2666960..bfdd2440ac 100644 --- a/packages/remote/src/upload-stream.ts +++ b/packages/remote/src/upload-stream.ts @@ -1,23 +1,21 @@ import fs from 'node:fs'; -import type { ClientRequest, IncomingHttpHeaders, IncomingMessage } from 'node:http'; +import type { IncomingHttpHeaders, IncomingMessage } from 'node:http'; import path from 'node:path'; import { pipeline } from 'node:stream/promises'; import { addAbortSignal } from 'node:stream'; import { AppError } from '@agent-device/kernel/errors'; -import { loadNodeHttpRequester, readNodeHttpResponseBody } from '@agent-device/host-kit/transport'; +import { + loadNodeHttpRequester, + observeCompletedRequestWrites, + readNodeHttpResponseBody, +} from '@agent-device/host-kit/transport'; import { createUploadProgressTransform, type UploadProgressSink, type UploadProgressStage, } from './upload-progress.ts'; -/** - * Ceiling for establishing the connection one artifact-transfer attempt runs on. - * - * Kept beside its domain rather than on the socket: the same number governs every hop of an upload, - * and every offset resume, so "how long may one hop take to connect" - * has exactly one answer. - */ +/** Ceiling for establishing the connection for each upload hop or offset resume. */ export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; /** @@ -205,15 +203,12 @@ async function streamFileToHttpRequestAttempt(options: { const headers = buildUploadRequestHeaders(options.headers, options.startOffset, payloadSize); emitUploadAttemptStarted(options.progress, options.startOffset, payloadSize); - // One watchdog for this attempt: connecting, then writing the body, then waiting for headers, then - // reading the response. A redirect or a resume is a NEW attempt and starts fresh phase timers, but - // it inherits `options.signal`, so the preparation aggregate the caller opened still expires it. + // A redirect or a resume is a new attempt with fresh phase timers under the same `options.signal`. const watchdog = createTransferPhaseWatchdog(options.signal); watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); - // The attempt's owned outgoing pipeline: artifact file → progress transform → request. Every exit - // from this attempt joins it, so no owned byte is still moving when the attempt resolves, fails, - // or hands the body to a redirect/resume attempt. + // The owned pipeline (file → progress → request). Every exit joins it, so no owned byte overlaps + // the outcome or the next hop. let upload: Promise | undefined; let uploadSettled = false; let responseReceived = false; @@ -229,21 +224,12 @@ async function streamFileToHttpRequestAttempt(options: { const fail = (error: unknown) => { stopObservingWrites(); watchdog.dispose(); - // Whatever is still queued on this attempt's own outgoing pipeline has to settle before the - // join in `finally`, so a failure that ends the attempt destroys the request too. Once the - // response has been consumed this cannot cut an accepted answer short; before it, the outcome - // is already decided by this failure. + // The queued pipeline has to settle before the join in `finally`. req.destroy(); reject(error); }; - // The phase this attempt is actually in, from the two events that can end the connect ceiling: - // the transport carrying the protocol, and every body byte reaching the socket. A pooled socket - // is already connected; a fresh one is not, and a TLS socket is not until its handshake ends. - // Until then the connect ceiling keeps counting even when the whole body already sits in the - // write buffer — buffered bytes are not transport progress. Once connected, a body the peer - // stopped taking is timed as body inactivity, never as a connect timeout; a body that flushed - // before the connection existed (a zero-length artifact) is already owed a response, so the - // header wait is what it waits on. + // Buffered bytes are not transport progress: the connect ceiling counts until the transport is + // carried. After that, an unflushed body is timed as inactivity and a flushed one waits on headers. const advanceOutgoingPhase = () => { if (responseReceived || watchdog.signal.aborted) return; if (!transportEstablished) return; @@ -262,14 +248,12 @@ async function streamFileToHttpRequestAttempt(options: { path: options.url.pathname + options.url.search, headers, // Aborting destroys the request mid-stream, which is the only way to stop bytes that are - // already piped at a device this client no longer holds the lease on. - // This is the attempt's phase budget combined with the caller's signal, so an expired phase - // destroys the socket exactly like a cancellation does. + // already piped at a device this client no longer holds the lease on. An expired phase + // destroys it exactly like a cancellation does. signal: watchdog.signal, }, (res) => { responseReceived = true; - // The response has started; from here the budget that matters is silence between body bytes. watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); addAbortSignal(watchdog.signal, res); void handleUploadResponse(res) @@ -282,34 +266,25 @@ async function streamFileToHttpRequestAttempt(options: { req.on('socket', (socket) => { if (socket.connecting) { - // A fresh socket is not a connection yet: plain HTTP is carried when it connects, TLS only - // when the handshake ends. Until then the connect ceiling keeps counting, whatever the - // request's write buffer already holds. + // Plain HTTP is carried on `connect`, TLS only once the handshake ends. const secured = (socket as { encrypted?: boolean }).encrypted === true; socket.once(secured ? 'secureConnect' : 'connect', () => { transportEstablished = true; advanceOutgoingPhase(); }); } else { - // A pooled socket — keep-alive reuse — is already carrying the protocol. transportEstablished = true; advanceOutgoingPhase(); } }); - // Body progress is a completed body write: its bytes reached the local kernel, not the peer. - // Read-ahead chunks, queued writes and UI progress are not progress, and a completion after the - // response, an abort or a destroy belongs to an exchange with no request-body phase left. - const stopObservingWrites = observeCompletedWrites(req, () => { - if (responseReceived || req.destroyed || watchdog.signal.aborted) return; - watchdog.touchBody(); - }); + const stopObservingWrites = observeCompletedRequestWrites(req, watchdog.signal, () => + watchdog.touchBody(), + ); req.on('error', (err) => { if (responseReceived) return; - // A caller that canceled owns this outcome: the signal's reason is the answer, and wrapping it - // as an ordinary transport failure would let a retry path treat "we stopped" as "it broke". - // An expired phase answers the same way — it is this client's own deadline, not a transport - // failure a retry or a legacy fallback could fix. + // A cancellation or an expired phase is this client's own outcome, never a transport failure + // that a retry or a legacy fallback could fix. if (watchdog.signal.aborted) { fail(watchdog.signal.reason); return; @@ -327,9 +302,6 @@ async function streamFileToHttpRequestAttempt(options: { ); }); - // The owned outgoing work: the artifact file and the progress transform feeding the request. - // `endOutgoingWork` stops it once the response has been consumed, and `upload` is the join that - // proves the owned file handle and request are closed before this attempt reports its outcome. const fileStream = fs.createReadStream(options.payloadPath, { start: options.startOffset }); const progressTransform = createUploadProgressTransform({ stage: options.progress?.stage ?? 'legacy', @@ -341,27 +313,19 @@ async function streamFileToHttpRequestAttempt(options: { upload = (async () => { await pipeline(fileStream, progressTransform, req, { signal: watchdog.signal }); uploadSettled = true; - // `pipeline` settles on the request's `finish`: every body byte has been flushed into the - // transport. The server may still be reading them — normal HTTP buffering — so this is this - // side's flush, not the peer's consumption; it is still the point where the server owes an - // answer. If the TLS handshake has not completed under that flush, the connect ceiling is - // still the live budget and the header wait is armed when it does. + // `finish`: this side's flush, not the peer's consumption, so the server owes an answer. bodyFlushed = true; advanceOutgoingPhase(); })().catch((err: unknown) => { - // After the response has been consumed to its end, this pipeline can only have been ended by - // the teardown that consumed it: the attempt destroys its own request at that point. The - // accepted answer is the outcome, and a rejection from our own destroy must not overwrite the - // server's verdict — a 413 that the client read completely is a 413, not a transport break. + // After a response, only this attempt's own teardown ends the pipeline, and the server's + // verdict stays the outcome: a 413 read completely is a 413, not a transport break. if (responseReceived) return; req.destroy(); if (watchdog.signal.aborted) { fail(watchdog.signal.reason); return; } - // Before any response, a rejection here is the artifact failing to reach the socket: the real - // read failure the caller has to see, never a swallowed transport error, because a transport - // break reports through the request's own `error` handler. + // A transport break reports through `error`, so this is the artifact read failing. const error = err instanceof Error ? err : new Error(String(err)); fail(new AppError('COMMAND_FAILED', 'Failed to read local artifact', {}, error)); }); @@ -369,11 +333,8 @@ async function streamFileToHttpRequestAttempt(options: { const endOutgoingWork = (): Promise => { if (!outgoingEnded) { outgoingEnded = true; - // The response has been consumed to its end, so destroying the request can no longer cut it - // short. Destroying it is what settles a pipeline still blocked on a peer that stopped - // reading: the owned file handle and the socket close now, not whenever the peer catches - // up. A pipeline that already flushed everything has nothing left to stop, and its socket - // stays in the pool exactly as it did before. + // The response is consumed, so a destroy cannot cut it short; it settles a pipeline blocked + // on a peer that stopped reading. A flushed pipeline keeps its pooled socket. if (!uploadSettled) req.destroy(); } return upload!.then( @@ -383,9 +344,8 @@ async function streamFileToHttpRequestAttempt(options: { }; async function handleUploadResponse(res: IncomingMessage): Promise { - // Read the accepted response to its end first, under the response-body budget; only then may - // the outgoing work be destroyed, and it must be joined before this attempt resolves or a - // redirect/resume starts — old bytes may never overlap the next hop. + // Read the accepted answer to its end, then end and join the outgoing work before this attempt + // resolves or the next hop starts. const body = await readNodeHttpResponseBody(res, () => watchdog.touchBody()); await endOutgoingWork(); stopObservingWrites(); @@ -445,44 +405,6 @@ async function streamFileToHttpRequestAttempt(options: { }); } -/** - * Reports each non-empty `req.write` chunk that Node completes without an error, until stopped. - * - * `pipeline` writes without a callback, and `drain` fires only after a write returned `false`. The - * wrapper passes the arguments, `this` and the backpressure result through, and runs the caller's - * callback exactly once with the same error. Stopping restores `write` only while this wrapper is - * still installed, and completions after the stop are not reported. - */ -function observeCompletedWrites(req: ClientRequest, onCompleted: () => void): () => void { - const ownWrite = Object.getOwnPropertyDescriptor(req, 'write'); - const write = req.write; - let observing = true; - const observedWrite = function (this: ClientRequest, ...args: unknown[]): boolean { - const [chunk, encoding] = args; - const callbackIndex = typeof encoding === 'function' ? 1 : 2; - const callback = args[callbackIndex]; - const hasBytes = - typeof chunk === 'string' - ? chunk.length > 0 - : chunk instanceof Uint8Array && chunk.byteLength > 0; - if (observing && hasBytes) { - args[callbackIndex] = (error?: Error | null) => { - if (!error && observing) onCompleted(); - if (typeof callback === 'function') callback(error); - }; - } - const accepted: boolean = Reflect.apply(write, this, args); - return accepted; - }; - req.write = observedWrite; - return () => { - observing = false; - if (req.write !== observedWrite) return; - if (ownWrite) Object.defineProperty(req, 'write', ownWrite); - else Reflect.deleteProperty(req, 'write'); - }; -} - function retryableUploadStreamDetail(options: { retryable?: boolean }): Record { return options.retryable === true ? { [UPLOAD_STREAM_RETRYABLE_DETAIL]: true } : {}; } From f295243608a7c78966927ac16f13734905528e79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 18:30:26 +0100 Subject: [PATCH 13/27] chore(gates): acknowledge the host-kit write owner in the upload wire digests Co-Authored-By: Claude Opus 5.5 --- test/wire-compat/ledger.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index ab27ac2820..b3ad98a522 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -169,7 +169,7 @@ "packages/remote/src/upload-client.ts#parseUploadPreflightResult": "sha256:7ce3f43b45c03df8b7f2690432beedc7a537c1bba540961716862f8a632d5623", "packages/remote/src/upload-client.ts#requestUploadPreflight": "sha256:e5f67a9a7799913f04cc9386291296a3d9fe13658776d2f4812a582c2e9befd5", "packages/remote/src/upload-client.ts#shouldRetryDirectUpload": "sha256:29059f076ff7f2b9540ad15408be48792b2d63a601ef2b9fb2d54f5576d49bbd", - "packages/remote/src/upload-client.ts#tryDirectUploadWithResume": "sha256:8b1b35bcac4977f79fdc126de1fe476fd1439e2b1a65b56f53b82366421e6986", + "packages/remote/src/upload-client.ts#tryDirectUploadWithResume": "sha256:cc5d33f38e047cf3be4bd412b64958c7495eb9a45e58eefadaa00eefa0cb6b53", "packages/remote/src/upload-client.ts#uploadDirectArtifact": "sha256:93a36a9d3dea88064569517f3cdc34f27b33df6bf5f0c0ecc9efc29b47f24b77", "packages/remote/src/upload-client.ts#uploadLegacyArtifact": "sha256:1546c696134b47bfaaaffcf95deb176cccdce243818b658ae0494dafb57fafb6", "packages/remote/src/upload-stream.ts#MAX_UPLOAD_REDIRECTS": "sha256:12ff46dfe33a1dce95bdb79c90c17a3ba4f3919e7dc4b3a1b04590fff430f79f", @@ -181,7 +181,7 @@ "packages/remote/src/upload-stream.ts#parseNonNegativeIntegerHeader": "sha256:5cbe9c6994459d673d988a51407e65556e28175b187ce0a9d27cbb6331962834", "packages/remote/src/upload-stream.ts#parseUploadResumeOffset": "sha256:297184ea22a6dc1cda9aee17034122ae0ea2f43d0355647f6fcf02f3f4b5ca0e", "packages/remote/src/upload-stream.ts#streamFileToHttpRequest": "sha256:c2b5e1369c8b1c9420036e23163d21e43fa2ac8b889c6a8f028cc6095055a16f", - "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:38cf753a537423c93e48d89b1fc45eb0237180ce4f3886173660718d1fcfb79e", + "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:3447b4c74b58d84c2cc785572ea14374ae80df24902e666fd642a537c41af33e", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_HEADER": "sha256:d2bb3f4846470188a0fc1724271da0f0bcbdf2337bd6bf331508142cadcb65d8", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_UPSTREAM_INSTANCE_HEADER": "sha256:4564c2ce437d30d7a823e3326f4375c0750ba8d42bec22b4d02e8eb7f7db2973", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_MISMATCH_HEADER": "sha256:a197edaf7f9fa94a6aab269856aa852da3204cc586f5fd23198689eb11908b61", @@ -440,8 +440,8 @@ }, { "declaration": "packages/remote/src/upload-client.ts#tryDirectUploadWithResume", - "digest": "sha256:8b1b35bcac4977f79fdc126de1fe476fd1439e2b1a65b56f53b82366421e6986", - "rationale": "#4441 ends the direct-upload sequence when a transfer phase's own budget expires, instead of re-preflighting for a fresh ticket or falling back, because a second attempt would be permission that budget already refused. Preflight, ticket PUT and finalize requests are unchanged in route, method, headers and body, so a protocol-2 daemon parses everything it parsed before; it just receives no further legs once the caller's aggregate has expired. Its signal is now required, because the preparation aggregate is always the one passed down; no request changes." + "digest": "sha256:cc5d33f38e047cf3be4bd412b64958c7495eb9a45e58eefadaa00eefa0cb6b53", + "rationale": "#4441 ends the direct-upload sequence when a transfer phase's own budget expires, instead of re-preflighting for a fresh ticket or falling back, because a second attempt would be permission that budget already refused. Preflight, ticket PUT and finalize requests are unchanged in route, method, headers and body, so a protocol-2 daemon parses everything it parsed before; it just receives no further legs once the caller's aggregate has expired. Its signal is now required, because the preparation aggregate is always the one passed down, and it is read directly; no request changes." }, { "declaration": "packages/remote/src/upload-client.ts#uploadLegacyArtifact", @@ -495,8 +495,8 @@ }, { "declaration": "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt", - "digest": "sha256:38cf753a537423c93e48d89b1fc45eb0237180ce4f3886173660718d1fcfb79e", - "rationale": "#4441 keeps each attempt under its sequential phase watchdog and corrects its transport lifecycle: the connect ceiling now ends at actual socket or TLS-handshake establishment, body progress counts only transport writes, and an early response reads the accepted answer to its end, then destroys and joins the owned file/request pipeline and retires this attempt's phase watchdog before resolving or starting the redirect/resume attempt, so a completed hop's timers never reach the next hop. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own sequencing and deadlines moved, so a protocol-2 peer parses exactly what it parsed before. Request-body progress is now each non-empty request write that completes without an error, which Node reports once the bytes reach the local kernel, instead of drain, which fires only after a write returned false. The write wrapper passes every argument and the backpressure result through, so the bytes and framing on the wire do not change." + "digest": "sha256:3447b4c74b58d84c2cc785572ea14374ae80df24902e666fd642a537c41af33e", + "rationale": "#4441 keeps each attempt under its sequential phase watchdog and corrects its transport lifecycle: the connect ceiling now ends at actual socket or TLS-handshake establishment, body progress counts only transport writes, and an early response reads the accepted answer to its end, then destroys and joins the owned file/request pipeline and retires this attempt's phase watchdog before resolving or starting the redirect/resume attempt, so a completed hop's timers never reach the next hop. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own sequencing and deadlines moved, so a protocol-2 peer parses exactly what it parsed before. Request-body progress is now each non-empty request write that completes without an error, which Node reports once the bytes reach the local kernel, instead of drain, which fires only after a write returned false. The write wrapper passes every argument and the backpressure result through, so the bytes and framing on the wire do not change. That completed-write observer is host-kit's transport owner, which also stops reporting after the response starts, the signal aborts or the request is destroyed; no request changes." }, { "declaration": "packages/remote/src/daemon-artifacts.ts#resolveMaterializedArtifactPath", From 1777e1dc066c0523852aa401683fc06004ede91a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 18:39:32 +0100 Subject: [PATCH 14/27] test(remote): tie loopback fixture cleanup to the test lifecycle Each loopback server, held-open producer and trusted certificate is released when its test finishes, so a failed assertion or a timeout cannot leave one running into the next test. Co-Authored-By: Claude Opus 5.5 --- .../support/upload-transfer-fixtures.ts | 34 +- .../__tests__/upload-stream-lifecycle.test.ts | 416 ++++++++---------- .../__tests__/upload-stream-phases.test.ts | 183 ++++---- 3 files changed, 278 insertions(+), 355 deletions(-) diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts index 063ef35b90..ad1fc8e313 100644 --- a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -295,6 +295,9 @@ export function holdArtifactSourceOpen(bytes = 0): { readonly destroyedAtNextOpen: boolean | undefined; } { const source = new PassThrough(); + onTestFinished(() => { + source.destroy(); + }); if (bytes > 0) source.write(Buffer.alloc(bytes, 'x')); const open = fs.createReadStream; const held = { source, destroyedAtNextOpen: undefined as boolean | undefined }; @@ -322,7 +325,10 @@ export function createArtifactFile(bytes: number, fileName = 'app.apk'): string return artifactPath; } -/** Listens on loopback without server ceilings, which would fire on the controlled clock. */ +/** + * Listens on loopback until its test finishes, without server ceilings, which would fire on the + * controlled clock. + */ async function listenUntimed(server: net.Server): Promise { if (server instanceof http.Server) { server.requestTimeout = 0; @@ -330,6 +336,7 @@ async function listenUntimed(server: net.Server): Promise { server.keepAliveTimeout = 0; } server.unref(); + onTestFinished(() => closeLoopbackServer(server)); return await listenOnLoopback(server); } @@ -435,7 +442,6 @@ export async function startParkedUploadDaemon( assert.ok(newestParkedLeg, 'no parked leg to reset'); newestParkedLeg.socket.resetAndDestroy(); }, - close: () => closeLoopbackServer(server), }; } @@ -452,7 +458,6 @@ export type ParkedUploadDaemon = { readToEnd(): void; /** Resets the connection of the newest parked leg: a real peer reset, not a clean close. */ resetParkedLeg(): void; - close(): Promise; }; /** @@ -482,7 +487,6 @@ export async function startAnswerUploadDaemon(options: { return { baseUrl: `http://127.0.0.1:${port}`, requests, - close: () => closeLoopbackServer(server), }; } @@ -505,7 +509,6 @@ export async function startConnectedStalledHttpOrigin() { get bytesTaken() { return bytesTaken; }, - close: () => closeLoopbackServer(server), }; } @@ -518,7 +521,7 @@ export async function startHalfAnsweredOrigin(options: { firstChunk: string }) { res.write(options.firstChunk); }); const port = await listenUntimed(server); - return { baseUrl: `http://127.0.0.1:${port}`, close: () => closeLoopbackServer(server) }; + return { baseUrl: `http://127.0.0.1:${port}` }; } /** A daemon that reads one parked chunk per release and answers once the body has ended. */ @@ -560,7 +563,6 @@ export async function startSlowReadingUploadDaemon(options: { status: number; bo releaseAllParked() { for (const entry of parked.splice(0)) entry.resume(); }, - close: () => closeLoopbackServer(server), }; } @@ -574,7 +576,7 @@ export async function startUnfinishedHandshakeTlsOrigin() { socket.on('error', () => {}); }); const port = await listenUntimed(server); - return { port, close: () => closeLoopbackServer(server) }; + return { port }; } /** A TLS origin that completes the handshake and then reads nothing and answers nothing. */ @@ -587,21 +589,20 @@ export async function startConnectedStalledTlsOrigin() { }, ); const port = await listenUntimed(server); - return { port, close: () => closeLoopbackServer(server) }; + return { port }; } /** * Trusts the repository's self-signed localhost certificate in the `node:https` global agent the - * subject uses, until restored; never `NODE_TLS_REJECT_UNAUTHORIZED`, which a test cannot put back. + * subject uses, until its test finishes; never `NODE_TLS_REJECT_UNAUTHORIZED`, which a test cannot + * put back. */ -export function trustSelfSignedLocalhostCertificate(): { restore(): void } { +export function trustSelfSignedLocalhostCertificate(): void { const savedCa = https.globalAgent.options.ca; https.globalAgent.options.ca = readRepoFixture('localhost-cert.pem'); - return { - restore() { - https.globalAgent.options.ca = savedCa; - }, - }; + onTestFinished(() => { + https.globalAgent.options.ca = savedCa; + }); } /** A daemon whose legs follow a script: whether the peer reads the whole body, and answers. */ @@ -656,7 +657,6 @@ export async function startUploadLegsDaemon( get connections() { return connections; }, - close: () => closeLoopbackServer(server), }; } diff --git a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts index 06b20c8792..bbd0dca018 100644 --- a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts +++ b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts @@ -52,19 +52,15 @@ test('an early 413 from a peer that stops reading is consumed and its owned work artifactPath: createArtifactFile(4 * MEBIBYTE), }); const atSettle = attempt.then(() => ({ destroyed: source.destroyed, finished: writes.finished })); - try { - assert.equal(await settleAfterBudget(attempt), true, 'the attempt ended on the early answer'); - // The body never finished, so only the attempt's own destroy-and-join can have ended it. - assert.deepEqual(await atSettle, { destroyed: true, finished: false }); - const result = await attempt; - - assert.equal(result.error, undefined, `got ${String(result.error)}`); - assert.equal(result.value?.statusCode, 413); - assert.equal(result.value?.body, 'payload too large'); - assert.deepEqual(daemon.requests, [LEGACY_UPLOAD_REQUEST]); - } finally { - await daemon.close(); - } + assert.equal(await settleAfterBudget(attempt), true, 'the attempt ended on the early answer'); + // The body never finished, so only the attempt's own destroy-and-join can have ended it. + assert.deepEqual(await atSettle, { destroyed: true, finished: false }); + const result = await attempt; + + assert.equal(result.error, undefined, `got ${String(result.error)}`); + assert.equal(result.value?.statusCode, 413); + assert.equal(result.value?.body, 'payload too large'); + assert.deepEqual(daemon.requests, [LEGACY_UPLOAD_REQUEST]); }); test('a resume response ends the stalled hop before the next attempt is dialed', async () => { @@ -82,17 +78,13 @@ test('a resume response ends the stalled hop before the next attempt is dialed', url: `${daemon.baseUrl}/upload`, artifactPath: createArtifactFile(4 * MEBIBYTE), }); - try { - // Leg two answers the same offset again: a typed terminal failure, not a third hop. - assert.equal(await settleAfterBudget(attempt), true); - const result = await attempt; - - assertDetailedError(result.error, { offset: resumeOffset, previousOffset: resumeOffset }); - assert.equal(daemon.requests.filter((entry) => entry === LEGACY_UPLOAD_REQUEST).length, 2); - assert.equal(held.destroyedAtNextOpen, true, 'hop one was ended before hop two opened'); - } finally { - await daemon.close(); - } + // Leg two answers the same offset again: a typed terminal failure, not a third hop. + assert.equal(await settleAfterBudget(attempt), true); + const result = await attempt; + + assertDetailedError(result.error, { offset: resumeOffset, previousOffset: resumeOffset }); + assert.equal(daemon.requests.filter((entry) => entry === LEGACY_UPLOAD_REQUEST).length, 2); + assert.equal(held.destroyedAtNextOpen, true, 'hop one was ended before hop two opened'); }); test('a resumed hop that makes real progress past the first hop ceiling keeps its own retryable reset', async () => { @@ -110,34 +102,30 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it method: 'POST', retryable: true, }); - try { - // No fake time is charged until hop two has really taken bytes. - await untilIo(() => hopTwoTookBytes(daemon) || attempt.settled); - assert.ok(hopTwoTookBytes(daemon), 'the resumed hop was dialed and took bytes'); - - for (let window = 0; window < 6; window += 1) { - const before = { bytes: daemon.receivedBytes, completed: writes.completed }; - // Hop one was joined before hop two was dialed, so every new completion is hop two's. - const progressed = await awaitBodyProgress(daemon, writes, before, attempt); - assert.equal(progressed, true, `window ${window}`); - await vi.advanceTimersByTimeAsync(progressWindowMs); - assert.equal(attempt.settled, false); - assert.equal(daemon.bodyEnded, false); - } - assert.ok(6 * progressWindowMs > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); - - daemon.resetParkedLeg(); - assert.equal(await settleAfterBudget(attempt), true); - const result = await attempt; - - // Hop one's retired budget must not relabel hop two's reset, and the retry marker survives. - assert.equal(result.value, undefined); - assert.equal(isTransferPhaseTimeoutError(result.error), false, JSON.stringify(result.error)); - assert.equal(isRetryableUploadStreamError(result.error), true); - assert.equal(daemon.requests.filter((entry) => entry === LEGACY_UPLOAD_REQUEST).length, 2); - } finally { - await daemon.close(); + // No fake time is charged until hop two has really taken bytes. + await untilIo(() => hopTwoTookBytes(daemon) || attempt.settled); + assert.ok(hopTwoTookBytes(daemon), 'the resumed hop was dialed and took bytes'); + + for (let window = 0; window < 6; window += 1) { + const before = { bytes: daemon.receivedBytes, completed: writes.completed }; + // Hop one was joined before hop two was dialed, so every new completion is hop two's. + const progressed = await awaitBodyProgress(daemon, writes, before, attempt); + assert.equal(progressed, true, `window ${window}`); + await vi.advanceTimersByTimeAsync(progressWindowMs); + assert.equal(attempt.settled, false); + assert.equal(daemon.bodyEnded, false); } + assert.ok(6 * progressWindowMs > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + + daemon.resetParkedLeg(); + assert.equal(await settleAfterBudget(attempt), true); + const result = await attempt; + + // Hop one's retired budget must not relabel hop two's reset, and the retry marker survives. + assert.equal(result.value, undefined); + assert.equal(isTransferPhaseTimeoutError(result.error), false, JSON.stringify(result.error)); + assert.equal(isRetryableUploadStreamError(result.error), true); + assert.equal(daemon.requests.filter((entry) => entry === LEGACY_UPLOAD_REQUEST).length, 2); }); function hopTwoTookBytes(daemon: ParkedUploadDaemon): boolean { @@ -160,18 +148,14 @@ test('a redirect response ends the stalled hop before the next hop is dialed', a url: `${daemon.baseUrl}/upload`, artifactPath: createArtifactFile(4 * MEBIBYTE), }); - try { - assert.equal(await settleAfterBudget(attempt), true); - const result = await attempt; - - assert.equal(result.error, undefined, `got ${String(result.error)}`); - assert.equal(result.value?.statusCode, 413); - assert.equal(result.value?.body, 'moved and refused'); - assert.deepEqual(daemon.requests, ['POST /upload', 'POST /moved']); - assert.equal(held.destroyedAtNextOpen, true, 'the redirected-from hop ended before the next'); - } finally { - await daemon.close(); - } + assert.equal(await settleAfterBudget(attempt), true); + const result = await attempt; + + assert.equal(result.error, undefined, `got ${String(result.error)}`); + assert.equal(result.value?.statusCode, 413); + assert.equal(result.value?.body, 'moved and refused'); + assert.deepEqual(daemon.requests, ['POST /upload', 'POST /moved']); + assert.equal(held.destroyedAtNextOpen, true, 'the redirected-from hop ended before the next'); }); test('a response body that stops mid-answer expires the response-body phase after what was read', async () => { @@ -181,21 +165,17 @@ test('a response body that stops mid-answer expires the response-body phase afte url: `${origin.baseUrl}/upload`, artifactPath: createArtifactFile(64), }); - try { - await pumpRealWork(); - await spendBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); - assert.equal(await settleAfterBudget(attempt), true); - const result = await attempt; - - assert.equal(result.value, undefined); - assertDetailedError(result.error, { - reason: 'transfer_phase_timeout', - transferPhase: 'response-body', - timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, - }); - } finally { - await origin.close(); - } + await pumpRealWork(); + await spendBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); + assert.equal(await settleAfterBudget(attempt), true); + const result = await attempt; + + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'response-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); }); test('a connected peer that never takes a body byte expires body inactivity, not connect', async () => { @@ -206,23 +186,19 @@ test('a connected peer that never takes a body byte expires body inactivity, not url: `http://127.0.0.1:${origin.port}/upload`, artifactPath: createArtifactFile(4 * MEBIBYTE), }); - try { - await spendSinceLastWrite(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, writes, attempt); - assert.equal(attempt.settled, false); - await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, attempt); - assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); - assert.equal(await settleAfterBudget(attempt), true); - const result = await attempt; - - assert.equal(result.value, undefined); - assertDetailedError(result.error, { - reason: 'transfer_phase_timeout', - transferPhase: 'request-body', - timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, - }); - } finally { - await origin.close(); - } + await spendSinceLastWrite(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, writes, attempt); + assert.equal(attempt.settled, false); + await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, attempt); + assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); + assert.equal(await settleAfterBudget(attempt), true); + const result = await attempt; + + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'request-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); }); test('a TLS handshake that never completes expires connect even with the whole body buffered', async () => { @@ -232,52 +208,40 @@ test('a TLS handshake that never completes expires connect even with the whole b url: `https://127.0.0.1:${origin.port}/upload`, artifactPath: createArtifactFile(4 * MEBIBYTE), }); - try { - await spendBudget(TRANSFER_CONNECT_TIMEOUT_MS + 5_000, attempt); - assert.equal(await settleAfterBudget(attempt), true); - const result = await attempt; - - assert.equal(result.value, undefined); - assertDetailedError(result.error, { - reason: 'transfer_phase_timeout', - transferPhase: 'connect', - timeoutMs: TRANSFER_CONNECT_TIMEOUT_MS, - }); - } finally { - await origin.close(); - } + await spendBudget(TRANSFER_CONNECT_TIMEOUT_MS + 5_000, attempt); + assert.equal(await settleAfterBudget(attempt), true); + const result = await attempt; + + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'connect', + timeoutMs: TRANSFER_CONNECT_TIMEOUT_MS, + }); }); test('a completed TLS handshake with a stalled body expires body inactivity, not connect', async () => { useTransferClock(); - const trusted = trustSelfSignedLocalhostCertificate(); - try { - const origin = await startConnectedStalledTlsOrigin(); - try { - const writes = recordOutgoingWrites(); - const attempt = startUploadAttempt({ - url: `https://127.0.0.1:${origin.port}/upload`, - artifactPath: createArtifactFile(4 * MEBIBYTE), - }); - await spendSinceLastWrite(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, writes, attempt); - assert.equal(attempt.settled, false, 'secureConnect ended the connect phase'); - await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, attempt); - assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); - assert.equal(await settleAfterBudget(attempt), true); - const result = await attempt; - - assert.equal(result.value, undefined); - assertDetailedError(result.error, { - reason: 'transfer_phase_timeout', - transferPhase: 'request-body', - timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, - }); - } finally { - await origin.close(); - } - } finally { - trusted.restore(); - } + trustSelfSignedLocalhostCertificate(); + const origin = await startConnectedStalledTlsOrigin(); + const writes = recordOutgoingWrites(); + const attempt = startUploadAttempt({ + url: `https://127.0.0.1:${origin.port}/upload`, + artifactPath: createArtifactFile(4 * MEBIBYTE), + }); + await spendSinceLastWrite(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, writes, attempt); + assert.equal(attempt.settled, false, 'secureConnect ended the connect phase'); + await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, attempt); + assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); + assert.equal(await settleAfterBudget(attempt), true); + const result = await attempt; + + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'request-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); }); test('a zero-length artifact on a peer that never answers waits on its header budget', async () => { @@ -287,23 +251,19 @@ test('a zero-length artifact on a peer that never answers waits on its header bu url: `http://127.0.0.1:${origin.port}/upload`, artifactPath: createArtifactFile(0), }); - try { - await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 5_000, attempt); - assert.equal(attempt.settled, false); - await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, attempt); - assert.equal(await settleAfterBudget(attempt), true); - const result = await attempt; - - assert.equal(result.value, undefined); - assertDetailedError(result.error, { - reason: 'transfer_phase_timeout', - transferPhase: 'response-headers', - timeoutMs: TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, - }); - assert.equal(origin.bytesTaken, 0); - } finally { - await origin.close(); - } + await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 5_000, attempt); + assert.equal(attempt.settled, false); + await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, attempt); + assert.equal(await settleAfterBudget(attempt), true); + const result = await attempt; + + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + }); + assert.equal(origin.bytesTaken, 0); }); test('an upload the peer keeps taking finishes while every phase ceiling is unexpired', async () => { @@ -317,35 +277,31 @@ test('an upload the peer keeps taking finishes while every phase ceiling is unex url: `${daemon.baseUrl}/upload`, artifactPath: createArtifactFile(MEBIBYTE), }); - try { - // One second is charged per release only while the client still writes: after its flush the - // peer reads the kernel's copy, and charging that would measure socket buffer depth. - let spent = 0; - while ( - !attempt.settled && - spent < TRANSFER_CONNECT_TIMEOUT_MS + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - ) { - const bytesBefore = daemon.receivedBytes; - daemon.releaseAllParked(); - if (!writes.finished) { - await vi.advanceTimersByTimeAsync(1_000); - spent += 1_000; - } - await untilIo(() => attempt.settled || daemon.receivedBytes > bytesBefore); + // One second is charged per release only while the client still writes: after its flush the + // peer reads the kernel's copy, and charging that would measure socket buffer depth. + let spent = 0; + while ( + !attempt.settled && + spent < TRANSFER_CONNECT_TIMEOUT_MS + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + ) { + const bytesBefore = daemon.receivedBytes; + daemon.releaseAllParked(); + if (!writes.finished) { + await vi.advanceTimersByTimeAsync(1_000); + spent += 1_000; } - const result = await attempt; - - assert.equal(result.error, undefined, `upload failed: ${String(result.error)}`); - assert.equal(result.value?.statusCode, 200); - assert.ok( - spent < TRANSFER_CONNECT_TIMEOUT_MS + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, - `finished in ${spent}ms of budget, inside both ceilings a progressing upload must not exceed`, - ); - assert.equal(daemon.bodyEnded, true); - assert.equal(daemon.receivedBytes, MEBIBYTE); - } finally { - await daemon.close(); + await untilIo(() => attempt.settled || daemon.receivedBytes > bytesBefore); } + const result = await attempt; + + assert.equal(result.error, undefined, `upload failed: ${String(result.error)}`); + assert.equal(result.value?.statusCode, 200); + assert.ok( + spent < TRANSFER_CONNECT_TIMEOUT_MS + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + `finished in ${spent}ms of budget, inside both ceilings a progressing upload must not exceed`, + ); + assert.equal(daemon.bodyEnded, true); + assert.equal(daemon.receivedBytes, MEBIBYTE); }, 30_000); test('a caller cancel during a stalled connect stays caller cancellation, distinct from the connect expiry', async () => { @@ -357,19 +313,15 @@ test('a caller cancel during a stalled connect stays caller cancellation, distin artifactPath: createArtifactFile(64 * 1024), signal: control.signal, }); - try { - // Well inside the connect ceiling: only the caller can be the reason this stops. - await spendBudget(5_000, canceled); - control.abort(); - const result = await canceled; - - assert.equal(result.value, undefined); - assert.ok(result.error instanceof DOMException, `got ${String(result.error)}`); - assert.equal((result.error as DOMException).name, 'AbortError'); - assert.equal(control.signal.aborted, true); - } finally { - await origin.close(); - } + // Well inside the connect ceiling: only the caller can be the reason this stops. + await spendBudget(5_000, canceled); + control.abort(); + const result = await canceled; + + assert.equal(result.value, undefined); + assert.ok(result.error instanceof DOMException, `got ${String(result.error)}`); + assert.equal((result.error as DOMException).name, 'AbortError'); + assert.equal(control.signal.aborted, true); }); test('an enclosing aggregate deadline ends a redirect hop with its own reason, not a phase timeout', async () => { @@ -387,20 +339,16 @@ test('an enclosing aggregate deadline ends a redirect hop with its own reason, n artifactPath: createArtifactFile(64 * 1024), signal: aggregate.signal, }); - try { - await pumpRealWork(); - // No phase budget is charged here, so nothing but the caller's deadline can end this. - const reason = new Error('preparation aggregate expired'); - aggregate.abort(reason); - assert.equal(await settleAfterBudget(attempt), true); - const result = await attempt; - - assert.equal(result.value, undefined); - assert.equal(result.error, reason); - assert.equal(isTransferPhaseTimeoutError(result.error), false); - } finally { - await daemon.close(); - } + await pumpRealWork(); + // No phase budget is charged here, so nothing but the caller's deadline can end this. + const reason = new Error('preparation aggregate expired'); + aggregate.abort(reason); + assert.equal(await settleAfterBudget(attempt), true); + const result = await attempt; + + assert.equal(result.value, undefined); + assert.equal(result.error, reason); + assert.equal(isTransferPhaseTimeoutError(result.error), false); }); test('an attempt on a reused connection is already past its connect phase', async () => { @@ -414,32 +362,28 @@ test('an attempt on a reused connection is already past its connect phase', asyn }, { readAllBody: true }, ]); - try { - const first = startUploadAttempt({ - url: `${daemon.baseUrl}/upload`, - artifactPath: createArtifactFile(64 * 1024), - }); - const firstResult = await first; - assert.equal(firstResult.error, undefined, `leg one failed: ${String(firstResult.error)}`); - - const second = startUploadAttempt({ - url: `${daemon.baseUrl}/upload`, - artifactPath: createArtifactFile(64 * 1024), - }); - // The header wait starts at leg two's flush; no budget is charged before it. - await untilIo(() => (daemon.requests.length === 2 && writes.finished) || second.settled); - await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, second); - assert.equal(await settleAfterBudget(second), true); - const result = await second; - - assert.equal(result.value, undefined); - assertDetailedError(result.error, { - reason: 'transfer_phase_timeout', - transferPhase: 'response-headers', - timeoutMs: TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, - }); - assert.equal(daemon.connections, 1, 'the second attempt reused the pooled socket'); - } finally { - await daemon.close(); - } + const first = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(64 * 1024), + }); + const firstResult = await first; + assert.equal(firstResult.error, undefined, `leg one failed: ${String(firstResult.error)}`); + + const second = startUploadAttempt({ + url: `${daemon.baseUrl}/upload`, + artifactPath: createArtifactFile(64 * 1024), + }); + // The header wait starts at leg two's flush; no budget is charged before it. + await untilIo(() => (daemon.requests.length === 2 && writes.finished) || second.settled); + await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, second); + assert.equal(await settleAfterBudget(second), true); + const result = await second; + + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + }); + assert.equal(daemon.connections, 1, 'the second attempt reused the pooled socket'); }); diff --git a/packages/remote/src/__tests__/upload-stream-phases.test.ts b/packages/remote/src/__tests__/upload-stream-phases.test.ts index 36d4135b25..686d4a1fb3 100644 --- a/packages/remote/src/__tests__/upload-stream-phases.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phases.test.ts @@ -40,22 +40,18 @@ test('a body that keeps moving outlives the body-inactivity ceiling', async () = baseUrl: daemon.baseUrl, artifactPath: createArtifactFile(16 * MEBIBYTE), }); - try { - const progress = await spendProgressWindows(daemon, upload, { - windows: 8, - windowMs: TRANSFER_CONNECT_TIMEOUT_MS, - writes, - }); + const progress = await spendProgressWindows(daemon, upload, { + windows: 8, + windowMs: TRANSFER_CONNECT_TIMEOUT_MS, + writes, + }); - assert.equal(progress.progressedInEveryWindow, true); - assert.ok( - progress.budgetMs > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, - `the transfer stayed alive past the body-inactivity ceiling, at ${progress.budgetMs}ms of budget`, - ); - assert.deepEqual(daemon.requests, [PREFLIGHT_REQUEST, LEGACY_UPLOAD_REQUEST]); - } finally { - await daemon.close(); - } + assert.equal(progress.progressedInEveryWindow, true); + assert.ok( + progress.budgetMs > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + `the transfer stayed alive past the body-inactivity ceiling, at ${progress.budgetMs}ms of budget`, + ); + assert.deepEqual(daemon.requests, [PREFLIGHT_REQUEST, LEGACY_UPLOAD_REQUEST]); }); test('a body the peer stops taking expires the request-body phase on its own budget', async () => { @@ -66,24 +62,20 @@ test('a body the peer stops taking expires the request-body phase on its own bud baseUrl: daemon.baseUrl, artifactPath: createArtifactFile(2 * MEBIBYTE), }); - try { - // The peer stops reading without answering or disconnecting. - await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, upload); - assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); - assert.equal(await settleAfterBudget(upload), true); - const result = await upload; + // The peer stops reading without answering or disconnecting. + await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, upload); + assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); + assert.equal(await settleAfterBudget(upload), true); + const result = await upload; - assert.equal(result.value, undefined); - assertDetailedError(result.error, { - reason: 'transfer_phase_timeout', - transferPhase: 'request-body', - timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, - }); - assert.deepEqual(daemon.requests, [PREFLIGHT_REQUEST, LEGACY_UPLOAD_REQUEST]); - assert.ok(daemon.receivedBytes > 0, 'the attempt had a body in flight to stall'); - } finally { - await daemon.close(); - } + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'request-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); + assert.deepEqual(daemon.requests, [PREFLIGHT_REQUEST, LEGACY_UPLOAD_REQUEST]); + assert.ok(daemon.receivedBytes > 0, 'the attempt had a body in flight to stall'); }); test('a body admitted past four connect ceilings still waits only its own header budget', async () => { @@ -94,38 +86,34 @@ test('a body admitted past four connect ceilings still waits only its own header baseUrl: daemon.baseUrl, artifactPath: createArtifactFile(16 * MEBIBYTE), }); - try { - const progress = await spendProgressWindows(daemon, upload, { - windows: 5, - windowMs: TRANSFER_CONNECT_TIMEOUT_MS, - writes, - }); - assert.equal(progress.progressedInEveryWindow, true); - assert.ok(progress.budgetMs > 4 * TRANSFER_CONNECT_TIMEOUT_MS, `${progress.budgetMs}ms`); - assert.equal(writes.finished, false, 'the body was still being written'); - assert.equal(daemon.bodyEnded, false); - assert.equal(upload.settled, false); + const progress = await spendProgressWindows(daemon, upload, { + windows: 5, + windowMs: TRANSFER_CONNECT_TIMEOUT_MS, + writes, + }); + assert.equal(progress.progressedInEveryWindow, true); + assert.ok(progress.budgetMs > 4 * TRANSFER_CONNECT_TIMEOUT_MS, `${progress.budgetMs}ms`); + assert.equal(writes.finished, false, 'the body was still being written'); + assert.equal(daemon.bodyEnded, false); + assert.equal(upload.settled, false); - // The peer takes the rest with no fake time passing; `finish` starts the only wait that expires. - daemon.readToEnd(); - await untilIo(() => (writes.finished && daemon.bodyEnded) || upload.settled); - assert.equal(writes.finished && daemon.bodyEnded, true, 'the whole body reached the peer'); - await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1); - await pumpRealWork(); - assert.equal(upload.settled, false, 'the header wait counts from the flush'); - await vi.advanceTimersByTimeAsync(1); - assert.equal(await settleAfterBudget(upload), true); - const result = await upload; + // The peer takes the rest with no fake time passing; `finish` starts the only wait that expires. + daemon.readToEnd(); + await untilIo(() => (writes.finished && daemon.bodyEnded) || upload.settled); + assert.equal(writes.finished && daemon.bodyEnded, true, 'the whole body reached the peer'); + await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1); + await pumpRealWork(); + assert.equal(upload.settled, false, 'the header wait counts from the flush'); + await vi.advanceTimersByTimeAsync(1); + assert.equal(await settleAfterBudget(upload), true); + const result = await upload; - assert.equal(result.value, undefined); - assertDetailedError(result.error, { - reason: 'transfer_phase_timeout', - transferPhase: 'response-headers', - timeoutMs: TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, - }); - } finally { - await daemon.close(); - } + assert.equal(result.value, undefined); + assertDetailedError(result.error, { + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + }); }); test('writes below the high-water mark keep the body budget on each completed write', async () => { @@ -138,36 +126,31 @@ test('writes below the high-water mark keep the body budget on each completed wr url: `${daemon.baseUrl}/upload`, artifactPath: createArtifactFile(64 * 1024), }); - try { - for (let window = 0; window < 6; window += 1) { - const before = { bytes: daemon.receivedBytes, completed: writes.completed }; - source.write(Buffer.alloc(1024, 'x')); - const progressed = await awaitBodyProgress(daemon, writes, before, attempt); - assert.equal(progressed, true, `window ${window}`); - await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS); - assert.equal(attempt.settled, false, `window ${window}`); - } - assert.ok(6 * TRANSFER_CONNECT_TIMEOUT_MS > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); - assert.equal(writes.drains, 0); - assert.equal(writes.falseReturns, 0); - - // The producer stops; the budget expires one ceiling after the last completed write. - await vi.advanceTimersByTimeAsync( - TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - TRANSFER_CONNECT_TIMEOUT_MS - 1, - ); - await pumpRealWork(); - assert.equal(attempt.settled, false); - await vi.advanceTimersByTimeAsync(1); - assert.equal(await settleAfterBudget(attempt), true); - assertDetailedError((await attempt).error, { - reason: 'transfer_phase_timeout', - transferPhase: 'request-body', - timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, - }); - } finally { - source.destroy(); - await daemon.close(); + for (let window = 0; window < 6; window += 1) { + const before = { bytes: daemon.receivedBytes, completed: writes.completed }; + source.write(Buffer.alloc(1024, 'x')); + const progressed = await awaitBodyProgress(daemon, writes, before, attempt); + assert.equal(progressed, true, `window ${window}`); + await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS); + assert.equal(attempt.settled, false, `window ${window}`); } + assert.ok(6 * TRANSFER_CONNECT_TIMEOUT_MS > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + assert.equal(writes.drains, 0); + assert.equal(writes.falseReturns, 0); + + // The producer stops; the budget expires one ceiling after the last completed write. + await vi.advanceTimersByTimeAsync( + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - TRANSFER_CONNECT_TIMEOUT_MS - 1, + ); + await pumpRealWork(); + assert.equal(attempt.settled, false); + await vi.advanceTimersByTimeAsync(1); + assert.equal(await settleAfterBudget(attempt), true); + assertDetailedError((await attempt).error, { + reason: 'transfer_phase_timeout', + transferPhase: 'request-body', + timeoutMs: TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + }); }); test('the caller ending the upload is reported as cancellation, not as an expired phase', async () => { @@ -179,16 +162,12 @@ test('the caller ending the upload is reported as cancellation, not as an expire artifactPath: createArtifactFile(8 * MEBIBYTE), signal: control.signal, }); - try { - // Well inside every phase budget: only the caller can be the reason this stops. - await spendBudget(30_000, upload); - control.abort(); - const result = await upload; + // Well inside every phase budget: only the caller can be the reason this stops. + await spendBudget(30_000, upload); + control.abort(); + const result = await upload; - assert.equal(result.value, undefined); - assert.ok(result.error instanceof DOMException, `got ${String(result.error)}`); - assert.equal((result.error as DOMException).name, 'AbortError'); - } finally { - await daemon.close(); - } + assert.equal(result.value, undefined); + assert.ok(result.error instanceof DOMException, `got ${String(result.error)}`); + assert.equal((result.error as DOMException).name, 'AbortError'); }); From ad4603bdb849159dd7cd2c8135969e6fa8df5898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 18:46:15 +0100 Subject: [PATCH 15/27] test(remote): end fixture teardown waits on settlement, not loop turns An ended transfer still gets at most 20 s of budget, less than any phase ceiling, and then waits for its settlement event, so a slow host's real teardown I/O cannot fail the test while the test timeout stays the bound. Co-Authored-By: Claude Opus 5.5 --- .../support/upload-transfer-fixtures.ts | 9 +++++--- .../__tests__/upload-stream-lifecycle.test.ts | 22 +++++++++---------- .../__tests__/upload-stream-phases.test.ts | 6 ++--- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts index ad1fc8e313..96afe2c684 100644 --- a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -276,13 +276,16 @@ function controlledNow(): number { return now.getTime(); } -/** Gives an ended transfer the clock and loop turns its teardown needs; reports settlement. */ -export async function settleAfterBudget(transfer: Settles): Promise { +/** + * Lets an ended transfer tear down: at most 20 s of budget, less than any phase ceiling, then the + * settlement itself, however long its real I/O takes on this host. + */ +export async function settleAfterBudget(transfer: Settles): Promise { for (let round = 0; round < 200 && !transfer.settled; round += 1) { await vi.advanceTimersByTimeAsync(100); await pumpRealWork(5); } - return transfer.settled; + await untilIo(() => transfer.settled); } /** diff --git a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts index bbd0dca018..3715d13146 100644 --- a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts +++ b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts @@ -52,7 +52,7 @@ test('an early 413 from a peer that stops reading is consumed and its owned work artifactPath: createArtifactFile(4 * MEBIBYTE), }); const atSettle = attempt.then(() => ({ destroyed: source.destroyed, finished: writes.finished })); - assert.equal(await settleAfterBudget(attempt), true, 'the attempt ended on the early answer'); + await settleAfterBudget(attempt); // The body never finished, so only the attempt's own destroy-and-join can have ended it. assert.deepEqual(await atSettle, { destroyed: true, finished: false }); const result = await attempt; @@ -79,7 +79,7 @@ test('a resume response ends the stalled hop before the next attempt is dialed', artifactPath: createArtifactFile(4 * MEBIBYTE), }); // Leg two answers the same offset again: a typed terminal failure, not a third hop. - assert.equal(await settleAfterBudget(attempt), true); + await settleAfterBudget(attempt); const result = await attempt; assertDetailedError(result.error, { offset: resumeOffset, previousOffset: resumeOffset }); @@ -118,7 +118,7 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it assert.ok(6 * progressWindowMs > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); daemon.resetParkedLeg(); - assert.equal(await settleAfterBudget(attempt), true); + await settleAfterBudget(attempt); const result = await attempt; // Hop one's retired budget must not relabel hop two's reset, and the retry marker survives. @@ -148,7 +148,7 @@ test('a redirect response ends the stalled hop before the next hop is dialed', a url: `${daemon.baseUrl}/upload`, artifactPath: createArtifactFile(4 * MEBIBYTE), }); - assert.equal(await settleAfterBudget(attempt), true); + await settleAfterBudget(attempt); const result = await attempt; assert.equal(result.error, undefined, `got ${String(result.error)}`); @@ -167,7 +167,7 @@ test('a response body that stops mid-answer expires the response-body phase afte }); await pumpRealWork(); await spendBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); - assert.equal(await settleAfterBudget(attempt), true); + await settleAfterBudget(attempt); const result = await attempt; assert.equal(result.value, undefined); @@ -190,7 +190,7 @@ test('a connected peer that never takes a body byte expires body inactivity, not assert.equal(attempt.settled, false); await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, attempt); assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); - assert.equal(await settleAfterBudget(attempt), true); + await settleAfterBudget(attempt); const result = await attempt; assert.equal(result.value, undefined); @@ -209,7 +209,7 @@ test('a TLS handshake that never completes expires connect even with the whole b artifactPath: createArtifactFile(4 * MEBIBYTE), }); await spendBudget(TRANSFER_CONNECT_TIMEOUT_MS + 5_000, attempt); - assert.equal(await settleAfterBudget(attempt), true); + await settleAfterBudget(attempt); const result = await attempt; assert.equal(result.value, undefined); @@ -233,7 +233,7 @@ test('a completed TLS handshake with a stalled body expires body inactivity, not assert.equal(attempt.settled, false, 'secureConnect ended the connect phase'); await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, attempt); assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); - assert.equal(await settleAfterBudget(attempt), true); + await settleAfterBudget(attempt); const result = await attempt; assert.equal(result.value, undefined); @@ -254,7 +254,7 @@ test('a zero-length artifact on a peer that never answers waits on its header bu await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 5_000, attempt); assert.equal(attempt.settled, false); await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, attempt); - assert.equal(await settleAfterBudget(attempt), true); + await settleAfterBudget(attempt); const result = await attempt; assert.equal(result.value, undefined); @@ -343,7 +343,7 @@ test('an enclosing aggregate deadline ends a redirect hop with its own reason, n // No phase budget is charged here, so nothing but the caller's deadline can end this. const reason = new Error('preparation aggregate expired'); aggregate.abort(reason); - assert.equal(await settleAfterBudget(attempt), true); + await settleAfterBudget(attempt); const result = await attempt; assert.equal(result.value, undefined); @@ -376,7 +376,7 @@ test('an attempt on a reused connection is already past its connect phase', asyn // The header wait starts at leg two's flush; no budget is charged before it. await untilIo(() => (daemon.requests.length === 2 && writes.finished) || second.settled); await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, second); - assert.equal(await settleAfterBudget(second), true); + await settleAfterBudget(second); const result = await second; assert.equal(result.value, undefined); diff --git a/packages/remote/src/__tests__/upload-stream-phases.test.ts b/packages/remote/src/__tests__/upload-stream-phases.test.ts index 686d4a1fb3..bddccfc4af 100644 --- a/packages/remote/src/__tests__/upload-stream-phases.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phases.test.ts @@ -65,7 +65,7 @@ test('a body the peer stops taking expires the request-body phase on its own bud // The peer stops reading without answering or disconnecting. await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, upload); assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); - assert.equal(await settleAfterBudget(upload), true); + await settleAfterBudget(upload); const result = await upload; assert.equal(result.value, undefined); @@ -105,7 +105,7 @@ test('a body admitted past four connect ceilings still waits only its own header await pumpRealWork(); assert.equal(upload.settled, false, 'the header wait counts from the flush'); await vi.advanceTimersByTimeAsync(1); - assert.equal(await settleAfterBudget(upload), true); + await settleAfterBudget(upload); const result = await upload; assert.equal(result.value, undefined); @@ -145,7 +145,7 @@ test('writes below the high-water mark keep the body budget on each completed wr await pumpRealWork(); assert.equal(attempt.settled, false); await vi.advanceTimersByTimeAsync(1); - assert.equal(await settleAfterBudget(attempt), true); + await settleAfterBudget(attempt); assertDetailedError((await attempt).error, { reason: 'transfer_phase_timeout', transferPhase: 'request-body', From 22cf1d4993ffbf7d7f29f8dd4825e64b6a5f52ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 19:21:04 +0100 Subject: [PATCH 16/27] fix(transport): make remaining progress tests independent of scheduling --- .../src/internal/transport-writes.test.ts | 14 +++- packages/host-kit/src/internal/transport.ts | 3 +- .../support/upload-transfer-fixtures.ts | 64 ++++++++----------- .../__tests__/upload-stream-lifecycle.test.ts | 39 ++++++----- .../__tests__/upload-stream-phases.test.ts | 15 +++-- 5 files changed, 73 insertions(+), 62 deletions(-) diff --git a/packages/host-kit/src/internal/transport-writes.test.ts b/packages/host-kit/src/internal/transport-writes.test.ts index fbbe8c3161..0770c7bbc8 100644 --- a/packages/host-kit/src/internal/transport-writes.test.ts +++ b/packages/host-kit/src/internal/transport-writes.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { EventEmitter } from 'node:events'; +import { runInNewContext } from 'node:vm'; import type { ClientRequest } from 'node:http'; import { test } from 'vitest'; import { observeCompletedRequestWrites } from '../transport.ts'; @@ -53,10 +54,11 @@ test('only a non-empty write that completes without an error is reported', () => for (const [text, encoding] of [ ['a', 'hex'], ['zz', 'hex'], + ['zz', 'HEX'], ['=', 'base64'], ['=', 'base64url'], ] as const) { - req.write(text, encoding, () => {}); + req.write(text, encoding as BufferEncoding, () => {}); fake.complete(fake.writes.length - 1); } assert.equal( @@ -82,6 +84,16 @@ test.each([ dispose(); }); +test('a Uint8Array from another realm is reported', () => { + const { fake, req, completed, dispose } = observe(); + const chunk = runInNewContext('new Uint8Array([1])') as Uint8Array; + assert.equal(chunk instanceof Uint8Array, false); + req.write(chunk); + fake.complete(0); + assert.equal(completed.count, 1); + dispose(); +}); + test('the observed write keeps every argument, this, the result and one callback', () => { const { fake, req, dispose } = observe(); const calls: unknown[] = []; diff --git a/packages/host-kit/src/internal/transport.ts b/packages/host-kit/src/internal/transport.ts index f80b58637d..c7223d9144 100644 --- a/packages/host-kit/src/internal/transport.ts +++ b/packages/host-kit/src/internal/transport.ts @@ -1,4 +1,5 @@ import crypto from 'node:crypto'; +import { types } from 'node:util'; import type { ClientRequest, IncomingMessage } from 'node:http'; import { AppError } from '@agent-device/kernel/errors'; @@ -117,7 +118,7 @@ export function observeCompletedRequestWrites( ? encodingName === 'hex' || encodingName === 'base64' || encodingName === 'base64url' ? Buffer.from(chunk, encodingName).byteLength > 0 : chunk.length > 0 - : chunk instanceof Uint8Array && chunk.byteLength > 0; + : types.isUint8Array(chunk) && chunk.byteLength > 0; if (observing && hasBytes) { args[callbackIndex] = (error?: Error | null) => { if (!error && observing && !req.destroyed && !signal.aborted) onCompleted(); diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts index 96afe2c684..2449fc74c2 100644 --- a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -199,6 +199,7 @@ export type OutgoingWrites = { readonly lastCompletedAt: number | undefined; /** True once the newest leg emitted `finish`: its whole body reached the transport. */ readonly finished: boolean; + readonly responseChunkRead: boolean; }; /** Records each outgoing request's writes, wrapped before the subject observes them. */ @@ -209,10 +210,12 @@ export function recordOutgoingWrites(): OutgoingWrites { drains: 0, finished: false, lastCompletedAt: undefined, + responseChunkRead: false, }; const observe = (req: http.ClientRequest): http.ClientRequest => { log.finished = false; log.lastCompletedAt = undefined; + log.responseChunkRead = false; const write = req.write; req.write = function (this: http.ClientRequest, ...writeArgs: unknown[]): boolean { const callbackIndex = typeof writeArgs[1] === 'function' ? 1 : 2; @@ -230,6 +233,12 @@ export function recordOutgoingWrites(): OutgoingWrites { noteIo(); return accepted; }; + req.on('response', (res) => { + res.once('data', () => { + log.responseChunkRead = true; + noteIo(); + }); + }); req.on('drain', () => { log.drains += 1; noteIo(); @@ -288,11 +297,7 @@ export async function settleAfterBudget(transfer: Settles): Promise { await untilIo(() => transfer.settled); } -/** - * Feeds the next `fs.createReadStream` from a source that holds `bytes` and never ends, so the - * attempt's pipeline can stop only when the attempt destroys it, whatever the kernel buffers. The - * open after it reads the real file and records whether this source was destroyed by then. - */ +/** Holds the first upload-body read open; hashing stays real. Later body reads record teardown. */ export function holdArtifactSourceOpen(bytes = 0): { readonly source: PassThrough; readonly destroyedAtNextOpen: boolean | undefined; @@ -304,12 +309,18 @@ export function holdArtifactSourceOpen(bytes = 0): { if (bytes > 0) source.write(Buffer.alloc(bytes, 'x')); const open = fs.createReadStream; const held = { source, destroyedAtNextOpen: undefined as boolean | undefined }; - vi.spyOn(fs, 'createReadStream') - .mockImplementationOnce(() => source as unknown as fs.ReadStream) - .mockImplementationOnce((...args) => { - held.destroyedAtNextOpen = source.destroyed; - return Reflect.apply(open, fs, args); - }); + let opened = false; + vi.spyOn(fs, 'createReadStream').mockImplementation((...args) => { + const options = args[1]; + if (options && typeof options === 'object' && typeof options.start === 'number') { + if (!opened) { + opened = true; + return source as unknown as fs.ReadStream; + } + held.destroyedAtNextOpen ??= source.destroyed; + } + return Reflect.apply(open, fs, args); + }); return held; } @@ -493,26 +504,14 @@ export async function startAnswerUploadDaemon(options: { }; } -/** - * An HTTP origin that accepts the TCP connection and then reads nothing and never speaks: the - * connection is carried, so the budget that has to expire is body inactivity. - */ -export async function startConnectedStalledHttpOrigin() { - let bytesTaken = 0; +/** A silent TCP peer: HTTP receives no answer, and TLS receives no handshake. */ +export async function startSilentTcpOrigin() { const server = net.createServer((socket) => { socket.pause(); - socket.on('data', (chunk: Buffer) => { - bytesTaken += chunk.length; - }); socket.on('error', () => {}); }); const port = await listenUntimed(server); - return { - port, - get bytesTaken() { - return bytesTaken; - }, - }; + return { port }; } /** An HTTP origin that answers with one response-body chunk and then goes silent mid-body. */ @@ -569,19 +568,6 @@ export async function startSlowReadingUploadDaemon(options: { status: number; bo }; } -/** - * An origin that accepts the TCP connection and never answers the TLS handshake: the client's socket - * stays encrypted and connecting while the artifact piles up in its write queue. - */ -export async function startUnfinishedHandshakeTlsOrigin() { - const server = net.createServer((socket) => { - socket.pause(); - socket.on('error', () => {}); - }); - const port = await listenUntimed(server); - return { port }; -} - /** A TLS origin that completes the handshake and then reads nothing and answers nothing. */ export async function startConnectedStalledTlsOrigin() { const server = tls.createServer( diff --git a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts index 3715d13146..c9737ae7e9 100644 --- a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts +++ b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts @@ -14,18 +14,16 @@ import { awaitBodyProgress, createArtifactFile, holdArtifactSourceOpen, - pumpRealWork, recordOutgoingWrites, settleAfterBudget, spendBudget, spendSinceLastWrite, startAnswerUploadDaemon, - startConnectedStalledHttpOrigin, + startSilentTcpOrigin, startConnectedStalledTlsOrigin, startHalfAnsweredOrigin, startParkedUploadDaemon, startSlowReadingUploadDaemon, - startUnfinishedHandshakeTlsOrigin, startUploadAttempt, startUploadLegsDaemon, trustSelfSignedLocalhostCertificate, @@ -160,12 +158,14 @@ test('a redirect response ends the stalled hop before the next hop is dialed', a test('a response body that stops mid-answer expires the response-body phase after what was read', async () => { useTransferClock(); + const writes = recordOutgoingWrites(); const origin = await startHalfAnsweredOrigin({ firstChunk: 'first chunk' }); const attempt = startUploadAttempt({ url: `${origin.baseUrl}/upload`, artifactPath: createArtifactFile(64), }); - await pumpRealWork(); + await untilIo(() => attempt.settled || writes.responseChunkRead); + assert.equal(writes.responseChunkRead, true); await spendBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); await settleAfterBudget(attempt); const result = await attempt; @@ -178,18 +178,19 @@ test('a response body that stops mid-answer expires the response-body phase afte }); }); -test('a connected peer that never takes a body byte expires body inactivity, not connect', async () => { +test('a connected HTTP upload with an unfinished source expires body inactivity, not connect', async () => { useTransferClock(); const writes = recordOutgoingWrites(); - const origin = await startConnectedStalledHttpOrigin(); + const origin = await startSilentTcpOrigin(); + holdArtifactSourceOpen(1024); const attempt = startUploadAttempt({ url: `http://127.0.0.1:${origin.port}/upload`, - artifactPath: createArtifactFile(4 * MEBIBYTE), + artifactPath: createArtifactFile(8 * 1024), }); await spendSinceLastWrite(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, writes, attempt); assert.equal(attempt.settled, false); await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, attempt); - assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); + assert.equal(writes.finished, false, 'the source still holds the body open'); await settleAfterBudget(attempt); const result = await attempt; @@ -203,7 +204,7 @@ test('a connected peer that never takes a body byte expires body inactivity, not test('a TLS handshake that never completes expires connect even with the whole body buffered', async () => { useTransferClock(); - const origin = await startUnfinishedHandshakeTlsOrigin(); + const origin = await startSilentTcpOrigin(); const attempt = startUploadAttempt({ url: `https://127.0.0.1:${origin.port}/upload`, artifactPath: createArtifactFile(4 * MEBIBYTE), @@ -220,19 +221,20 @@ test('a TLS handshake that never completes expires connect even with the whole b }); }); -test('a completed TLS handshake with a stalled body expires body inactivity, not connect', async () => { +test('a completed TLS handshake with an unfinished source expires body inactivity, not connect', async () => { useTransferClock(); trustSelfSignedLocalhostCertificate(); const origin = await startConnectedStalledTlsOrigin(); const writes = recordOutgoingWrites(); + holdArtifactSourceOpen(1024); const attempt = startUploadAttempt({ url: `https://127.0.0.1:${origin.port}/upload`, - artifactPath: createArtifactFile(4 * MEBIBYTE), + artifactPath: createArtifactFile(8 * 1024), }); await spendSinceLastWrite(TRANSFER_CONNECT_TIMEOUT_MS + 1_000, writes, attempt); assert.equal(attempt.settled, false, 'secureConnect ended the connect phase'); await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, attempt); - assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); + assert.equal(writes.finished, false, 'the source still holds the body open'); await settleAfterBudget(attempt); const result = await attempt; @@ -246,11 +248,14 @@ test('a completed TLS handshake with a stalled body expires body inactivity, not test('a zero-length artifact on a peer that never answers waits on its header budget', async () => { useTransferClock(); - const origin = await startConnectedStalledHttpOrigin(); + const writes = recordOutgoingWrites(); + const origin = await startSilentTcpOrigin(); const attempt = startUploadAttempt({ url: `http://127.0.0.1:${origin.port}/upload`, artifactPath: createArtifactFile(0), }); + await untilIo(() => attempt.settled || writes.finished); + assert.equal(writes.finished, true); await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 5_000, attempt); assert.equal(attempt.settled, false); await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, attempt); @@ -263,7 +268,7 @@ test('a zero-length artifact on a peer that never answers waits on its header bu transferPhase: 'response-headers', timeoutMs: TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, }); - assert.equal(origin.bytesTaken, 0); + assert.equal(writes.completed, 0); }); test('an upload the peer keeps taking finishes while every phase ceiling is unexpired', async () => { @@ -307,7 +312,7 @@ test('an upload the peer keeps taking finishes while every phase ceiling is unex test('a caller cancel during a stalled connect stays caller cancellation, distinct from the connect expiry', async () => { useTransferClock(); const control = new AbortController(); - const origin = await startUnfinishedHandshakeTlsOrigin(); + const origin = await startSilentTcpOrigin(); const canceled = startUploadAttempt({ url: `https://127.0.0.1:${origin.port}/upload`, artifactPath: createArtifactFile(64 * 1024), @@ -339,7 +344,9 @@ test('an enclosing aggregate deadline ends a redirect hop with its own reason, n artifactPath: createArtifactFile(64 * 1024), signal: aggregate.signal, }); - await pumpRealWork(); + await untilIo(() => attempt.settled || daemon.requests.includes('POST /moved')); + assert.deepEqual(daemon.requests, ['POST /upload', 'POST /moved']); + assert.equal(attempt.settled, false); // No phase budget is charged here, so nothing but the caller's deadline can end this. const reason = new Error('preparation aggregate expired'); aggregate.abort(reason); diff --git a/packages/remote/src/__tests__/upload-stream-phases.test.ts b/packages/remote/src/__tests__/upload-stream-phases.test.ts index bddccfc4af..fa06ffee11 100644 --- a/packages/remote/src/__tests__/upload-stream-phases.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phases.test.ts @@ -54,17 +54,18 @@ test('a body that keeps moving outlives the body-inactivity ceiling', async () = assert.deepEqual(daemon.requests, [PREFLIGHT_REQUEST, LEGACY_UPLOAD_REQUEST]); }); -test('a body the peer stops taking expires the request-body phase on its own budget', async () => { +test('an unfinished source with no new bytes expires the request-body phase on its own budget', async () => { useTransferClock(); const writes = recordOutgoingWrites(); const daemon = await startParkedUploadDaemon(); + holdArtifactSourceOpen(1024); const upload = startUpload({ baseUrl: daemon.baseUrl, - artifactPath: createArtifactFile(2 * MEBIBYTE), + artifactPath: createArtifactFile(8 * 1024), }); - // The peer stops reading without answering or disconnecting. + // The source remains open without yielding more bytes; no kernel-buffer assumption is needed. await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, upload); - assert.equal(writes.finished, false, 'the stalled body was still queued at the client'); + assert.equal(writes.finished, false, 'the source still holds the body open'); await settleAfterBudget(upload); const result = await upload; @@ -155,6 +156,7 @@ test('writes below the high-water mark keep the body budget on each completed wr test('the caller ending the upload is reported as cancellation, not as an expired phase', async () => { useTransferClock(); + const writes = recordOutgoingWrites(); const daemon = await startParkedUploadDaemon(); const control = new AbortController(); const upload = startUpload({ @@ -163,7 +165,10 @@ test('the caller ending the upload is reported as cancellation, not as an expire signal: control.signal, }); // Well inside every phase budget: only the caller can be the reason this stops. - await spendBudget(30_000, upload); + await untilIo(() => upload.settled || (writes.completed > 0 && daemon.receivedBytes > 0)); + assert.equal(upload.settled, false); + assert.ok(writes.completed > 0 && daemon.receivedBytes > 0); + await spendBudget(5_000, upload); control.abort(); const result = await upload; From cffddd7254f5be257345922f841762bd6feb9db3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 20:06:49 +0100 Subject: [PATCH 17/27] fix(remote): stop control request expiry before upload fallback --- .../support/upload-transfer-fixtures.ts | 149 +++++++----------- .../upload-client-cancellation.test.ts | 148 +++++++++++++---- .../__tests__/upload-stream-lifecycle.test.ts | 30 ++-- .../upload-stream-phase-budget.test.ts | 13 -- .../__tests__/upload-stream-phases.test.ts | 25 +-- packages/remote/src/upload-client.ts | 106 +++++++------ packages/remote/src/upload-stream.ts | 51 ++---- test/wire-compat/ledger.json | 16 +- 8 files changed, 280 insertions(+), 258 deletions(-) diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts index 2449fc74c2..8604faf759 100644 --- a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -1,7 +1,3 @@ -// Support for the tests that put real budget pressure on a real artifact upload (#4441). The peer -// holds the body open chunk by chunk and the test drives the clock; sockets, framing and the client's -// own stream stay real, because a mocked transport cannot show what an owned pipeline was doing. - import assert from 'node:assert/strict'; import fs from 'node:fs'; import http from 'node:http'; @@ -10,12 +6,12 @@ import net from 'node:net'; import path from 'node:path'; import { PassThrough } from 'node:stream'; import tls from 'node:tls'; +import { setTimeout as realSetTimeout, clearTimeout as realClearTimeout } from 'node:timers'; import { onTestFinished, vi } from 'vitest'; import { closeLoopbackServer, listenOnLoopback, mkdtempForTestSync } from '@agent-device/test-kit'; import { uploadArtifact } from '../../upload-client.ts'; import { type UploadStreamResponse, streamFileToHttpRequest } from '../../upload-stream.ts'; - -// Adopted phase budgets are independent expectations for the loopback tests. +// Adopted budgets are literal expectations, independent of the subject. export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; export const TRANSFER_RESPONSE_HEADER_TIMEOUT_MS = 30_000; export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; @@ -23,60 +19,69 @@ export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; export const PREFLIGHT_REQUEST = 'POST /upload/preflight'; export const LEGACY_UPLOAD_REQUEST = 'POST /upload'; -/** Fine enough that no phase boundary is stepped over, cheap enough for minutes of budget. */ const BUDGET_STEP_MS = 5_000; -/** What the parked daemon takes per release: megabytes outlast the windows a test runs. */ const RELEASE_QUOTA_BYTES = 256 * 1024; -/** Anything whose settlement a test observes without awaiting it. */ export type Settles = { readonly settled: boolean }; -/** The pending `untilIo` waits, each with the rejection the end of its test delivers. */ const ioWaiters = new Map<() => void, () => void>(); let ioWaitersEndWithTest = false; +let ioDeadline: number | undefined; -/** Called by every owned I/O callback below, after it records what it observed. */ function noteIo(): void { for (const wake of ioWaiters.keys()) wake(); } -/** - * Resolves once `ready` holds, rechecked after each owned I/O event below. It charges no budget and - * counts no loop turns; a wait still pending when its test finishes is rejected with its stack. - */ -export async function untilIo(ready: () => boolean): Promise { +/** Charges no fake time; all pending waits share a 3s real diagnostic deadline per test. */ +export async function untilIo( + ready: () => boolean, + label = 'I/O prerequisite', + snapshot: () => unknown = () => ({}), +): Promise { if (ready()) return; const unmet = new Error('untilIo: the I/O this wait needs never happened before its test ended'); if (!ioWaitersEndWithTest) { ioWaitersEndWithTest = true; + ioDeadline = performance.now() + 3_000; onTestFinished(() => { ioWaitersEndWithTest = false; + ioDeadline = undefined; for (const release of ioWaiters.values()) release(); ioWaiters.clear(); }); } await new Promise((resolve, reject) => { + const finish = (error?: unknown) => { + realClearTimeout(timer); + ioWaiters.delete(wake); + if (error) reject(error); + else resolve(); + }; const wake = () => { try { - if (!ready()) return; - ioWaiters.delete(wake); - resolve(); + if (ready()) finish(); } catch (error) { - ioWaiters.delete(wake); - reject(error); + finish(error); } }; - ioWaiters.set(wake, () => reject(unmet)); + const timer = realSetTimeout( + () => { + const error = new Error(`${label}: real I/O deadline; ${JSON.stringify(snapshot())}`, { + cause: unmet, + }); + finish(error); + }, + Math.max(0, (ioDeadline ?? performance.now()) - performance.now()), + ); + ioWaiters.set(wake, () => finish(unmet)); }); } -/** Starts the clock every transfer budget in these tests is measured against. */ export function useTransferClock(): void { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); } -/** An outcome observable without awaiting it, which a controlled clock needs. */ export type ObservedOutcome = Promise<{ value: T | undefined; error: unknown }> & Settles; function observeOutcome(work: Promise): ObservedOutcome { @@ -108,7 +113,6 @@ export function startUpload(options: { ); } -/** One attempt straight at `url`: the attempt lifecycle belongs to `streamFileToHttpRequest`. */ export function startUploadAttempt(options: { url: string; artifactPath: string; @@ -130,10 +134,6 @@ export function startUploadAttempt(options: { ); } -/** - * Spends `ms` of budget in `BUDGET_STEP_MS` steps, letting real socket work run between them, until - * the transfer is over. A budget that fired is an answer, not a reason to keep ticking. - */ export async function spendBudget(ms: number, transfer: Settles): Promise { for (let elapsed = 0; elapsed < ms && !transfer.settled; elapsed += BUDGET_STEP_MS) { await vi.advanceTimersByTimeAsync(BUDGET_STEP_MS); @@ -141,16 +141,21 @@ export async function spendBudget(ms: number, transfer: Settles): Promise } } -/** - * Charges `windows` windows of `windowMs`, each only after one more completed write reached the - * peer, and nothing before the first. Once the peer has the whole body, no growth is required. - */ export async function spendProgressWindows( daemon: ParkedUploadDaemon, upload: Settles, options: { windows: number; windowMs: number; writes: OutgoingWrites }, ): Promise<{ progressedInEveryWindow: boolean; budgetMs: number }> { - await untilIo(() => upload.settled || (options.writes.completed > 0 && daemon.receivedBytes > 0)); + await untilIo( + () => upload.settled || (options.writes.completed > 0 && daemon.receivedBytes > 0), + 'first body progress', + () => ({ + writes: options.writes, + receivedBytes: daemon.receivedBytes, + requests: daemon.requests, + settled: upload.settled, + }), + ); let progressedInEveryWindow = true; let spent = 0; for (let window = 0; window < options.windows; window += 1) { @@ -168,10 +173,6 @@ export async function spendProgressWindows( return { progressedInEveryWindow, budgetMs: spent }; } -/** - * Waits until the client completed one more body write and the peer took one more byte, or the - * transfer settled. The parked peer is released again after each I/O event. - */ export async function awaitBodyProgress( daemon: { readonly receivedBytes: number; releaseAllParked(): void }, writes: OutgoingWrites, @@ -180,29 +181,28 @@ export async function awaitBodyProgress( ): Promise { const progressed = () => writes.completed > before.completed && daemon.receivedBytes > before.bytes; - await untilIo(() => { - if (progressed() || transfer.settled) return true; - daemon.releaseAllParked(); - return false; - }); + await untilIo( + () => { + if (progressed() || transfer.settled) return true; + daemon.releaseAllParked(); + return false; + }, + 'next body progress', + () => ({ writes, receivedBytes: daemon.receivedBytes, before, settled: transfer.settled }), + ); return progressed(); } -/** The body writes of every outgoing request, totalled over the legs of one upload. */ export type OutgoingWrites = { - /** `write` calls that returned `false`. */ readonly falseReturns: number; - /** Writes whose callback reported success: bytes handed to the local kernel. */ + /** Successful write callbacks mean local-kernel acceptance, not peer receipt. */ readonly completed: number; readonly drains: number; - /** Controlled-clock time of the newest leg's latest completed write. */ readonly lastCompletedAt: number | undefined; - /** True once the newest leg emitted `finish`: its whole body reached the transport. */ readonly finished: boolean; readonly responseChunkRead: boolean; }; -/** Records each outgoing request's writes, wrapped before the subject observes them. */ export function recordOutgoingWrites(): OutgoingWrites { const log: { -readonly [K in keyof OutgoingWrites]: OutgoingWrites[K] } = { falseReturns: 0, @@ -260,11 +260,6 @@ export function recordOutgoingWrites(): OutgoingWrites { return log; } -/** - * Spends budget until `ms` of it has passed since the newest leg's last completed write, charging - * nothing before the first one. The kernel may still take bytes late after a stall began; each - * completion comes from the callback the subject observes, so it moves both deadlines alike. - */ export async function spendSinceLastWrite( ms: number, writes: OutgoingWrites, @@ -285,19 +280,18 @@ function controlledNow(): number { return now.getTime(); } -/** - * Lets an ended transfer tear down: at most 20 s of budget, less than any phase ceiling, then the - * settlement itself, however long its real I/O takes on this host. - */ -export async function settleAfterBudget(transfer: Settles): Promise { +export async function settleAfterBudget( + transfer: Settles, + snapshot: () => unknown = () => transfer, +): Promise { for (let round = 0; round < 200 && !transfer.settled; round += 1) { await vi.advanceTimersByTimeAsync(100); await pumpRealWork(5); } - await untilIo(() => transfer.settled); + await untilIo(() => transfer.settled, 'settlement after phase expiry', snapshot); } -/** Holds the first upload-body read open; hashing stays real. Later body reads record teardown. */ +/** Holds upload reads selected by start offset; metadata hashing remains real. */ export function holdArtifactSourceOpen(bytes = 0): { readonly source: PassThrough; readonly destroyedAtNextOpen: boolean | undefined; @@ -324,14 +318,12 @@ export function holdArtifactSourceOpen(bytes = 0): { return held; } -/** Lets real socket work, such as a destroyed connection's close, run without spending budget. */ export async function pumpRealWork(rounds = 40): Promise { for (let round = 0; round < rounds; round += 1) { await new Promise((resolve) => setImmediate(resolve)); } } -/** Writes an artifact of exactly `bytes`, so a test can size a body against a release cadence. */ export function createArtifactFile(bytes: number, fileName = 'app.apk'): string { const dir = mkdtempForTestSync('agent-device-upload-phases-'); const artifactPath = path.join(dir, fileName); @@ -339,10 +331,6 @@ export function createArtifactFile(bytes: number, fileName = 'app.apk'): string return artifactPath; } -/** - * Listens on loopback until its test finishes, without server ceilings, which would fire on the - * controlled clock. - */ async function listenUntimed(server: net.Server): Promise { if (server instanceof http.Server) { server.requestTimeout = 0; @@ -354,7 +342,6 @@ async function listenUntimed(server: net.Server): Promise { return await listenOnLoopback(server); } -/** Refuses preflight, so the upload runs on the legacy POST route that pipes the file. */ function refusedPreflight(req: http.IncomingMessage, res: http.ServerResponse): boolean { if (`${req.method} ${req.url}` !== PREFLIGHT_REQUEST) return false; req.resume(); @@ -365,13 +352,8 @@ function refusedPreflight(req: http.IncomingMessage, res: http.ServerResponse): return true; } -/** - * A loopback daemon that parks the upload body a quota at a time and never answers it, so a test - * decides how much the peer takes between two budget windows under real backpressure. - */ export async function startParkedUploadDaemon( options: { - /** Answers the first upload leg and stops reading it; later legs are parked at the quota. */ answerFirstLeg?: { status: number; headers?: Record }; } = {}, ): Promise { @@ -392,7 +374,6 @@ export async function startParkedUploadDaemon( if (refusedPreflight(req, res)) return; if (key === LEGACY_UPLOAD_REQUEST) { uploadLegs += 1; - // The client destroys a leg it has moved past, and a reset leg errors on this side too. req.on('error', () => {}); const answer = options.answerFirstLeg; if (answer && uploadLegs === 1) { @@ -462,22 +443,13 @@ export async function startParkedUploadDaemon( export type ParkedUploadDaemon = { readonly baseUrl: string; readonly requests: readonly string[]; - /** Bytes the daemon has accepted for the upload leg. */ readonly receivedBytes: number; - /** True once the daemon has taken the whole upload body and only a response is owed. */ readonly bodyEnded: boolean; - /** Lets the parked body take up to one more quota of bytes. */ releaseAllParked(): void; - /** Stops parking: the daemon reads every remaining body byte as it arrives, still never answering. */ readToEnd(): void; - /** Resets the connection of the newest parked leg: a real peer reset, not a clean close. */ resetParkedLeg(): void; }; -/** - * An origin that answers each upload leg at once — an early 4xx, a redirect or an offset resume — - * and then stops reading its body while the client still has bytes queued. - */ export async function startAnswerUploadDaemon(options: { respond(req: http.IncomingMessage): { status: number; @@ -504,7 +476,6 @@ export async function startAnswerUploadDaemon(options: { }; } -/** A silent TCP peer: HTTP receives no answer, and TLS receives no handshake. */ export async function startSilentTcpOrigin() { const server = net.createServer((socket) => { socket.pause(); @@ -514,7 +485,6 @@ export async function startSilentTcpOrigin() { return { port }; } -/** An HTTP origin that answers with one response-body chunk and then goes silent mid-body. */ export async function startHalfAnsweredOrigin(options: { firstChunk: string }) { const server = http.createServer((req, res) => { req.resume(); @@ -526,7 +496,6 @@ export async function startHalfAnsweredOrigin(options: { firstChunk: string }) { return { baseUrl: `http://127.0.0.1:${port}` }; } -/** A daemon that reads one parked chunk per release and answers once the body has ended. */ export async function startSlowReadingUploadDaemon(options: { status: number; body: string }) { const parked: http.IncomingMessage[] = []; let receivedBytes = 0; @@ -568,7 +537,6 @@ export async function startSlowReadingUploadDaemon(options: { status: number; bo }; } -/** A TLS origin that completes the handshake and then reads nothing and answers nothing. */ export async function startConnectedStalledTlsOrigin() { const server = tls.createServer( { key: readRepoFixture('localhost-key.pem'), cert: readRepoFixture('localhost-cert.pem') }, @@ -581,11 +549,7 @@ export async function startConnectedStalledTlsOrigin() { return { port }; } -/** - * Trusts the repository's self-signed localhost certificate in the `node:https` global agent the - * subject uses, until its test finishes; never `NODE_TLS_REJECT_UNAUTHORIZED`, which a test cannot - * put back. - */ +/** Restores the global agent even when later origin setup fails. */ export function trustSelfSignedLocalhostCertificate(): void { const savedCa = https.globalAgent.options.ca; https.globalAgent.options.ca = readRepoFixture('localhost-cert.pem'); @@ -594,7 +558,6 @@ export function trustSelfSignedLocalhostCertificate(): void { }); } -/** A daemon whose legs follow a script: whether the peer reads the whole body, and answers. */ export async function startUploadLegsDaemon( behaviors: readonly { readAllBody: boolean; diff --git a/packages/remote/src/__tests__/upload-client-cancellation.test.ts b/packages/remote/src/__tests__/upload-client-cancellation.test.ts index 74795076f5..f5dce572c8 100644 --- a/packages/remote/src/__tests__/upload-client-cancellation.test.ts +++ b/packages/remote/src/__tests__/upload-client-cancellation.test.ts @@ -11,6 +11,8 @@ import { createHash } from 'node:crypto'; import fs from 'node:fs'; import http, { type IncomingMessage, type ServerResponse } from 'node:http'; import path from 'node:path'; +import { PassThrough } from 'node:stream'; +import { AppError } from '@agent-device/kernel/errors'; import { once } from 'node:events'; import { uploadArtifact } from '../upload-client.ts'; import { mkdtempForTestSync } from '@agent-device/test-kit'; @@ -47,50 +49,140 @@ test('cancellation refuses preparation before it reads a missing artifact', asyn assert.equal(fetch.mock.calls.length, 0); }); -test('a standalone upload expires its preparation aggregate without starting a fallback', async () => { +test('a standalone upload expires its preparation aggregate during hashing without starting a request', async () => { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); const artifactPath = createTempFile('app.apk', 'payload'); - let preflightEntered = () => {}; - const entered = new Promise((resolve) => { - preflightEntered = resolve; + const hashSource = new PassThrough(); + vi.spyOn(fs, 'createReadStream').mockReturnValue(hashSource as unknown as fs.ReadStream); + const fetch = vi.spyOn(globalThis, 'fetch'); + const outcome = uploadArtifact({ + env: process.env, + localPath: artifactPath, + baseUrl: 'http://127.0.0.1:1', + token: TEST_TOKEN, + }).catch((error: unknown) => error); + try { + await vi.advanceTimersByTimeAsync(600_001); + const error = await outcome; + assert.ok(error instanceof AppError); + assert.deepEqual(error.details, { + reason: 'artifact_preparation_deadline_exceeded', + transferPhase: 'preparation', + timeoutMs: 600_000, + }); + assert.equal(hashSource.destroyed, true); + assert.equal(fetch.mock.calls.length, 0); + } finally { + hashSource.destroy(); + await outcome; + } +}); + +test.each([ + 'preflight', + 'finalize', + 'retry-preflight', + 'retry-finalize', + 'prose-preflight', + 'prose-finalize', +])('control request %s distinguishes a deadline from matching error prose', async (stage) => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const caller = new AbortController(); + const artifactPath = createTempFile('app.apk', 'control-plane-payload'); + const retry = stage.startsWith('retry-'); + const prose = stage.startsWith('prose-'); + const phase = stage.endsWith('preflight') ? 'preflight' : 'finalize'; + const events: string[] = []; + const routes: string[] = []; + let uploads = 0; + const server = await startServer(async (req, res) => { + routes.push(`${req.method} ${req.url}`); + if (retry && ++uploads === 1) { + req.destroy(); + return; + } + await readRequestBody(req); + sendJson(res, { ok: true, uploadId: 'control-upload' }); + }); + let entered = () => {}; + const stalled = new Promise((resolve) => { + entered = resolve; }); - let preflightSignal: AbortSignal | undefined; - let refusePreflight: ((reason: Error) => void) | undefined; - const fetch = vi.spyOn(globalThis, 'fetch').mockImplementation(async (_url, init) => { - const signal = init?.signal; - assert.ok(signal); - preflightSignal = signal; - preflightEntered(); - return await new Promise((_resolve, reject) => { - refusePreflight = reject; - signal.addEventListener('abort', () => reject(signal.reason), { once: true }); + let stalledSignal: AbortSignal | undefined; + let preflights = 0; + const fetch = vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { + const route = new URL(String(input)).pathname; + if (route.endsWith('/preflight')) preflights += 1; + if (route.endsWith('/' + phase) && (!retry || preflights === 2)) { + const signal = init?.signal; + assert.ok(signal); + stalledSignal = signal; + entered(); + if (prose) throw new Error(`Artifact transfer ${phase} timed out`); + return await new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }); + }); + } + return Response.json({ + ok: true, + uploadId: 'control-upload', + upload: { url: `${server.baseUrl}/ticket`, headers: {} }, }); }); + let settled = false; const outcome = uploadArtifact({ env: process.env, localPath: artifactPath, - baseUrl: 'http://127.0.0.1:1', + baseUrl: server.baseUrl, token: TEST_TOKEN, - }).then( - (value) => ({ value, error: undefined }), - (error: unknown) => ({ value: undefined, error }), - ); + signal: caller.signal, + onProgress: (event) => events.push(event.type), + }) + .then( + (value) => ({ value, error: undefined as unknown }), + (error: unknown) => ({ value: undefined, error }), + ) + .finally(() => { + settled = true; + }); try { - await entered; - await vi.advanceTimersByTimeAsync(600_001); - assert.equal(preflightSignal?.aborted, true); + await stalled; + if (prose) { + const result = await outcome; + assert.equal(result.error, undefined); + assert.equal(result.value, 'control-upload'); + assert.ok(routes.includes('POST /upload')); + assert.equal(stalledSignal?.aborted, false); + return; + } + await vi.advanceTimersByTimeAsync(29_999); + assert.equal(stalledSignal?.aborted, false); + assert.equal(settled, false); + await vi.advanceTimersByTimeAsync(1); + assert.equal(stalledSignal?.aborted, true); + assert.equal(settled, true, 'expiry settles without starting another transfer'); const result = await outcome; assert.equal(result.value, undefined); - assert.ok(result.error && typeof result.error === 'object' && 'details' in result.error); + assert.ok(result.error instanceof AppError); assert.deepEqual(result.error.details, { - reason: 'artifact_preparation_deadline_exceeded', - transferPhase: 'preparation', - timeoutMs: 600_000, + reason: 'transfer_phase_timeout', + transferPhase: phase, + timeoutMs: 30_000, }); - assert.equal(fetch.mock.calls.length, 1); + assert.equal(caller.signal.aborted, false); + assert.equal(events.includes('fallback'), false); + assert.deepEqual( + routes, + Array.from( + { length: phase === 'preflight' ? (retry ? 1 : 0) : retry ? 2 : 1 }, + () => 'PUT /ticket', + ), + ); + assert.equal(fetch.mock.calls.length, phase === 'preflight' ? (retry ? 2 : 1) : retry ? 3 : 2); } finally { - refusePreflight?.(new Error('fixture cleanup')); + caller.abort(new Error('fixture cleanup')); await outcome; + await server.close(); } }); diff --git a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts index c9737ae7e9..8d4b169051 100644 --- a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts +++ b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts @@ -1,7 +1,3 @@ -// One upload attempt against a misbehaving loopback peer (#4441 review), on a controlled clock. An -// early answer is read to its end, and the owned work is destroyed and joined before the attempt -// resolves or the next hop starts. The connect ceiling ends only when the transport is carried. - import { afterEach, assert, test, vi } from 'vitest'; import { isRetryableUploadStreamError, isTransferPhaseTimeoutError } from '../upload-stream.ts'; import { @@ -51,7 +47,6 @@ test('an early 413 from a peer that stops reading is consumed and its owned work }); const atSettle = attempt.then(() => ({ destroyed: source.destroyed, finished: writes.finished })); await settleAfterBudget(attempt); - // The body never finished, so only the attempt's own destroy-and-join can have ended it. assert.deepEqual(await atSettle, { destroyed: true, finished: false }); const result = await attempt; @@ -76,7 +71,6 @@ test('a resume response ends the stalled hop before the next attempt is dialed', url: `${daemon.baseUrl}/upload`, artifactPath: createArtifactFile(4 * MEBIBYTE), }); - // Leg two answers the same offset again: a typed terminal failure, not a third hop. await settleAfterBudget(attempt); const result = await attempt; @@ -100,13 +94,11 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it method: 'POST', retryable: true, }); - // No fake time is charged until hop two has really taken bytes. await untilIo(() => hopTwoTookBytes(daemon) || attempt.settled); assert.ok(hopTwoTookBytes(daemon), 'the resumed hop was dialed and took bytes'); for (let window = 0; window < 6; window += 1) { const before = { bytes: daemon.receivedBytes, completed: writes.completed }; - // Hop one was joined before hop two was dialed, so every new completion is hop two's. const progressed = await awaitBodyProgress(daemon, writes, before, attempt); assert.equal(progressed, true, `window ${window}`); await vi.advanceTimersByTimeAsync(progressWindowMs); @@ -118,8 +110,6 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it daemon.resetParkedLeg(); await settleAfterBudget(attempt); const result = await attempt; - - // Hop one's retired budget must not relabel hop two's reset, and the retry marker survives. assert.equal(result.value, undefined); assert.equal(isTransferPhaseTimeoutError(result.error), false, JSON.stringify(result.error)); assert.equal(isRetryableUploadStreamError(result.error), true); @@ -254,12 +244,13 @@ test('a zero-length artifact on a peer that never answers waits on its header bu url: `http://127.0.0.1:${origin.port}/upload`, artifactPath: createArtifactFile(0), }); - await untilIo(() => attempt.settled || writes.finished); + const snapshot = () => ({ writes, settled: attempt.settled }); + await untilIo(() => attempt.settled || writes.finished, 'empty artifact flush', snapshot); assert.equal(writes.finished, true); await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 5_000, attempt); assert.equal(attempt.settled, false); await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, attempt); - await settleAfterBudget(attempt); + await settleAfterBudget(attempt, snapshot); const result = await attempt; assert.equal(result.value, undefined); @@ -282,8 +273,6 @@ test('an upload the peer keeps taking finishes while every phase ceiling is unex url: `${daemon.baseUrl}/upload`, artifactPath: createArtifactFile(MEBIBYTE), }); - // One second is charged per release only while the client still writes: after its flush the - // peer reads the kernel's copy, and charging that would measure socket buffer depth. let spent = 0; while ( !attempt.settled && @@ -318,7 +307,6 @@ test('a caller cancel during a stalled connect stays caller cancellation, distin artifactPath: createArtifactFile(64 * 1024), signal: control.signal, }); - // Well inside the connect ceiling: only the caller can be the reason this stops. await spendBudget(5_000, canceled); control.abort(); const result = await canceled; @@ -347,7 +335,6 @@ test('an enclosing aggregate deadline ends a redirect hop with its own reason, n await untilIo(() => attempt.settled || daemon.requests.includes('POST /moved')); assert.deepEqual(daemon.requests, ['POST /upload', 'POST /moved']); assert.equal(attempt.settled, false); - // No phase budget is charged here, so nothing but the caller's deadline can end this. const reason = new Error('preparation aggregate expired'); aggregate.abort(reason); await settleAfterBudget(attempt); @@ -360,7 +347,6 @@ test('an enclosing aggregate deadline ends a redirect hop with its own reason, n test('an attempt on a reused connection is already past its connect phase', async () => { useTransferClock(); - // A pooled socket never emits `connect`; leg two reuses leg one's socket and is never answered. const writes = recordOutgoingWrites(); const daemon = await startUploadLegsDaemon([ { @@ -380,10 +366,14 @@ test('an attempt on a reused connection is already past its connect phase', asyn url: `${daemon.baseUrl}/upload`, artifactPath: createArtifactFile(64 * 1024), }); - // The header wait starts at leg two's flush; no budget is charged before it. - await untilIo(() => (daemon.requests.length === 2 && writes.finished) || second.settled); + const snapshot = () => ({ writes, requests: daemon.requests, settled: second.settled }); + await untilIo( + () => (daemon.requests.length === 2 && writes.finished) || second.settled, + 'reused request flush', + snapshot, + ); await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, second); - await settleAfterBudget(second); + await settleAfterBudget(second, snapshot); const result = await second; assert.equal(result.value, undefined); diff --git a/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts b/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts index bdb65016a5..788c7e3ce5 100644 --- a/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phase-budget.test.ts @@ -1,6 +1,3 @@ -// The phase budget of one artifact-transfer attempt (#4441), on a controlled clock. Each ceiling ends a -// stalled attempt on its own, and none of them touches the aggregate the attempt was admitted under. - import assert from 'node:assert/strict'; import { afterEach, test, vi } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; @@ -27,8 +24,6 @@ test('a body phase restarts its ceiling on every byte that moves', async () => { const watchdog = createTransferPhaseWatchdog(); try { watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); - // Three full ceilings' worth of wall clock, each broken by real progress: 300 seconds of a slow - // artifact is admitted work, and nothing here may end it. for (const _ of [0, 1, 2]) { await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - 20_000); watchdog.touchBody(); @@ -52,8 +47,6 @@ test('a phase that carries no bytes is not extended by progress reported in anot try { watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 10_000); - // Bytes moving elsewhere — an unrelated header, a keepalive, a timer callback — are not the - // response-header wait making progress. watchdog.touchBody(); await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); assertPhaseDetails( @@ -73,8 +66,6 @@ test('arming the next phase stops counting the previous one', async () => { watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS - 1_000); watchdog.arm('response-headers', TRANSFER_RESPONSE_HEADER_TIMEOUT_MS); - // 58 seconds of total attempt time: past the connect ceiling, which the completed hop may spend, - // and inside the header ceiling the attempt is actually waiting on. await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1_000); assert.equal(watchdog.signal.aborted, false); await vi.advanceTimersByTimeAsync(1_000); @@ -97,8 +88,6 @@ test('an enclosing deadline wins and is never extended by a phase timer', async try { watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); await vi.advanceTimersByTimeAsync(5_001); - // The caller's reason is the answer, not a phase's own evidence, so a consumer can still tell - // "the work this upload served is gone" from "this transfer stalled". assert.equal(watchdog.signal.aborted, true); assert.equal(watchdog.signal.reason, reason); assert.equal(isTransferPhaseTimeoutError(watchdog.signal.reason), false); @@ -115,8 +104,6 @@ test('a phase expiry is its own typed evidence and never caller cancellation', a watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); await vi.advanceTimersByTimeAsync(TRANSFER_CONNECT_TIMEOUT_MS + 1); assertPhaseDetails(watchdog.signal.reason, 'connect', TRANSFER_CONNECT_TIMEOUT_MS); - // Distinctness the retry policy depends on: nobody canceled, so the caller's own signal stays - // live and an enclosing aggregate is free to reach a different outcome. assert.equal(caller.signal.aborted, false); } finally { watchdog.dispose(); diff --git a/packages/remote/src/__tests__/upload-stream-phases.test.ts b/packages/remote/src/__tests__/upload-stream-phases.test.ts index fa06ffee11..e602b8e9ed 100644 --- a/packages/remote/src/__tests__/upload-stream-phases.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phases.test.ts @@ -1,6 +1,3 @@ -// Phase budgets on a real loopback upload (#4441): a transfer that keeps moving survives budget after -// budget, and one that stops is ended by the one budget that ran out. - import { afterEach, assert, test, vi } from 'vitest'; import { LEGACY_UPLOAD_REQUEST, @@ -63,7 +60,6 @@ test('an unfinished source with no new bytes expires the request-body phase on i baseUrl: daemon.baseUrl, artifactPath: createArtifactFile(8 * 1024), }); - // The source remains open without yielding more bytes; no kernel-buffer assumption is needed. await spendSinceLastWrite(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, writes, upload); assert.equal(writes.finished, false, 'the source still holds the body open'); await settleAfterBudget(upload); @@ -97,16 +93,25 @@ test('a body admitted past four connect ceilings still waits only its own header assert.equal(writes.finished, false, 'the body was still being written'); assert.equal(daemon.bodyEnded, false); assert.equal(upload.settled, false); - - // The peer takes the rest with no fake time passing; `finish` starts the only wait that expires. daemon.readToEnd(); - await untilIo(() => (writes.finished && daemon.bodyEnded) || upload.settled); + const snapshot = () => ({ + writes, + bodyEnded: daemon.bodyEnded, + receivedBytes: daemon.receivedBytes, + requests: daemon.requests, + settled: upload.settled, + }); + await untilIo( + () => (writes.finished && daemon.bodyEnded) || upload.settled, + 'header proof flush', + snapshot, + ); assert.equal(writes.finished && daemon.bodyEnded, true, 'the whole body reached the peer'); await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1); await pumpRealWork(); assert.equal(upload.settled, false, 'the header wait counts from the flush'); await vi.advanceTimersByTimeAsync(1); - await settleAfterBudget(upload); + await settleAfterBudget(upload, snapshot); const result = await upload; assert.equal(result.value, undefined); @@ -119,7 +124,6 @@ test('a body admitted past four connect ceilings still waits only its own header test('writes below the high-water mark keep the body budget on each completed write', async () => { useTransferClock(); - // 1 KiB writes return true and never `drain`: only their completion reports kernel progress. const { source } = holdArtifactSourceOpen(); const writes = recordOutgoingWrites(); const daemon = await startParkedUploadDaemon(); @@ -138,8 +142,6 @@ test('writes below the high-water mark keep the body budget on each completed wr assert.ok(6 * TRANSFER_CONNECT_TIMEOUT_MS > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); assert.equal(writes.drains, 0); assert.equal(writes.falseReturns, 0); - - // The producer stops; the budget expires one ceiling after the last completed write. await vi.advanceTimersByTimeAsync( TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - TRANSFER_CONNECT_TIMEOUT_MS - 1, ); @@ -164,7 +166,6 @@ test('the caller ending the upload is reported as cancellation, not as an expire artifactPath: createArtifactFile(8 * MEBIBYTE), signal: control.signal, }); - // Well inside every phase budget: only the caller can be the reason this stops. await untilIo(() => upload.settled || (writes.completed > 0 && daemon.receivedBytes > 0)); assert.equal(upload.settled, false); assert.ok(writes.completed > 0 && daemon.receivedBytes > 0); diff --git a/packages/remote/src/upload-client.ts b/packages/remote/src/upload-client.ts index 3cd9978aa1..67918dea5a 100644 --- a/packages/remote/src/upload-client.ts +++ b/packages/remote/src/upload-client.ts @@ -7,6 +7,7 @@ import { } from '@agent-device/contracts/daemon-http'; import { prepareUploadArtifact, type PreparedUploadArtifact } from './upload-client-artifact.ts'; import { + createTransferPhaseWatchdog, isRetryableUploadStreamError, isTransferPhaseTimeoutError, streamFileToHttpRequest, @@ -287,47 +288,60 @@ async function requestUploadPreflight(options: { buildDaemonHttpLeaseHeaders(options.leaseId), ); - const response = await fetch(preflightUrl, { - method: 'POST', - // The daemon never redirects this route. A redirect would carry x-agent-device-token, which - // fetch keeps on a cross-origin hop, so any redirect fails the preflight instead. - redirect: 'error', - headers, - signal: combineUploadSignals(options.signal, UPLOAD_CONTROL_PLANE_TIMEOUT_MS), - body: JSON.stringify({ - uploadAttemptId: options.uploadAttemptId, - sha256: options.artifact.sha256, - fileName: options.artifact.fileName, - sizeBytes: options.artifact.sizeBytes, - artifactType: options.artifact.artifactType, - ...(options.artifact.platform ? { platform: options.artifact.platform } : {}), - contentType: options.artifact.contentType, - }), - }).catch((error: unknown) => { + const result = await requestUploadControl( + preflightUrl, + { + method: 'POST', + // The daemon never redirects this route. A redirect would carry x-agent-device-token, which + // fetch keeps on a cross-origin hop, so any redirect fails the preflight instead. + redirect: 'error', + headers, + body: JSON.stringify({ + uploadAttemptId: options.uploadAttemptId, + sha256: options.artifact.sha256, + fileName: options.artifact.fileName, + sizeBytes: options.artifact.sizeBytes, + artifactType: options.artifact.artifactType, + ...(options.artifact.platform ? { platform: options.artifact.platform } : {}), + contentType: options.artifact.contentType, + }), + }, + 'preflight', + options.signal, + ).catch((error: unknown) => { // A failed preflight is ordinary here — the caller falls back to the legacy upload. A canceled // one is not: there is nothing to fall back to when the lease that paid for this upload is gone. - if (options.signal?.aborted) throw error; + if (options.signal?.aborted || isTransferPhaseTimeoutError(error)) throw error; return undefined; }); - if (!response?.ok) { + if (!result?.response.ok) { return undefined; } - return parseUploadPreflightResult(await response.json().catch(() => undefined)); + return parseUploadPreflightResult(result.body); } -/** - * The signal one small control-plane request runs under: the caller's cancellation and this - * request's own ceiling, whichever fires first. - * - * `preflight` and `finalize` carry a few hundred bytes of JSON that the transport writes with the - * request, so their ceiling is the same 30 seconds the artifact transfer spends getting connected and - * reading headers, and there is no separate body phase to time. - */ -function combineUploadSignals(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal { - const timeout = AbortSignal.timeout(timeoutMs); - return signal ? AbortSignal.any([signal, timeout]) : timeout; +async function requestUploadControl( + url: URL, + init: RequestInit, + phase: 'preflight' | 'finalize', + callerSignal?: AbortSignal, +): Promise<{ response: Response; body: unknown }> { + const watchdog = createTransferPhaseWatchdog(callerSignal); + watchdog.signal.throwIfAborted(); + watchdog.arm(phase, UPLOAD_CONTROL_PLANE_TIMEOUT_MS); + try { + const response = await fetch(url, { ...init, signal: watchdog.signal }); + const body: unknown = await response.json().catch(() => undefined); + watchdog.signal.throwIfAborted(); + return { response, body }; + } catch (error) { + watchdog.signal.throwIfAborted(); + throw error; + } finally { + watchdog.dispose(); + } } function parseUploadPreflightResult(value: unknown): UploadPreflightResult | undefined { @@ -425,28 +439,32 @@ async function finalizeDirectUpload(options: { buildDaemonHttpLeaseHeaders(options.leaseId), ); - const response = await fetch(finalizeUrl, { - method: 'POST', - // As with preflight: the route never redirects, and a redirect must not carry the token. - redirect: 'error', - headers, - signal: combineUploadSignals(options.signal, UPLOAD_CONTROL_PLANE_TIMEOUT_MS), - body: JSON.stringify({ uploadId: options.uploadId }), - }).catch((error: unknown) => { + const result = await requestUploadControl( + finalizeUrl, + { + method: 'POST', + // As with preflight: the route never redirects, and a redirect must not carry the token. + redirect: 'error', + headers, + body: JSON.stringify({ uploadId: options.uploadId }), + }, + 'finalize', + options.signal, + ).catch((error: unknown) => { // As with preflight: a canceled finalize is not a transport failure to report, it is the caller // saying this upload is no longer worth finishing. - if (options.signal?.aborted) throw error; + if (options.signal?.aborted || isTransferPhaseTimeoutError(error)) throw error; throw new AppError('COMMAND_FAILED', 'Failed to finalize direct artifact upload', {}, error); }); - if (!response.ok) { + if (!result.response.ok) { throw new AppError('COMMAND_FAILED', 'Direct artifact upload finalize failed', { - status: response.status, - statusText: response.statusText, + status: result.response.status, + statusText: result.response.statusText, }); } - const parsed = (await response.json().catch(() => undefined)) as UploadResponse | undefined; + const parsed = result.body as UploadResponse | undefined; if (!parsed?.ok || !parsed.uploadId) { throw new AppError('COMMAND_FAILED', 'Invalid upload finalize response'); } diff --git a/packages/remote/src/upload-stream.ts b/packages/remote/src/upload-stream.ts index bfdd2440ac..195c1428f5 100644 --- a/packages/remote/src/upload-stream.ts +++ b/packages/remote/src/upload-stream.ts @@ -18,36 +18,26 @@ import { /** Ceiling for establishing the connection for each upload hop or offset resume. */ export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; -/** - * Ceiling for waiting on response headers once this side has written everything the server is - * waiting for. For an upload that is the moment its outgoing request body is fully flushed, never - * the moment the request was created; for a bodyless request it is connection establishment. - */ +/** Header wait starts after connection and outgoing flush, never while the body progresses. */ export const TRANSFER_RESPONSE_HEADER_TIMEOUT_MS = 30_000; -/** - * Ceiling for silence between body bytes in either direction. Bytes actually written or read reset - * it, so an artifact that keeps moving keeps its budget while a connection that stopped carrying - * bytes does not. Progress never touches the aggregate budget the attempt runs under. - */ +/** Body-byte inactivity does not extend the enclosing aggregate. */ export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; -export type TransferPhase = 'connect' | 'request-body' | 'response-headers' | 'response-body'; +export type TransferPhase = + | 'connect' + | 'request-body' + | 'response-headers' + | 'response-body' + | 'preflight' + | 'finalize'; const BODY_PHASES: ReadonlySet = new Set(['request-body', 'response-body']); /** The `details.reason` every phase expiry carries, so no consumer has to match error text. */ const TRANSFER_PHASE_TIMEOUT_REASON = 'transfer_phase_timeout'; -/** - * Watches the sequential phases of ONE transfer attempt. - * - * One attempt gets one watchdog: arming a later phase stops counting the earlier one, so the ceiling - * that applies is always the phase the transfer is actually in. Progress re-arms only the two body - * phases. Nothing here extends an enclosing aggregate budget — the caller's signal is combined in, - * never replaced, so the earliest deadline still wins and every attempt, redirect and resume keeps - * sharing the original aggregate. - */ +/** One sequential phase under the earliest caller/aggregate deadline; progress resets body phases only. */ export type TransferPhaseWatchdog = { /** The signal the attempt runs on: the caller's cancellation plus every phase deadline. */ signal: AbortSignal; @@ -95,10 +85,6 @@ export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): Transfe }; } -/** - * Typed evidence that a transfer phase, rather than the caller or the whole preparation aggregate, - * ended the attempt. The expired phase and its effective budget travel in `details`. - */ function createTransferPhaseTimeoutError(phase: TransferPhase, timeoutMs: number): AppError { return new AppError('COMMAND_FAILED', `Artifact transfer ${phase} timed out`, { reason: TRANSFER_PHASE_TIMEOUT_REASON, @@ -167,10 +153,7 @@ export async function streamFileToHttpRequest(options: { errorMessage: string; errorHint?: string; retryable?: boolean; - /** - * The preparation aggregate `uploadArtifact` opened, combined with its caller's cancellation. Every - * attempt, redirect and resume runs under it; none of them opens a deadline of its own. - */ + /** Every attempt and hop shares the enclosing aggregate/caller signal. */ signal: AbortSignal; progress?: UploadStreamProgressOptions; }): Promise { @@ -203,12 +186,9 @@ async function streamFileToHttpRequestAttempt(options: { const headers = buildUploadRequestHeaders(options.headers, options.startOffset, payloadSize); emitUploadAttemptStarted(options.progress, options.startOffset, payloadSize); - // A redirect or a resume is a new attempt with fresh phase timers under the same `options.signal`. const watchdog = createTransferPhaseWatchdog(options.signal); watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); - // The owned pipeline (file → progress → request). Every exit joins it, so no owned byte overlaps - // the outcome or the next hop. let upload: Promise | undefined; let uploadSettled = false; let responseReceived = false; @@ -228,8 +208,6 @@ async function streamFileToHttpRequestAttempt(options: { req.destroy(); reject(error); }; - // Buffered bytes are not transport progress: the connect ceiling counts until the transport is - // carried. After that, an unflushed body is timed as inactivity and a flushed one waits on headers. const advanceOutgoingPhase = () => { if (responseReceived || watchdog.signal.aborted) return; if (!transportEstablished) return; @@ -266,7 +244,6 @@ async function streamFileToHttpRequestAttempt(options: { req.on('socket', (socket) => { if (socket.connecting) { - // Plain HTTP is carried on `connect`, TLS only once the handshake ends. const secured = (socket as { encrypted?: boolean }).encrypted === true; socket.once(secured ? 'secureConnect' : 'connect', () => { transportEstablished = true; @@ -283,8 +260,6 @@ async function streamFileToHttpRequestAttempt(options: { req.on('error', (err) => { if (responseReceived) return; - // A cancellation or an expired phase is this client's own outcome, never a transport failure - // that a retry or a legacy fallback could fix. if (watchdog.signal.aborted) { fail(watchdog.signal.reason); return; @@ -313,7 +288,6 @@ async function streamFileToHttpRequestAttempt(options: { upload = (async () => { await pipeline(fileStream, progressTransform, req, { signal: watchdog.signal }); uploadSettled = true; - // `finish`: this side's flush, not the peer's consumption, so the server owes an answer. bodyFlushed = true; advanceOutgoingPhase(); })().catch((err: unknown) => { @@ -325,7 +299,6 @@ async function streamFileToHttpRequestAttempt(options: { fail(watchdog.signal.reason); return; } - // A transport break reports through `error`, so this is the artifact read failing. const error = err instanceof Error ? err : new Error(String(err)); fail(new AppError('COMMAND_FAILED', 'Failed to read local artifact', {}, error)); }); @@ -344,8 +317,6 @@ async function streamFileToHttpRequestAttempt(options: { }; async function handleUploadResponse(res: IncomingMessage): Promise { - // Read the accepted answer to its end, then end and join the outgoing work before this attempt - // resolves or the next hop starts. const body = await readNodeHttpResponseBody(res, () => watchdog.touchBody()); await endOutgoingWork(); stopObservingWrites(); diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index b3ad98a522..ede67dcf74 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -164,10 +164,10 @@ "packages/remote/src/upload-client.ts#UploadPreflightResponse": "sha256:196a1d9ef41a1aec192589fa7ccb30cf71d971a005349d5d73eed42e6ab21327", "packages/remote/src/upload-client.ts#UploadPreflightResult": "sha256:bd8fc6d617c0630325455fd64d0fe319daaced198ee6745e4673a8ace3df460d", "packages/remote/src/upload-client.ts#UploadResponse": "sha256:499e01466b5e0e03413f8d3f1986f5af6bc120704c50a3747d84de422b3f0314", - "packages/remote/src/upload-client.ts#finalizeDirectUpload": "sha256:e8966f79d0518aa8d592113ff290efbde16120fd00ce8bb8648f243c38d0d173", + "packages/remote/src/upload-client.ts#finalizeDirectUpload": "sha256:3fd5ac061bbc16b0dd821db5e5c3e03daf53091cc93caa10845256e078779735", "packages/remote/src/upload-client.ts#isStringRecord": "sha256:b6d6b571b90baf0976d9c347c4a45a4363a4df5ad184ea1087cb3b9febd4b624", "packages/remote/src/upload-client.ts#parseUploadPreflightResult": "sha256:7ce3f43b45c03df8b7f2690432beedc7a537c1bba540961716862f8a632d5623", - "packages/remote/src/upload-client.ts#requestUploadPreflight": "sha256:e5f67a9a7799913f04cc9386291296a3d9fe13658776d2f4812a582c2e9befd5", + "packages/remote/src/upload-client.ts#requestUploadPreflight": "sha256:471a307f501c849cccea1036af1685187d72ca55d425bc5a8afe6f30d575ddb8", "packages/remote/src/upload-client.ts#shouldRetryDirectUpload": "sha256:29059f076ff7f2b9540ad15408be48792b2d63a601ef2b9fb2d54f5576d49bbd", "packages/remote/src/upload-client.ts#tryDirectUploadWithResume": "sha256:cc5d33f38e047cf3be4bd412b64958c7495eb9a45e58eefadaa00eefa0cb6b53", "packages/remote/src/upload-client.ts#uploadDirectArtifact": "sha256:93a36a9d3dea88064569517f3cdc34f27b33df6bf5f0c0ecc9efc29b47f24b77", @@ -221,7 +221,7 @@ { "declaration": "packages/remote/src/daemon-artifacts.ts#DownloadRemoteArtifactParams", "digest": "sha256:ba6d2f6ac51e05ccc5e0c940b355d62374e018b1ae6f6f744becf6e1760cbb63", - "rationale": "#2246 adds the optional isDirectory field. It is pure client-local state — never serialized, never sent to the daemon — that tells the CLIENT'S OWN download logic to extract a tar body instead of writing it verbatim; the GET /artifacts/:id request and response framing are unchanged. Every existing call site (screenshot, recording) omits it and keeps writing a single file exactly as before. The signature also names the client request as DaemonWireRequest and the response as the kernel DaemonResponse instead of the daemon package aliases; only compile-time type names moved." + "rationale": "#2246 adds the optional isDirectory field. It is pure client-local state \u2014 never serialized, never sent to the daemon \u2014 that tells the CLIENT'S OWN download logic to extract a tar body instead of writing it verbatim; the GET /artifacts/:id request and response framing are unchanged. Every existing call site (screenshot, recording) omits it and keeps writing a single file exactly as before. The signature also names the client request as DaemonWireRequest and the response as the kernel DaemonResponse instead of the daemon package aliases; only compile-time type names moved." }, { "declaration": "packages/remote/src/daemon-artifacts.ts#downloadRemoteArtifact", @@ -296,7 +296,7 @@ { "declaration": "packages/daemon/src/server/http/http-authorization.ts#authorizeAuxiliaryHttpRequest", "digest": "sha256:1c323f97c3cefec8f95633b6abd127342bf75494bfb109d1cc6eadf3ceacaeae", - "rationale": "#2198 context: adds one optional field to the value this DAEMON-INTERNAL gate hands its own route handlers — sessionNamespace, the caller's tenant plus whether the daemon partitioned that caller's session names. Nothing about it is serialized: no request header is read that was not read before, no response field is added, and a refusal keeps the same 401 {ok:false,error,code} REST body sendRestJsonError already sent. What it enables is a refusal DROPPED, never one added — an unattested (client-declared) tenant no longer has the : prefix rule applied to it, because scopeRequestSession never applied that prefix to its sessions either. A released peer that now gets 200 where it got 401 is getting the ndjson record it asked for and already parses." + "rationale": "#2198 context: adds one optional field to the value this DAEMON-INTERNAL gate hands its own route handlers \u2014 sessionNamespace, the caller's tenant plus whether the daemon partitioned that caller's session names. Nothing about it is serialized: no request header is read that was not read before, no response field is added, and a refusal keeps the same 401 {ok:false,error,code} REST body sendRestJsonError already sent. What it enables is a refusal DROPPED, never one added \u2014 an unattested (client-declared) tenant no longer has the : prefix rule applied to it, because scopeRequestSession never applied that prefix to its sessions either. A released peer that now gets 200 where it got 401 is getting the ndjson record it asked for and already parses." }, { "declaration": "packages/daemon/src/server/request-diagnostics-http.ts#RequestDiagnosticsHttpAuthorizer", @@ -510,13 +510,13 @@ }, { "declaration": "packages/remote/src/upload-client.ts#requestUploadPreflight", - "digest": "sha256:e5f67a9a7799913f04cc9386291296a3d9fe13658776d2f4812a582c2e9befd5", - "rationale": "#4441 renames the constant naming this call's ceiling from UPLOAD_PREFLIGHT_TIMEOUT_MS to UPLOAD_CONTROL_PLANE_TIMEOUT_MS because it governs finalize as well. The value is the same 30 seconds the released client used, and the request body, headers, route and redirect refusal are byte-for-byte unchanged, so a released daemon or proxy parses the identical request." + "digest": "sha256:471a307f501c849cccea1036af1685187d72ca55d425bc5a8afe6f30d575ddb8", + "rationale": "#4441 preserves the 30-second control-request ceiling but reports its typed phase timeout instead of retrying or falling back after expiry. The shared Fetch/JSON owner retains route, method, authentication, request body, redirect refusal and successful response parsing, so protocol-2 peers parse the same messages; only the local timeout outcome stops subsequent upload legs." }, { "declaration": "packages/remote/src/upload-client.ts#finalizeDirectUpload", - "digest": "sha256:e8966f79d0518aa8d592113ff290efbde16120fd00ce8bb8648f243c38d0d173", - "rationale": "#4441 renames the constant naming this call's 30-second ceiling and annotates a caught error; neither reaches the wire. The request body, headers, route and redirect refusal are unchanged, and a released daemon or proxy never redirects this route, so it parses the identical request." + "digest": "sha256:3fd5ac061bbc16b0dd821db5e5c3e03daf53091cc93caa10845256e078779735", + "rationale": "#4441 preserves the 30-second control-request ceiling but reports its typed phase timeout instead of retrying or falling back after expiry. The shared Fetch/JSON owner retains route, method, authentication, request body, redirect refusal and successful response parsing, so protocol-2 peers parse the same messages; only the local timeout outcome stops subsequent upload legs." }, { "declaration": "packages/daemon/src/server/http/http-rpc-methods.ts#toInstallFromSourceDaemonRequest", From dc5e0245c3694e14361a1884784fc844ae373e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 20:14:26 +0100 Subject: [PATCH 18/27] refactor(test): simplify shared upload fixture expectations --- .../src/internal/transport-writes.test.ts | 14 +------ .../support/upload-transfer-fixtures.ts | 42 ++++++------------- .../__tests__/upload-stream-lifecycle.test.ts | 15 ------- .../__tests__/upload-stream-phases.test.ts | 4 -- 4 files changed, 15 insertions(+), 60 deletions(-) diff --git a/packages/host-kit/src/internal/transport-writes.test.ts b/packages/host-kit/src/internal/transport-writes.test.ts index 0770c7bbc8..62737827a4 100644 --- a/packages/host-kit/src/internal/transport-writes.test.ts +++ b/packages/host-kit/src/internal/transport-writes.test.ts @@ -5,10 +5,6 @@ import type { ClientRequest } from 'node:http'; import { test } from 'vitest'; import { observeCompletedRequestWrites } from '../transport.ts'; -/** - * A request whose `write` the test completes by hand, as Node does once a chunk reaches the kernel. - * Every write, empty or not, holds its callback until `complete` runs it. - */ class FakeRequest extends EventEmitter { destroyed = false; accepts = true; @@ -24,15 +20,9 @@ class FakeRequest extends EventEmitter { } } -type Observed = { - fake: FakeRequest; - req: ClientRequest; - caller: AbortController; - completed: { count: number }; - dispose(): void; -}; +type Observed = ReturnType; -function observe(prepare?: (fake: FakeRequest) => void): Observed { +function observe(prepare?: (fake: FakeRequest) => void) { const fake = new FakeRequest(); prepare?.(fake); const caller = new AbortController(); diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts index 8604faf759..b77eae4610 100644 --- a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -1,4 +1,5 @@ import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; import fs from 'node:fs'; import http from 'node:http'; import https from 'node:https'; @@ -11,7 +12,6 @@ import { onTestFinished, vi } from 'vitest'; import { closeLoopbackServer, listenOnLoopback, mkdtempForTestSync } from '@agent-device/test-kit'; import { uploadArtifact } from '../../upload-client.ts'; import { type UploadStreamResponse, streamFileToHttpRequest } from '../../upload-stream.ts'; -// Adopted budgets are literal expectations, independent of the subject. export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; export const TRANSFER_RESPONSE_HEADER_TIMEOUT_MS = 30_000; export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; @@ -366,7 +366,6 @@ export async function startParkedUploadDaemon( let uploadLegs = 0; let newestParkedLeg: http.IncomingMessage | undefined; let readingToEnd = false; - const server = http.createServer((req, res) => { const key = `${req.method} ${req.url}`; requests.push(key); @@ -377,10 +376,7 @@ export async function startParkedUploadDaemon( req.on('error', () => {}); const answer = options.answerFirstLeg; if (answer && uploadLegs === 1) { - res.statusCode = answer.status; - for (const [name, value] of Object.entries(answer.headers ?? {})) { - res.setHeader(name, value); - } + res.writeHead(answer.status, answer.headers); res.end(); req.pause(); return; @@ -462,10 +458,7 @@ export async function startAnswerUploadDaemon(options: { requests.push(`${req.method} ${req.url}`); req.on('error', () => {}); const answer = options.respond(req); - res.statusCode = answer.status; - for (const [name, value] of Object.entries(answer.headers ?? {})) { - res.setHeader(name, value); - } + res.writeHead(answer.status, answer.headers); res.end(answer.body); req.pause(); }); @@ -580,10 +573,7 @@ export async function startUploadLegsDaemon( const answer = () => { if (!behavior.respond) return; const reply = behavior.respond(); - res.statusCode = reply.status; - for (const [name, value] of Object.entries(reply.headers ?? {})) { - res.setHeader(name, value); - } + res.writeHead(reply.status, reply.headers); res.end(reply.body); }; if (behavior.readAllBody) { @@ -613,26 +603,20 @@ export async function startUploadLegsDaemon( } function readRepoFixture(fileName: string): Buffer { - let dir = process.cwd(); - for (let depth = 0; depth < 8; depth += 1) { - const candidate = path.join(dir, 'test/integration/provider-scenarios/fixtures', fileName); - if (fs.existsSync(candidate)) return fs.readFileSync(candidate); - dir = path.dirname(dir); - } - throw new Error(`missing ${fileName} under test/integration/provider-scenarios/fixtures`); + return fs.readFileSync( + new URL( + `../../../../../test/integration/provider-scenarios/fixtures/${fileName}`, + import.meta.url, + ), + ); } export function assertDetailedError( error: unknown, details: Record, -): asserts error is { details?: Record } { - assert.ok(error && typeof error === 'object', `expected an error, got ${String(error)}`); - const actual = JSON.stringify((error as { details?: unknown }).details ?? {}); +): asserts error is AppError { + assert.ok(error instanceof AppError, `expected AppError, got ${String(error)}`); for (const [key, value] of Object.entries(details)) { - assert.equal( - (error as { details?: Record }).details?.[key], - value, - `${key} of ${actual}`, - ); + assert.equal(error.details?.[key], value, `${key} of ${JSON.stringify(error.details)}`); } } diff --git a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts index 8d4b169051..32b4f74c4d 100644 --- a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts +++ b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts @@ -49,7 +49,6 @@ test('an early 413 from a peer that stops reading is consumed and its owned work await settleAfterBudget(attempt); assert.deepEqual(await atSettle, { destroyed: true, finished: false }); const result = await attempt; - assert.equal(result.error, undefined, `got ${String(result.error)}`); assert.equal(result.value?.statusCode, 413); assert.equal(result.value?.body, 'payload too large'); @@ -73,7 +72,6 @@ test('a resume response ends the stalled hop before the next attempt is dialed', }); await settleAfterBudget(attempt); const result = await attempt; - assertDetailedError(result.error, { offset: resumeOffset, previousOffset: resumeOffset }); assert.equal(daemon.requests.filter((entry) => entry === LEGACY_UPLOAD_REQUEST).length, 2); assert.equal(held.destroyedAtNextOpen, true, 'hop one was ended before hop two opened'); @@ -96,7 +94,6 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it }); await untilIo(() => hopTwoTookBytes(daemon) || attempt.settled); assert.ok(hopTwoTookBytes(daemon), 'the resumed hop was dialed and took bytes'); - for (let window = 0; window < 6; window += 1) { const before = { bytes: daemon.receivedBytes, completed: writes.completed }; const progressed = await awaitBodyProgress(daemon, writes, before, attempt); @@ -106,7 +103,6 @@ test('a resumed hop that makes real progress past the first hop ceiling keeps it assert.equal(daemon.bodyEnded, false); } assert.ok(6 * progressWindowMs > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); - daemon.resetParkedLeg(); await settleAfterBudget(attempt); const result = await attempt; @@ -138,7 +134,6 @@ test('a redirect response ends the stalled hop before the next hop is dialed', a }); await settleAfterBudget(attempt); const result = await attempt; - assert.equal(result.error, undefined, `got ${String(result.error)}`); assert.equal(result.value?.statusCode, 413); assert.equal(result.value?.body, 'moved and refused'); @@ -159,7 +154,6 @@ test('a response body that stops mid-answer expires the response-body phase afte await spendBudget(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 5_000, attempt); await settleAfterBudget(attempt); const result = await attempt; - assert.equal(result.value, undefined); assertDetailedError(result.error, { reason: 'transfer_phase_timeout', @@ -183,7 +177,6 @@ test('a connected HTTP upload with an unfinished source expires body inactivity, assert.equal(writes.finished, false, 'the source still holds the body open'); await settleAfterBudget(attempt); const result = await attempt; - assert.equal(result.value, undefined); assertDetailedError(result.error, { reason: 'transfer_phase_timeout', @@ -202,7 +195,6 @@ test('a TLS handshake that never completes expires connect even with the whole b await spendBudget(TRANSFER_CONNECT_TIMEOUT_MS + 5_000, attempt); await settleAfterBudget(attempt); const result = await attempt; - assert.equal(result.value, undefined); assertDetailedError(result.error, { reason: 'transfer_phase_timeout', @@ -227,7 +219,6 @@ test('a completed TLS handshake with an unfinished source expires body inactivit assert.equal(writes.finished, false, 'the source still holds the body open'); await settleAfterBudget(attempt); const result = await attempt; - assert.equal(result.value, undefined); assertDetailedError(result.error, { reason: 'transfer_phase_timeout', @@ -252,7 +243,6 @@ test('a zero-length artifact on a peer that never answers waits on its header bu await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, attempt); await settleAfterBudget(attempt, snapshot); const result = await attempt; - assert.equal(result.value, undefined); assertDetailedError(result.error, { reason: 'transfer_phase_timeout', @@ -287,7 +277,6 @@ test('an upload the peer keeps taking finishes while every phase ceiling is unex await untilIo(() => attempt.settled || daemon.receivedBytes > bytesBefore); } const result = await attempt; - assert.equal(result.error, undefined, `upload failed: ${String(result.error)}`); assert.equal(result.value?.statusCode, 200); assert.ok( @@ -310,7 +299,6 @@ test('a caller cancel during a stalled connect stays caller cancellation, distin await spendBudget(5_000, canceled); control.abort(); const result = await canceled; - assert.equal(result.value, undefined); assert.ok(result.error instanceof DOMException, `got ${String(result.error)}`); assert.equal((result.error as DOMException).name, 'AbortError'); @@ -339,7 +327,6 @@ test('an enclosing aggregate deadline ends a redirect hop with its own reason, n aggregate.abort(reason); await settleAfterBudget(attempt); const result = await attempt; - assert.equal(result.value, undefined); assert.equal(result.error, reason); assert.equal(isTransferPhaseTimeoutError(result.error), false); @@ -361,7 +348,6 @@ test('an attempt on a reused connection is already past its connect phase', asyn }); const firstResult = await first; assert.equal(firstResult.error, undefined, `leg one failed: ${String(firstResult.error)}`); - const second = startUploadAttempt({ url: `${daemon.baseUrl}/upload`, artifactPath: createArtifactFile(64 * 1024), @@ -375,7 +361,6 @@ test('an attempt on a reused connection is already past its connect phase', asyn await spendBudget(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS + 5_000, second); await settleAfterBudget(second, snapshot); const result = await second; - assert.equal(result.value, undefined); assertDetailedError(result.error, { reason: 'transfer_phase_timeout', diff --git a/packages/remote/src/__tests__/upload-stream-phases.test.ts b/packages/remote/src/__tests__/upload-stream-phases.test.ts index e602b8e9ed..81a79fbc8c 100644 --- a/packages/remote/src/__tests__/upload-stream-phases.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phases.test.ts @@ -42,7 +42,6 @@ test('a body that keeps moving outlives the body-inactivity ceiling', async () = windowMs: TRANSFER_CONNECT_TIMEOUT_MS, writes, }); - assert.equal(progress.progressedInEveryWindow, true); assert.ok( progress.budgetMs > TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, @@ -64,7 +63,6 @@ test('an unfinished source with no new bytes expires the request-body phase on i assert.equal(writes.finished, false, 'the source still holds the body open'); await settleAfterBudget(upload); const result = await upload; - assert.equal(result.value, undefined); assertDetailedError(result.error, { reason: 'transfer_phase_timeout', @@ -113,7 +111,6 @@ test('a body admitted past four connect ceilings still waits only its own header await vi.advanceTimersByTimeAsync(1); await settleAfterBudget(upload, snapshot); const result = await upload; - assert.equal(result.value, undefined); assertDetailedError(result.error, { reason: 'transfer_phase_timeout', @@ -172,7 +169,6 @@ test('the caller ending the upload is reported as cancellation, not as an expire await spendBudget(5_000, upload); control.abort(); const result = await upload; - assert.equal(result.value, undefined); assert.ok(result.error instanceof DOMException, `got ${String(result.error)}`); assert.equal((result.error as DOMException).name, 'AbortError'); From 049d1160a0254867c1fadf0744282c62886dee62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 20:35:45 +0100 Subject: [PATCH 19/27] fix(test): preserve declared upload budgets and caller cancellation proof --- .../support/upload-transfer-fixtures.ts | 55 ++++++----------- .../upload-client-cancellation.test.ts | 60 ++++++++++--------- .../__tests__/upload-stream-lifecycle.test.ts | 6 +- packages/remote/src/upload-client.ts | 2 +- 4 files changed, 57 insertions(+), 66 deletions(-) diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts index b77eae4610..4b98850b26 100644 --- a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -8,7 +8,7 @@ import path from 'node:path'; import { PassThrough } from 'node:stream'; import tls from 'node:tls'; import { setTimeout as realSetTimeout, clearTimeout as realClearTimeout } from 'node:timers'; -import { onTestFinished, vi } from 'vitest'; +import { beforeEach, onTestFinished, vi } from 'vitest'; import { closeLoopbackServer, listenOnLoopback, mkdtempForTestSync } from '@agent-device/test-kit'; import { uploadArtifact } from '../../upload-client.ts'; import { type UploadStreamResponse, streamFileToHttpRequest } from '../../upload-stream.ts'; @@ -20,20 +20,26 @@ export const PREFLIGHT_REQUEST = 'POST /upload/preflight'; export const LEGACY_UPLOAD_REQUEST = 'POST /upload'; const BUDGET_STEP_MS = 5_000; - const RELEASE_QUOTA_BYTES = 256 * 1024; export type Settles = { readonly settled: boolean }; const ioWaiters = new Map<() => void, () => void>(); -let ioWaitersEndWithTest = false; -let ioDeadline: number | undefined; +let ioDeadline: number; +beforeEach(({ task }) => { + assert.ok(task.timeout > 100, 'transfer diagnostics require a finite test timeout'); + ioDeadline = performance.now() + task.timeout - 100; + onTestFinished(() => { + for (const release of ioWaiters.values()) release(); + ioWaiters.clear(); + }); +}); function noteIo(): void { for (const wake of ioWaiters.keys()) wake(); } -/** Charges no fake time; all pending waits share a 3s real diagnostic deadline per test. */ +/** Charges no fake time; diagnostics fire 100ms before the declared test timeout. */ export async function untilIo( ready: () => boolean, label = 'I/O prerequisite', @@ -41,16 +47,6 @@ export async function untilIo( ): Promise { if (ready()) return; const unmet = new Error('untilIo: the I/O this wait needs never happened before its test ended'); - if (!ioWaitersEndWithTest) { - ioWaitersEndWithTest = true; - ioDeadline = performance.now() + 3_000; - onTestFinished(() => { - ioWaitersEndWithTest = false; - ioDeadline = undefined; - for (const release of ioWaiters.values()) release(); - ioWaiters.clear(); - }); - } await new Promise((resolve, reject) => { const finish = (error?: unknown) => { realClearTimeout(timer); @@ -72,7 +68,7 @@ export async function untilIo( }); finish(error); }, - Math.max(0, (ioDeadline ?? performance.now()) - performance.now()), + Math.max(0, ioDeadline - performance.now()), ); ioWaiters.set(wake, () => finish(unmet)); }); @@ -84,7 +80,7 @@ export function useTransferClock(): void { export type ObservedOutcome = Promise<{ value: T | undefined; error: unknown }> & Settles; -function observeOutcome(work: Promise): ObservedOutcome { +export function observeOutcome(work: Promise): ObservedOutcome { let settled = false; const outcome = work.then( (value) => ({ value, error: undefined as unknown }), @@ -282,7 +278,7 @@ function controlledNow(): number { export async function settleAfterBudget( transfer: Settles, - snapshot: () => unknown = () => transfer, + snapshot: () => unknown = () => ({ settled: transfer.settled }), ): Promise { for (let round = 0; round < 200 && !transfer.settled; round += 1) { await vi.advanceTimersByTimeAsync(100); @@ -346,8 +342,7 @@ function refusedPreflight(req: http.IncomingMessage, res: http.ServerResponse): if (`${req.method} ${req.url}` !== PREFLIGHT_REQUEST) return false; req.resume(); req.on('end', () => { - res.statusCode = 404; - res.end('not found'); + res.writeHead(404).end('not found'); }); return true; } @@ -481,8 +476,7 @@ export async function startSilentTcpOrigin() { export async function startHalfAnsweredOrigin(options: { firstChunk: string }) { const server = http.createServer((req, res) => { req.resume(); - res.statusCode = 200; - res.setHeader('content-type', 'text/plain'); + res.writeHead(200, { 'content-type': 'text/plain' }); res.write(options.firstChunk); }); const port = await listenUntimed(server); @@ -493,7 +487,6 @@ export async function startSlowReadingUploadDaemon(options: { status: number; bo const parked: http.IncomingMessage[] = []; let receivedBytes = 0; let bodyEnded = false; - let connections = 0; const server = http.createServer((req, res) => { if (refusedPreflight(req, res)) return; req.on('data', (chunk: Buffer) => { @@ -504,14 +497,10 @@ export async function startSlowReadingUploadDaemon(options: { status: number; bo }); req.on('end', () => { bodyEnded = true; - res.statusCode = options.status; - res.end(options.body); + res.writeHead(options.status).end(options.body); noteIo(); }); }); - server.on('connection', () => { - connections += 1; - }); const port = await listenUntimed(server); return { baseUrl: `http://127.0.0.1:${port}`, @@ -521,9 +510,6 @@ export async function startSlowReadingUploadDaemon(options: { status: number; bo get bodyEnded() { return bodyEnded; }, - get connections() { - return connections; - }, releaseAllParked() { for (const entry of parked.splice(0)) entry.resume(); }, @@ -581,12 +567,7 @@ export async function startUploadLegsDaemon( req.on('end', answer); return; } - let tookOneChunk = false; - req.on('data', () => { - if (tookOneChunk) return; - tookOneChunk = true; - req.pause(); - }); + req.once('data', () => req.pause()); answer(); }); server.on('connection', () => { diff --git a/packages/remote/src/__tests__/upload-client-cancellation.test.ts b/packages/remote/src/__tests__/upload-client-cancellation.test.ts index f5dce572c8..011f6bbe01 100644 --- a/packages/remote/src/__tests__/upload-client-cancellation.test.ts +++ b/packages/remote/src/__tests__/upload-client-cancellation.test.ts @@ -16,6 +16,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { once } from 'node:events'; import { uploadArtifact } from '../upload-client.ts'; import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { observeOutcome } from './support/upload-transfer-fixtures.ts'; const TEST_TOKEN = 'agent-device-upload-cancel-token'; const tempDirs: string[] = []; @@ -83,6 +84,8 @@ test.each([ 'finalize', 'retry-preflight', 'retry-finalize', + 'cancel-preflight', + 'cancel-finalize', 'prose-preflight', 'prose-finalize', ])('control request %s distinguishes a deadline from matching error prose', async (stage) => { @@ -91,6 +94,8 @@ test.each([ const artifactPath = createTempFile('app.apk', 'control-plane-payload'); const retry = stage.startsWith('retry-'); const prose = stage.startsWith('prose-'); + const canceled = stage.startsWith('cancel-'); + const callerReason = new Error('lease ended during control request'); const phase = stage.endsWith('preflight') ? 'preflight' : 'finalize'; const events: string[] = []; const routes: string[] = []; @@ -110,6 +115,7 @@ test.each([ }); let stalledSignal: AbortSignal | undefined; let preflights = 0; + let refuseStall: ((error: Error) => void) | undefined; const fetch = vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { const route = new URL(String(input)).pathname; if (route.endsWith('/preflight')) preflights += 1; @@ -120,6 +126,7 @@ test.each([ entered(); if (prose) throw new Error(`Artifact transfer ${phase} timed out`); return await new Promise((_resolve, reject) => { + refuseStall = reject; signal.addEventListener('abort', () => reject(signal.reason), { once: true }); }); } @@ -129,22 +136,16 @@ test.each([ upload: { url: `${server.baseUrl}/ticket`, headers: {} }, }); }); - let settled = false; - const outcome = uploadArtifact({ - env: process.env, - localPath: artifactPath, - baseUrl: server.baseUrl, - token: TEST_TOKEN, - signal: caller.signal, - onProgress: (event) => events.push(event.type), - }) - .then( - (value) => ({ value, error: undefined as unknown }), - (error: unknown) => ({ value: undefined, error }), - ) - .finally(() => { - settled = true; - }); + const outcome = observeOutcome( + uploadArtifact({ + env: process.env, + localPath: artifactPath, + baseUrl: server.baseUrl, + token: TEST_TOKEN, + signal: caller.signal, + onProgress: (event) => events.push(event.type), + }), + ); try { await stalled; if (prose) { @@ -155,21 +156,25 @@ test.each([ assert.equal(stalledSignal?.aborted, false); return; } - await vi.advanceTimersByTimeAsync(29_999); + await vi.advanceTimersByTimeAsync(canceled ? 10_000 : 29_999); assert.equal(stalledSignal?.aborted, false); - assert.equal(settled, false); - await vi.advanceTimersByTimeAsync(1); + assert.equal(outcome.settled, false); + if (canceled) caller.abort(callerReason); + await vi.advanceTimersByTimeAsync(canceled ? 0 : 1); assert.equal(stalledSignal?.aborted, true); - assert.equal(settled, true, 'expiry settles without starting another transfer'); + assert.equal(outcome.settled, true, 'expiry settles without starting another transfer'); const result = await outcome; assert.equal(result.value, undefined); - assert.ok(result.error instanceof AppError); - assert.deepEqual(result.error.details, { - reason: 'transfer_phase_timeout', - transferPhase: phase, - timeoutMs: 30_000, - }); - assert.equal(caller.signal.aborted, false); + if (canceled) assert.equal(result.error, callerReason); + else { + assert.ok(result.error instanceof AppError); + assert.deepEqual(result.error.details, { + reason: 'transfer_phase_timeout', + transferPhase: phase, + timeoutMs: 30_000, + }); + } + assert.equal(caller.signal.aborted, canceled); assert.equal(events.includes('fallback'), false); assert.deepEqual( routes, @@ -181,6 +186,7 @@ test.each([ assert.equal(fetch.mock.calls.length, phase === 'preflight' ? (retry ? 2 : 1) : retry ? 3 : 2); } finally { caller.abort(new Error('fixture cleanup')); + refuseStall?.(new Error('fixture cleanup')); await outcome; await server.close(); } diff --git a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts index 32b4f74c4d..ed1cd9a47a 100644 --- a/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts +++ b/packages/remote/src/__tests__/upload-stream-lifecycle.test.ts @@ -274,7 +274,11 @@ test('an upload the peer keeps taking finishes while every phase ceiling is unex await vi.advanceTimersByTimeAsync(1_000); spent += 1_000; } - await untilIo(() => attempt.settled || daemon.receivedBytes > bytesBefore); + await untilIo( + () => attempt.settled || daemon.receivedBytes > bytesBefore, + 'slow upload next chunk', + () => ({ writes, receivedBytes: daemon.receivedBytes, settled: attempt.settled }), + ); } const result = await attempt; assert.equal(result.error, undefined, `upload failed: ${String(result.error)}`); diff --git a/packages/remote/src/upload-client.ts b/packages/remote/src/upload-client.ts index 67918dea5a..9820750489 100644 --- a/packages/remote/src/upload-client.ts +++ b/packages/remote/src/upload-client.ts @@ -333,7 +333,7 @@ async function requestUploadControl( watchdog.arm(phase, UPLOAD_CONTROL_PLANE_TIMEOUT_MS); try { const response = await fetch(url, { ...init, signal: watchdog.signal }); - const body: unknown = await response.json().catch(() => undefined); + const body: unknown = response.ok ? await response.json().catch(() => undefined) : undefined; watchdog.signal.throwIfAborted(); return { response, body }; } catch (error) { From e4863396836b7c307b642b60c2d1859f6d458f9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 20:35:45 +0100 Subject: [PATCH 20/27] chore(gates): track the shared upload control request owner --- test/wire-compat/ledger.json | 10 ++++++++-- test/wire-compat/surface.ts | 1 + test/wire-compat/wire-mutations.test.ts | 7 +++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index ede67dcf74..0d40fa8b68 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -167,6 +167,7 @@ "packages/remote/src/upload-client.ts#finalizeDirectUpload": "sha256:3fd5ac061bbc16b0dd821db5e5c3e03daf53091cc93caa10845256e078779735", "packages/remote/src/upload-client.ts#isStringRecord": "sha256:b6d6b571b90baf0976d9c347c4a45a4363a4df5ad184ea1087cb3b9febd4b624", "packages/remote/src/upload-client.ts#parseUploadPreflightResult": "sha256:7ce3f43b45c03df8b7f2690432beedc7a537c1bba540961716862f8a632d5623", + "packages/remote/src/upload-client.ts#requestUploadControl": "sha256:05c7bd5090eade7b163456c10489899842c452b5633730827539d4adcb94c85d", "packages/remote/src/upload-client.ts#requestUploadPreflight": "sha256:471a307f501c849cccea1036af1685187d72ca55d425bc5a8afe6f30d575ddb8", "packages/remote/src/upload-client.ts#shouldRetryDirectUpload": "sha256:29059f076ff7f2b9540ad15408be48792b2d63a601ef2b9fb2d54f5576d49bbd", "packages/remote/src/upload-client.ts#tryDirectUploadWithResume": "sha256:cc5d33f38e047cf3be4bd412b64958c7495eb9a45e58eefadaa00eefa0cb6b53", @@ -221,7 +222,7 @@ { "declaration": "packages/remote/src/daemon-artifacts.ts#DownloadRemoteArtifactParams", "digest": "sha256:ba6d2f6ac51e05ccc5e0c940b355d62374e018b1ae6f6f744becf6e1760cbb63", - "rationale": "#2246 adds the optional isDirectory field. It is pure client-local state \u2014 never serialized, never sent to the daemon \u2014 that tells the CLIENT'S OWN download logic to extract a tar body instead of writing it verbatim; the GET /artifacts/:id request and response framing are unchanged. Every existing call site (screenshot, recording) omits it and keeps writing a single file exactly as before. The signature also names the client request as DaemonWireRequest and the response as the kernel DaemonResponse instead of the daemon package aliases; only compile-time type names moved." + "rationale": "#2246 adds the optional isDirectory field. It is pure client-local state — never serialized, never sent to the daemon — that tells the CLIENT'S OWN download logic to extract a tar body instead of writing it verbatim; the GET /artifacts/:id request and response framing are unchanged. Every existing call site (screenshot, recording) omits it and keeps writing a single file exactly as before. The signature also names the client request as DaemonWireRequest and the response as the kernel DaemonResponse instead of the daemon package aliases; only compile-time type names moved." }, { "declaration": "packages/remote/src/daemon-artifacts.ts#downloadRemoteArtifact", @@ -296,7 +297,7 @@ { "declaration": "packages/daemon/src/server/http/http-authorization.ts#authorizeAuxiliaryHttpRequest", "digest": "sha256:1c323f97c3cefec8f95633b6abd127342bf75494bfb109d1cc6eadf3ceacaeae", - "rationale": "#2198 context: adds one optional field to the value this DAEMON-INTERNAL gate hands its own route handlers \u2014 sessionNamespace, the caller's tenant plus whether the daemon partitioned that caller's session names. Nothing about it is serialized: no request header is read that was not read before, no response field is added, and a refusal keeps the same 401 {ok:false,error,code} REST body sendRestJsonError already sent. What it enables is a refusal DROPPED, never one added \u2014 an unattested (client-declared) tenant no longer has the : prefix rule applied to it, because scopeRequestSession never applied that prefix to its sessions either. A released peer that now gets 200 where it got 401 is getting the ndjson record it asked for and already parses." + "rationale": "#2198 context: adds one optional field to the value this DAEMON-INTERNAL gate hands its own route handlers — sessionNamespace, the caller's tenant plus whether the daemon partitioned that caller's session names. Nothing about it is serialized: no request header is read that was not read before, no response field is added, and a refusal keeps the same 401 {ok:false,error,code} REST body sendRestJsonError already sent. What it enables is a refusal DROPPED, never one added — an unattested (client-declared) tenant no longer has the : prefix rule applied to it, because scopeRequestSession never applied that prefix to its sessions either. A released peer that now gets 200 where it got 401 is getting the ndjson record it asked for and already parses." }, { "declaration": "packages/daemon/src/server/request-diagnostics-http.ts#RequestDiagnosticsHttpAuthorizer", @@ -508,6 +509,11 @@ "digest": "sha256:eed6cb154db2b67ffe5f058ef0570fec247a5d2a43af2d42952dee0b08870a87", "rationale": "The /rpc body limit is now DAEMON_REQUEST_MAX_BYTES (64 MiB), the one request limit the socket transport, the proxy default, and client writers share, raised from 1 MiB. A larger limit refuses nothing a released client sends; a new client refuses a request over 64 MiB before sending it, and a released daemon still answers anything between 1 MiB and 64 MiB with its own -32700 refusal, as before." }, + { + "declaration": "packages/remote/src/upload-client.ts#requestUploadControl", + "digest": "sha256:05c7bd5090eade7b163456c10489899842c452b5633730827539d4adcb94c85d", + "rationale": "Lists the shared control Fetch/JSON owner so changes to method, headers, body or redirect forwarding remain wire-gated. Existing preflight/finalize routes and protocol-2 payloads are preserved; the original 30-second ceiling now retains typed local phase expiry and stops later legs. Non-success status handling is unchanged." + }, { "declaration": "packages/remote/src/upload-client.ts#requestUploadPreflight", "digest": "sha256:471a307f501c849cccea1036af1685187d72ca55d425bc5a8afe6f30d575ddb8", diff --git a/test/wire-compat/surface.ts b/test/wire-compat/surface.ts index a434384d33..61d685487b 100644 --- a/test/wire-compat/surface.ts +++ b/test/wire-compat/surface.ts @@ -393,6 +393,7 @@ export const WIRE_SURFACE: readonly WireSurfaceGroup[] = [ 'parseUploadPreflightResult', 'isStringRecord', 'requestUploadPreflight', + 'requestUploadControl', 'uploadDirectArtifact', 'tryDirectUploadWithResume', 'shouldRetryDirectUpload', diff --git a/test/wire-compat/wire-mutations.test.ts b/test/wire-compat/wire-mutations.test.ts index 92340a37c4..3674ba25a0 100644 --- a/test/wire-compat/wire-mutations.test.ts +++ b/test/wire-compat/wire-mutations.test.ts @@ -49,6 +49,13 @@ type WireMutation = { }; const MUTATIONS: readonly WireMutation[] = [ + { + breakClass: 'upload control: Fetch drops the method, headers and body', + file: 'packages/remote/src/upload-client.ts', + name: 'requestUploadControl', + from: '...init, signal:', + to: 'signal:', + }, { breakClass: 'upload ticket: the response wrapper drops the configured origin', file: 'packages/proxy/src/daemon-proxy.ts', From 4de08a9eb2f2a9975a76191cf096c5404c0132dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 21:00:27 +0100 Subject: [PATCH 21/27] refactor(remote): constrain control requests at their sending owner --- packages/remote/src/upload-client.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/remote/src/upload-client.ts b/packages/remote/src/upload-client.ts index 9820750489..62291f5fe1 100644 --- a/packages/remote/src/upload-client.ts +++ b/packages/remote/src/upload-client.ts @@ -291,10 +291,6 @@ async function requestUploadPreflight(options: { const result = await requestUploadControl( preflightUrl, { - method: 'POST', - // The daemon never redirects this route. A redirect would carry x-agent-device-token, which - // fetch keeps on a cross-origin hop, so any redirect fails the preflight instead. - redirect: 'error', headers, body: JSON.stringify({ uploadAttemptId: options.uploadAttemptId, @@ -324,7 +320,7 @@ async function requestUploadPreflight(options: { async function requestUploadControl( url: URL, - init: RequestInit, + init: { headers: Record; body: string }, phase: 'preflight' | 'finalize', callerSignal?: AbortSignal, ): Promise<{ response: Response; body: unknown }> { @@ -332,7 +328,13 @@ async function requestUploadControl( watchdog.signal.throwIfAborted(); watchdog.arm(phase, UPLOAD_CONTROL_PLANE_TIMEOUT_MS); try { - const response = await fetch(url, { ...init, signal: watchdog.signal }); + // Redirects could forward the daemon token, so these control requests never follow them. + const response = await fetch(url, { + ...init, + method: 'POST', + redirect: 'error', + signal: watchdog.signal, + }); const body: unknown = response.ok ? await response.json().catch(() => undefined) : undefined; watchdog.signal.throwIfAborted(); return { response, body }; @@ -442,9 +444,6 @@ async function finalizeDirectUpload(options: { const result = await requestUploadControl( finalizeUrl, { - method: 'POST', - // As with preflight: the route never redirects, and a redirect must not carry the token. - redirect: 'error', headers, body: JSON.stringify({ uploadId: options.uploadId }), }, From d33183d83b850b23925e77056ed3dc8156f275cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 21:00:28 +0100 Subject: [PATCH 22/27] chore(gates): acknowledge fixed upload control request construction --- test/wire-compat/ledger.json | 12 ++++++------ test/wire-compat/wire-mutations.test.ts | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index 0d40fa8b68..42ae8f3cbf 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -164,11 +164,11 @@ "packages/remote/src/upload-client.ts#UploadPreflightResponse": "sha256:196a1d9ef41a1aec192589fa7ccb30cf71d971a005349d5d73eed42e6ab21327", "packages/remote/src/upload-client.ts#UploadPreflightResult": "sha256:bd8fc6d617c0630325455fd64d0fe319daaced198ee6745e4673a8ace3df460d", "packages/remote/src/upload-client.ts#UploadResponse": "sha256:499e01466b5e0e03413f8d3f1986f5af6bc120704c50a3747d84de422b3f0314", - "packages/remote/src/upload-client.ts#finalizeDirectUpload": "sha256:3fd5ac061bbc16b0dd821db5e5c3e03daf53091cc93caa10845256e078779735", + "packages/remote/src/upload-client.ts#finalizeDirectUpload": "sha256:ca3fa667ab2a60fb8af095bddeacb4875dcd2d12db783585a0fb045db35da212", "packages/remote/src/upload-client.ts#isStringRecord": "sha256:b6d6b571b90baf0976d9c347c4a45a4363a4df5ad184ea1087cb3b9febd4b624", "packages/remote/src/upload-client.ts#parseUploadPreflightResult": "sha256:7ce3f43b45c03df8b7f2690432beedc7a537c1bba540961716862f8a632d5623", - "packages/remote/src/upload-client.ts#requestUploadControl": "sha256:05c7bd5090eade7b163456c10489899842c452b5633730827539d4adcb94c85d", - "packages/remote/src/upload-client.ts#requestUploadPreflight": "sha256:471a307f501c849cccea1036af1685187d72ca55d425bc5a8afe6f30d575ddb8", + "packages/remote/src/upload-client.ts#requestUploadControl": "sha256:5689da0c0cd18d0293e5bb05ddc0d8a07d1bca9ab1d94f8ca185b03bc5b8d758", + "packages/remote/src/upload-client.ts#requestUploadPreflight": "sha256:6976942601980c80ef069a0e1cbd7058840d0384e9e87c48f5dfd8d1c774703c", "packages/remote/src/upload-client.ts#shouldRetryDirectUpload": "sha256:29059f076ff7f2b9540ad15408be48792b2d63a601ef2b9fb2d54f5576d49bbd", "packages/remote/src/upload-client.ts#tryDirectUploadWithResume": "sha256:cc5d33f38e047cf3be4bd412b64958c7495eb9a45e58eefadaa00eefa0cb6b53", "packages/remote/src/upload-client.ts#uploadDirectArtifact": "sha256:93a36a9d3dea88064569517f3cdc34f27b33df6bf5f0c0ecc9efc29b47f24b77", @@ -511,17 +511,17 @@ }, { "declaration": "packages/remote/src/upload-client.ts#requestUploadControl", - "digest": "sha256:05c7bd5090eade7b163456c10489899842c452b5633730827539d4adcb94c85d", + "digest": "sha256:5689da0c0cd18d0293e5bb05ddc0d8a07d1bca9ab1d94f8ca185b03bc5b8d758", "rationale": "Lists the shared control Fetch/JSON owner so changes to method, headers, body or redirect forwarding remain wire-gated. Existing preflight/finalize routes and protocol-2 payloads are preserved; the original 30-second ceiling now retains typed local phase expiry and stops later legs. Non-success status handling is unchanged." }, { "declaration": "packages/remote/src/upload-client.ts#requestUploadPreflight", - "digest": "sha256:471a307f501c849cccea1036af1685187d72ca55d425bc5a8afe6f30d575ddb8", + "digest": "sha256:6976942601980c80ef069a0e1cbd7058840d0384e9e87c48f5dfd8d1c774703c", "rationale": "#4441 preserves the 30-second control-request ceiling but reports its typed phase timeout instead of retrying or falling back after expiry. The shared Fetch/JSON owner retains route, method, authentication, request body, redirect refusal and successful response parsing, so protocol-2 peers parse the same messages; only the local timeout outcome stops subsequent upload legs." }, { "declaration": "packages/remote/src/upload-client.ts#finalizeDirectUpload", - "digest": "sha256:3fd5ac061bbc16b0dd821db5e5c3e03daf53091cc93caa10845256e078779735", + "digest": "sha256:ca3fa667ab2a60fb8af095bddeacb4875dcd2d12db783585a0fb045db35da212", "rationale": "#4441 preserves the 30-second control-request ceiling but reports its typed phase timeout instead of retrying or falling back after expiry. The shared Fetch/JSON owner retains route, method, authentication, request body, redirect refusal and successful response parsing, so protocol-2 peers parse the same messages; only the local timeout outcome stops subsequent upload legs." }, { diff --git a/test/wire-compat/wire-mutations.test.ts b/test/wire-compat/wire-mutations.test.ts index 3674ba25a0..a88c4e027a 100644 --- a/test/wire-compat/wire-mutations.test.ts +++ b/test/wire-compat/wire-mutations.test.ts @@ -50,11 +50,11 @@ type WireMutation = { const MUTATIONS: readonly WireMutation[] = [ { - breakClass: 'upload control: Fetch drops the method, headers and body', + breakClass: 'upload control: Fetch drops authentication headers and the JSON body', file: 'packages/remote/src/upload-client.ts', name: 'requestUploadControl', - from: '...init, signal:', - to: 'signal:', + from: '...init,', + to: '', }, { breakClass: 'upload ticket: the response wrapper drops the configured origin', From 77a5e167795e9e227fd5fc539412f9bbc5603908 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 21:15:13 +0100 Subject: [PATCH 23/27] test(remote): simplify control request count expectations --- .../src/__tests__/upload-client-cancellation.test.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/remote/src/__tests__/upload-client-cancellation.test.ts b/packages/remote/src/__tests__/upload-client-cancellation.test.ts index 011f6bbe01..573e8c2911 100644 --- a/packages/remote/src/__tests__/upload-client-cancellation.test.ts +++ b/packages/remote/src/__tests__/upload-client-cancellation.test.ts @@ -176,14 +176,12 @@ test.each([ } assert.equal(caller.signal.aborted, canceled); assert.equal(events.includes('fallback'), false); + const expectedTicketLegs = Number(retry) + Number(phase === 'finalize'); assert.deepEqual( routes, - Array.from( - { length: phase === 'preflight' ? (retry ? 1 : 0) : retry ? 2 : 1 }, - () => 'PUT /ticket', - ), + Array.from({ length: expectedTicketLegs }, () => 'PUT /ticket'), ); - assert.equal(fetch.mock.calls.length, phase === 'preflight' ? (retry ? 2 : 1) : retry ? 3 : 2); + assert.equal(fetch.mock.calls.length, 1 + expectedTicketLegs); } finally { caller.abort(new Error('fixture cleanup')); refuseStall?.(new Error('fixture cleanup')); From 16a12af9ae731de85306bc165b30ba6265ba2e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 21:58:52 +0100 Subject: [PATCH 24/27] fix(remote): start the header deadline at request finish --- packages/host-kit/src/internal/transport.ts | 3 +-- .../support/upload-transfer-fixtures.ts | 6 ++++++ .../src/__tests__/upload-stream-phases.test.ts | 12 +++++++++++- packages/remote/src/upload-stream.ts | 18 +++++------------- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/packages/host-kit/src/internal/transport.ts b/packages/host-kit/src/internal/transport.ts index c7223d9144..8e4a1351b1 100644 --- a/packages/host-kit/src/internal/transport.ts +++ b/packages/host-kit/src/internal/transport.ts @@ -125,8 +125,7 @@ export function observeCompletedRequestWrites( if (typeof callback === 'function') callback(error); }; } - const accepted: boolean = Reflect.apply(write, this, args); - return accepted; + return Reflect.apply(write, this, args); }; req.write = observedWrite; req.prependListener('response', onResponse); diff --git a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts index 4b98850b26..83dd4dfc2c 100644 --- a/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts +++ b/packages/remote/src/__tests__/support/upload-transfer-fixtures.ts @@ -196,19 +196,25 @@ export type OutgoingWrites = { readonly drains: number; readonly lastCompletedAt: number | undefined; readonly finished: boolean; + readonly destroyed: boolean; readonly responseChunkRead: boolean; }; export function recordOutgoingWrites(): OutgoingWrites { + let currentRequest: http.ClientRequest | undefined; const log: { -readonly [K in keyof OutgoingWrites]: OutgoingWrites[K] } = { falseReturns: 0, completed: 0, drains: 0, finished: false, + get destroyed() { + return currentRequest?.destroyed ?? false; + }, lastCompletedAt: undefined, responseChunkRead: false, }; const observe = (req: http.ClientRequest): http.ClientRequest => { + currentRequest = req; log.finished = false; log.lastCompletedAt = undefined; log.responseChunkRead = false; diff --git a/packages/remote/src/__tests__/upload-stream-phases.test.ts b/packages/remote/src/__tests__/upload-stream-phases.test.ts index 81a79fbc8c..118c9404cb 100644 --- a/packages/remote/src/__tests__/upload-stream-phases.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phases.test.ts @@ -1,4 +1,5 @@ -import { afterEach, assert, test, vi } from 'vitest'; +import fs from 'node:fs'; +import { afterEach, assert, onTestFinished, test, vi } from 'vitest'; import { LEGACY_UPLOAD_REQUEST, TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, @@ -91,6 +92,12 @@ test('a body admitted past four connect ceilings still waits only its own header assert.equal(writes.finished, false, 'the body was still being written'); assert.equal(daemon.bodyEnded, false); assert.equal(upload.settled, false); + const close = fs.close; + const closeSpy = vi.spyOn(fs, 'close').mockImplementation((fd, callback) => { + const release = vi.fn().mockImplementationOnce(() => close(fd, callback)); + onTestFinished(release); + setTimeout(release, 20_000); + }); daemon.readToEnd(); const snapshot = () => ({ writes, @@ -105,10 +112,13 @@ test('a body admitted past four connect ceilings still waits only its own header snapshot, ); assert.equal(writes.finished && daemon.bodyEnded, true, 'the whole body reached the peer'); + assert.equal(closeSpy.mock.calls.length, 1, 'source close is held after the outgoing flush'); await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1); await pumpRealWork(); assert.equal(upload.settled, false, 'the header wait counts from the flush'); await vi.advanceTimersByTimeAsync(1); + await pumpRealWork(); + assert.equal(writes.destroyed, true, 'source close cannot reset the header deadline'); await settleAfterBudget(upload, snapshot); const result = await upload; assert.equal(result.value, undefined); diff --git a/packages/remote/src/upload-stream.ts b/packages/remote/src/upload-stream.ts index 195c1428f5..8b112650ac 100644 --- a/packages/remote/src/upload-stream.ts +++ b/packages/remote/src/upload-stream.ts @@ -15,13 +15,11 @@ import { type UploadProgressStage, } from './upload-progress.ts'; -/** Ceiling for establishing the connection for each upload hop or offset resume. */ export const TRANSFER_CONNECT_TIMEOUT_MS = 30_000; /** Header wait starts after connection and outgoing flush, never while the body progresses. */ export const TRANSFER_RESPONSE_HEADER_TIMEOUT_MS = 30_000; -/** Body-byte inactivity does not extend the enclosing aggregate. */ export const TRANSFER_BODY_INACTIVITY_TIMEOUT_MS = 120_000; export type TransferPhase = @@ -34,16 +32,12 @@ export type TransferPhase = const BODY_PHASES: ReadonlySet = new Set(['request-body', 'response-body']); -/** The `details.reason` every phase expiry carries, so no consumer has to match error text. */ const TRANSFER_PHASE_TIMEOUT_REASON = 'transfer_phase_timeout'; /** One sequential phase under the earliest caller/aggregate deadline; progress resets body phases only. */ export type TransferPhaseWatchdog = { - /** The signal the attempt runs on: the caller's cancellation plus every phase deadline. */ signal: AbortSignal; - /** Starts watching `phase`. The phase being watched before stops counting. */ arm(phase: TransferPhase, timeoutMs: number): void; - /** Records body bytes moving, so only the silence since the last byte counts against a body phase. */ touchBody(): void; /** Stops the armed timer. Call it once the attempt is settled. */ dispose(): void; @@ -72,8 +66,6 @@ export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): Transfe rearm(); }, touchBody() { - // Unrelated headers, keepalive traffic and timer callbacks are not body progress, and - // progress in one phase buys no time in another: only a body phase already armed re-arms. if (!armed || !BODY_PHASES.has(armed.phase)) return; rearm(); }, @@ -98,7 +90,6 @@ export function isTransferPhaseTimeoutError(error: unknown): boolean { return error instanceof AppError && error.details?.reason === TRANSFER_PHASE_TIMEOUT_REASON; } -/** The one artifact-preparation aggregate: ten minutes from admission to the last finalize. */ const UPLOAD_PREPARATION_TIMEOUT_MS = 600_000; export async function withUploadPreparationDeadline( @@ -226,8 +217,7 @@ async function streamFileToHttpRequestAttempt(options: { path: options.url.pathname + options.url.search, headers, // Aborting destroys the request mid-stream, which is the only way to stop bytes that are - // already piped at a device this client no longer holds the lease on. An expired phase - // destroys it exactly like a cancellation does. + // already piped at a device this client no longer holds the lease on. signal: watchdog.signal, }, (res) => { @@ -254,6 +244,10 @@ async function streamFileToHttpRequestAttempt(options: { advanceOutgoingPhase(); } }); + req.on('finish', () => { + bodyFlushed = true; + advanceOutgoingPhase(); + }); const stopObservingWrites = observeCompletedRequestWrites(req, watchdog.signal, () => watchdog.touchBody(), ); @@ -288,8 +282,6 @@ async function streamFileToHttpRequestAttempt(options: { upload = (async () => { await pipeline(fileStream, progressTransform, req, { signal: watchdog.signal }); uploadSettled = true; - bodyFlushed = true; - advanceOutgoingPhase(); })().catch((err: unknown) => { // After a response, only this attempt's own teardown ends the pipeline, and the server's // verdict stays the outcome: a 413 read completely is a 413, not a transport break. From ee7735b2a3cd1fb900103d2c0a436820a6ca04db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 21:59:00 +0100 Subject: [PATCH 25/27] chore(gates): acknowledge request-finish header timing --- test/wire-compat/ledger.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index 42ae8f3cbf..98ffb5dd36 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -182,7 +182,7 @@ "packages/remote/src/upload-stream.ts#parseNonNegativeIntegerHeader": "sha256:5cbe9c6994459d673d988a51407e65556e28175b187ce0a9d27cbb6331962834", "packages/remote/src/upload-stream.ts#parseUploadResumeOffset": "sha256:297184ea22a6dc1cda9aee17034122ae0ea2f43d0355647f6fcf02f3f4b5ca0e", "packages/remote/src/upload-stream.ts#streamFileToHttpRequest": "sha256:c2b5e1369c8b1c9420036e23163d21e43fa2ac8b889c6a8f028cc6095055a16f", - "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:3447b4c74b58d84c2cc785572ea14374ae80df24902e666fd642a537c41af33e", + "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt": "sha256:1a4b994c3ac2cf89e01e9c56526a5c9a0356fb8de425a248d062bd1cd67686b8", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_HEADER": "sha256:d2bb3f4846470188a0fc1724271da0f0bcbdf2337bd6bf331508142cadcb65d8", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_UPSTREAM_INSTANCE_HEADER": "sha256:4564c2ce437d30d7a823e3326f4375c0750ba8d42bec22b4d02e8eb7f7db2973", "packages/contracts/src/daemon-http.ts#DAEMON_HTTP_INSTANCE_MISMATCH_HEADER": "sha256:a197edaf7f9fa94a6aab269856aa852da3204cc586f5fd23198689eb11908b61", @@ -496,8 +496,8 @@ }, { "declaration": "packages/remote/src/upload-stream.ts#streamFileToHttpRequestAttempt", - "digest": "sha256:3447b4c74b58d84c2cc785572ea14374ae80df24902e666fd642a537c41af33e", - "rationale": "#4441 keeps each attempt under its sequential phase watchdog and corrects its transport lifecycle: the connect ceiling now ends at actual socket or TLS-handshake establishment, body progress counts only transport writes, and an early response reads the accepted answer to its end, then destroys and joins the owned file/request pipeline and retires this attempt's phase watchdog before resolving or starting the redirect/resume attempt, so a completed hop's timers never reach the next hop. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own sequencing and deadlines moved, so a protocol-2 peer parses exactly what it parsed before. Request-body progress is now each non-empty request write that completes without an error, which Node reports once the bytes reach the local kernel, instead of drain, which fires only after a write returned false. The write wrapper passes every argument and the backpressure result through, so the bytes and framing on the wire do not change. That completed-write observer is host-kit's transport owner, which also stops reporting after the response starts, the signal aborts or the request is destroyed; no request changes." + "digest": "sha256:1a4b994c3ac2cf89e01e9c56526a5c9a0356fb8de425a248d062bd1cd67686b8", + "rationale": "#4441 keeps each attempt under its sequential phase watchdog and corrects its transport lifecycle: the connect ceiling now ends at actual socket or TLS-handshake establishment, body progress counts only transport writes, and an early response reads the accepted answer to its end, then destroys and joins the owned file/request pipeline and retires this attempt's phase watchdog before resolving or starting the redirect/resume attempt, so a completed hop's timers never reach the next hop. Bytes on the wire are unchanged: same route, method, headers, body framing, resume-offset handling and redirect limit. Only this client's own sequencing and deadlines moved, so a protocol-2 peer parses exactly what it parsed before. Request-body progress is now each non-empty request write that completes without an error, which Node reports once the bytes reach the local kernel, instead of drain, which fires only after a write returned false. The write wrapper passes every argument and the backpressure result through, so the bytes and framing on the wire do not change. That completed-write observer is host-kit's transport owner, which also stops reporting after the response starts, the signal aborts or the request is destroyed; no request changes. The response-header ceiling starts at connected request finish, without waiting for or rearming after source-file close. This changes local deadline timing only; URL, method, credentials, body, redirects, resume headers and response framing are unchanged." }, { "declaration": "packages/remote/src/daemon-artifacts.ts#resolveMaterializedArtifactPath", From 0d5790efd64b57b8f8b4f917eb9acc90d377b449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 22:23:46 +0100 Subject: [PATCH 26/27] test(remote): pin the upload source close before advancing time --- .../src/__tests__/upload-stream-phases.test.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/remote/src/__tests__/upload-stream-phases.test.ts b/packages/remote/src/__tests__/upload-stream-phases.test.ts index 118c9404cb..94d2dcf5af 100644 --- a/packages/remote/src/__tests__/upload-stream-phases.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phases.test.ts @@ -93,8 +93,17 @@ test('a body admitted past four connect ceilings still waits only its own header assert.equal(daemon.bodyEnded, false); assert.equal(upload.settled, false); const close = fs.close; + let sourceClosed: Promise | undefined; const closeSpy = vi.spyOn(fs, 'close').mockImplementation((fd, callback) => { - const release = vi.fn().mockImplementationOnce(() => close(fd, callback)); + const release = vi.fn().mockImplementationOnce(() => { + sourceClosed = new Promise((resolve) => { + close(fd, (error) => { + callback(error); + resolve(); + }); + }); + return sourceClosed; + }); onTestFinished(release); setTimeout(release, 20_000); }); @@ -113,7 +122,10 @@ test('a body admitted past four connect ceilings still waits only its own header ); assert.equal(writes.finished && daemon.bodyEnded, true, 'the whole body reached the peer'); assert.equal(closeSpy.mock.calls.length, 1, 'source close is held after the outgoing flush'); + assert.equal(fs.fstatSync(closeSpy.mock.calls[0][0]).size, 16 * MEBIBYTE); await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1); + assert.ok(sourceClosed, 'source close was released before the header deadline'); + await sourceClosed; await pumpRealWork(); assert.equal(upload.settled, false, 'the header wait counts from the flush'); await vi.advanceTimersByTimeAsync(1); From 13a4c1d9576777989ab3849cda9fb78661a33988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 22:25:32 +0100 Subject: [PATCH 27/27] test(remote): narrow the held-close callback and descriptor --- packages/remote/src/__tests__/upload-stream-phases.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/remote/src/__tests__/upload-stream-phases.test.ts b/packages/remote/src/__tests__/upload-stream-phases.test.ts index 94d2dcf5af..d668f97cd6 100644 --- a/packages/remote/src/__tests__/upload-stream-phases.test.ts +++ b/packages/remote/src/__tests__/upload-stream-phases.test.ts @@ -95,6 +95,7 @@ test('a body admitted past four connect ceilings still waits only its own header const close = fs.close; let sourceClosed: Promise | undefined; const closeSpy = vi.spyOn(fs, 'close').mockImplementation((fd, callback) => { + assert.ok(callback); const release = vi.fn().mockImplementationOnce(() => { sourceClosed = new Promise((resolve) => { close(fd, (error) => { @@ -122,7 +123,7 @@ test('a body admitted past four connect ceilings still waits only its own header ); assert.equal(writes.finished && daemon.bodyEnded, true, 'the whole body reached the peer'); assert.equal(closeSpy.mock.calls.length, 1, 'source close is held after the outgoing flush'); - assert.equal(fs.fstatSync(closeSpy.mock.calls[0][0]).size, 16 * MEBIBYTE); + assert.equal(fs.fstatSync(closeSpy.mock.calls[0]![0]).size, 16 * MEBIBYTE); await vi.advanceTimersByTimeAsync(TRANSFER_RESPONSE_HEADER_TIMEOUT_MS - 1); assert.ok(sourceClosed, 'source close was released before the header deadline'); await sourceClosed;