From 04a5f66553413430bbee26bbbbed88f01e7625c1 Mon Sep 17 00:00:00 2001 From: Janic Duplessis Date: Sun, 15 Nov 2026 11:40:44 -0500 Subject: [PATCH] feat: record how long a refused report child outlives its cancel --- .../report-runtime.integration.test.ts | 49 +++++++++++++ packages/stim-cli/src/exec.ts | 70 ++++++++++++++++--- packages/stim-cli/src/guide/logs.ts | 7 ++ packages/stim-cli/src/report-child-log.ts | 14 ++++ packages/stim-cli/src/stimd/main.ts | 2 + packages/stim-cli/src/stimd/threads.ts | 2 + 6 files changed, 134 insertions(+), 10 deletions(-) create mode 100644 packages/stim-cli/src/report-child-log.ts diff --git a/packages/stim-cli/src/__tests__/report-runtime.integration.test.ts b/packages/stim-cli/src/__tests__/report-runtime.integration.test.ts index 714084c8b..9d9f297c6 100644 --- a/packages/stim-cli/src/__tests__/report-runtime.integration.test.ts +++ b/packages/stim-cli/src/__tests__/report-runtime.integration.test.ts @@ -8,6 +8,7 @@ import { tmpdir } from 'node:os'; import { basename, join } from 'node:path'; import { withAmbientContext, type StimContext } from '@stim-cli/core'; import { getExecutor } from '../exec.ts'; +import { setReportChildLog } from '../report-child-log.ts'; import { runReport as executeReport, trackReportResource } from '../report-runtime.ts'; vi.mock('../process-group-exit.ts', async (importOriginal) => ({ @@ -234,6 +235,54 @@ test.each([true])('failed initial birth capture never signals the unverifiable c } }); +test('a refused uncaptured child is recorded with the time it outlived its cancel', async () => { + const controller = new AbortController(); + let ready: () => void = () => {}; + const started = new Promise((resolve) => { + ready = resolve; + }); + const records: { event: string; fields: Record }[] = []; + setReportChildLog((_level, event, _msg, fields) => void records.push({ event, fields })); + const capture = vi + .spyOn(identities, 'captureProcessIdentity') + .mockReturnValue({ ok: false, reason: 'ACCESS_DENIED' }); + let nativeClosed: Promise = Promise.resolve(); + try { + const reading = runReport(context(controller.signal), async () => + getExecutor().runFileAsync(process.execPath, ['-e', 'console.log("ready"); setTimeout(() => {}, 3000)'], { + timeoutMs: 20_000, + signal: controller.signal, + onSpawn: (child) => { + nativeClosed = new Promise((resolve) => child.once('close', resolve)); + child.stdout?.once('data', () => ready()); + }, + }), + ).catch((error: unknown) => error); + await started; + controller.abort(new Error('cancel fixture')); + expect(await reading).toMatchObject({ code: 'STIM_CLAIM_UNAVAILABLE' }); + await vi.waitFor(() => expect(records).toHaveLength(1), { timeout: 10_000 }); + expect(records[0]).toMatchObject({ + event: 'daemon_report_child_exited', + fields: { + cancelReason: 'abort', + refusal: 'unverified', + timeoutMs: 20_000, + captured: false, + captureReason: 'ACCESS_DENIED', + exitCode: 0, + exitSignal: null, + }, + }); + expect(records[0]?.fields.command).toContain(`${basename(process.execPath)} -e`); + expect(records[0]?.fields.cancelToExitMs).toBeGreaterThanOrEqual(1500); + } finally { + setReportChildLog(undefined); + capture.mockRestore(); + await nativeClosed; + } +}); + test('report cancellation refuses a verified child that does not close after termination', async () => { const controller = new AbortController(); let owned: { pid: number; processToken: string } | undefined; diff --git a/packages/stim-cli/src/exec.ts b/packages/stim-cli/src/exec.ts index a06ffa906..b0be5327f 100644 --- a/packages/stim-cli/src/exec.ts +++ b/packages/stim-cli/src/exec.ts @@ -15,6 +15,7 @@ import { groupExitBudgetMs, waitForGroupExit } from './process-group-exit.ts'; import { redactArgv, redactCommandLine, tailscaleEnv, trace } from '@stim-cli/core/state'; import { debugLog } from './debug-log.ts'; import { reportSignal, trackReportResource } from './report-runtime.ts'; +import { logReportChild } from './report-child-log.ts'; import { captureProcessIdentity, inspectProcessIdentity } from './process-identity.ts'; import { processGroupAlive } from './ownership-claim.ts'; @@ -179,6 +180,9 @@ function describeReportCommand(file: string, args: readonly string[], redact?: r return text.length > REPORT_COMMAND_MAX_LENGTH ? `${text.slice(0, REPORT_COMMAND_MAX_LENGTH - 3)}...` : text; } +type CancelReason = 'timeout' | 'overflow' | 'exit' | 'close' | 'report_budget' | 'abort' | 'unknown'; +type Refusal = 'unverified' | 'did_not_close' | 'descendants'; + function reportChild( child: ChildProcess, signal?: AbortSignal, @@ -186,7 +190,8 @@ function reportChild( contained = false, selectedContext?: StimContext, command?: string, -): { cancel: () => Promise; closed: Promise } | undefined { + timeoutMs?: number, +): { cancel: (reason?: CancelReason) => Promise; closed: Promise } | undefined { const context = selectedContext ?? ambientContext(); const report = reportSignal(context); signal ??= report; @@ -202,10 +207,19 @@ function reportChild( child.once('close', () => { ended = true; finishNativeClose(); - void cancel().then(resolve, reject); + void cancel('close').then(resolve, reject); }); }); const pid = child.pid; + const spawnedAt = performance.now(); + let exitedAt: number | undefined; + let exitStatus: { code: number | null; signal: NodeJS.Signals | null } | undefined; + child.once('exit', (code, exitSignal) => { + exitedAt = performance.now(); + exitStatus = { code, signal: exitSignal }; + }); + let cancelledAt = spawnedAt; + let cancelReason: CancelReason = 'unknown'; const named = `${pid}${command ? ` running \`${command}\`` : ''}`; const cleanupSettled = () => ended && (pid === undefined || (process.platform === 'win32' ? contained : !processGroupAlive(pid))); @@ -221,7 +235,32 @@ function reportChild( } return 'unknown'; }; - const refuse = (message: string): never => { + const recordRefused = (refusal: Refusal) => { + if (pid === undefined) return; + const write = () => { + if (exitedAt === undefined) return; + const cancelToExitMs = Math.round(exitedAt - cancelledAt); + logReportChild( + `report child ${named} exited ${cancelToExitMs} ms after its cancel (${cancelReason}, refused: ${refusal})`, + { + pid, + command, + cancelReason, + refusal, + ...(timeoutMs === undefined ? {} : { timeoutMs }), + spawnToCancelMs: Math.round(cancelledAt - spawnedAt), + cancelToExitMs, + exitCode: exitStatus?.code ?? null, + exitSignal: exitStatus?.signal ?? null, + captured: captured?.ok === true, + ...(captured && !captured.ok ? { captureReason: captured.reason } : {}), + }, + ); + }; + if (exitedAt === undefined) child.once('exit', write); + else write(); + }; + const refuse = (message: string, refusal: Refusal): never => { const failure = Object.assign(new Error(message), { code: 'STIM_CLAIM_UNAVAILABLE', reportCleanup: true, @@ -235,12 +274,21 @@ function reportChild( child.stdout?.destroy(); child.stderr?.destroy(); child.unref(); + recordRefused(refusal); rejectClosed(error); throw error; }; let cancelled: Promise | undefined; - const cancel = () => + const cancel = (reason?: CancelReason) => (cancelled ??= (async () => { + cancelledAt = performance.now(); + cancelReason = + reason ?? + (signal.aborted + ? signal.reason instanceof Error && 'code' in signal.reason && signal.reason.code === 'STIM_REPORT_TIMEOUT' + ? 'report_budget' + : 'abort' + : 'unknown'); let identity = observe(); if (pid !== undefined) { const deadline = performance.now() + 1000; @@ -251,6 +299,7 @@ function reportChild( if (identity === 'unknown') { refuse( `Cannot verify cleanup for report child ${named}${captured && !captured.ok ? ` (its identity was not captured at spawn: ${captured.reason})` : ''}; its process or descendants were left untouched.`, + 'unverified', ); } } @@ -302,14 +351,14 @@ function reportChild( await awaitClose(); if (!ended) await new Promise((resolve) => setImmediate(resolve)); if (!ended && observe() === 'gone') await awaitClose(); - if (!ended) refuse(`Report child ${named} did not close within the cleanup budget.`); + if (!ended) refuse(`Report child ${named} did not close within the cleanup budget.`, 'did_not_close'); } if ( pid !== undefined && process.platform !== 'win32' && !(await waitForGroupExit(pid, groupExitBudgetMs(REPORT_CHILD_EXIT_CAP_MS))) ) - refuse(`Report child ${named} still has descendants after cleanup.`); + refuse(`Report child ${named} still has descendants after cleanup.`, 'descendants'); })()); if (report) trackReportResource(context, closed, cancel, signal, undefined, cleanupSettled); else { @@ -543,7 +592,7 @@ const defaultExecutor: Executor = { else quietly(() => observed.started(watch, pid)); } let lifetime: ReturnType; - let cancel: (() => Promise) | undefined; + let cancel: ((reason?: Parameters['cancel']>[0]) => Promise) | undefined; let timer: ReturnType | undefined; let poll: ReturnType | undefined; let phase: 'starting' | 'running' | 'closing' = markers ? 'starting' : 'running'; @@ -581,12 +630,13 @@ const defaultExecutor: Executor = { contained, context, describeReportCommand(file, args, redact), + timeout, ); cancel = lifetime?.cancel; if (cleanupOnExit && cancel) child.once('exit', () => { stopTimers(); - void cancel?.().catch(reject); + void cancel?.('exit').catch(reject); }); void lifetime?.closed.catch(reject); if (observed && watch !== undefined && child.pid !== undefined) @@ -627,7 +677,7 @@ const defaultExecutor: Executor = { if (remaining > 0) chunks.push(chunk.subarray(0, remaining)); bufferedBytes = MAX_BUFFER; overflowed = true; - if (cancel) void cancel().catch(reject); + if (cancel) void cancel('overflow').catch(reject); else { child.kill('SIGKILL'); child.stdin?.destroy(); @@ -641,7 +691,7 @@ const defaultExecutor: Executor = { timer = setTimeout(() => { timedOut = true; clearInterval(poll); - if (cancel) void cancel().catch(reject); + if (cancel) void cancel('timeout').catch(reject); else { child.kill(killSignal ?? 'SIGTERM'); child.stdin?.destroy(); diff --git a/packages/stim-cli/src/guide/logs.ts b/packages/stim-cli/src/guide/logs.ts index cec957e01..7e505c499 100644 --- a/packages/stim-cli/src/guide/logs.ts +++ b/packages/stim-cli/src/guide/logs.ts @@ -381,6 +381,13 @@ dependency step adds about 7 an hour. The records change nothing a pass does. Each disk-usage cache entry written after a walk also records walkMs, in milliseconds, from just before the install signature and target reads to the end of the folder walk; nothing reads it yet. +When a report child's cleanup is refused (STIM_CLAIM_UNAVAILABLE) inside stimd, the daemon log also gets +one info record, event daemon_report_child_exited, when that child's own exit is seen. It never appears for a +child that has not exited. Fields: pid, command (program and redacted, bounded arguments), cancelReason (timeout, +report_budget, abort, overflow, exit, close or unknown; report_budget is the report's own budget, abort is any +other abort), timeoutMs (when the call had one), spawnToCancelMs, cancelToExitMs (negative when the exit came before +the cancel), exitCode, exitSignal, refusal (unverified, did_not_close or descendants), captured and captureReason. +It changes nothing a pass does. BUILD LEDGER stim logs --source builds reads $STIM_HOME/builds/builds.ndjson from any diff --git a/packages/stim-cli/src/report-child-log.ts b/packages/stim-cli/src/report-child-log.ts new file mode 100644 index 000000000..fd7f39879 --- /dev/null +++ b/packages/stim-cli/src/report-child-log.ts @@ -0,0 +1,14 @@ +type ReportChildLog = (level: 'info', event: string, msg: string, fields: Record) => void; + +let sink: ReportChildLog | undefined; + +/** Routes the records about a refused report child to the daemon log. Only stimd and its threads set it. */ +export function setReportChildLog(log: ReportChildLog | undefined): void { + sink = log; +} + +export function logReportChild(msg: string, fields: Record): void { + try { + sink?.('info', 'daemon_report_child_exited', msg, fields); + } catch {} +} diff --git a/packages/stim-cli/src/stimd/main.ts b/packages/stim-cli/src/stimd/main.ts index f0e367967..8075381a5 100644 --- a/packages/stim-cli/src/stimd/main.ts +++ b/packages/stim-cli/src/stimd/main.ts @@ -124,6 +124,7 @@ import { import { invocation, traceInvocation, withInvocation, type Invocation } from './forwarded.ts'; import { createPassesLane, createThreadTable } from './threads.ts'; import { createCleanupWatch, isUnverifiedCleanup } from './cleanup-watch.ts'; +import { setReportChildLog } from '../report-child-log.ts'; import { createStreams } from './logs/streams.ts'; import { handleLogRequest } from './logs/protocol.ts'; import { writeDesktopKey } from './desktop-key.ts'; @@ -774,6 +775,7 @@ async function runStimdInner( const inDaemonContext = AsyncLocalStorage.snapshot(); const log = (level: Level, event: string, msg: string, fields: Record = {}) => writer.write({ src: 'stimd', level, event, msg, ...fields }); + setReportChildLog(log); const path = stimdSocket(); const bootAt = Date.now(); const leftover = readClaimSet(stimdClaimRoot()).dead; diff --git a/packages/stim-cli/src/stimd/threads.ts b/packages/stim-cli/src/stimd/threads.ts index dc943f8f5..4f762566d 100644 --- a/packages/stim-cli/src/stimd/threads.ts +++ b/packages/stim-cli/src/stimd/threads.ts @@ -3,6 +3,7 @@ import type { DaemonLadderAction } from '@stim-cli/core/state'; import type { MaintenanceLane } from '../daemon/passes.ts'; import { spawnEntry } from '../spawn-entry.ts'; import { childKey } from './cleanup-watch.ts'; +import { setReportChildLog } from '../report-child-log.ts'; import { serveThreadJobs } from './thread-jobs.ts'; type ThreadName = 'passes' | 'measure'; @@ -232,6 +233,7 @@ else if (thread && parentPort) { const port = parentPort; const dependencyCadence: BigInt64Array | undefined = workerData.dependencyCadence; const log: Log = (level, event, msg, fields = {}) => port.postMessage({ type: 'log', level, event, msg, fields }); + setReportChildLog(log); const { processContext } = await import('../cli-operation.ts'); let lane: MaintenanceLane | undefined; serveThreadJobs(