Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions packages/stim-cli/src/__tests__/report-runtime.integration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { tmpdir } from 'node:os';
import { basename, join } from 'node:path';
import { withAmbientContext, type StimContext } from '@stim-cli/core';
import { getExecutor } from '../exec.ts';
import { setReportChildLog } from '../report-child-log.ts';
import { runReport as executeReport, trackReportResource } from '../report-runtime.ts';

vi.mock('../process-group-exit.ts', async (importOriginal) => ({
Expand Down Expand Up @@ -234,6 +235,54 @@ test.each([true])('failed initial birth capture never signals the unverifiable c
}
});

test('a refused uncaptured child is recorded with the time it outlived its cancel', async () => {
const controller = new AbortController();
let ready: () => void = () => {};
const started = new Promise<void>((resolve) => {
ready = resolve;
});
const records: { event: string; fields: Record<string, unknown> }[] = [];
setReportChildLog((_level, event, _msg, fields) => void records.push({ event, fields }));
const capture = vi
.spyOn(identities, 'captureProcessIdentity')
.mockReturnValue({ ok: false, reason: 'ACCESS_DENIED' });
let nativeClosed: Promise<void> = Promise.resolve();
try {
const reading = runReport(context(controller.signal), async () =>
getExecutor().runFileAsync(process.execPath, ['-e', 'console.log("ready"); setTimeout(() => {}, 3000)'], {
timeoutMs: 20_000,
signal: controller.signal,
onSpawn: (child) => {
nativeClosed = new Promise<void>((resolve) => child.once('close', resolve));
child.stdout?.once('data', () => ready());
},
}),
).catch((error: unknown) => error);
await started;
controller.abort(new Error('cancel fixture'));
expect(await reading).toMatchObject({ code: 'STIM_CLAIM_UNAVAILABLE' });
await vi.waitFor(() => expect(records).toHaveLength(1), { timeout: 10_000 });
expect(records[0]).toMatchObject({
event: 'daemon_report_child_exited',
fields: {
cancelReason: 'abort',
refusal: 'unverified',
timeoutMs: 20_000,
captured: false,
captureReason: 'ACCESS_DENIED',
exitCode: 0,
exitSignal: null,
},
});
expect(records[0]?.fields.command).toContain(`${basename(process.execPath)} -e`);
expect(records[0]?.fields.cancelToExitMs).toBeGreaterThanOrEqual(1500);
} finally {
setReportChildLog(undefined);
capture.mockRestore();
await nativeClosed;
}
});

test('report cancellation refuses a verified child that does not close after termination', async () => {
const controller = new AbortController();
let owned: { pid: number; processToken: string } | undefined;
Expand Down
70 changes: 60 additions & 10 deletions packages/stim-cli/src/exec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { groupExitBudgetMs, waitForGroupExit } from './process-group-exit.ts';
import { redactArgv, redactCommandLine, tailscaleEnv, trace } from '@stim-cli/core/state';
import { debugLog } from './debug-log.ts';
import { reportSignal, trackReportResource } from './report-runtime.ts';
import { logReportChild } from './report-child-log.ts';
import { captureProcessIdentity, inspectProcessIdentity } from './process-identity.ts';
import { processGroupAlive } from './ownership-claim.ts';

Expand Down Expand Up @@ -179,14 +180,18 @@ function describeReportCommand(file: string, args: readonly string[], redact?: r
return text.length > REPORT_COMMAND_MAX_LENGTH ? `${text.slice(0, REPORT_COMMAND_MAX_LENGTH - 3)}...` : text;
}

type CancelReason = 'timeout' | 'overflow' | 'exit' | 'close' | 'report_budget' | 'abort' | 'unknown';
type Refusal = 'unverified' | 'did_not_close' | 'descendants';

function reportChild(
child: ChildProcess,
signal?: AbortSignal,
failed?: (error: Error) => Error,
contained = false,
selectedContext?: StimContext,
command?: string,
): { cancel: () => Promise<void>; closed: Promise<void> } | undefined {
timeoutMs?: number,
): { cancel: (reason?: CancelReason) => Promise<void>; closed: Promise<void> } | undefined {
const context = selectedContext ?? ambientContext();
const report = reportSignal(context);
signal ??= report;
Expand All @@ -202,10 +207,19 @@ function reportChild(
child.once('close', () => {
ended = true;
finishNativeClose();
void cancel().then(resolve, reject);
void cancel('close').then(resolve, reject);
});
});
const pid = child.pid;
const spawnedAt = performance.now();
let exitedAt: number | undefined;
let exitStatus: { code: number | null; signal: NodeJS.Signals | null } | undefined;
child.once('exit', (code, exitSignal) => {
exitedAt = performance.now();
exitStatus = { code, signal: exitSignal };
});
let cancelledAt = spawnedAt;
let cancelReason: CancelReason = 'unknown';
const named = `${pid}${command ? ` running \`${command}\`` : ''}`;
const cleanupSettled = () =>
ended && (pid === undefined || (process.platform === 'win32' ? contained : !processGroupAlive(pid)));
Expand All @@ -221,7 +235,32 @@ function reportChild(
}
return 'unknown';
};
const refuse = (message: string): never => {
const recordRefused = (refusal: Refusal) => {
if (pid === undefined) return;
const write = () => {
if (exitedAt === undefined) return;
const cancelToExitMs = Math.round(exitedAt - cancelledAt);
logReportChild(
`report child ${named} exited ${cancelToExitMs} ms after its cancel (${cancelReason}, refused: ${refusal})`,
{
pid,
command,
cancelReason,
refusal,
...(timeoutMs === undefined ? {} : { timeoutMs }),
spawnToCancelMs: Math.round(cancelledAt - spawnedAt),
cancelToExitMs,
exitCode: exitStatus?.code ?? null,
exitSignal: exitStatus?.signal ?? null,
captured: captured?.ok === true,
...(captured && !captured.ok ? { captureReason: captured.reason } : {}),
},
);
};
if (exitedAt === undefined) child.once('exit', write);
else write();
};
const refuse = (message: string, refusal: Refusal): never => {
const failure = Object.assign(new Error(message), {
code: 'STIM_CLAIM_UNAVAILABLE',
reportCleanup: true,
Expand All @@ -235,12 +274,21 @@ function reportChild(
child.stdout?.destroy();
child.stderr?.destroy();
child.unref();
recordRefused(refusal);
rejectClosed(error);
throw error;
};
let cancelled: Promise<void> | undefined;
const cancel = () =>
const cancel = (reason?: CancelReason) =>
(cancelled ??= (async () => {
cancelledAt = performance.now();
cancelReason =
reason ??
(signal.aborted
? signal.reason instanceof Error && 'code' in signal.reason && signal.reason.code === 'STIM_REPORT_TIMEOUT'
? 'report_budget'
: 'abort'
: 'unknown');
let identity = observe();
if (pid !== undefined) {
const deadline = performance.now() + 1000;
Expand All @@ -251,6 +299,7 @@ function reportChild(
if (identity === 'unknown') {
refuse(
`Cannot verify cleanup for report child ${named}${captured && !captured.ok ? ` (its identity was not captured at spawn: ${captured.reason})` : ''}; its process or descendants were left untouched.`,
'unverified',
);
}
}
Expand Down Expand Up @@ -302,14 +351,14 @@ function reportChild(
await awaitClose();
if (!ended) await new Promise<void>((resolve) => setImmediate(resolve));
if (!ended && observe() === 'gone') await awaitClose();
if (!ended) refuse(`Report child ${named} did not close within the cleanup budget.`);
if (!ended) refuse(`Report child ${named} did not close within the cleanup budget.`, 'did_not_close');
}
if (
pid !== undefined &&
process.platform !== 'win32' &&
!(await waitForGroupExit(pid, groupExitBudgetMs(REPORT_CHILD_EXIT_CAP_MS)))
)
refuse(`Report child ${named} still has descendants after cleanup.`);
refuse(`Report child ${named} still has descendants after cleanup.`, 'descendants');
})());
if (report) trackReportResource(context, closed, cancel, signal, undefined, cleanupSettled);
else {
Expand Down Expand Up @@ -543,7 +592,7 @@ const defaultExecutor: Executor = {
else quietly(() => observed.started(watch, pid));
}
let lifetime: ReturnType<typeof reportChild>;
let cancel: (() => Promise<void>) | undefined;
let cancel: ((reason?: Parameters<NonNullable<typeof lifetime>['cancel']>[0]) => Promise<void>) | undefined;
let timer: ReturnType<typeof setTimeout> | undefined;
let poll: ReturnType<typeof setInterval> | undefined;
let phase: 'starting' | 'running' | 'closing' = markers ? 'starting' : 'running';
Expand Down Expand Up @@ -581,12 +630,13 @@ const defaultExecutor: Executor = {
contained,
context,
describeReportCommand(file, args, redact),
timeout,
);
cancel = lifetime?.cancel;
if (cleanupOnExit && cancel)
child.once('exit', () => {
stopTimers();
void cancel?.().catch(reject);
void cancel?.('exit').catch(reject);
});
void lifetime?.closed.catch(reject);
if (observed && watch !== undefined && child.pid !== undefined)
Expand Down Expand Up @@ -627,7 +677,7 @@ const defaultExecutor: Executor = {
if (remaining > 0) chunks.push(chunk.subarray(0, remaining));
bufferedBytes = MAX_BUFFER;
overflowed = true;
if (cancel) void cancel().catch(reject);
if (cancel) void cancel('overflow').catch(reject);
else {
child.kill('SIGKILL');
child.stdin?.destroy();
Expand All @@ -641,7 +691,7 @@ const defaultExecutor: Executor = {
timer = setTimeout(() => {
timedOut = true;
clearInterval(poll);
if (cancel) void cancel().catch(reject);
if (cancel) void cancel('timeout').catch(reject);
else {
child.kill(killSignal ?? 'SIGTERM');
child.stdin?.destroy();
Expand Down
7 changes: 7 additions & 0 deletions packages/stim-cli/src/guide/logs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,13 @@ dependency step adds about 7 an hour. The records change nothing a pass does.
Each disk-usage cache entry written after a walk also records walkMs, in milliseconds, from just
before the install signature and target reads to the end of the folder walk;
nothing reads it yet.
When a report child's cleanup is refused (STIM_CLAIM_UNAVAILABLE) inside stimd, the daemon log also gets
one info record, event daemon_report_child_exited, when that child's own exit is seen. It never appears for a
child that has not exited. Fields: pid, command (program and redacted, bounded arguments), cancelReason (timeout,
report_budget, abort, overflow, exit, close or unknown; report_budget is the report's own budget, abort is any
other abort), timeoutMs (when the call had one), spawnToCancelMs, cancelToExitMs (negative when the exit came before
the cancel), exitCode, exitSignal, refusal (unverified, did_not_close or descendants), captured and captureReason.
It changes nothing a pass does.

BUILD LEDGER
stim logs --source builds reads $STIM_HOME/builds/builds.ndjson from any
Expand Down
14 changes: 14 additions & 0 deletions packages/stim-cli/src/report-child-log.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
type ReportChildLog = (level: 'info', event: string, msg: string, fields: Record<string, unknown>) => void;

let sink: ReportChildLog | undefined;

/** Routes the records about a refused report child to the daemon log. Only stimd and its threads set it. */
export function setReportChildLog(log: ReportChildLog | undefined): void {
sink = log;
}

export function logReportChild(msg: string, fields: Record<string, unknown>): void {
try {
sink?.('info', 'daemon_report_child_exited', msg, fields);
} catch {}
}
2 changes: 2 additions & 0 deletions packages/stim-cli/src/stimd/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ import {
import { invocation, traceInvocation, withInvocation, type Invocation } from './forwarded.ts';
import { createPassesLane, createThreadTable } from './threads.ts';
import { createCleanupWatch, isUnverifiedCleanup } from './cleanup-watch.ts';
import { setReportChildLog } from '../report-child-log.ts';
import { createStreams } from './logs/streams.ts';
import { handleLogRequest } from './logs/protocol.ts';
import { writeDesktopKey } from './desktop-key.ts';
Expand Down Expand Up @@ -774,6 +775,7 @@ async function runStimdInner(
const inDaemonContext = AsyncLocalStorage.snapshot();
const log = (level: Level, event: string, msg: string, fields: Record<string, unknown> = {}) =>
writer.write({ src: 'stimd', level, event, msg, ...fields });
setReportChildLog(log);
const path = stimdSocket();
const bootAt = Date.now();
const leftover = readClaimSet(stimdClaimRoot()).dead;
Expand Down
2 changes: 2 additions & 0 deletions packages/stim-cli/src/stimd/threads.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type { DaemonLadderAction } from '@stim-cli/core/state';
import type { MaintenanceLane } from '../daemon/passes.ts';
import { spawnEntry } from '../spawn-entry.ts';
import { childKey } from './cleanup-watch.ts';
import { setReportChildLog } from '../report-child-log.ts';
import { serveThreadJobs } from './thread-jobs.ts';

type ThreadName = 'passes' | 'measure';
Expand Down Expand Up @@ -232,6 +233,7 @@ else if (thread && parentPort) {
const port = parentPort;
const dependencyCadence: BigInt64Array | undefined = workerData.dependencyCadence;
const log: Log = (level, event, msg, fields = {}) => port.postMessage({ type: 'log', level, event, msg, fields });
setReportChildLog(log);
const { processContext } = await import('../cli-operation.ts');
let lane: MaintenanceLane | undefined;
serveThreadJobs<ThreadInput>(
Expand Down
Loading