diff --git a/packages/stim-cli/src/__tests__/daemon-ladder.test.ts b/packages/stim-cli/src/__tests__/daemon-ladder.test.ts index b0b265fc1..894cdfd0f 100644 --- a/packages/stim-cli/src/__tests__/daemon-ladder.test.ts +++ b/packages/stim-cli/src/__tests__/daemon-ladder.test.ts @@ -24,6 +24,7 @@ import { dormantCandidates, pendingDormant, runDormantPass } from '../daemon/sta import { recordCreatedDevice } from '../devices/created-devices.ts'; import { parkSim, readParked } from '../devices/sim-pool.ts'; import * as android from '../devices/android.ts'; +import * as simPool from '../devices/sim-pool.ts'; import * as ios from '../devices/ios.ts'; import type { IosSimRecord } from '../devices/ios.ts'; import * as teardown from '../devices/teardown.ts'; @@ -1226,3 +1227,77 @@ describe.skipIf(process.platform === 'win32')( }, 20_000); }, ); + +describe('rung timing records', () => { + const steps = () => + log.mock.calls + .filter(([, event]) => event === 'daemon_step') + .map(([logLevel, , , fields]) => Object.assign({ logLevel }, fields)); + const run = (operation?: ReturnType) => { + const config = loadConfig(); + if (!config) throw new Error('missing fixture config'); + return runLadder( + { + config, + act: true, + log, + devices: [], + simsFailed: false, + remeasure: async () => pressure('high'), + fromRung: 2, + lastRung: 2, + ...(operation ? { operation } : {}), + }, + pressure('high'), + ); + }; + + beforeEach(() => { + let clock = 0; + vi.spyOn(performance, 'now').mockImplementation(() => (clock += 100)); + }); + + test('a rung that finishes records its start, duration, actions and outcome', async () => { + vi.spyOn(simPool, 'readParked').mockReturnValue([]); + + await run(); + + expect(steps()).toEqual([ + { + logLevel: 'debug', + step: 'rung', + rung: 2, + pressureLevel: 'high', + startedAt: expect.stringMatching(/^\d{4}-\d\d-\d\dT[\d:.]+Z$/), + durationMs: 100, + items: 0, + outcome: 'finished', + }, + ]); + }); + + test('a rung that throws records the failure, and the ladder goes on as before', async () => { + vi.spyOn(simPool, 'readParked').mockImplementation(() => { + throw new Error('boom'); + }); + + const { actions } = await run(); + + expect(steps()).toMatchObject([{ step: 'rung', rung: 2, outcome: 'threw', error: 'boom' }]); + expect(actions).toMatchObject([{ rung: 2, kind: 'rung', outcome: 'failed', detail: 'boom' }]); + expect(log).toHaveBeenCalledWith('warn', 'pressure_rung_failed', 'rung 2 failed: boom'); + }); + + test('a rung stopped by an abort records it and the abort still reaches the caller', async () => { + const controller = new AbortController(); + vi.spyOn(simPool, 'readParked').mockImplementation(() => { + controller.abort(); + throw new Error('stopped'); + }); + + await expect(run(testContext({ signal: controller.signal }))).rejects.toMatchObject({ name: 'AbortError' }); + + expect(steps()).toMatchObject([{ step: 'rung', rung: 2, outcome: 'aborted' }]); + expect(steps()[0]).not.toHaveProperty('error'); + }); +}); diff --git a/packages/stim-cli/src/__tests__/daemon-step-timing.test.ts b/packages/stim-cli/src/__tests__/daemon-step-timing.test.ts new file mode 100644 index 000000000..1a5875f33 --- /dev/null +++ b/packages/stim-cli/src/__tests__/daemon-step-timing.test.ts @@ -0,0 +1,130 @@ +import { dormantCandidates } from '../daemon/stages.ts'; +import { timedStep, type DaemonLog } from '../daemon/step-timing.ts'; + +const log = vi.fn(); + +beforeEach(() => { + log.mockReset(); +}); + +test('the step returns what it returned and records the items it visited', async () => { + const result = { rows: [1, 2, 3] }; + + const returned = await timedStep(log, 'idle_stage_candidates', async (tally) => ((tally.items = 7), result)); + + expect(returned).toBe(result); + expect(log).toHaveBeenCalledWith( + 'debug', + 'daemon_step', + expect.any(String), + expect.objectContaining({ step: 'idle_stage_candidates', items: 7, outcome: 'finished' }), + ); +}); + +test('the step rethrows the error object it threw, with its properties, and a failing log does not replace it', async () => { + const failure = Object.assign(new Error('unverified cleanup'), { + code: 'STIM_CLAIM_UNAVAILABLE', + reportCleanup: true, + }); + log.mockImplementation(() => { + throw new Error('log full'); + }); + + await expect( + timedStep(log, 'plan_history', () => Promise.reject(failure), { items: (planned) => planned?.length }), + ).rejects.toBe(failure); + + expect(log).toHaveBeenCalledTimes(1); +}); + +test('without a log the step only runs the function', async () => { + await expect(timedStep(undefined, 'plan_rung6', async () => 'plan')).resolves.toBe('plan'); +}); + +test('a throw is recorded as threw with its message, and the same error object is rethrown', async () => { + const failure = new Error('boom'); + + await expect(timedStep(log, 'plan_rung6', () => Promise.reject(failure))).rejects.toBe(failure); + + expect(log.mock.calls[0]?.[3]).toMatchObject({ step: 'plan_rung6', outcome: 'threw', error: 'boom' }); +}); + +test('a throw while the signal is aborted is recorded as aborted, without an error field', async () => { + const controller = new AbortController(); + const failure = new Error('stopped'); + + await expect( + timedStep( + log, + 'plan_worktrees', + async () => { + controller.abort(); + throw failure; + }, + { signal: controller.signal }, + ), + ).rejects.toBe(failure); + + expect(log.mock.calls[0]?.[3]).toMatchObject({ outcome: 'aborted' }); + expect(log.mock.calls[0]?.[3]).not.toHaveProperty('error'); +}); + +test('a thrown value that cannot be described is still the value that is rethrown', async () => { + const odd = { + toString() { + throw new Error('no string'); + }, + }; + + await expect(timedStep(log, 'plan_history', () => Promise.reject(odd))).rejects.toBe(odd); + + expect(log.mock.calls[0]?.[3]).toMatchObject({ outcome: 'threw' }); +}); + +test('through the shape every daemon log writer builds, the record keeps its severity and the pressure level', async () => { + const written: Record[] = []; + const writer: DaemonLog = (level, event, msg, fields = {}) => + written.push({ src: 'stimd', level, event, msg, ...fields }); + + await timedStep(writer, 'rung', async () => undefined, { + fields: { rung: 5, pressureLevel: 'critical', level: 'critical', src: 'x', event: 'x', msg: 'x', ts: 1 }, + }); + + expect(written).toEqual([ + expect.objectContaining({ + src: 'stimd', + level: 'debug', + event: 'daemon_step', + step: 'rung', + rung: 5, + pressureLevel: 'critical', + outcome: 'finished', + }), + ]); + expect(written[0]).not.toHaveProperty('ts'); + expect(written[0]?.msg).toMatch(/^rung 5 finished in \d+ms$/); +}); + +test('an evidence loop returns what it returned without a log, and writes its record with one', async () => { + const config = { version: 2 as const, projects: {}, repos: {} }; + const reading = { devices: [], simsFailed: false }; + + const plain = await dormantCandidates(config, reading, 1000); + const timed = await dormantCandidates(config, { ...reading, log }, 1000); + + expect(timed).toEqual(plain); + expect(log.mock.calls[0]?.[3]).toMatchObject({ step: 'dormant_candidates', items: 0, outcome: 'finished' }); +}); + +test('an error whose name cannot be read is still the error that is rethrown, recorded as threw', async () => { + const odd = new Error('odd'); + Object.defineProperty(odd, 'name', { + get() { + throw new Error('no name'); + }, + }); + + await expect(timedStep(log, 'plan_history', () => Promise.reject(odd))).rejects.toBe(odd); + + expect(log.mock.calls[0]?.[3]).toMatchObject({ outcome: 'threw' }); +}); diff --git a/packages/stim-cli/src/__tests__/disk-usage.integration.test.ts b/packages/stim-cli/src/__tests__/disk-usage.integration.test.ts index 2c0aa4ce6..942866d27 100644 --- a/packages/stim-cli/src/__tests__/disk-usage.integration.test.ts +++ b/packages/stim-cli/src/__tests__/disk-usage.integration.test.ts @@ -1,8 +1,14 @@ -import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { releaseClaim, tryAcquireClaim } from '@stim-cli/core/ownership-claim'; -import { diskUsageWalkClaims, readDiskUsage, readDiskUsageFailure } from '@stim-cli/core/state'; +import { + diskUsageCacheFile, + diskUsageWalkClaims, + readDiskUsage, + readDiskUsageFailure, + readJsonObject, +} from '@stim-cli/core/state'; import { knownPrivateBytes, refreshMeasure, trustedMeasure } from '../disk-usage.ts'; import { getExecutor, resetExecutor, setExecutor } from '../exec.ts'; @@ -130,3 +136,30 @@ test('a claimed walk whose child exits before its identity can be recorded is st expect(result.outcome).toBe('measured'); }); + +test('a walk records how long it took, and an entry written without it still reads', async () => { + const plain = join(scratch, 'timed'); + mkdirSync(plain); + writeFileSync(join(plain, 'blob'), Buffer.alloc(10_000)); + let clock = Date.parse('2026-11-04T10:00:00.000Z'); + const real = getExecutor(); + setExecutor({ + ...real, + runFileAsync: (file, args, opts) => { + if (file === 'nice') clock += 1234; + return real.runFileAsync(file, args, opts); + }, + }); + + const walked = await refreshMeasure(plain, { ...options, now: () => clock, claim: true }); + + expect(walked.outcome).toBe('measured'); + expect(readJsonObject(diskUsageCacheFile(plain))?.walkMs).toBe(1234); + expect(readDiskUsage(plain)).toEqual(walked.measure); + + const { walkMs: _walkMs, ...legacy } = JSON.parse(readFileSync(diskUsageCacheFile(plain), 'utf8')); + writeFileSync(diskUsageCacheFile(plain), JSON.stringify(legacy)); + expect(readJsonObject(diskUsageCacheFile(plain))?.walkMs).toBeUndefined(); + expect(readDiskUsage(plain)).toEqual(walked.measure); + expect((await refreshMeasure(plain, { ...options, now: () => clock })).outcome).toBe('fresh'); +}); diff --git a/packages/stim-cli/src/daemon/dependencies.ts b/packages/stim-cli/src/daemon/dependencies.ts index 956e775b1..20e92cd96 100644 --- a/packages/stim-cli/src/daemon/dependencies.ts +++ b/packages/stim-cli/src/daemon/dependencies.ts @@ -31,6 +31,7 @@ import { import { sourceDependenciesDue } from '../policy/checkout-order.ts'; import { warmedFromAt } from '../workspace/source-dependencies.ts'; import type { Config } from '../workspace/config.ts'; +import { timedStep, type DaemonLog } from './step-timing.ts'; /** * The private bytes above which deleting a node_modules or Pods tree is worth a `stim worktree warm` or reinstall. @@ -105,6 +106,7 @@ export async function rankDependencies( protectedRoot: string, operation?: StimContext, source = false, + log?: DaemonLog, ): Promise { const rank = new Map( idle.flatMap((candidate, index) => { @@ -113,11 +115,17 @@ export async function rankDependencies( }), ); if (!rank.size) return []; - const found = await collectArtifactsAsync( - 'dependencies', - { now: Date.now(), windowMs: 0, protectedRoot, pinned: maintenancePinnedReader(operation), source }, - operation?.signal ?? new AbortController().signal, - operation, + const found = await timedStep( + log, + 'collect_artifacts', + () => + collectArtifactsAsync( + 'dependencies', + { now: Date.now(), windowMs: 0, protectedRoot, pinned: maintenancePinnedReader(operation), source }, + operation?.signal ?? new AbortController().signal, + operation, + ), + { signal: operation?.signal, items: (result) => result && result.candidates.length + result.skips.length }, ); return found.candidates .flatMap((candidate) => { @@ -162,6 +170,7 @@ export async function rankSourceDependencies( protectedRoot: string, rule: SourceRule, operation?: StimContext, + log?: DaemonLog, ): Promise<{ ranked: RankedDependency[]; kept: { target: string; workspace?: string; reason: string }[] }> { const rank = new Map( idle.flatMap((candidate, index) => { @@ -169,11 +178,17 @@ export async function rankSourceDependencies( return key ? [[key, index] as const] : []; }), ); - const found = await collectArtifactsAsync( - 'dependencies', - { now: Date.now(), windowMs: 0, protectedRoot, pinned: maintenancePinnedReader(operation), source: true }, - operation?.signal ?? new AbortController().signal, - operation, + const found = await timedStep( + log, + 'collect_artifacts', + () => + collectArtifactsAsync( + 'dependencies', + { now: Date.now(), windowMs: 0, protectedRoot, pinned: maintenancePinnedReader(operation), source: true }, + operation?.signal ?? new AbortController().signal, + operation, + ), + { signal: operation?.signal, items: (result) => result && result.candidates.length + result.skips.length }, ); const kept: { target: string; workspace?: string; reason: string }[] = found.skips.map(({ target, reason }) => ({ target, @@ -414,10 +429,17 @@ export async function runDormantDependencies( if (!input.force && checkedAt !== 0n && now - checkedAt < BigInt(HOUR_MS)) return []; Atomics.store(cadence, 0, now); } - const actions: DaemonLadderAction[] = []; - if (afterMs > 0) actions.push(...(await takeIdleDependencies(input, config, afterMs, false))); - if (sourceMs > 0) actions.push(...(await takeIdleDependencies(input, config, sourceMs, true))); - return actions; + return timedStep( + input.log, + 'dormant_dependencies', + async () => { + const actions: DaemonLadderAction[] = []; + if (afterMs > 0) actions.push(...(await takeIdleDependencies(input, config, afterMs, false))); + if (sourceMs > 0) actions.push(...(await takeIdleDependencies(input, config, sourceMs, true))); + return actions; + }, + { signal: input.operation?.signal, items: (actions) => actions?.length }, + ); } async function takeIdleDependencies( @@ -445,18 +467,25 @@ async function takeIdleDependencies( const actions: DaemonLadderAction[] = []; let take: RankedDependency[]; if (source) { - const { ranked, kept } = await rankSourceDependencies( - idle, - keeps, - input.protectedRoot ?? '', - { - level: 'normal', - dormantMs: afterMs, - criticalIdleMs: Infinity, - thresholdMs: afterMs, - setting: 'idle.sourceDormantDays', - }, - operation, + const { ranked, kept } = await timedStep( + input.log, + 'rank_source_dependencies', + () => + rankSourceDependencies( + idle, + keeps, + input.protectedRoot ?? '', + { + level: 'normal', + dormantMs: afterMs, + criticalIdleMs: Infinity, + thresholdMs: afterMs, + setting: 'idle.sourceDormantDays', + }, + operation, + input.log, + ), + { signal: operation?.signal, items: (found) => (found ? found.ranked.length + found.kept.length : undefined) }, ); if (!input.act || input.force) for (const entry of kept) @@ -486,7 +515,15 @@ async function takeIdleDependencies( }); } else { if (!idle.length) return []; - take = selectDependencies(await rankDependencies(idle, input.protectedRoot ?? '', operation), 'take').take; + take = selectDependencies( + await timedStep( + input.log, + 'rank_dependencies', + () => rankDependencies(idle, input.protectedRoot ?? '', operation, false, input.log), + { signal: operation?.signal, items: (ranked) => ranked?.length }, + ), + 'take', + ).take; } const worktrees = take.length ? await artifactWorktreesFor(operation) : []; const why = `${source ? 'idle.sourceDormantDays' : 'idle.dormantDays'} is ${afterMs / (24 * HOUR_MS)}`; diff --git a/packages/stim-cli/src/daemon/idle.ts b/packages/stim-cli/src/daemon/idle.ts index 8e767e72e..cf8562d30 100644 --- a/packages/stim-cli/src/daemon/idle.ts +++ b/packages/stim-cli/src/daemon/idle.ts @@ -49,6 +49,7 @@ import { resolveProjectSettingsAsync, } from '../workspace/settings.ts'; import { readWorkspaceState, updateWorkspaceStateAsync } from '../workspace/workspace-state.ts'; +import type { DaemonLog } from './step-timing.ts'; const MINUTE_MS = 60_000; const SIZE_TIMEOUT_MS = 60_000; @@ -69,7 +70,7 @@ export interface IdlePassInput { protectedRoot?: string; /** The daemon tick's context, whose reads run without blocking its event loop. */ operation?: StimContext; - log: (level: 'info' | 'warn' | 'error', event: string, msg: string, fields?: Record) => void; + log: DaemonLog; } type Candidate = diff --git a/packages/stim-cli/src/daemon/ladder.ts b/packages/stim-cli/src/daemon/ladder.ts index 96c6f11ec..1b8b1bfbb 100644 --- a/packages/stim-cli/src/daemon/ladder.ts +++ b/packages/stim-cli/src/daemon/ladder.ts @@ -23,6 +23,7 @@ import { readParked } from '../devices/sim-pool.ts'; import { restoreProblem } from '../workspace/lightened.ts'; import { launchingOf, type Launching } from '../workspace/launchers.ts'; import { formatBytes, measuredDirectorySizeAsync } from '../fs-util.ts'; +import { timedStep } from './step-timing.ts'; import { executeAction, type ActContext, type ActionOutcome } from '../stimd/passes/act.ts'; import { artifactWorktreesFor } from '../stimd/passes/artifacts.ts'; import { cacheBlocked } from '../stimd/passes/measure.ts'; @@ -578,7 +579,15 @@ async function dependenciesRung(context: RungContext): Promise { const idleMs = context.level === 'critical' ? PRESSURE_IDLE_MS : DEPENDENCIES_HIGH_IDLE_MS; const idle = await idleStageCandidates(context.config, context, idleMs); const protectedRoot = context.protectedRoot ? (canonicalOrNull(context.protectedRoot) ?? '') : ''; - const { take, kept } = selectDependencies(await rankDependencies(idle, protectedRoot, context.operation), 'take'); + const { take, kept } = selectDependencies( + await timedStep( + context.log, + 'rank_dependencies', + () => rankDependencies(idle, protectedRoot, context.operation, false, context.log), + { signal: context.operation?.signal, items: (ranked) => ranked?.length }, + ), + 'take', + ); for (const { dependency, detail } of kept) recordAction( context, @@ -667,12 +676,18 @@ async function dependenciesRung(context: RungContext): Promise { * worktree trash. At normal the daily sweep runs the same plan. */ async function sweepRung(context: RungContext, causes: readonly Cause[]): Promise { - const plan = await planRung6(context.settings, { - disk: causes.includes('disk'), - memory: causes.includes('memory'), - everyGradleVersion: context.level === 'critical', - operation: context.operation, - }); + const plan = await timedStep( + context.log, + 'plan_rung6', + () => + planRung6(context.settings, { + disk: causes.includes('disk'), + memory: causes.includes('memory'), + everyGradleVersion: context.level === 'critical', + operation: context.operation, + }), + { signal: context.operation?.signal, items: (planned) => planned?.actions.length }, + ); for (const action of plan.actions) { if (!(await context.stillShort())) return; await run(context, action); @@ -690,19 +705,26 @@ async function worktreesRung(context: RungContext): Promise { if (!context.settings.removeFinishedWorktrees) return; const protectedRoot = context.protectedRoot ? (canonicalOrNull(context.protectedRoot) ?? '') : ''; const pins = new Map(); - const planned = await planWorktrees({ - protectedRoot, - pinned: (workspace) => { - let value = pins.get(workspace); - if (value === undefined) pins.set(workspace, (value = maintenancePinned(workspace))); - return value; - }, - now: Date.now(), - ...(context.level === 'critical' ? { graceMs: 0 } : {}), - refresh: context.act && Boolean(context.force), - idleDays: context.level === 'critical' && context.settings.removeIdleWorktrees ? DEFAULT_WORKTREE_IDLE_DAYS : null, - operation: context.operation, - }); + const planned = await timedStep( + context.log, + 'plan_worktrees', + () => + planWorktrees({ + protectedRoot, + pinned: (workspace) => { + let value = pins.get(workspace); + if (value === undefined) pins.set(workspace, (value = maintenancePinned(workspace))); + return value; + }, + now: Date.now(), + ...(context.level === 'critical' ? { graceMs: 0 } : {}), + refresh: context.act && Boolean(context.force), + idleDays: + context.level === 'critical' && context.settings.removeIdleWorktrees ? DEFAULT_WORKTREE_IDLE_DAYS : null, + operation: context.operation, + }), + { signal: context.operation?.signal, items: (result) => result?.sweep.worktrees.length }, + ); const clean = planned.sweep.worktrees.filter((worktree) => !worktree.dirty); const sweep = { ...planned.sweep, worktrees: clean }; for (const worktree of planned.sweep.worktrees) @@ -739,18 +761,28 @@ async function sourceDependenciesStep(context: RungContext): Promise { onKept: (path, reason) => keeps.set(canonicalOrNull(path) ?? path, reason), }); const protectedRoot = context.protectedRoot ? (canonicalOrNull(context.protectedRoot) ?? '') : ''; - const { ranked, kept } = await rankSourceDependencies( - idle, - keeps, - protectedRoot, + const { ranked, kept } = await timedStep( + context.log, + 'rank_source_dependencies', + () => + rankSourceDependencies( + idle, + keeps, + protectedRoot, + { + level: 'critical', + dormantMs: 0, + criticalIdleMs: PRESSURE_IDLE_MS, + thresholdMs: PRESSURE_IDLE_MS, + setting: 'the pressure idle threshold', + }, + context.operation, + context.log, + ), { - level: 'critical', - dormantMs: 0, - criticalIdleMs: PRESSURE_IDLE_MS, - thresholdMs: PRESSURE_IDLE_MS, - setting: 'the pressure idle threshold', + signal: context.operation?.signal, + items: (found) => (found ? found.ranked.length + found.kept.length : undefined), }, - context.operation, ); if (!context.act || context.force) for (const entry of kept) @@ -843,7 +875,11 @@ async function sourceDependenciesStep(context: RungContext): Promise { async function historyRung(context: RungContext): Promise { if (context.level !== 'critical') return; await sourceDependenciesStep(context); - for (const action of await planHistory(context.operation)) { + const history = await timedStep(context.log, 'plan_history', () => planHistory(context.operation), { + signal: context.operation?.signal, + items: (planned) => planned?.length, + }); + for (const action of history) { if (!(await context.stillShort())) return; await run(context, action); } @@ -959,8 +995,13 @@ export async function runLadder( const settings = resolveMaintenanceSettings(input.config); const { force } = input; const runRung = async (rung: Rung, context: RungContext, causes: Cause[]) => { + const before = actions.length; try { - await RUNG_ACTS[rung](context, causes); + await timedStep(input.log, 'rung', () => RUNG_ACTS[rung](context, causes), { + signal: input.operation?.signal, + fields: { rung, pressureLevel: context.level }, + items: () => actions.length - before, + }); } catch (error) { if (isClaimUnavailable(error)) throw error; input.operation?.signal.throwIfAborted(); diff --git a/packages/stim-cli/src/daemon/stages.ts b/packages/stim-cli/src/daemon/stages.ts index ee90658c7..2bf4ec879 100644 --- a/packages/stim-cli/src/daemon/stages.ts +++ b/packages/stim-cli/src/daemon/stages.ts @@ -53,6 +53,7 @@ import { } from '../workspace/settings.ts'; import { readWorkspaceState, updateWorkspaceStateAsync } from '../workspace/workspace-state.ts'; import { deleteOutputs, idleOutputs, revivable, writeWorkspaceEvent, type IdlePassInput } from './idle.ts'; +import { timedStep, type DaemonLog, type StepTally } from './step-timing.ts'; const DAY_MS = 86_400_000; const MINUTE_MS = 60_000; @@ -89,6 +90,8 @@ export interface DeviceReading { protectedRoot?: string; /** The context whose reads run without blocking the event loop: the daemon tick's or a command's. */ operation?: StimContext; + /** Where the evidence loops write their `daemon_step` record; absent, they write none. */ + log?: DaemonLog; } function projectSettings(path: string, operation: StimContext | undefined) { @@ -233,7 +236,25 @@ interface IdleSelection { onKept?: (path: string, reason: string) => void; } -async function idleWorkspaces( +function idleWorkspaces( + step: string, + config: Config, + reading: DeviceReading, + thresholdMs: number, + setting: string, + worthIt: (path: string, project: ProjectRecord) => boolean | Promise, + selection: Omit & { discover?: boolean } = {}, +): Promise { + return timedStep( + reading.log, + step, + (tally) => readIdleWorkspaces(tally, config, reading, thresholdMs, setting, worthIt, selection), + { signal: reading.operation?.signal }, + ); +} + +async function readIdleWorkspaces( + tally: StepTally, config: Config, reading: DeviceReading, thresholdMs: number, @@ -246,7 +267,9 @@ async function idleWorkspaces( const found = discover ? await discoverCheckouts(config, listedRoots(), reading.operation) : []; const discovered = found.map(({ path }): [string, ProjectRecord] => [path, {}]); const reduceOnly = new Set(found.filter((entry) => entry.reduceOnly).map(({ path }) => path)); - for (const [path, project] of [...Object.entries(config.projects), ...discovered]) { + const checkouts = [...Object.entries(config.projects), ...discovered]; + tally.items = checkouts.length; + for (const [path, project] of checkouts) { if (sameRoot(path, reading.protectedRoot) || !(await worthIt(path, project))) continue; const used = Date.parse(String(readWorkspaceState(path)?.lastUsedAt ?? '')); if (Number.isFinite(used) && Date.now() - used < thresholdMs) { @@ -311,7 +334,10 @@ export function idleStageCandidates( selection: IdleSelection = {}, ): Promise { const { setting = PRESSURE_THRESHOLD, ...rest } = selection; - return idleWorkspaces(config, reading, thresholdMs, setting, () => true, { discover: true, ...rest }); + return idleWorkspaces('idle_stage_candidates', config, reading, thresholdMs, setting, () => true, { + discover: true, + ...rest, + }); } /** Workspaces with nothing running and regenerable build outputs on disk, idle `thresholdMs`, longest-idle first. */ @@ -320,7 +346,9 @@ export function stoppedOutputCandidates( reading: DeviceReading, thresholdMs: number, ): Promise { - return idleWorkspaces(config, reading, thresholdMs, PRESSURE_THRESHOLD, hasOutputs, { discover: true }); + return idleWorkspaces('stopped_output_candidates', config, reading, thresholdMs, PRESSURE_THRESHOLD, hasOutputs, { + discover: true, + }); } type StageResult = Pick; @@ -467,6 +495,7 @@ export function dormantCandidates( thresholdMs: number, ): Promise { return idleWorkspaces( + 'dormant_candidates', config, reading, thresholdMs, diff --git a/packages/stim-cli/src/daemon/step-timing.ts b/packages/stim-cli/src/daemon/step-timing.ts new file mode 100644 index 000000000..55e8fe810 --- /dev/null +++ b/packages/stim-cli/src/daemon/step-timing.ts @@ -0,0 +1,79 @@ +export type DaemonLog = ( + level: 'debug' | 'info' | 'warn' | 'error', + event: string, + msg: string, + fields?: Record, +) => void; + +type StepOutcome = 'finished' | 'threw' | 'aborted'; + +/** A count a step fills in while it runs, for a loop whose visited items its result does not show. */ +export interface StepTally { + items?: number; +} + +const ERROR_MAX = 200; +const WRITER_FIELDS = new Set(['src', 'level', 'event', 'msg', 'ts']); + +function classify(error: unknown, signal: AbortSignal | undefined): { outcome: StepOutcome; failure?: string } { + try { + if (signal?.aborted || (error instanceof Error && error.name === 'AbortError')) return { outcome: 'aborted' }; + return { outcome: 'threw', failure: (error instanceof Error ? error.message : String(error)).slice(0, ERROR_MAX) }; + } catch { + return { outcome: 'threw' }; + } +} + +/** + * Runs `run` and writes one `daemon_step` record to the daemon log: when it started, how long it took, how many items + * it visited and how it ended. It returns `run`'s value, rethrows its error unchanged and changes nothing else; with + * no `log` it only calls `run`. `items` is read after the step ends, with the result when it finished. + */ +export async function timedStep( + log: DaemonLog | undefined, + step: string, + run: (tally: StepTally) => Promise, + { + signal, + fields, + items, + }: { + signal?: AbortSignal; + fields?: Record; + items?: (result: T | undefined) => number | undefined; + } = {}, +): Promise { + const tally: StepTally = {}; + if (!log) return run(tally); + const startedAt = Date.now(); + const started = performance.now(); + let outcome: StepOutcome = 'finished'; + let result: T | undefined; + let failure: string | undefined; + try { + result = await run(tally); + return result; + } catch (error) { + ({ outcome, failure } = classify(error, signal)); + throw error; + } finally { + try { + const durationMs = Math.round(performance.now() - started); + const count = tally.items ?? items?.(result); + log( + 'debug', + 'daemon_step', + `${step}${fields?.rung === undefined ? '' : ` ${String(fields.rung)}`} ${outcome} in ${durationMs}ms${count === undefined ? '' : `; ${count} items`}`, + { + step, + ...Object.fromEntries(Object.entries(fields ?? {}).filter(([key]) => !WRITER_FIELDS.has(key))), + startedAt: new Date(startedAt).toISOString(), + durationMs, + ...(count === undefined ? {} : { items: count }), + outcome, + ...(failure ? { error: failure } : {}), + }, + ); + } catch {} + } +} diff --git a/packages/stim-cli/src/disk-usage.ts b/packages/stim-cli/src/disk-usage.ts index 16fba01db..15c1ce683 100644 --- a/packages/stim-cli/src/disk-usage.ts +++ b/packages/stim-cli/src/disk-usage.ts @@ -146,19 +146,25 @@ export async function refreshMeasure( const skip = () => freshMeasure(path, options) !== null || failedRecently(); const fresh = () => ({ outcome: 'fresh' as const, measure: freshMeasure(path, options) }); if (skip()) return fresh(); - const record = (bytes: number, privateBytes: number | null, signature: string | null, at: number): DiskMeasure => { + const record = ( + bytes: number, + privateBytes: number | null, + signature: string | null, + at: number, + walkMs: number | null, + ): DiskMeasure => { const measure: DiskMeasure = { bytes, measuredAt: new Date(at).toISOString(), ...(privateBytes !== null ? { privateBytes } : {}), ...(signature ? { signature } : {}), }; - writeCacheFile(diskUsageCacheFile(path), { path, ...measure }); + writeCacheFile(diskUsageCacheFile(path), { path, ...measure, ...(walkMs !== null ? { walkMs } : {}) }); return measure; }; if (!existsSync(path)) { if (!dependency) return { outcome: 'fresh', measure: null }; - return { outcome: 'measured', measure: record(0, 0, ABSENT, now()) }; + return { outcome: 'measured', measure: record(0, 0, ABSENT, now(), null) }; } const claim = options.claim ? takeWalkClaim() : null; if (options.claim && !claim) return { outcome: 'busy', measure: null }; @@ -190,7 +196,10 @@ export async function refreshMeasure( writeCacheFile(diskUsageFailureFile(path), { path, failedAt: new Date(now()).toISOString() }); return { outcome: 'failed', measure: null }; } - return { outcome: 'measured', measure: record(size.bytes, size.privateBytes, signature, startedAt) }; + return { + outcome: 'measured', + measure: record(size.bytes, size.privateBytes, signature, startedAt, Math.max(0, now() - startedAt)), + }; } catch (error) { cleanupUnverified = error instanceof Error && 'reportCleanup' in error; throw error; diff --git a/packages/stim-cli/src/guide/logs.ts b/packages/stim-cli/src/guide/logs.ts index 98b349f27..cec957e01 100644 --- a/packages/stim-cli/src/guide/logs.ts +++ b/packages/stim-cli/src/guide/logs.ts @@ -352,6 +352,36 @@ Each record has src: gc, level (info, or error for a failed item), msg and and the reason a kept or failed item was left Records of one run share runId. See stim guide cleanup, WHAT WAS CLEARED. +DAEMON LOG TIMING +$STIM_HOME/daemon/daemon.ndjson (rotated at 8 MiB, one older file kept as +daemon.ndjson.1) carries one debug record, event daemon_step, for each timed +step. It is written when the step ends, whether it finished, threw or was +aborted, so a step that took hours shows where the time went. Fields: step, +startedAt (ISO time), durationMs, items (when the step counts any), outcome +(finished, threw or aborted), error (threw only, at most 200 characters), and +for a rung: rung and pressureLevel (the level the rung ran at). A loop's record falls +between its rung's startedAt and its end. Steps and what items counts: + rung a pressure ladder rung; the actions it recorded + idle_stage_candidates, + stopped_output_candidates, + dormant_candidates the workspace and checkout evidence loops; + the checkouts visited + collect_artifacts the artifact scan of a dependency step; the + candidates and skips it found + rank_dependencies, + rank_source_dependencies the dependencies ranked, and for the source + step also those kept + plan_rung6, plan_worktrees, + plan_history the actions or worktrees the plan returned + dormant_dependencies the dormant dependency step, when it runs (at + most hourly); the actions it took or kept +A step writes one record, never one per item. A tick at normal writes one +(dormant_candidates); at high up to 16 and at critical up to 22; the dormant +dependency step adds about 7 an hour. The records change nothing a pass does. +Each disk-usage cache entry written after a walk also records walkMs, in milliseconds, from just +before the install signature and target reads to the end of the folder walk; +nothing reads it yet. + BUILD LEDGER stim logs --source builds reads $STIM_HOME/builds/builds.ndjson from any directory: one record per native build run (ios, android or macos; compiled diff --git a/website/docs/dev-server-and-logs.md b/website/docs/dev-server-and-logs.md index 66d1b17bc..65e7188c8 100644 --- a/website/docs/dev-server-and-logs.md +++ b/website/docs/dev-server-and-logs.md @@ -530,6 +530,18 @@ rotated during it, look in `daemon.ndjson.1` too. Include records from both sides of a restart or engine switch. Preserve timestamps, event names, error codes, build versions, and the run IDs needed to join those records. +Each step of the daemon's cleanup passes writes one `daemon_step` record when +it ends: every pressure ladder rung, and the loops that read evidence for a +checkout (`idle_stage_candidates`, `stopped_output_candidates`, +`dormant_candidates`, `collect_artifacts`, `rank_dependencies`, +`rank_source_dependencies`, `plan_rung6`, `plan_worktrees`, `plan_history`, +`dormant_dependencies`). A record has `step`, `startedAt`, `durationMs`, `items` +(the items it visited or returned, when it counts any), and `outcome` +(`finished`, `threw` or `aborted`); a rung adds `rung` and `pressureLevel`, and a step +that threw adds `error`. A tick at normal writes one, up to 16 at high and up +to 22 at critical. Use them to see which step a slow tick or maintenance run +spent its time in. The records change nothing a pass does. + For local device lease contention, look for `device_lease_acquire` when the request was refused or spent at least five seconds waiting. `waitMs` counts time spent waiting for another holder to release the lease. `outcome` records