From ff881ff1b37914aa7ebe2fbf0176e3325f271290 Mon Sep 17 00:00:00 2001 From: Janic Duplessis Date: Sun, 15 Nov 2026 09:24:04 -0500 Subject: [PATCH 01/12] feat: measure the checkouts the home folder scan found that Stim does not fully manage --- packages/core/state/discovery-census.ts | 137 +++++++++ packages/core/state/index.ts | 1 + packages/core/state/paths.ts | 4 + .../src/__tests__/discovery-census.test.ts | 176 ++++++++++++ .../src/__tests__/discovery-report.test.ts | 261 ++++++++++++++++++ packages/stim-cli/src/stimd/passes/run.ts | 4 +- .../src/workspace/discovery-census.ts | 225 +++++++++++++++ .../src/workspace/discovery-report.ts | 260 +++++++++++++++++ packages/stim-cli/src/workspace/discovery.ts | 61 +++- 9 files changed, 1123 insertions(+), 6 deletions(-) create mode 100644 packages/core/state/discovery-census.ts create mode 100644 packages/stim-cli/src/__tests__/discovery-census.test.ts create mode 100644 packages/stim-cli/src/__tests__/discovery-report.test.ts create mode 100644 packages/stim-cli/src/workspace/discovery-census.ts create mode 100644 packages/stim-cli/src/workspace/discovery-report.ts diff --git a/packages/core/state/discovery-census.ts b/packages/core/state/discovery-census.ts new file mode 100644 index 000000000..68ab8b2a2 --- /dev/null +++ b/packages/core/state/discovery-census.ts @@ -0,0 +1,137 @@ +import { isJsonObject, readStateFile } from './json-file.ts'; +import { discoveryCensusFile } from './paths.ts'; + +/** + * A checkout the last full scan found that Stim does not fully manage: `reduce-only` ones are reduced and never removed, + * `dot-folder` ones lie under a home dot-folder, where Stim does nothing with them. + */ +export interface CensusFind { + path: string; + kind: 'reduce-only' | 'dot-folder'; + /** The folder whose direct children include the checkout, the one `maintenance.discoverRoots` would take. */ + folder: string; + /** Whether `maintenance.discoverRoots` can name `folder`: the home folder and the folders holding it cannot. */ + adoptable: boolean; + /** When its build output was last measured, epoch ms; 0 when it was not measured yet. */ + evaluatedAt: number; + /** The recognised build directories in it, the ones Stim would clear. */ + dirs: number; + /** How many of them were measured, and how many were tried and failed or timed out (the rest are not measured yet). */ + measuredDirs: number; + failedDirs: number; + /** The bytes of the measured directories; never an estimate of the others. */ + bytes: number; + /** + * The newest activity in it by the idle stage's full evidence, epoch ms, set only when that evidence was complete, + * nothing used the checkout, and the newest activity was older than `idleDays` at `evaluatedAt`. Null otherwise, + * which is not "active": it is "not known to be idle". Always null for `dot-folder`. + */ + lastActivityAt: number | null; +} + +/** + * What the daemon derived from the last full scan for the doctor listing and the low-disk line: which finds Stim does + * not fully manage, and the size of their build output. Commands read it and never walk the disk. + */ +export interface DiscoveryCensus { + version: 1; + /** When the daemon last wrote it, epoch ms. */ + at: number; + /** The `ranAt` of the scan it was derived from. */ + scanRanAt: number; + /** The `maintenance.discoverRoots` entries it was classified under, as absolute paths. */ + roots: string[]; + /** The days without activity (`maintenance.olderThanDays`) the idle evidence was taken for; null when it was not. */ + idleDays: number | null; + /** Some find has a directory not measured yet, so another run continues. */ + pending: boolean; + finds: CensusFind[]; +} + +const finite = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value); +const strings = (value: unknown): value is string[] => + Array.isArray(value) && value.every((entry) => typeof entry === 'string'); + +function parseFind(value: unknown): CensusFind | null { + if ( + !isJsonObject(value) || + typeof value.path !== 'string' || + (value.kind !== 'reduce-only' && value.kind !== 'dot-folder') || + typeof value.folder !== 'string' || + typeof value.adoptable !== 'boolean' || + !finite(value.evaluatedAt) || + !finite(value.dirs) || + !finite(value.measuredDirs) || + !finite(value.failedDirs) || + !finite(value.bytes) || + (value.lastActivityAt !== null && !finite(value.lastActivityAt)) + ) + return null; + return { + path: value.path, + kind: value.kind, + folder: value.folder, + adoptable: value.adoptable, + evaluatedAt: value.evaluatedAt, + dirs: value.dirs, + measuredDirs: value.measuredDirs, + failedDirs: value.failedDirs, + bytes: value.bytes, + lastActivityAt: value.lastActivityAt, + }; +} + +/** Rejects missing, corrupt or unsupported payloads without throwing. */ +export function parseDiscoveryCensus(value: unknown): DiscoveryCensus | null { + if ( + !isJsonObject(value) || + value.version !== 1 || + !finite(value.at) || + !finite(value.scanRanAt) || + !strings(value.roots) || + (value.idleDays !== null && !finite(value.idleDays)) || + typeof value.pending !== 'boolean' || + !Array.isArray(value.finds) + ) + return null; + const finds: CensusFind[] = []; + for (const entry of value.finds) { + const find = parseFind(entry); + if (!find) return null; + finds.push(find); + } + return { + version: 1, + at: value.at, + scanRanAt: value.scanRanAt, + roots: value.roots, + idleDays: value.idleDays, + pending: value.pending, + finds, + }; +} + +/** + * The census file: `absent` (the daemon has not written one), `unreadable` (it exists and could not be read), or `read` + * with the census, which is null when the contents do not parse or are another version. + */ +export type DiscoveryCensusFile = + | { status: 'absent' } + | { status: 'unreadable' } + | { status: 'read'; census: DiscoveryCensus | null }; + +export function readDiscoveryCensusFile(): DiscoveryCensusFile { + let text: string; + try { + text = readStateFile(discoveryCensusFile()); + } catch (error) { + return error instanceof Error && 'code' in error && error.code === 'ENOENT' + ? { status: 'absent' } + : { status: 'unreadable' }; + } + try { + return { status: 'read', census: parseDiscoveryCensus(JSON.parse(text)) }; + } catch { + return { status: 'read', census: null }; + } +} diff --git a/packages/core/state/index.ts b/packages/core/state/index.ts index a710b757f..67c3f74ee 100644 --- a/packages/core/state/index.ts +++ b/packages/core/state/index.ts @@ -53,6 +53,7 @@ export * from './server-service.ts'; export * from './hosted-logs.ts'; export * from './maintenance.ts'; export * from './discovery-scan.ts'; +export * from './discovery-census.ts'; export * from './gc-log.ts'; export * from './build-ledger.ts'; export * from './agent-device.ts'; diff --git a/packages/core/state/paths.ts b/packages/core/state/paths.ts index 1e2a4996a..cd8ade398 100644 --- a/packages/core/state/paths.ts +++ b/packages/core/state/paths.ts @@ -250,6 +250,10 @@ export function discoveryScanFile(): string { return join(maintenanceDir(), 'discovery-scan.json'); } +export function discoveryCensusFile(): string { + return join(maintenanceDir(), 'discovery-census.json'); +} + export function gcLogDir(): string { return join(configDir(), 'gc'); } diff --git a/packages/stim-cli/src/__tests__/discovery-census.test.ts b/packages/stim-cli/src/__tests__/discovery-census.test.ts new file mode 100644 index 000000000..cf31312db --- /dev/null +++ b/packages/stim-cli/src/__tests__/discovery-census.test.ts @@ -0,0 +1,176 @@ +import { mkdirSync, mkdtempSync, realpathSync, rmSync, utimesSync, writeFileSync } from 'node:fs'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { readDiscoveryCensusFile } from '@stim-cli/core/state'; +import * as diskUsage from '../disk-usage.ts'; +import { censusDue, runDiscoveryCensusIfDue } from '../workspace/discovery-census.ts'; +import { runDiscoveryScanIfDue } from '../workspace/discovery-scan.ts'; +import { saveConfig } from '../workspace/config.ts'; +import { testContext } from './_context.ts'; + +vi.mock('../workspace/protected-locations.ts', async (original) => { + const real = await original(); + return { + ...real, + isProtectedLocation: (path: string) => real.isProtectedLocation(path, { home: homedir(), temps: [] }), + }; +}); + +const hour = 3_600_000; +const day = 24 * hour; +const t0 = Date.UTC(2026, 10, 14); +const reads = { + processes: async () => ({ commands: [], cwds: [] }), + launchers: async () => ({ targets: [], incomplete: [] }), + sessions: async () => [], +}; + +describe('censusDue', () => { + const last = { at: t0, scanRanAt: 5, roots: ['/a'], pending: false }; + it('is due for no census, another scan, other roots, a day, or an hour while something waits', () => { + expect(censusDue(null, 5, ['/a'], t0)).toBe(true); + expect(censusDue(last, 6, ['/a'], t0 + hour)).toBe(true); + expect(censusDue(last, 5, ['/a', '/b'], t0 + hour)).toBe(true); + expect(censusDue(last, 5, ['/a'], t0 + hour)).toBe(false); + expect(censusDue(last, 5, ['/a'], t0 + day)).toBe(true); + expect(censusDue({ ...last, pending: true }, 5, ['/a'], t0 + hour)).toBe(true); + expect(censusDue({ ...last, pending: true }, 5, ['/a'], t0 + hour - 1)).toBe(false); + }); +}); + +describe('runDiscoveryCensusIfDue', () => { + let base: string; + let home: string; + const tick = (env: NodeJS.ProcessEnv = {}) => ({ + operation: testContext({ env }), + measured: { measurement: { disk: [{ volume: '/', freeMb: 500_000 }] } }, + }); + + function checkout(path: string, output = 4096) { + mkdirSync(join(path, 'android', 'build'), { recursive: true }); + writeFileSync(join(path, 'package.json'), JSON.stringify({ dependencies: { expo: '*' } })); + writeFileSync(join(path, 'android', 'build', 'out.bin'), Buffer.alloc(output)); + } + + function age(path: string, at: number) { + const date = new Date(at); + utimesSync(join(path, 'package.json'), date, date); + utimesSync(join(path, 'android', 'build', 'out.bin'), date, date); + utimesSync(join(path, 'android', 'build'), date, date); + utimesSync(join(path, 'android'), date, date); + utimesSync(path, date, date); + } + + async function run(now: number, options: Parameters[1] = {}) { + await runDiscoveryCensusIfDue(tick(), { now, attempted: { at: 0 }, reads, ...options }); + const file = readDiscoveryCensusFile(); + return file.status === 'read' ? file.census : null; + } + + beforeEach(async () => { + base = realpathSync(mkdtempSync(join(tmpdir(), 'stim-census-'))); + home = join(base, 'home'); + mkdirSync(home); + vi.stubEnv('STIM_HOME', ''); + vi.stubEnv('HOME', home); + vi.stubEnv('USERPROFILE', home); + saveConfig({ version: 2, projects: {}, repos: {}, maintenance: { discoverRoots: [join(home, 'dev', 'tools')] } }); + checkout(join(home, 'dev', 'old'), 8192); + checkout(join(home, 'dev', 'fresh')); + checkout(join(home, 'dev', 'tools', 'managed')); + checkout(join(home, '.exact', 'exact2', 'a'), 16384); + checkout(join(home, '.exact', 'exact2', 'b')); + checkout(join(home, '.mytool')); + age(join(home, 'dev', 'old'), t0 - 40 * day); + await runDiscoveryScanIfDue(tick(), { home, now: t0, attempted: { at: 0 } }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + rmSync(base, { recursive: true, force: true }); + }); + + it('lists what no other route manages, with each folder, and measures the build output of both kinds', async () => { + const census = await run(t0); + expect(census?.scanRanAt).toBe(t0); + const found = (census?.finds ?? []).map(({ path, kind, folder, adoptable }) => [ + path.slice(home.length), + kind, + folder.slice(home.length), + adoptable, + ]); + expect(found).toEqual([ + ['/dev/fresh', 'reduce-only', '/dev', true], + ['/dev/old', 'reduce-only', '/dev', true], + ['/.exact/exact2/a', 'dot-folder', '/.exact/exact2', true], + ['/.exact/exact2/b', 'dot-folder', '/.exact/exact2', true], + ['/.mytool', 'dot-folder', '', false], + ]); + for (const find of census?.finds ?? []) expect(find).toMatchObject({ dirs: 1, measuredDirs: 1, failedDirs: 0 }); + expect(census?.finds.find(({ path }) => path.endsWith('/a'))?.bytes).toBeGreaterThanOrEqual(16384); + expect(census?.pending).toBe(false); + }); + + it('records the newest activity only of a reduce-only checkout proven idle past maintenance.olderThanDays', async () => { + const census = await run(t0); + const activity = Object.fromEntries( + (census?.finds ?? []).map(({ path, lastActivityAt }) => [path.slice(home.length), lastActivityAt]), + ); + expect(activity['/dev/old']).toBe(t0 - 40 * day); + expect(activity['/dev/fresh']).toBeNull(); + expect(activity['/.exact/exact2/a']).toBeNull(); + }); + + it('keeps a checkout that something runs from out of the idle list', async () => { + const census = await run(t0, { + reads: { + ...reads, + processes: async () => ({ commands: [{ pid: 7, line: `${join(home, 'dev', 'old')}/bin/tool` }], cwds: [] }), + }, + }); + expect(census?.finds.find(({ path }) => path.endsWith('/old'))?.lastActivityAt).toBeNull(); + }); + + it('never reports a walk that failed as zero bytes, and a walk that waits as unmeasured yet', async () => { + const refresh = vi.spyOn(diskUsage, 'refreshMeasure'); + refresh.mockImplementation(async (path) => + path.includes('/old/') ? { outcome: 'failed', measure: null } : { outcome: 'busy', measure: null }, + ); + const census = await run(t0); + const of = (name: string) => census?.finds.find(({ path }) => path.endsWith(name)); + expect(of('/dev/old')).toMatchObject({ dirs: 1, measuredDirs: 0, failedDirs: 1, bytes: 0 }); + expect(of('/dev/fresh')).toMatchObject({ dirs: 1, measuredDirs: 0, failedDirs: 0, bytes: 0 }); + expect(census?.pending).toBe(true); + }); + + it('measures at most the bound per run, leaves the rest not measured, and the next run continues without repeating', async () => { + const first = await run(t0, { maxCheckouts: 2 }); + expect(first?.finds.filter(({ evaluatedAt }) => evaluatedAt === 0)).toHaveLength(3); + expect(first?.pending).toBe(true); + const refresh = vi.spyOn(diskUsage, 'refreshMeasure'); + const second = await run(t0 + hour, { maxCheckouts: 2 }); + expect(second?.finds.filter(({ evaluatedAt }) => evaluatedAt === 0)).toHaveLength(1); + expect(refresh).toHaveBeenCalledTimes(2); + const third = await run(t0 + 2 * hour, { maxCheckouts: 2 }); + expect(third?.pending).toBe(false); + }); + + it('classifies again after the roots change, so a folder just added is no longer listed', async () => { + await run(t0); + saveConfig({ + version: 2, + projects: {}, + repos: {}, + maintenance: { discoverRoots: [join(home, 'dev', 'tools'), join(home, 'dev')] }, + }); + const census = await run(t0 + hour); + expect(census?.finds.filter(({ kind }) => kind === 'reduce-only')).toEqual([]); + }); + + it('is never run in a scoped home and writes nothing', async () => { + await runDiscoveryCensusIfDue(tick({ STIM_HOME: join(base, 'state') }), { now: t0, attempted: { at: 0 }, reads }); + expect(readDiscoveryCensusFile()).toEqual({ status: 'absent' }); + }); +}); diff --git a/packages/stim-cli/src/__tests__/discovery-report.test.ts b/packages/stim-cli/src/__tests__/discovery-report.test.ts new file mode 100644 index 000000000..cd5acd51c --- /dev/null +++ b/packages/stim-cli/src/__tests__/discovery-report.test.ts @@ -0,0 +1,261 @@ +import { describe, expect, it } from 'vitest'; +import type { CensusFind, DiscoveryCensusFile, DiscoveryScan, DiscoveryScanFile } from '@stim-cli/core/state'; +import { + adoptCommand, + discoveryLines, + discoveryReport, + lowDiskLine, + type DiscoverRootsValue, +} from '../workspace/discovery-report.ts'; + +const home = '/Users/me'; +const now = Date.UTC(2026, 10, 20); +const day = 24 * 60 * 60_000; +const GB = 1024 ** 3; + +const scan: DiscoveryScan = { + version: 2, + ranAt: Date.UTC(2026, 10, 14), + completedAt: Date.UTC(2026, 10, 14), + lowScanAt: 0, + durationMs: 1000, + trigger: 'weekly', + visited: 4200, + found: [], + dotFolderFound: [], + unreadable: [], + truncated: false, + truncation: 'none', +}; + +function find(over: Partial & Pick): CensusFind { + return { + kind: 'reduce-only', + folder: over.path.slice(0, over.path.lastIndexOf('/')), + adoptable: true, + evaluatedAt: now, + dirs: 1, + measuredDirs: 1, + failedDirs: 0, + bytes: GB, + lastActivityAt: null, + ...over, + }; +} + +const census = (finds: CensusFind[]): DiscoveryCensusFile => ({ + status: 'read', + census: { + version: 1, + at: now, + scanRanAt: scan.ranAt, + roots: [], + idleDays: 3, + pending: false, + finds, + }, +}); + +const noRoots: DiscoverRootsValue = { value: [], error: null }; + +function report( + finds: CensusFind[], + setting: DiscoverRootsValue = noRoots, + scanFile: DiscoveryScanFile = { status: 'read', scan }, +) { + return discoveryReport({ + scan: scanFile, + census: census(finds), + setting, + idleAfterDays: 3, + home, + now, + }); +} + +describe('adoptCommand', () => { + it('builds the pasteable command from the current value, quoting a path with a single quote for the shell', () => { + expect(adoptCommand({ value: ['/a'], error: null }, "/Users/me/it's here")).toBe( + `stim settings set maintenance.discoverRoots '["/a","/Users/me/it'\\''s here"]' --scope machine`, + ); + }); + + it('offers none when the current value does not read', () => { + expect(adoptCommand({ value: undefined, error: 'maintenance is not an object' }, '/a')).toBeNull(); + expect(adoptCommand({ value: 'x', error: null }, '/a')).toBeNull(); + }); +}); + +describe('discoveryReport', () => { + it('groups by containing folder, reduce-only first, each with its own sum and adopt command', () => { + const result = report([ + find({ + path: '/Users/me/.exact/exact2/a', + kind: 'dot-folder', + folder: '/Users/me/.exact/exact2', + bytes: 80 * GB, + }), + find({ path: '/Users/me/dev/app1', bytes: 2 * GB }), + find({ path: '/Users/me/dev/app2', bytes: 3 * GB }), + ]); + expect(result.groups.map(({ folder, kind, checkouts, bytes }) => ({ folder, kind, checkouts, bytes }))).toEqual([ + { folder: '/Users/me/dev', kind: 'reduce-only', checkouts: 2, bytes: 5 * GB }, + { folder: '/Users/me/.exact/exact2', kind: 'dot-folder', checkouts: 1, bytes: 80 * GB }, + ]); + expect(result.groups[1]?.command).toBe( + `stim settings set maintenance.discoverRoots '["/Users/me/.exact/exact2"]' --scope machine`, + ); + }); + + it('leaves out a folder that maintenance.discoverRoots names since the daemon classified', () => { + const result = report([find({ path: '/Users/me/dev/app1' }), find({ path: '/Users/me/work/app2' })], { + value: ['~/dev'], + error: null, + }); + expect(result.groups.map(({ folder }) => folder)).toEqual(['/Users/me/work']); + expect(result.groups[0]?.command).toContain(`'["~/dev","/Users/me/work"]'`); + }); + + it('offers no command for the home folder, and none when the setting does not read, and says which', () => { + const [direct] = report([ + find({ path: '/Users/me/.mytool', kind: 'dot-folder', folder: '/Users/me', adoptable: false }), + ]).groups; + expect(direct).toMatchObject({ command: null, commandProblem: 'too-broad' }); + const [broken] = report([find({ path: '/Users/me/dev/a' })], { value: undefined, error: 'bad' }).groups; + expect(broken).toMatchObject({ command: null, commandProblem: 'setting-unreadable' }); + }); + + it('counts a checkout that failed and one not measured yet apart from the measured, never as zero bytes', () => { + const [group] = report([ + find({ path: '/Users/me/dev/a', bytes: GB }), + find({ path: '/Users/me/dev/b', bytes: 0, measuredDirs: 0, failedDirs: 1 }), + find({ path: '/Users/me/dev/c', bytes: 0, measuredDirs: 0, failedDirs: 0, evaluatedAt: 0, dirs: 0 }), + ]).groups; + expect(group).toMatchObject({ measured: 1, failed: 1, notMeasuredYet: 1, bytes: GB }); + expect( + discoveryLines(report([find({ path: '/Users/me/dev/b', bytes: 0, measuredDirs: 0, failedDirs: 1 })]), home).join( + '\n', + ), + ).toContain('build output not measured'); + }); + + it('lists a reduce-only checkout idle at least the setting, never a dot-folder one', () => { + const [group] = report([ + find({ path: '/Users/me/dev/old', lastActivityAt: now - 40 * day }), + find({ path: '/Users/me/dev/recent', lastActivityAt: now - 1 * day }), + find({ path: '/Users/me/dev/unknown', lastActivityAt: null }), + ]).groups; + expect(group?.idle).toEqual([{ path: '/Users/me/dev/old', idleDays: 40 }]); + const [dot] = report([ + find({ path: '/Users/me/.t/x', kind: 'dot-folder', folder: '/Users/me/.t', lastActivityAt: now - 40 * day }), + ]).groups; + expect(dot?.idle).toEqual([]); + }); + + it('keeps an unreadable, absent and unusable scan or census apart', () => { + const base = { setting: noRoots, idleAfterDays: 3, home, now }; + const missing: DiscoveryCensusFile = { status: 'absent' }; + expect(discoveryReport({ ...base, scan: { status: 'absent' }, census: missing }).scan).toEqual({ + status: 'absent', + }); + expect(discoveryReport({ ...base, scan: { status: 'unreadable' }, census: missing }).scan).toEqual({ + status: 'unreadable', + }); + expect(discoveryReport({ ...base, scan: { status: 'read', scan: null }, census: missing }).scan).toEqual({ + status: 'unusable', + }); + const scanFile: DiscoveryScanFile = { status: 'read', scan }; + const withScan = { ...base, scan: scanFile }; + expect(discoveryReport({ ...withScan, census: missing }).census).toBe('absent'); + expect(discoveryReport({ ...withScan, census: { status: 'unreadable' } }).census).toBe('unreadable'); + expect(discoveryReport({ ...withScan, census: { status: 'read', census: null } }).census).toBe('unusable'); + }); +}); + +describe('discoveryLines', () => { + it('words the two kinds differently, shows the command, when the scan ran and what it could not read', () => { + const lines = discoveryLines( + discoveryReport({ + scan: { + status: 'read', + scan: { ...scan, truncated: true, truncation: 'budget', unreadable: ['/Users/me/Documents/x'] }, + }, + census: census([ + find({ + path: '/Users/me/.exact/exact2/a', + kind: 'dot-folder', + folder: '/Users/me/.exact/exact2', + bytes: 80 * GB, + }), + find({ path: '/Users/me/dev/old', bytes: 2 * GB, lastActivityAt: now - 40 * day }), + ]), + setting: noRoots, + idleAfterDays: 3, + home, + now, + }), + home, + ).join('\n'); + expect(lines).toContain('last scanned the home folder on 2026-11-14'); + expect(lines).toContain('stopped early (time budget)'); + expect(lines).toContain('could not read ~/Documents/x'); + expect(lines).toContain('~/dev: 1 checkout, 2.0 GB of build output'); + expect(lines).toContain('~/.exact/exact2: 1 checkout, 80.0 GB of build output'); + expect(lines).toContain('Stim reduces their build output when idle and never removes them'); + expect(lines).toContain('Stim cannot tell a checkout from an installed tool, so it does nothing with them'); + expect(lines).toContain(`'["/Users/me/.exact/exact2"]'`); + expect(lines).toContain( + 'Idle for at least 40 days and nothing uses it: ~/dev/old. Stim never removes it; delete it yourself', + ); + }); +}); + +describe('lowDiskLine', () => { + it('names the total and the largest reduce-only folder, though a dot-folder group is larger', () => { + const line = lowDiskLine( + report([ + find({ + path: '/Users/me/.exact/exact2/a', + kind: 'dot-folder', + folder: '/Users/me/.exact/exact2', + bytes: 80 * GB, + }), + find({ path: '/Users/me/dev/app1', bytes: 2 * GB }), + ]), + home, + ); + expect(line).toBe( + `Stim: checkouts it does not fully manage hold 82.0 GB of build output. Add the largest folder, ~/dev (2.0 GB), to what Stim manages with: stim settings set maintenance.discoverRoots '["/Users/me/dev"]' --scope machine; stim doctor lists the rest.`, + ); + }); + + it('says at least when some output is unmeasured, and adds no command when only a dot-folder holds output', () => { + const line = lowDiskLine( + report([ + find({ path: '/Users/me/.t/a', kind: 'dot-folder', folder: '/Users/me/.t', bytes: 5 * GB }), + find({ + path: '/Users/me/.t/b', + kind: 'dot-folder', + folder: '/Users/me/.t', + bytes: 0, + measuredDirs: 0, + failedDirs: 1, + }), + ]), + home, + ); + expect(line).toBe( + 'Stim: checkouts it does not fully manage hold at least 5.0 GB of build output; stim doctor lists them.', + ); + }); + + it('adds nothing when nothing is measured or nothing is unmanaged', () => { + expect(lowDiskLine(report([]), home)).toBeNull(); + expect( + lowDiskLine( + report([find({ path: '/Users/me/dev/a', bytes: 0, measuredDirs: 0, evaluatedAt: 0, dirs: 0 })]), + home, + ), + ).toBeNull(); + }); +}); diff --git a/packages/stim-cli/src/stimd/passes/run.ts b/packages/stim-cli/src/stimd/passes/run.ts index 227bd130e..dbd232802 100644 --- a/packages/stim-cli/src/stimd/passes/run.ts +++ b/packages/stim-cli/src/stimd/passes/run.ts @@ -38,6 +38,7 @@ import { executeAction, type ActionOutcome } from './act.ts'; import { plannedMaintenance, triggeringRoot, type PlannedMaintenance } from './preview.ts'; import { maintenanceLogger } from './log.ts'; import { executedActionName } from '../../maintenance-kind.ts'; +import { runDiscoveryCensusIfDue } from '../../workspace/discovery-census.ts'; import { runDiscoveryScanIfDue } from '../../workspace/discovery-scan.ts'; import { maintenanceItem, openGcLog, type GcLog } from '../../commands/gc/log.ts'; @@ -752,13 +753,14 @@ export async function runMaintenance( } /** - * The discovery scan, after the pass has released its claim. Not after a failed pass or a cancellation, and nothing it + * The discovery scan and the census of what it found, after the pass has released its claim. Not after a failed pass or a cancellation, and nothing it * throws can reach the caller: the pass's own error, such as an unverified child cleanup, is the one that must surface. */ async function scanAfterPass(tick: MaintenanceTick | undefined, passFailed: boolean): Promise { if (passFailed || tick?.operation.signal.aborted) return; try { await runDiscoveryScanIfDue(tick); + await runDiscoveryCensusIfDue(tick); } catch {} } diff --git a/packages/stim-cli/src/workspace/discovery-census.ts b/packages/stim-cli/src/workspace/discovery-census.ts new file mode 100644 index 000000000..9a590680c --- /dev/null +++ b/packages/stim-cli/src/workspace/discovery-census.ts @@ -0,0 +1,225 @@ +import { existsSync, mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { isScopedHome, withDirLock, type StimContext } from '@stim-cli/core'; +import { + discoveryCensusFile, + machineSettingValue, + maintenanceDir, + readDiscoveryCensusFile, + readDiscoveryScanFile, + type CensusFind, + type DiscoveryCensus, +} from '@stim-cli/core/state'; +import { listedRoots } from '../commands/gc/worktrees.ts'; +import { refreshMeasure } from '../disk-usage.ts'; +import { appRoots } from '../stimd/passes/artifacts.ts'; +import { loadConfig } from './config.ts'; +import { discoveredEvidence } from './discovered-activity.ts'; +import { discoveredReads, type DiscoveredReads } from './discovered-reads.ts'; +import { DISCOVER_ROOTS_KEY, rootsOf } from './discovery-report.ts'; +import { isBroadFolder, unmanagedScanFinds } from './discovery.ts'; +import { existingBuildDirsAsync } from './regenerable-dirs.ts'; + +const HOUR_MS = 60 * 60_000; +const DAY_MS = 24 * HOUR_MS; +/** Per build directory: the walk of the one disk-usage walk the machine allows at a time. */ +const WALK_TIMEOUT_MS = 10_000; +/** The most checkouts one run measures, and the longest it starts new walks for; the rest wait for the next run. */ +const MAX_CHECKOUTS = 25; +const BUDGET_MS = 90_000; + +/** When this process last tried a run, so a census that cannot be stored does not start walks on every tick. */ +const attempts = { at: 0 }; + +// @ref LLP 0004#the-doctor-listing-and-the-low-disk-line -- the daemon measures, commands read +/** + * Whether a census run is due: none exists, the scan it came from is not the last one, `maintenance.discoverRoots` + * changed, some directory is still waiting for a measurement and an hour passed, or a day passed. + */ +export function censusDue( + last: Pick | null, + scanRanAt: number, + roots: readonly string[], + now: number, +): boolean { + if (!last || last.at > now || last.scanRanAt !== scanRanAt) return true; + if (last.roots.length !== roots.length || last.roots.some((root, at) => root !== roots[at])) return true; + return now - last.at >= (last.pending ? HOUR_MS : DAY_MS); +} + +function writeCensus(census: DiscoveryCensus): void { + mkdirSync(maintenanceDir(), { recursive: true }); + withDirLock(join(maintenanceDir(), 'discovery-census.lock'), () => { + const file = discoveryCensusFile(); + const tmp = `${file}.${process.pid}.tmp`; + writeFileSync(tmp, `${JSON.stringify(census)}\n`); + try { + renameSync(tmp, file); + } finally { + try { + unlinkSync(tmp); + } catch {} + } + }); +} + +interface Budget { + deadline: number; + checkouts: number; +} + +type Measurement = Pick; + +/** + * The build output Stim recognises in a checkout, measured one directory at a time through `refreshMeasure`: the disk + * usage cache while an entry is under a day old, else one walk under the machine's walk claim, with the walk timeout. + * A walk that failed or timed out counts as failed, never as zero; a directory not reached before the budget ran out, or + * while another walk held the claim, is not measured yet. + */ +async function measureBuildOutput(path: string, operation: StimContext, budget: Budget): Promise { + const dirs = new Set(); + for (const root of appRoots({ path, keys: [], linked: false, locked: false, reduceOnly: true }, 'build')) + for (const dir of await existingBuildDirsAsync(root)) dirs.add(dir); + const result: Measurement = { dirs: dirs.size, measuredDirs: 0, failedDirs: 0, bytes: 0 }; + for (const dir of dirs) { + operation.signal.throwIfAborted(); + if (Date.now() > budget.deadline || !existsSync(dir)) continue; + const { outcome, measure } = await refreshMeasure(dir, { + timeoutMs: WALK_TIMEOUT_MS, + maxAgeMs: DAY_MS, + claim: true, + operation, + private: false, + failures: true, + }); + if (measure) { + result.measuredDirs++; + result.bytes += measure.bytes; + } else if (outcome !== 'busy') result.failedDirs++; + } + return result; +} + +/** + * The newest activity of a reduce-only checkout when the idle stage's own evidence proves it idle for `idleMs` and nothing + * uses it, else null. The evidence stops at the first activity newer than the threshold, so a value is always the + * checkout's real newest activity, never a bound. + */ +async function idleSince( + path: string, + idleMs: number, + now: number, + reads: DiscoveredReads, + operation: StimContext, +): Promise { + const { activity, keep } = await discoveredEvidence(path, { now, idleMs, reads, operation, reduceOnly: true }); + if (keep.length || !activity.length) return null; + const newest = Math.max(...activity.map(({ at }) => at)); + return newest <= now - idleMs ? newest : null; +} + +/** + * Classifies the finds of the last full scan that Stim does not fully manage, and measures their recognised build + * output, for the doctor listing and the low-disk line. It runs in the daemon after the scan, never in a command, and + * a scoped STIM_HOME never does. It reads the scan result the daemon wrote and writes `maintenance/discovery-census.json`. + * + * Bounds: at most `MAX_CHECKOUTS` checkouts are measured per run, none after `BUDGET_MS`, one walk at a time machine-wide + * (`refreshMeasure` with a claim), each with `WALK_TIMEOUT_MS`. A checkout measured within a day is carried over without a + * walk; one left over stays unmeasured and the next run, within the hour, continues. Only an abort escapes. + */ +export async function runDiscoveryCensusIfDue( + tick: { operation: StimContext } | undefined, + { + now = Date.now(), + attempted = attempts, + reads, + maxCheckouts = MAX_CHECKOUTS, + budgetMs = BUDGET_MS, + }: { + now?: number; + attempted?: { at: number }; + /** The process and agent-session readers; tests inject them. */ + reads?: DiscoveredReads; + maxCheckouts?: number; + budgetMs?: number; + } = {}, +): Promise { + if (!tick || isScopedHome(tick.operation.env)) return; + const { operation } = tick; + try { + const scanFile = readDiscoveryScanFile(); + const file = readDiscoveryCensusFile(); + if (scanFile.status !== 'read' || !scanFile.scan || file.status === 'unreadable') return; + const { scan } = scanFile; + const config = loadConfig(); + const setting = machineSettingValue(DISCOVER_ROOTS_KEY, { config, env: operation.env }); + const roots = rootsOf(setting, homedir()); + const last = file.status === 'read' ? file.census : null; + if (!censusDue(last, scan.ranAt, roots, now) || (now >= attempted.at && now - attempted.at < HOUR_MS)) return; + attempted.at = now; + const { value: days, error } = machineSettingValue('maintenance.olderThanDays', { config, env: operation.env }); + const idleDays = !error && typeof days === 'number' && days > 0 ? days : null; + const finds = await unmanagedScanFinds(config, listedRoots(), operation); + const previous = new Map((last?.finds ?? []).map((find) => [`${find.kind}\0${find.path}`, find])); + const budget: Budget = { deadline: Date.now() + budgetMs, checkouts: 0 }; + const sharedReads = reads ?? discoveredReads(operation.signal, operation); + const entries: CensusFind[] = []; + const groups: Array<[CensusFind['kind'], string[]]> = [ + ['reduce-only', finds.reduceOnly], + ['dot-folder', finds.dotFolder], + ]; + for (const [kind, paths] of groups) + for (const path of paths) { + operation.signal.throwIfAborted(); + const folder = dirname(path); + const earlier = previous.get(`${kind}\0${path}`); + const placement = { path, kind, folder, adoptable: !isBroadFolder(folder) }; + const complete = + earlier && earlier.measuredDirs === earlier.dirs && earlier.failedDirs === 0 && earlier.evaluatedAt > 0; + if (earlier && complete && now - earlier.evaluatedAt < DAY_MS && last?.idleDays === idleDays) { + entries.push({ ...earlier, ...placement }); + continue; + } + if (budget.checkouts >= maxCheckouts || Date.now() > budget.deadline) { + entries.push( + earlier + ? { ...earlier, ...placement } + : { + ...placement, + evaluatedAt: 0, + dirs: 0, + measuredDirs: 0, + failedDirs: 0, + bytes: 0, + lastActivityAt: null, + }, + ); + continue; + } + budget.checkouts++; + const measured = await measureBuildOutput(path, operation, budget); + entries.push({ + ...placement, + ...measured, + evaluatedAt: now, + lastActivityAt: + kind === 'reduce-only' && idleDays !== null + ? await idleSince(path, idleDays * DAY_MS, now, sharedReads, operation) + : null, + }); + } + operation.signal.throwIfAborted(); + writeCensus({ + version: 1, + at: now, + scanRanAt: scan.ranAt, + roots, + idleDays, + pending: entries.some((find) => find.evaluatedAt === 0 || find.measuredDirs + find.failedDirs < find.dirs), + finds: entries, + }); + } catch { + operation.signal.throwIfAborted(); + } +} diff --git a/packages/stim-cli/src/workspace/discovery-report.ts b/packages/stim-cli/src/workspace/discovery-report.ts new file mode 100644 index 000000000..59e017e32 --- /dev/null +++ b/packages/stim-cli/src/workspace/discovery-report.ts @@ -0,0 +1,260 @@ +import { resolve, sep } from 'node:path'; +import type { CensusFind, DiscoveryCensusFile, DiscoveryScanFile, ScanTruncation } from '@stim-cli/core/state'; + +const DAY_MS = 24 * 60 * 60_000; + +export const DISCOVER_ROOTS_KEY = 'maintenance.discoverRoots'; + +// @ref LLP 0004#the-doctor-listing-and-the-low-disk-line -- the daemon measures, commands read +/** One folder of checkouts Stim does not fully manage, and what adopting it would take. */ +interface UnmanagedGroup { + folder: string; + /** `reduce-only`: Stim reduces their build output and never removes them. `dot-folder`: Stim does nothing with them. */ + kind: 'reduce-only' | 'dot-folder'; + checkouts: number; + /** The bytes of the build directories that were measured; a lower bound while `failed` or `notMeasuredYet` is above 0. */ + bytes: number; + /** Checkouts whose every build directory was measured. */ + measured: number; + /** Checkouts with a build directory that was tried and could not be measured in time. */ + failed: number; + /** Checkouts the daemon has not measured yet, wholly or in part. */ + notMeasuredYet: number; + /** The command that adds `folder` to the machine's `maintenance.discoverRoots`; null with `commandProblem` set. */ + command: string | null; + commandProblem: 'too-broad' | 'setting-unreadable' | null; + /** Reduce-only checkouts idle at least `idleAfterDays` that nothing uses, for the person to delete; never `dot-folder`. */ + idle: Array<{ path: string; idleDays: number }>; +} + +type ScanSummary = + | { + status: 'read'; + ranAt: number; + trigger: 'weekly' | 'low-disk'; + truncation: ScanTruncation; + visited: number; + unreadable: string[]; + } + | { status: 'absent' | 'unreadable' | 'unusable' }; + +/** The doctor listing, from the two files the daemon wrote and the settings. It reads and walks nothing. */ +export interface DiscoveryReport { + scan: ScanSummary; + /** `none`: no scan result, so no census is expected. */ + census: 'read' | 'absent' | 'unreadable' | 'unusable' | 'none'; + /** When the daemon last classified and measured the finds, epoch ms. */ + censusAt: number | null; + /** `maintenance.olderThanDays` when idle checkouts are listed; null when it is 0 or does not read. */ + idleAfterDays: number | null; + groups: UnmanagedGroup[]; +} + +export interface DiscoverRootsValue { + value: unknown; + error: string | null; +} + +const isStrings = (value: unknown): value is string[] => + Array.isArray(value) && value.every((entry) => typeof entry === 'string'); + +function expandHome(path: string, home: string): string { + return path === '~' || path.startsWith('~/') ? `${home}${path.slice(1)}` : path; +} + +/** The `maintenance.discoverRoots` entries as absolute paths, a leading `~` expanded; empty when the value does not read. */ +export function rootsOf(setting: DiscoverRootsValue, home: string): string[] { + if (setting.error || !isStrings(setting.value)) return []; + return setting.value.map((entry) => resolve(expandHome(entry, home))).toSorted(); +} + +function shellQuote(text: string): string { + return `'${text.replaceAll("'", `'\\''`)}'`; +} + +/** The command that adds `folder` to the machine's roots, or null when the current value does not read. */ +export function adoptCommand(setting: DiscoverRootsValue, folder: string): string | null { + if (setting.error || !isStrings(setting.value)) return null; + return `stim settings set ${DISCOVER_ROOTS_KEY} ${shellQuote(JSON.stringify([...setting.value, folder]))} --scope machine`; +} + +function groupOf( + first: CensusFind, + finds: readonly CensusFind[], + setting: DiscoverRootsValue, + idleAfterDays: number | null, + now: number, +): UnmanagedGroup { + let bytes = 0; + let measured = 0; + let failed = 0; + let notMeasuredYet = 0; + const idle: UnmanagedGroup['idle'] = []; + for (const find of finds) { + bytes += find.bytes; + if (find.evaluatedAt === 0 || (find.failedDirs === 0 && find.measuredDirs < find.dirs)) notMeasuredYet++; + else if (find.failedDirs > 0) failed++; + else measured++; + if (find.kind === 'reduce-only' && idleAfterDays !== null && find.lastActivityAt !== null) { + const idleDays = Math.floor((now - find.lastActivityAt) / DAY_MS); + if (idleDays >= idleAfterDays) idle.push({ path: find.path, idleDays }); + } + } + const command = first.adoptable ? adoptCommand(setting, first.folder) : null; + return { + folder: first.folder, + kind: first.kind, + checkouts: finds.length, + bytes, + measured, + failed, + notMeasuredYet, + command, + commandProblem: !first.adoptable ? 'too-broad' : command === null ? 'setting-unreadable' : null, + idle: idle.toSorted((a, b) => b.idleDays - a.idleDays), + }; +} + +/** + * The listing for doctor and the low-disk line. The census is the daemon's classification as of its last run, so a find + * whose folder `maintenance.discoverRoots` names now is left out, and the listing is cut to what is still unmanaged. + */ +export function discoveryReport({ + scan, + census, + setting, + idleAfterDays, + home, + now, +}: { + scan: DiscoveryScanFile; + census: DiscoveryCensusFile; + setting: DiscoverRootsValue; + idleAfterDays: number | null; + home: string; + now: number; +}): DiscoveryReport { + const summary: ScanSummary = + scan.status === 'read' + ? scan.scan + ? { + status: 'read', + ranAt: scan.scan.ranAt, + trigger: scan.scan.trigger, + truncation: scan.scan.truncation, + visited: scan.scan.visited, + unreadable: scan.scan.unreadable, + } + : { status: 'unusable' } + : { status: scan.status }; + const base = { scan: summary, idleAfterDays, groups: [] }; + if (summary.status !== 'read') return { ...base, census: 'none', censusAt: null }; + if (census.status !== 'read') return { ...base, census: census.status, censusAt: null }; + if (!census.census) return { ...base, census: 'unusable', censusAt: null }; + const roots = new Set(rootsOf(setting, home)); + const byFolder = new Map(); + for (const find of census.census.finds) { + if (roots.has(find.folder)) continue; + const key = `${find.kind}\0${find.folder}`; + const group = byFolder.get(key) ?? { first: find, finds: [] }; + group.finds.push(find); + byFolder.set(key, group); + } + const groups = [...byFolder.values()] + .map(({ first, finds }) => groupOf(first, finds, setting, idleAfterDays, now)) + .toSorted((a, b) => (a.kind === b.kind ? b.bytes - a.bytes : a.kind === 'reduce-only' ? -1 : 1)); + return { scan: summary, census: 'read', censusAt: census.census.at, idleAfterDays, groups }; +} + +const gb = (bytes: number): string => + bytes >= 1024 ** 3 ? `${(bytes / 1024 ** 3).toFixed(1)} GB` : `${Math.max(1, Math.round(bytes / 1024 ** 2))} MB`; + +const shorten = (path: string, home: string): string => + path === home ? '~' : path.startsWith(`${home}${sep}`) ? `~${path.slice(home.length)}` : path; + +const count = (n: number, word: string): string => `${n} ${word}${n === 1 ? '' : 's'}`; + +const day = (at: number): string => new Date(at).toISOString().slice(0, 10); + +function sizeText(group: UnmanagedGroup): string { + const unknown = group.failed + group.notMeasuredYet; + if (group.checkouts === unknown && group.bytes === 0) return 'build output not measured'; + const prefix = unknown ? 'at least ' : ''; + const rest = [ + group.failed ? `${count(group.failed, 'checkout')} could not be measured in time` : '', + group.notMeasuredYet ? `${count(group.notMeasuredYet, 'checkout')} not measured yet` : '', + ].filter(Boolean); + return `${prefix}${gb(group.bytes)} of build output${rest.length ? ` (${rest.join(', ')})` : ''}`; +} + +/** The doctor lines for the listing; empty when there is nothing to say. */ +export function discoveryLines(report: DiscoveryReport, home: string): string[] { + const { scan } = report; + if (scan.status !== 'read') + return [ + { + absent: 'The daemon has not scanned the home folder yet; it scans about weekly and when the disk is low.', + unreadable: 'The home folder scan result cannot be read, so Stim does not know what the scan found.', + unusable: + 'The home folder scan result is unusable (corrupt or from another Stim version); the next scan replaces it.', + }[scan.status], + ]; + const when = `The daemon last scanned the home folder on ${day(scan.ranAt)} (${scan.trigger}), ${count(scan.visited, 'folder')} read.`; + const cut = + scan.truncation === 'none' + ? '' + : scan.truncation === 'failed' + ? ' That scan failed, so it shows the finds of the earlier one.' + : ` That scan stopped early (${{ budget: 'time budget', folders: 'folder limit', read: 'a read that did not answer' }[scan.truncation]}), so it may have missed checkouts.`; + const unread = scan.unreadable.length + ? ` It could not read ${scan.unreadable.map((path) => shorten(path, home)).join(', ')}.` + : ''; + const lines = [`${when}${cut}${unread}`]; + if (report.census === 'absent') return [...lines, 'The daemon has not yet listed and measured what it found.']; + if (report.census === 'unreadable') + return [...lines, 'The list of what it found cannot be read, so Stim does not know what is unmanaged.']; + if (report.census === 'unusable') + return [ + ...lines, + 'The list of what it found is unusable (corrupt or from another Stim version); the daemon rewrites it.', + ]; + if (!report.groups.length) return [...lines, 'Every checkout it found is managed.']; + if (report.censusAt !== null) + lines.push(`Listed and measured by the daemon on ${day(report.censusAt)}; commands never walk the disk.`); + for (const group of report.groups) { + lines.push(''); + lines.push(`${shorten(group.folder, home)}: ${count(group.checkouts, 'checkout')}, ${sizeText(group)}`); + lines.push( + group.kind === 'reduce-only' + ? ' Stim reduces their build output when idle and never removes them. Adding the folder to maintenance.discoverRoots applies the full discovered-checkout rules, which include removing finished linked worktrees:' + : ' Under a home dot-folder, where Stim cannot tell a checkout from an installed tool, so it does nothing with them. Adding the folder to maintenance.discoverRoots lets Stim reduce their build output and, for linked worktrees, remove finished ones:', + ); + lines.push( + group.command + ? ` ${group.command}` + : group.commandProblem === 'too-broad' + ? ' This folder is the home folder or holds it, which maintenance.discoverRoots cannot name.' + : ' maintenance.discoverRoots does not read, so no command is offered; fix it with stim settings first.', + ); + for (const { path, idleDays } of group.idle) + lines.push( + ` Idle for at least ${count(idleDays, 'day')} and nothing uses it: ${shorten(path, home)}. Stim never removes it; delete it yourself if you no longer need it.`, + ); + } + return lines; +} + +/** + * The extra line of the low-disk warning: how much the unmanaged checkouts hold and the one command that adds the largest + * reduce-only folder. Null when no checkout has a measured build directory. Dot-folder finds count toward the total but + * are never the one named, since adopting them gives a tool home the full discovered rules. + */ +export function lowDiskLine(report: DiscoveryReport, home: string): string | null { + const total = report.groups.reduce((sum, group) => sum + group.bytes, 0); + const measuring = report.groups.some((group) => group.failed + group.notMeasuredYet > 0); + if (total === 0) return null; + const largest = report.groups.find((group) => group.kind === 'reduce-only' && group.command && group.bytes > 0); + const holding = `Stim: checkouts it does not fully manage hold ${measuring ? 'at least ' : ''}${gb(total)} of build output`; + if (!largest?.command) return `${holding}; stim doctor lists them.`; + return `${holding}. Add the largest folder, ${shorten(largest.folder, home)} (${gb(largest.bytes)}), to what Stim manages with: ${largest.command}; stim doctor lists the rest.`; +} diff --git a/packages/stim-cli/src/workspace/discovery.ts b/packages/stim-cli/src/workspace/discovery.ts index 82cee508d..860e45000 100644 --- a/packages/stim-cli/src/workspace/discovery.ts +++ b/packages/stim-cli/src/workspace/discovery.ts @@ -143,6 +143,14 @@ function configuredRoots(config: Config | null, env: NodeJS.ProcessEnv): { roots }; } +/** The existing `maintenance.discoverRoots` folders discovery uses: symlinks resolved, too broad ones left out. */ +function canonicalDiscoverRoots(config: Config | null, env: NodeJS.ProcessEnv): string[] { + return configuredRoots(config, env) + .roots.filter((root) => existsSync(root)) + .map(canonicalPath) + .filter((root) => !isBroadFolder(root)); +} + /** The `maintenance.discoverRoots` entries that do not name a folder, for doctor to report. */ export function missingDiscoverRoots(config: Config | null, env: NodeJS.ProcessEnv): string[] { return configuredRoots(config, env).roots.filter((root) => { @@ -247,11 +255,7 @@ async function computeDiscovered( ): Promise { const env = operation?.env ?? {}; const scoped = isScopedHome(operation?.env); - const { roots } = configuredRoots(config, env); - const canonicalRoots = roots - .filter((root) => existsSync(root)) - .map(canonicalPath) - .filter((root) => !isBroadFolder(root)); + const canonicalRoots = canonicalDiscoverRoots(config, env); const repositories: { entries: { path: string }[]; main: string }[] = []; const folders = new Map(); const children: DiscoveryChild[] = []; @@ -320,3 +324,50 @@ async function computeDiscovered( outside: outside.filter(({ path }) => !listed.has(path) && !coveredByKnown(path)), }; } + +/** What the last full scan found that this Stim home does not fully manage. */ +export interface UnmanagedScanFinds { + /** Scan finds discovery selects and no other route reaches: Stim reduces their outputs and never removes them. */ + reduceOnly: string[]; + /** Scan finds under a home dot-folder that no route reaches: Stim does nothing with them. */ + dotFolder: string[]; +} + +const touches = (path: string, other: string): boolean => + path === other || path.startsWith(`${other}${sep}`) || other.startsWith(`${path}${sep}`); + +/** + * The scan finds that are not managed through another route, classified from the complete discovery list: the finds + * discovery selects with origin `scan` are reduce-only, and the find of a home dot-folder, which discovery never selects, + * is unmanaged unless a registered project, a folder of `maintenance.discoverRoots`, a tool folder or any discovered + * checkout covers it. Each dot-folder find is revalidated like a scan find, and one that does not answer is left out. + * A scoped STIM_HOME scans nothing and so finds nothing. + */ +export async function unmanagedScanFinds( + config: Config | null, + known: readonly string[], + operation: StimContext, +): Promise { + const { found } = await discovery(config, known, operation); + const reduceOnly = found.filter(({ origin }) => origin === 'scan').map(({ path }) => path); + if (isScopedHome(operation.env)) return { reduceOnly, dotFolder: [] }; + const home = canonicalPath(homedir()); + const roots = canonicalDiscoverRoots(config, operation.env); + const knownRoots = known.map(canonicalPath); + const rules = { home, isCheckout: recognises, skip: scanSkipper(home) }; + const ancestors = new Map(); + const dotFolder: string[] = []; + for (const path of readDiscoveryScan()?.dotFolderFound ?? []) { + operation.signal.throwIfAborted(); + if ( + !isAbsolute(path) || + roots.some((root) => inside(path, root)) || + inToolFolder(path, home) || + knownRoots.some((root) => touches(path, root)) || + found.some((checkout) => touches(path, checkout.path)) + ) + continue; + if ((await scanEntryValid(path, { ...rules, ancestors })) === 'valid') dotFolder.push(path); + } + return { reduceOnly, dotFolder: dotFolder.toSorted() }; +} From 5b311104585f6cdadc09679e6ecb24784ed8826c Mon Sep 17 00:00:00 2001 From: Janic Duplessis Date: Sun, 15 Nov 2026 09:24:04 -0500 Subject: [PATCH 02/12] feat: list unmanaged checkouts in doctor and on the critical disk line --- packages/stim-cli/api.api.txt | 30 ++++++++ .../src/__tests__/maintenance-rung6.test.ts | 74 +++++++++++++++++++ packages/stim-cli/src/commands/doctor.ts | 45 ++++++++++- packages/stim-cli/src/stimd/passes/trigger.ts | 29 +++++++- 4 files changed, 175 insertions(+), 3 deletions(-) diff --git a/packages/stim-cli/api.api.txt b/packages/stim-cli/api.api.txt index 10df1fb61..bf45b5e09 100644 --- a/packages/stim-cli/api.api.txt +++ b/packages/stim-cli/api.api.txt @@ -361,8 +361,15 @@ [*] interface DiscoveredCheckout { @@1 origin: DiscoveryOrigin; [*] interface DiscoveredCheckout { @@1 path: string; [*] interface DiscoveredCheckout { @@1 reduceOnly?: boolean; +[*] interface DiscoveryReport { +[*] interface DiscoveryReport { @@1 census: "read" | "absent" | "unreadable" | "unusable" | "none"; +[*] interface DiscoveryReport { @@1 censusAt: number | null; +[*] interface DiscoveryReport { @@1 groups: UnmanagedGroup[]; +[*] interface DiscoveryReport { @@1 idleAfterDays: number | null; +[*] interface DiscoveryReport { @@1 scan: ScanSummary; [*] interface DoctorReport { [*] interface DoctorReport { @@1 @internal daemon: DaemonStatus & { +[*] interface DoctorReport { @@1 @internal discovery: DiscoveryReport; [*] interface DoctorReport { @@1 @internal maintenance: ReturnType & { [*] interface DoctorReport { @@1 deviceHosts: Array>["machines"][number] & { [*] interface DoctorReport { @@1 findings: Finding[]; @@ -1410,6 +1417,19 @@ [*] interface TunnelOutcome { @@1 provider?: string; [*] interface TunnelOutcome { @@1 reason?: string; [*] interface TunnelOutcome { @@1 status: string; +[*] interface UnmanagedGroup { +[*] interface UnmanagedGroup { @@1 bytes: number; +[*] interface UnmanagedGroup { @@1 checkouts: number; +[*] interface UnmanagedGroup { @@1 command: string | null; +[*] interface UnmanagedGroup { @@1 commandProblem: "too-broad" | "setting-unreadable" | null; +[*] interface UnmanagedGroup { @@1 failed: number; +[*] interface UnmanagedGroup { @@1 folder: string; +[*] interface UnmanagedGroup { @@1 idle: Array<{ +[*] interface UnmanagedGroup { @@1 kind: "reduce-only" | "dot-folder"; +[*] interface UnmanagedGroup { @@1 measured: number; +[*] interface UnmanagedGroup { @@1 notMeasuredYet: number; +[*] interface UnmanagedGroup { @@2 idleDays: number; +[*] interface UnmanagedGroup { @@2 path: string; [*] interface UnverifiedCleanup { [*] interface UnverifiedCleanup { @@1 child: number; [*] interface UnverifiedCleanup { @@1 message: string; @@ -1662,6 +1682,16 @@ [*] type RunSelectors = PlatformRunOptions extends (infer Options) ? Options extends { @branch1 @@0 } ? Omit & { [*] type RunSelectors = PlatformRunOptions extends (infer Options) ? Options extends { @branch1 @@1 waitSeconds?: number | false; [*] type RunSelectors = PlatformRunOptions extends (infer Options) ? Options extends { @branch2 @@0 } : Options : never; +[*] type ScanSummary = { +[*] type ScanSummary = { @@1 ranAt: number; +[*] type ScanSummary = { @@1 status: "read"; +[*] type ScanSummary = { @@1 trigger: "weekly" | "low-disk"; +[*] type ScanSummary = { @@1 truncation: ScanTruncation; +[*] type ScanSummary = { @@1 unreadable: string[]; +[*] type ScanSummary = { @@1 visited: number; +[*] type ScanSummary = { @branch1 @@0 } | { +[*] type ScanSummary = { @branch1 @@1 status: "absent" | "unreadable" | "unusable"; +[*] type ScanTruncation = "none" | "budget" | "folders" | "read" | "failed"; [*] type SettingScope = "machine" | "workspace" | "repo" | "committed"; [*] type SettingsPayload = { [*] type SettingsPayload = { @@1 files: Partial>; diff --git a/packages/stim-cli/src/__tests__/maintenance-rung6.test.ts b/packages/stim-cli/src/__tests__/maintenance-rung6.test.ts index 0d31bcf0f..d1669ee9f 100644 --- a/packages/stim-cli/src/__tests__/maintenance-rung6.test.ts +++ b/packages/stim-cli/src/__tests__/maintenance-rung6.test.ts @@ -130,6 +130,80 @@ test('the next command, report-only gc included, warns on stderr while the daemo expect(errors).toEqual([expect.stringContaining('disk pressure is critical (4.0 GB free on /')]); }); +test('the critical-disk warning adds one line with what unmanaged checkouts hold and the command for the largest folder, only from a readable census', () => { + mkdirSync(join(home, 'daemon'), { recursive: true }); + mkdirSync(join(home, 'maintenance'), { recursive: true }); + writeFileSync( + join(home, 'daemon', 'snapshot.json'), + JSON.stringify({ + version: 1, + pid: 1, + startedAt: new Date().toISOString(), + mode: 'on', + tickAt: new Date().toISOString(), + tickMs: 1, + intervalMs: 60_000, + measurement: {}, + workspaces: [], + pressure: { + level: 'critical', + disk: 'critical', + memory: 'normal', + reasons: ['4.0 GB free on /'], + memoryWarningSince: null, + }, + }), + ); + const ranAt = Date.now(); + writeFileSync( + join(home, 'maintenance', 'discovery-scan.json'), + JSON.stringify({ + version: 2, + ranAt, + completedAt: ranAt, + lowScanAt: 0, + durationMs: 1, + trigger: 'weekly', + visited: 1, + found: [], + dotFolderFound: [], + unreadable: [], + truncated: false, + truncation: 'none', + }), + ); + const find = { + path: '/Users/x/dev/app', + kind: 'reduce-only', + folder: '/Users/x/dev', + adoptable: true, + evaluatedAt: ranAt, + dirs: 1, + measuredDirs: 1, + failedDirs: 0, + bytes: 3 * 1024 ** 3, + lastActivityAt: null, + }; + const census = join(home, 'maintenance', 'discovery-census.json'); + const errors: string[] = []; + vi.spyOn(console, 'error').mockImplementation((...args) => void errors.push(args.join(' '))); + + writeFileSync(census, '{'); + warnCriticalDisk(['ios']); + expect(errors).toHaveLength(1); + + errors.length = 0; + writeFileSync( + census, + JSON.stringify({ version: 1, at: ranAt, scanRanAt: ranAt, roots: [], idleDays: 3, pending: false, finds: [find] }), + ); + warnCriticalDisk(['ios']); + expect(errors).toHaveLength(2); + expect(errors[1]).toBe( + `Stim: checkouts it does not fully manage hold 3.0 GB of build output. Add the largest folder, /Users/x/dev (3.0 GB), to what Stim manages with: stim settings set maintenance.discoverRoots '["/Users/x/dev"]' --scope machine; stim doctor lists the rest.`, + ); +}); + const DAEMON_ROW = ' 101 /opt/jdk/bin/java -cp /opt/gradle-9.4.1/lib/gradle-daemon-main-9.4.1.jar org.gradle.launcher.daemon.bootstrap.GradleDaemon 9.4.1\n'; const WRAPPER_ROW = ' 202 /opt/jdk/bin/java -jar /app/gradle/wrapper/gradle-wrapper.jar assembleDebug\n'; diff --git a/packages/stim-cli/src/commands/doctor.ts b/packages/stim-cli/src/commands/doctor.ts index c27b1ab9f..23bd770bd 100644 --- a/packages/stim-cli/src/commands/doctor.ts +++ b/packages/stim-cli/src/commands/doctor.ts @@ -3,7 +3,10 @@ import { daemonNdjsonFile, daemonSnapshotFile, gcLogFile, + machineSettingValue, readDaemonSnapshot, + readDiscoveryCensusFile, + readDiscoveryScanFile, type DaemonPressure, type DaemonStatus, } from '@stim-cli/core/state'; @@ -20,6 +23,12 @@ import type { ProjectRegistry } from '../integrations/project-registry.ts'; import type { ProjectDoctor } from '../integrations/project-doctor.ts'; import { getProject, loadConfig } from '../workspace/config.ts'; import { broadDiscoverRoots, missingDiscoverRoots } from '../workspace/discovery.ts'; +import { + DISCOVER_ROOTS_KEY, + discoveryLines, + discoveryReport, + type DiscoveryReport, +} from '../workspace/discovery-report.ts'; import { projectSettingsContextAsync, resolveAndroidLayout, @@ -27,7 +36,7 @@ import { type ProjectSettingsContext, type SettingsObject, } from '../workspace/settings.ts'; -import type { StimContext } from '@stim-cli/core'; +import { isScopedHome, type StimContext } from '@stim-cli/core'; import { readMachineSettings, type MachineSettings } from '../diagnostics/doctor-config.ts'; import { checkWorktreeStorageLayout } from '../diagnostics/doctor-storage.ts'; import { @@ -219,6 +228,21 @@ function toolchainLines(report: ToolchainReport): string[] { ]; } +/** `maintenance.olderThanDays`, the age the doctor listing calls idle; null when it is 0 or does not read. */ +function idleAfterDays(env: NodeJS.ProcessEnv): number | null { + const { value, error } = machineSettingValue('maintenance.olderThanDays', { config: loadConfig(), env }); + return !error && typeof value === 'number' && value > 0 ? value : null; +} + +const DISCOVERY_TITLE = 'Checkouts Stim does not fully manage'; + +function discoverySection(report: DiscoveryReport, env: NodeJS.ProcessEnv): string[] { + const lines = isScopedHome(env) + ? ['A scoped Stim home never scans the home folder, so it lists nothing.'] + : discoveryLines(report, homedir()); + return lines.map((line) => (line ? ` ${line}` : line)); +} + function pressureLines(pressure: DaemonPressure | null): string[] { return pressure ? [phaseLine('pressure', `${pressure.level}${pressure.reasons.length ? `: ${pressure.reasons.join('; ')}` : ''}`)] @@ -398,6 +422,11 @@ export interface DoctorReport { maintenance: ReturnType & { logPath: string }; /** @internal Carries no compatibility: the stim-daemon state, as `status --json` prints it. */ daemon: DaemonStatus & { logPath: string }; + /** + * @internal Carries no compatibility: what the last home folder scan found that Stim does not fully manage, grouped + * by folder with the size of the build output, from the daemon's cached result. + */ + discovery: DiscoveryReport; remoteMachines: Array; deviceHosts: Array>['machines'][number] & { priority?: number }>; toolchain: ToolchainReport | null; @@ -758,6 +787,14 @@ async function assembleDoctorReport( pressure: pressure.pressure, maintenance: { ...maintenance, logPath: gcLogFile() }, daemon: { ...daemon, logPath: daemonNdjsonFile() }, + discovery: discoveryReport({ + scan: readDiscoveryScanFile(), + census: readDiscoveryCensusFile(), + setting: machineSettingValue(DISCOVER_ROOTS_KEY, { config: loadConfig(), env }), + idleAfterDays: idleAfterDays(env), + home: homedir(), + now: Date.now(), + }), remoteMachines: withPriority(remoteMachines.machines), deviceHosts: withPriority(deviceHostMachines.machines), toolchain, @@ -842,6 +879,9 @@ export function doctorOp( toolchain ? toolchainLines(toolchain) : [], ), ...maintenanceLines, + '', + DISCOVERY_TITLE, + ...discoverySection(report.discovery, process.env), ]; for (const [index, line] of lines.entries()) { if (index === 1) console.log(chalk.green(line)); @@ -872,6 +912,9 @@ export function doctorOp( if (f.fix) console.log(chalk.dim(` -> ${f.fix}`)); } + console.log(chalk.bold(`\n${DISCOVERY_TITLE}`)); + for (const line of discoverySection(report.discovery, process.env)) console.log(chalk.dim(line)); + const serverOnly = findings.every(isServerFinding); console.log( chalk.dim( diff --git a/packages/stim-cli/src/stimd/passes/trigger.ts b/packages/stim-cli/src/stimd/passes/trigger.ts index b7f76958a..2c7339810 100644 --- a/packages/stim-cli/src/stimd/passes/trigger.ts +++ b/packages/stim-cli/src/stimd/passes/trigger.ts @@ -1,8 +1,20 @@ -import { readDaemonSnapshot } from '@stim-cli/core/state'; +import { homedir } from 'node:os'; +import { + loadConfig, + machineSettingValue, + readDaemonSnapshot, + readDiscoveryCensusFile, + readDiscoveryScanFile, +} from '@stim-cli/core/state'; +import { DISCOVER_ROOTS_KEY, discoveryReport, lowDiskLine } from '../../workspace/discovery-report.ts'; const WARNING_FRESH_MS = 10 * 60_000; -/** Warns on a command's stderr while the stim-daemon's last tick found the disk level critical. */ +/** + * Warns on a command's stderr while the stim-daemon's last tick found the disk level critical, and, on a second line, + * what the checkouts Stim does not fully manage hold when the daemon's cached census measured any. It reads two small + * files and walks nothing; a census it cannot read adds no line. + */ export function warnCriticalDisk(argv: readonly string[] = process.argv.slice(2)): void { try { if ( @@ -22,5 +34,18 @@ export function warnCriticalDisk(argv: readonly string[] = process.argv.slice(2) console.error( `Stim: disk pressure is critical (${snapshot.pressure.reasons.join('; ')}). ${acting}; run \`stim status\` to see it, and free space outside Stim.`, ); + const home = homedir(); + const line = lowDiskLine( + discoveryReport({ + scan: readDiscoveryScanFile(), + census: readDiscoveryCensusFile(), + setting: machineSettingValue(DISCOVER_ROOTS_KEY, { config: loadConfig(), env: process.env }), + idleAfterDays: null, + home, + now: Date.now(), + }), + home, + ); + if (line) console.error(line); } catch {} } From c34b02d4dfcc51dffc7a9d184e806d682f4a3658 Mon Sep 17 00:00:00 2001 From: Janic Duplessis Date: Sun, 15 Nov 2026 09:24:09 -0500 Subject: [PATCH 03/12] docs: describe the unmanaged checkout listing and the low disk line --- llp/0004-discovered-checkouts.decision.md | 76 +++++++++++++++++++++-- packages/stim-cli/src/guide/cleanup.ts | 58 ++++++++++++++++- packages/stim-cli/src/guide/facts.ts | 9 +++ website/docs/commands.md | 5 ++ website/docs/worktrees.md | 6 ++ 5 files changed, 149 insertions(+), 5 deletions(-) diff --git a/llp/0004-discovered-checkouts.decision.md b/llp/0004-discovered-checkouts.decision.md index 77dee7a40..5a6a0c516 100644 --- a/llp/0004-discovered-checkouts.decision.md +++ b/llp/0004-discovered-checkouts.decision.md @@ -142,8 +142,8 @@ list. Reason: Stim cannot tell an installed tool from a checkout there, for example a Swift CLI cloned under `~/.mytool` whose launcher on PATH points into its ignored `.build/release`; clearing that output once idle would break the installed command. The record lets doctor report what the folder holds and the -command that adds it to `maintenance.discoverRoots`; that listing is not part of -this change. +command that adds it to `maintenance.discoverRoots`; the listing is described in +"The doctor listing and the low-disk line". The scan has a 45 second budget (and a 200 thousand folder cap, and an abort), checked before every read it makes, including every read of the safety checks @@ -288,8 +288,74 @@ retry after an hour of a scan that failed or ended on a read that timed out, whi after the pass, and the PATH launcher keep. The scan is not deferred under host load; it takes about a second and a half on one MacBook. -Not part of this change: points 4, 5 and 7, which belong to the doctor listing -and the low-disk line. +Points 4, 5 and 7 are implemented in "The doctor listing and the low-disk line". + +## The doctor listing and the low-disk line + +Maintainer direction, 2026-11-14 (#5762), verbatim, on the line Stim commands +print when the disk is low: "do stim commands when low on disk space already log +smt", then "could be part of that", then "smt like low disk space, found projects +not managed by stim taking x space , command to add to discover roots". On the +first of the two points it records for doctor, the decision above says: "doctor +lists them and suggest adding as managed path or delete if idle for long time". +Relayed by the coordinator, not on the issue: "like maybe not part of doctor since +it could be slow", said about the scan, so doctor never walks the disk. + +What these words state: doctor lists the finds Stim does not fully manage and +suggests adding the folder as a managed path, or deleting what has been idle a +long time; the low-disk line carries how much they hold and the command that adds +them; the scan stays out of doctor. + +Chosen by the implementing lane and the coordinator's brief, not by the +maintainer: + +- The daemon writes the result. After a scan it classifies the scan's finds from + the complete discovery list and measures their recognised build output into + `maintenance/discovery-census.json`. Doctor and the low-disk line read that + file, the scan file and the `maintenance.discoverRoots` setting, and walk nothing. + A group whose folder the setting names now is left out before the daemon + classifies again. +- Two kinds, worded differently because they are different facts: reduce-only + scan finds (Stim reduces their outputs and never removes them), and finds under + a home dot-folder (Stim does nothing with them). Both are grouped by the + folder that holds them, the only folder `maintenance.discoverRoots` can name to + adopt them, with a command built from the current value plus that folder and + quoted for the shell. The home folder and a folder holding it cannot be named, so + such a group gets words and no command. The command adds the folder to the full + discovered-checkout rules, finished-worktree removal included, and for a + dot-folder find lets Stim act where it did not; the listing says so. +- Size is the recognised build output only (the build directories maintenance + clears, a Cargo target with `CACHEDIR.TAG` or Exact's), not `node_modules` or + Pods, and not proof that a directory is deletable. It goes through + `refreshMeasure` and the disk-usage cache: one walk at a time on the machine, 10 + seconds per directory, at most 25 checkouts and 90 seconds per run; a checkout + measured within a day is carried over, and what a run leaves is measured by the + next one an hour later. A walk that failed or timed out is "could not be + measured" and a directory not reached is "not measured yet", never zero; a total + with either is "at least". +- Idle uses `maintenance.olderThanDays` (default 3, "Days unused before the age + sweep removes it"; 0 turns the listing off) as the threshold, with the idle + stage's own evidence for a reduce-only checkout (`discoveredEvidence`): the + daemon records the newest activity only when that evidence is complete, older + than the threshold and nothing uses the checkout. Doctor lists such a checkout by + path, as words for the person; Stim offers no command that deletes. A dot-folder + find gets no idle evidence and no suggestion to delete, since an installed tool + cannot be told from a checkout there. +- The low-disk line is a second stderr line after the existing one, at critical + only, never on stdout or in JSON. It gives the total (with "at least" while some + is unmeasured) and the command for the largest reduce-only folder; a dot-folder + group counts in the total and is never the one named. +- Doctor shows it as a section outside the findings, so it never changes a PASS or + the count, and carries it in `--json` as `discovery`, an internal field. Doctor + runs only inside a project, so the section does too. +- The worktrees in `~/.codex/worktrees` reach discovery through git as reduce-only + worktrees and are not scan finds, so they are not listed. + +Open for the maintainer: how long idle is "a long time" (the threshold borrows an +existing three-day setting, which is short for a deletion suggestion); whether +the low-disk line also appears at the high level; whether listing the +`~/.codex/worktrees` worktrees belongs here; whether `stim status` and Stim +Desktop show the listing. ## Removing a discovered worktree needs more evidence @@ -328,6 +394,8 @@ Chosen by the coordinator and the reviews of #5519, not by the maintainer. - Issues: #5512, #5515, #5757, #5762. Pull request: #5519. - Code: `packages/stim-cli/src/workspace/discovery.ts`, `packages/stim-cli/src/workspace/discovery-scan.ts`, + `packages/stim-cli/src/workspace/discovery-census.ts`, + `packages/stim-cli/src/workspace/discovery-report.ts`, `packages/stim-cli/src/workspace/discovered-removal.ts`, `packages/stim-cli/src/workspace/protected-locations.ts`, `packages/stim-cli/src/workspace/hidden-edits.ts`, diff --git a/packages/stim-cli/src/guide/cleanup.ts b/packages/stim-cli/src/guide/cleanup.ts index 595a65299..26726962e 100644 --- a/packages/stim-cli/src/guide/cleanup.ts +++ b/packages/stim-cli/src/guide/cleanup.ts @@ -259,7 +259,13 @@ DISK PRESSURE tool's own command, only at critical and only while the earlier rungs left the disk short (\`stim guide cleanup gc\`); Xcode DerivedData outside Stim is never deleted. Every command prints one stderr line while the daemon's last tick, at most 10 - minutes old, found the disk critical. + minutes old, found the disk critical. When the daemon's census (DISCOVERED + CHECKOUTS NOT FULLY MANAGED) holds checkouts with measured build output, a + second stderr line says how much they hold ("at least" while some are not + measured) and gives the command that adds the largest reduce-only folder to + maintenance.discoverRoots; a dot-folder group counts in the total and is never + the one named. It reads two small files, never walks the disk, and goes to + stderr only, never into a JSON payload. It appears at critical only. stim status last checks, plan and running pass stim status --json maintenance observations and recent records @@ -1283,6 +1289,56 @@ finished-worktree check inspects every discovered worktree with git (about 0.14 seconds each where measured). A folder named in maintenance.discoverRoots keeps the full rules. A scoped STIM_HOME uses neither source. + +DISCOVERED CHECKOUTS NOT FULLY MANAGED (the census, the doctor listing) + After a scan the daemon (also on the passes thread, after the scan) writes + maintenance/discovery-census.json: the finds Stim does not fully manage, + classified from the complete discovery list, and the size of their build + output. Reduce-only: scan finds that discovery selects with origin scan. + Dot-folder: scan finds under a home dot-folder that no route reaches (a + registered project, a maintenance.discoverRoots folder, ~/.codex/worktrees or + another discovered checkout) and that pass the same revalidation as a scan + find; Stim does nothing with them. A checkout another route already manages is + not listed, and the worktrees in ~/.codex/worktrees are not scan finds, so + they are not listed either. + Size is the recognised build output only: the directories maintenance clears + as build output for the checkout and for each folder directly in its apps and + packages (Android build and .gradle, .build, android .cxx, a .dd laid out as + DerivedData, a Cargo target that Exact owns or that carries CACHEDIR.TAG). + node_modules and Pods are not counted, and the size is not proof that the + directory could be deleted. It goes through the disk-usage cache, one walk at + a time on the machine, 10 seconds per directory, at most 25 checkouts and 90 + seconds per run; a checkout measured within a day is carried over without a + walk, and what a run leaves is measured by the next one, an hour later. A walk + that fails or times out is "could not be measured", never zero; a directory not + reached is "not measured yet"; a total with either is "at least". The census + is written again after a new scan, when the machine's maintenance.discoverRoots + changes, and after a day. A reduce-only checkout also gets the idle stage's + evidence (newest source write, git write and commit, process, agent session and + PATH launcher) with maintenance.olderThanDays as the threshold: the census + records its newest activity only when the evidence is complete and older than + that, and records nothing (not known to be idle) otherwise. A dot-folder find + gets no idle evidence and no delete suggestion, since an installed tool cannot + be told from a checkout there. + stim doctor prints it as the section "Checkouts Stim does not fully manage" + and carries it in --json as discovery (internal, no compatibility promise): + scan { status: read | absent | unreadable | unusable, ranAt, trigger, + truncation, visited, unreadable }, census (read | absent | unreadable | + unusable | none), censusAt, idleAfterDays, and groups, one per containing + folder and kind, each { folder, kind, checkouts, bytes, measured, failed, + notMeasuredYet, command, commandProblem, idle: [{ path, idleDays }] }. Doctor + reads the two files and the setting and walks nothing; a group whose folder + maintenance.discoverRoots names now is left out, though the daemon has not + classified again yet. The section shows only where doctor runs, which is inside + a project. command is stim settings set maintenance.discoverRoots '' + --scope machine with the current value plus the folder, quoted for the shell; + it is null for the home folder or a folder holding it (discoverRoots refuses + those) and when the current value does not read. Adding a folder gives its + checkouts the full discovered-checkout rules, finished-worktree removal + included; for a dot-folder find it also lets Stim act where it did not. + A reduce-only checkout idle at least maintenance.olderThanDays (0 turns this + off) is listed with its path as words for the person to delete it; Stim offers + no command for that and never removes it. stim settings set maintenance.discoverRoots '["~/Developer"]' --scope machine Idle, with no Stim activity record, means the newest of its source file writes (git-ignored files included; node_modules, Pods and nested worktrees skipped, diff --git a/packages/stim-cli/src/guide/facts.ts b/packages/stim-cli/src/guide/facts.ts index b22692995..4b8e952bd 100644 --- a/packages/stim-cli/src/guide/facts.ts +++ b/packages/stim-cli/src/guide/facts.ts @@ -562,6 +562,15 @@ leased until