From b4eb374d2914f1256a41dfe665a64d3148afb8ab Mon Sep 17 00:00:00 2001 From: Aaron Sachs <898627+asachs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:37:53 -0400 Subject: [PATCH] test(daemon): stop watchdog tests leaking poll loops and timing out start() runs an unawaited infinite poll loop. At the 1s default pollInterval, advancing 50 minutes of fake time replayed ~3,000 poll cycles of real fs work purely as overhead -- enough to blow the 10s timeout on a busy machine. Worse, stop()/wake() sat at the END of each test body, so a test that timed out never reached it and leaked a live loop; afterEach's useRealTimers() then turned that into a REAL 1s fs-polling loop for the remainder of the file, slowing later tests until they timed out too. That cascade is why WHICH watchdog test failed varied run to run. Coarse pollInterval for these tests, teardown moved into afterEach. 42s with 3-4 flaky failures -> 6.8s, 75/75 stable. --- tests/unit/daemon/fast-checker.test.ts | 64 +++++++++++++++----------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/tests/unit/daemon/fast-checker.test.ts b/tests/unit/daemon/fast-checker.test.ts index f5720b69d5..08f6a2b9de 100644 --- a/tests/unit/daemon/fast-checker.test.ts +++ b/tests/unit/daemon/fast-checker.test.ts @@ -831,20 +831,41 @@ describe('FastChecker', () => { }); describe('heartbeat watchdog', () => { - beforeEach(() => { vi.useFakeTimers(); }); - afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); + // These tests only exercise the 50-min heartbeat timer, but start() also + // drives an unawaited infinite poll loop. At the 1s default pollInterval, + // advancing 50 minutes of fake time replays ~3,000 poll cycles — each doing + // real fs work — purely as overhead, which is enough to blow the 10s test + // timeout on a busy machine. A coarse interval leaves the loop nearly idle + // without changing anything these tests assert. + const WATCHDOG_POLL_MS = 5 * 60 * 1000; + const startedCheckers: FastChecker[] = []; + + function startWatchdog(agent: ReturnType): FastChecker { + const checker = new FastChecker(agent, paths, '/tmp/framework', { + pollInterval: WATCHDOG_POLL_MS, + }); + startedCheckers.push(checker); + checker.start(); + return checker; + } + + beforeEach(() => { vi.useFakeTimers(); startedCheckers.length = 0; }); + afterEach(() => { + // Teardown lives here rather than at the end of each test body. A test + // that fails or times out never reaches its own stop()/wake(), leaking a + // still-running poll loop — which the useRealTimers() below then converts + // into a REAL 1s-interval loop doing fs I/O for the remainder of the + // file, slowing later tests until they time out in turn. That cascade is + // why WHICH watchdog test failed varied from run to run. + for (const checker of startedCheckers) { checker.stop(); checker.wake(); } + startedCheckers.length = 0; + vi.useRealTimers(); + vi.clearAllMocks(); + }); it('fires exec after bootstrap at 50-min interval', async () => { const { execFile } = await import('child_process'); - const agent = createMockAgent('my-agent'); - // pollInterval widened to 60s (vs. the 1s production default): advancing fake - // time by 50min at a 1s poll cadence forces vitest to simulate ~3000 poll-loop - // iterations, which is real CPU-bound work that can exceed this test's 10s - // wall-clock timeout under load — the exact source of this test's flakiness. - // The watchdog-fires-at-50min behavior under test is independent of poll - // cadence, so widening it here doesn't weaken the assertion. - const checker = new FastChecker(agent, paths, '/tmp/framework', { pollInterval: 60_000 }); - checker.start(); + startWatchdog(createMockAgent('my-agent')); await vi.advanceTimersByTimeAsync(50 * 60 * 1000); expect(execFile).toHaveBeenCalledWith( 'cortextos', @@ -852,8 +873,6 @@ describe('FastChecker', () => { expect.objectContaining({ env: expect.any(Object) }), expect.any(Function), ); - checker.stop(); - checker.wake(); }); // task_1785174835840: the daemon is a SINGLE PM2 process shared by every @@ -873,10 +892,7 @@ describe('FastChecker', () => { // state. it('passes the WATCHED agent name via explicit env (task_1785174835840)', async () => { const { execFile } = await import('child_process'); - const agent = createMockAgent('my-agent'); - // pollInterval widened, see the identical note on the preceding test. - const checker = new FastChecker(agent, paths, '/tmp/framework', { pollInterval: 60_000 }); - checker.start(); + startWatchdog(createMockAgent('my-agent')); await vi.advanceTimersByTimeAsync(50 * 60 * 1000); expect(execFile).toHaveBeenCalledWith( 'cortextos', @@ -884,20 +900,17 @@ describe('FastChecker', () => { expect.objectContaining({ env: expect.objectContaining({ CTX_AGENT_NAME: 'my-agent' }) }), expect.any(Function), ); - checker.stop(); - checker.wake(); }); it('clears timer on stop — no further exec calls after stop', async () => { const { execFile } = await import('child_process'); const execMock = execFile as ReturnType; - const agent = createMockAgent('my-agent'); - // pollInterval widened, see the identical note on the earlier watchdog tests. - const checker = new FastChecker(agent, paths, '/tmp/framework', { pollInterval: 60_000 }); - checker.start(); + const checker = startWatchdog(createMockAgent('my-agent')); await vi.advanceTimersByTimeAsync(50 * 60 * 1000); const callsBefore = execMock.mock.calls.length; expect(callsBefore).toBeGreaterThan(0); + // Stopping mid-test is the behavior under test here (not teardown — + // afterEach still stops it again, which is idempotent). checker.stop(); checker.wake(); await vi.advanceTimersByTimeAsync(50 * 60 * 1000); @@ -908,8 +921,7 @@ describe('FastChecker', () => { const { execFile } = await import('child_process'); const agent = createMockAgent('my-agent'); agent.isBootstrapped.mockReturnValue(false); - const checker = new FastChecker(agent, paths, '/tmp/framework'); - checker.start(); + startWatchdog(agent); await vi.advanceTimersByTimeAsync(20 * 1000); expect(execFile).not.toHaveBeenCalledWith( 'cortextos', @@ -917,8 +929,6 @@ describe('FastChecker', () => { expect.objectContaining({ env: expect.any(Object) }), expect.any(Function), ); - checker.stop(); - checker.wake(); }); });