From 5adaffccf5dfebf82ce0c64ea5cc62447ade3e12 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sat, 14 Nov 2026 09:33:01 +0800 Subject: [PATCH] feat(daemon): pressure-adaptive workdir GC TTL + spawn-rescue pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Incident 2026-11-13 ENOSPC (owner-reported 2026-11-14): the VM disk drained to 0MB in ~3h while a root-cause batch spawned 515 workdirs (66G). The built-in GC ran on schedule (5-min tight cadence) but could not delete anything: every workdir was younger than the fixed 2d TTL, so 60 messages failed the 1GB spawn watermark and stayed dead (failed messages do not retry). Fix (owner-approved 2026-11-14): - pressureTtlMs(): effective workdir TTL shortens as free space shrinks (free <5G -> 6h, <10G -> 12h, <20G -> 24h; configured TTL stays the ceiling; null free space never tightens) — a flood of fresh workdirs can no longer outrun the reclaim. - spawn-rescue GC: when the disk watermark is about to refuse a spawn, run one GC pass immediately (re-entrancy-guarded, shared with the observer interval) and re-measure before refusing; recovered spawns proceed. - GC log lines now carry the reason (interval/spawn-rescue) and the effective TTL in hours. Tests: pressureTtlMs pinned to the incident-approved literals (5G/6h, 10G/12h, 20G/24h, min-semantics, null passthrough, disabled passthrough). Full suite 498/0. --- packages/daemon/package.json | 2 +- packages/daemon/src/opencode.ts | 104 +++++++++++++++-------- packages/daemon/src/workdir-gc.ts | 26 ++++++ packages/daemon/tests/workdir-gc.test.ts | 43 +++++++++- 4 files changed, 139 insertions(+), 36 deletions(-) diff --git a/packages/daemon/package.json b/packages/daemon/package.json index ab0d26f..4fbc78e 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -1,6 +1,6 @@ { "name": "ework-daemon", - "version": "0.4.211", + "version": "0.4.216", "description": "Issue-driven AI development daemon. Spawns opencode subprocesses to resolve Gitea issues.", "module": "src/index.ts", "type": "module", diff --git a/packages/daemon/src/opencode.ts b/packages/daemon/src/opencode.ts index 6b55be0..1d1f0f7 100644 --- a/packages/daemon/src/opencode.ts +++ b/packages/daemon/src/opencode.ts @@ -3,7 +3,7 @@ import { mkdirSync, writeFileSync, unlinkSync, rmdirSync, readdirSync, existsSyn import { join, dirname, resolve, isAbsolute } from "path"; import { homedir, hostname } from "os"; import { log } from "./logger"; -import { freeSpaceMb, gcIntervalMs, listBusyOpencodeWorkdirs, purgeStaleNodeModules, purgeStaleWorkdirs, shouldBlockSpawn } from "./workdir-gc"; +import { freeSpaceMb, gcIntervalMs, listBusyOpencodeWorkdirs, pressureTtlMs, purgeStaleNodeModules, purgeStaleWorkdirs, shouldBlockSpawn } from "./workdir-gc"; import type { Config } from "./config"; import { isForeignKeyError, type Store } from "./op"; import type { IssueTracker, TrackerRef, TrackerEvent, TrackerComment, Issue, OpSession, Message } from "./trackers/types"; @@ -961,6 +961,7 @@ export class Engine { private observedIssues = new Set(); private lastWorkdirGcAt = 0; + private workdirGcPass: Promise | null = null; private lastWebReconcileAt = 0; private badgeWrites = new Map(); private observerTimer?: ReturnType; @@ -3271,17 +3272,26 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st if (this.cfg.opencode.minFreeMb > 0) { const freeMb = freeSpaceMb(workdir); if (shouldBlockSpawn(freeMb, this.cfg.opencode.minFreeMb)) { - log.warn(`engine: spawn blocked for ${k} — ${freeMb}MB free < ${this.cfg.opencode.minFreeMb}MB watermark`); - await tracker.createComment( - ref, - `[system] ⚠️ ${this.sessionRef(session)} spawn refused: only ${freeMb}MB free on disk (< ${this.cfg.opencode.minFreeMb}MB watermark). Free up space and post again.`, - ).catch((err) => log.error(`engine: watermark notice failed for ${k}:`, (err as Error).message)); - await this.store.updateMessageStatus(msg.id, "failed", `disk watermark: ${freeMb}MB < ${this.cfg.opencode.minFreeMb}MB`); - this.running.delete(k); - this.currentMessage.delete(k); - this.currentModel.delete(k); - void this.dequeueAfterGate(k, session, issue); - return; + // Rescue (incident 2026-11-13): the 5-min background cadence can sit + // idle against a disk that drained in one hour. Reclaim once NOW, + // with the pressure-adaptive TTL, then re-measure before refusing. + log.warn(`engine: spawn blocked for ${k} — ${freeMb}MB free < ${this.cfg.opencode.minFreeMb}MB watermark; running rescue GC pass`); + await this.runWorkdirGcPass("spawn-rescue"); + const freeMbAfter = freeSpaceMb(workdir); + if (shouldBlockSpawn(freeMbAfter, this.cfg.opencode.minFreeMb)) { + log.warn(`engine: spawn blocked for ${k} after rescue GC — ${freeMbAfter}MB free < ${this.cfg.opencode.minFreeMb}MB watermark`); + await tracker.createComment( + ref, + `[system] ⚠️ ${this.sessionRef(session)} spawn refused: only ${freeMbAfter}MB free on disk (< ${this.cfg.opencode.minFreeMb}MB watermark). Free up space and post again.`, + ).catch((err) => log.error(`engine: watermark notice failed for ${k}:`, (err as Error).message)); + await this.store.updateMessageStatus(msg.id, "failed", `disk watermark: ${freeMbAfter}MB < ${this.cfg.opencode.minFreeMb}MB`); + this.running.delete(k); + this.currentMessage.delete(k); + this.currentModel.delete(k); + void this.dequeueAfterGate(k, session, issue); + return; + } + log.info(`engine: rescue GC recovered ${k} — ${freeMbAfter}MB free >= watermark, spawning`); } } @@ -4830,30 +4840,28 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st } } - private async runObserverCycle() { - try { - await this.store.releaseDeadOwners(this.cfg.work.leaseTtlMs); - const dangling = await this.store.releaseDanglingOwners(); - if (dangling > 0) log.info(`engine: released ${dangling} issue(s) owned by vanished daemon rows`); - } catch (err) { - log.error("engine: releaseDeadOwners failed:", (err as Error).message); - } - // CI-lane post-fix verification sweep piggybacks the 5-min observer tick - // (audit RC3) — nothing to do when the queue is empty or undue. - void this.runCiVerifySweep(); - - const nmTtlMs = this.cfg.opencode.nodeModulesTtlDays * 24 * 60 * 60 * 1000; - const wdTtlMs = this.cfg.opencode.workdirTtlDays * 24 * 60 * 60 * 1000; - if (nmTtlMs > 0 || wdTtlMs > 0) { - const freeMb = freeSpaceMb(this.cfg.opencode.baseWorkdir); - if (Date.now() - this.lastWorkdirGcAt > gcIntervalMs(freeMb)) { - this.lastWorkdirGcAt = Date.now(); + /** + * One full workdir-GC pass (node_modules TTL + issue-workdir TTL). + * + * The workdir TTL is pressure-adaptive (owner-approved 2026-11-14, + * incident 2026-11-13 ENOSPC): when free space shrinks the effective TTL + * shortens, so a flood of brand-new workdirs cannot outrun the reclaim. + * Re-entrancy guard: the observer interval and the spawn-rescue path may + * overlap — a second caller piggybacks on the in-flight pass. + */ + private runWorkdirGcPass(reason: "interval" | "spawn-rescue"): Promise { + if (this.workdirGcPass) return this.workdirGcPass; + this.workdirGcPass = (async () => { + const nmTtlMs = this.cfg.opencode.nodeModulesTtlDays * 24 * 60 * 60 * 1000; + const wdTtlMs = this.cfg.opencode.workdirTtlDays * 24 * 60 * 60 * 1000; + if (nmTtlMs <= 0 && wdTtlMs <= 0) return; try { + const freeMb = freeSpaceMb(this.cfg.opencode.baseWorkdir); // busy cwds cover live processes from ANY daemon sharing baseWorkdir. const busy = await listBusyOpencodeWorkdirs(); if (nmTtlMs > 0) { const removed = await purgeStaleNodeModules(this.cfg.opencode.baseWorkdir, nmTtlMs, busy); - if (removed > 0) log.info(`workdir-gc: removed ${removed} node_modules dir(s) older than ${this.cfg.opencode.nodeModulesTtlDays}d`); + if (removed > 0) log.info(`workdir-gc(${reason}): removed ${removed} node_modules dir(s) older than ${this.cfg.opencode.nodeModulesTtlDays}d`); } if (wdTtlMs > 0) { // Running judgment comes from engine/session state, never mtime @@ -4863,12 +4871,40 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st for (const s of await this.store.listNonIdleSessions()) { if (s.state === "running" && s.workdir) protectedDirs.push(s.workdir); } - const removed = await purgeStaleWorkdirs(this.cfg.opencode.baseWorkdir, wdTtlMs, protectedDirs); - if (removed > 0) log.info(`workdir-gc: removed ${removed} issue workdir(s) older than ${this.cfg.opencode.workdirTtlDays}d`); + const effectiveTtlMs = pressureTtlMs(freeMb, wdTtlMs); + const removed = await purgeStaleWorkdirs(this.cfg.opencode.baseWorkdir, effectiveTtlMs, protectedDirs); + const ttlH = Math.round((effectiveTtlMs / 3_600_000) * 10) / 10; + const freeG = freeMb === null ? "n/a" : `${Math.round(freeMb / 1024)}G`; + if (removed > 0) log.info(`workdir-gc(${reason}): removed ${removed} issue workdir(s) older than ${ttlH}h (configured ${this.cfg.opencode.workdirTtlDays}d, free ${freeG})`); } } catch (err) { - log.warn("workdir-gc failed:", (err as Error).message); + log.warn(`workdir-gc(${reason}) failed:`, (err as Error).message); } + })().finally(() => { + this.workdirGcPass = null; + }); + return this.workdirGcPass; + } + + private async runObserverCycle() { + try { + await this.store.releaseDeadOwners(this.cfg.work.leaseTtlMs); + const dangling = await this.store.releaseDanglingOwners(); + if (dangling > 0) log.info(`engine: released ${dangling} issue(s) owned by vanished daemon rows`); + } catch (err) { + log.error("engine: releaseDeadOwners failed:", (err as Error).message); + } + // CI-lane post-fix verification sweep piggybacks the 5-min observer tick + // (audit RC3) — nothing to do when the queue is empty or undue. + void this.runCiVerifySweep(); + + const nmTtlMs = this.cfg.opencode.nodeModulesTtlDays * 24 * 60 * 60 * 1000; + const wdTtlMs = this.cfg.opencode.workdirTtlDays * 24 * 60 * 60 * 1000; + if (nmTtlMs > 0 || wdTtlMs > 0) { + const freeMb = freeSpaceMb(this.cfg.opencode.baseWorkdir); + if (Date.now() - this.lastWorkdirGcAt > gcIntervalMs(freeMb)) { + this.lastWorkdirGcAt = Date.now(); + void this.runWorkdirGcPass("interval"); } } diff --git a/packages/daemon/src/workdir-gc.ts b/packages/daemon/src/workdir-gc.ts index 2fb9365..9914afa 100644 --- a/packages/daemon/src/workdir-gc.ts +++ b/packages/daemon/src/workdir-gc.ts @@ -205,6 +205,32 @@ export function gcIntervalMs(freeMb: number | null): number { return freeMb !== null && freeMb < GC_TIGHT_FREE_MB ? GC_TIGHT_INTERVAL_MS : GC_BASELINE_INTERVAL_MS; } +/** Pressure tiers for the adaptive workdir TTL (in MB) — incident 2026-11-13 ENOSPC. */ +export const GC_TTL_P1_FREE_MB = 5 * 1024; // free < 5G → TTL floor 6h +export const GC_TTL_P2_FREE_MB = 10 * 1024; // free < 10G → TTL floor 12h +export const GC_TTL_P3_FREE_MB = 20 * 1024; // free < 20G → TTL floor 24h +export const GC_TTL_P1_MS = 6 * 60 * 60 * 1000; +export const GC_TTL_P2_MS = 12 * 60 * 60 * 1000; +export const GC_TTL_P3_MS = 24 * 60 * 60 * 1000; + +/** + * Pressure-adaptive workdir TTL (owner-approved 2026-11-14, incident + * 2026-11-13 ENOSPC): a flood of brand-new workdirs (e.g. a root-cause + * batch) leaves a fixed TTL with nothing eligible to delete while the disk + * drains. As free space shrinks the effective TTL shortens; the configured + * TTL remains the ceiling. null free space (df unavailable) keeps the + * configured TTL — never tighten blindly. + */ +export function pressureTtlMs(freeMb: number | null, configuredTtlMs: number): number { + if (freeMb === null || configuredTtlMs <= 0) return configuredTtlMs; + const floor = + freeMb < GC_TTL_P1_FREE_MB ? GC_TTL_P1_MS + : freeMb < GC_TTL_P2_FREE_MB ? GC_TTL_P2_MS + : freeMb < GC_TTL_P3_FREE_MB ? GC_TTL_P3_MS + : Number.MAX_SAFE_INTEGER; + return Math.min(configuredTtlMs, floor); +} + export async function purgeStaleNodeModules( root: string, ttlMs: number, diff --git a/packages/daemon/tests/workdir-gc.test.ts b/packages/daemon/tests/workdir-gc.test.ts index c1d9aa2..4380b44 100644 --- a/packages/daemon/tests/workdir-gc.test.ts +++ b/packages/daemon/tests/workdir-gc.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test, beforeAll, afterAll, afterEach } from "bun:test import { mkdirSync, writeFileSync, rmSync, existsSync } from "node:fs"; import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; -import { freeSpaceMb, gcIntervalMs, purgeStaleNodeModules, purgeStaleWorkdirs, shouldBlockSpawn } from "../src/workdir-gc"; +import { freeSpaceMb, gcIntervalMs, listBusyOpencodeWorkdirs, pressureTtlMs, purgeStaleNodeModules, purgeStaleWorkdirs, shouldBlockSpawn } from "../src/workdir-gc"; const ROOT = join(tmpdir(), `ework-gc-test-${Date.now()}`); const DAY = 24 * 60 * 60 * 1000; @@ -153,3 +153,44 @@ describe("pressure-aware GC cadence (gcIntervalMs)", () => { expect(gcIntervalMs(null)).toBe(24 * 60 * 60 * 1000); }); }); + +// Literals on purpose: these pin the incident-approved pressure-TTL policy +// (5G→6h / 10G→12h / 20G→24h, configured TTL stays the ceiling). +describe("pressure-adaptive workdir TTL (pressureTtlMs)", () => { + const H = 3_600_000; + const configured2d = 2 * 24 * H; + + test("free < 5G → 6h floor", () => { + expect(pressureTtlMs(5 * 1024 - 1, configured2d)).toBe(6 * H); + expect(pressureTtlMs(0, configured2d)).toBe(6 * H); + }); + + test("5G <= free < 10G → 12h floor", () => { + expect(pressureTtlMs(5 * 1024, configured2d)).toBe(12 * H); + expect(pressureTtlMs(10 * 1024 - 1, configured2d)).toBe(12 * H); + }); + + test("10G <= free < 20G → 24h floor", () => { + expect(pressureTtlMs(10 * 1024, configured2d)).toBe(24 * H); + expect(pressureTtlMs(20 * 1024 - 1, configured2d)).toBe(24 * H); + }); + + test("free >= 20G → configured TTL unchanged", () => { + expect(pressureTtlMs(20 * 1024, configured2d)).toBe(configured2d); + expect(pressureTtlMs(102_400, configured2d)).toBe(configured2d); + }); + + test("effective TTL is min(configured, pressure floor)", () => { + expect(pressureTtlMs(100, 24 * H)).toBe(6 * H); // 1d configured, disk < 5G → floor 6h wins + expect(pressureTtlMs(50_000, 6 * H)).toBe(6 * H); // 6h configured, healthy disk → configured wins + }); + + test("null free space → configured TTL (never tighten blindly)", () => { + expect(pressureTtlMs(null, configured2d)).toBe(configured2d); + }); + + test("disabled (ttl <= 0) passes through unchanged", () => { + expect(pressureTtlMs(100, 0)).toBe(0); + expect(pressureTtlMs(100, -5)).toBe(-5); + }); +});