Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/daemon/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "ework-daemon",
"version": "0.4.211",
"version": "0.4.216",
"description": "Issue-driven AI development daemon. Spawns opencode subprocesses to resolve Gitea issues.",
"module": "src/index.ts",
"type": "module",
Expand Down
104 changes: 70 additions & 34 deletions packages/daemon/src/opencode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { mkdirSync, writeFileSync, unlinkSync, rmdirSync, readdirSync, existsSyn
import { join, dirname, resolve, isAbsolute } from "path";
import { homedir, hostname } from "os";
import { log } from "./logger";
import { freeSpaceMb, gcIntervalMs, listBusyOpencodeWorkdirs, purgeStaleNodeModules, purgeStaleWorkdirs, shouldBlockSpawn } from "./workdir-gc";
import { freeSpaceMb, gcIntervalMs, listBusyOpencodeWorkdirs, pressureTtlMs, purgeStaleNodeModules, purgeStaleWorkdirs, shouldBlockSpawn } from "./workdir-gc";
import type { Config } from "./config";
import { isForeignKeyError, type Store } from "./op";
import type { IssueTracker, TrackerRef, TrackerEvent, TrackerComment, Issue, OpSession, Message } from "./trackers/types";
Expand Down Expand Up @@ -961,6 +961,7 @@ export class Engine {

private observedIssues = new Set<string>();
private lastWorkdirGcAt = 0;
private workdirGcPass: Promise<void> | null = null;
private lastWebReconcileAt = 0;
private badgeWrites = new Map<string, string>();
private observerTimer?: ReturnType<typeof setInterval>;
Expand Down Expand Up @@ -3271,17 +3272,26 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st
if (this.cfg.opencode.minFreeMb > 0) {
const freeMb = freeSpaceMb(workdir);
if (shouldBlockSpawn(freeMb, this.cfg.opencode.minFreeMb)) {
log.warn(`engine: spawn blocked for ${k} — ${freeMb}MB free < ${this.cfg.opencode.minFreeMb}MB watermark`);
await tracker.createComment(
ref,
`[system] ⚠️ ${this.sessionRef(session)} spawn refused: only ${freeMb}MB free on disk (< ${this.cfg.opencode.minFreeMb}MB watermark). Free up space and post again.`,
).catch((err) => log.error(`engine: watermark notice failed for ${k}:`, (err as Error).message));
await this.store.updateMessageStatus(msg.id, "failed", `disk watermark: ${freeMb}MB < ${this.cfg.opencode.minFreeMb}MB`);
this.running.delete(k);
this.currentMessage.delete(k);
this.currentModel.delete(k);
void this.dequeueAfterGate(k, session, issue);
return;
// Rescue (incident 2026-11-13): the 5-min background cadence can sit
// idle against a disk that drained in one hour. Reclaim once NOW,
// with the pressure-adaptive TTL, then re-measure before refusing.
log.warn(`engine: spawn blocked for ${k} — ${freeMb}MB free < ${this.cfg.opencode.minFreeMb}MB watermark; running rescue GC pass`);
await this.runWorkdirGcPass("spawn-rescue");
const freeMbAfter = freeSpaceMb(workdir);
if (shouldBlockSpawn(freeMbAfter, this.cfg.opencode.minFreeMb)) {
log.warn(`engine: spawn blocked for ${k} after rescue GC — ${freeMbAfter}MB free < ${this.cfg.opencode.minFreeMb}MB watermark`);
await tracker.createComment(
ref,
`[system] ⚠️ ${this.sessionRef(session)} spawn refused: only ${freeMbAfter}MB free on disk (< ${this.cfg.opencode.minFreeMb}MB watermark). Free up space and post again.`,
).catch((err) => log.error(`engine: watermark notice failed for ${k}:`, (err as Error).message));
await this.store.updateMessageStatus(msg.id, "failed", `disk watermark: ${freeMbAfter}MB < ${this.cfg.opencode.minFreeMb}MB`);
this.running.delete(k);
this.currentMessage.delete(k);
this.currentModel.delete(k);
void this.dequeueAfterGate(k, session, issue);
return;
}
log.info(`engine: rescue GC recovered ${k} — ${freeMbAfter}MB free >= watermark, spawning`);
}
}

Expand Down Expand Up @@ -4830,30 +4840,28 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st
}
}

private async runObserverCycle() {
try {
await this.store.releaseDeadOwners(this.cfg.work.leaseTtlMs);
const dangling = await this.store.releaseDanglingOwners();
if (dangling > 0) log.info(`engine: released ${dangling} issue(s) owned by vanished daemon rows`);
} catch (err) {
log.error("engine: releaseDeadOwners failed:", (err as Error).message);
}
// CI-lane post-fix verification sweep piggybacks the 5-min observer tick
// (audit RC3) — nothing to do when the queue is empty or undue.
void this.runCiVerifySweep();

const nmTtlMs = this.cfg.opencode.nodeModulesTtlDays * 24 * 60 * 60 * 1000;
const wdTtlMs = this.cfg.opencode.workdirTtlDays * 24 * 60 * 60 * 1000;
if (nmTtlMs > 0 || wdTtlMs > 0) {
const freeMb = freeSpaceMb(this.cfg.opencode.baseWorkdir);
if (Date.now() - this.lastWorkdirGcAt > gcIntervalMs(freeMb)) {
this.lastWorkdirGcAt = Date.now();
/**
* One full workdir-GC pass (node_modules TTL + issue-workdir TTL).
*
* The workdir TTL is pressure-adaptive (owner-approved 2026-11-14,
* incident 2026-11-13 ENOSPC): when free space shrinks the effective TTL
* shortens, so a flood of brand-new workdirs cannot outrun the reclaim.
* Re-entrancy guard: the observer interval and the spawn-rescue path may
* overlap — a second caller piggybacks on the in-flight pass.
*/
private runWorkdirGcPass(reason: "interval" | "spawn-rescue"): Promise<void> {
if (this.workdirGcPass) return this.workdirGcPass;
this.workdirGcPass = (async () => {
const nmTtlMs = this.cfg.opencode.nodeModulesTtlDays * 24 * 60 * 60 * 1000;
const wdTtlMs = this.cfg.opencode.workdirTtlDays * 24 * 60 * 60 * 1000;
if (nmTtlMs <= 0 && wdTtlMs <= 0) return;
try {
const freeMb = freeSpaceMb(this.cfg.opencode.baseWorkdir);
// busy cwds cover live processes from ANY daemon sharing baseWorkdir.
const busy = await listBusyOpencodeWorkdirs();
if (nmTtlMs > 0) {
const removed = await purgeStaleNodeModules(this.cfg.opencode.baseWorkdir, nmTtlMs, busy);
if (removed > 0) log.info(`workdir-gc: removed ${removed} node_modules dir(s) older than ${this.cfg.opencode.nodeModulesTtlDays}d`);
if (removed > 0) log.info(`workdir-gc(${reason}): removed ${removed} node_modules dir(s) older than ${this.cfg.opencode.nodeModulesTtlDays}d`);
}
if (wdTtlMs > 0) {
// Running judgment comes from engine/session state, never mtime
Expand All @@ -4863,12 +4871,40 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st
for (const s of await this.store.listNonIdleSessions()) {
if (s.state === "running" && s.workdir) protectedDirs.push(s.workdir);
}
const removed = await purgeStaleWorkdirs(this.cfg.opencode.baseWorkdir, wdTtlMs, protectedDirs);
if (removed > 0) log.info(`workdir-gc: removed ${removed} issue workdir(s) older than ${this.cfg.opencode.workdirTtlDays}d`);
const effectiveTtlMs = pressureTtlMs(freeMb, wdTtlMs);
const removed = await purgeStaleWorkdirs(this.cfg.opencode.baseWorkdir, effectiveTtlMs, protectedDirs);
const ttlH = Math.round((effectiveTtlMs / 3_600_000) * 10) / 10;
const freeG = freeMb === null ? "n/a" : `${Math.round(freeMb / 1024)}G`;
if (removed > 0) log.info(`workdir-gc(${reason}): removed ${removed} issue workdir(s) older than ${ttlH}h (configured ${this.cfg.opencode.workdirTtlDays}d, free ${freeG})`);
}
} catch (err) {
log.warn("workdir-gc failed:", (err as Error).message);
log.warn(`workdir-gc(${reason}) failed:`, (err as Error).message);
}
})().finally(() => {
this.workdirGcPass = null;
});
return this.workdirGcPass;
}

private async runObserverCycle() {
try {
await this.store.releaseDeadOwners(this.cfg.work.leaseTtlMs);
const dangling = await this.store.releaseDanglingOwners();
if (dangling > 0) log.info(`engine: released ${dangling} issue(s) owned by vanished daemon rows`);
} catch (err) {
log.error("engine: releaseDeadOwners failed:", (err as Error).message);
}
// CI-lane post-fix verification sweep piggybacks the 5-min observer tick
// (audit RC3) — nothing to do when the queue is empty or undue.
void this.runCiVerifySweep();

const nmTtlMs = this.cfg.opencode.nodeModulesTtlDays * 24 * 60 * 60 * 1000;
const wdTtlMs = this.cfg.opencode.workdirTtlDays * 24 * 60 * 60 * 1000;
if (nmTtlMs > 0 || wdTtlMs > 0) {
const freeMb = freeSpaceMb(this.cfg.opencode.baseWorkdir);
if (Date.now() - this.lastWorkdirGcAt > gcIntervalMs(freeMb)) {
this.lastWorkdirGcAt = Date.now();
void this.runWorkdirGcPass("interval");
}
}

Expand Down
26 changes: 26 additions & 0 deletions packages/daemon/src/workdir-gc.ts
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,32 @@ export function gcIntervalMs(freeMb: number | null): number {
return freeMb !== null && freeMb < GC_TIGHT_FREE_MB ? GC_TIGHT_INTERVAL_MS : GC_BASELINE_INTERVAL_MS;
}

/** Pressure tiers for the adaptive workdir TTL (in MB) — incident 2026-11-13 ENOSPC. */
export const GC_TTL_P1_FREE_MB = 5 * 1024; // free < 5G → TTL floor 6h
export const GC_TTL_P2_FREE_MB = 10 * 1024; // free < 10G → TTL floor 12h
export const GC_TTL_P3_FREE_MB = 20 * 1024; // free < 20G → TTL floor 24h
export const GC_TTL_P1_MS = 6 * 60 * 60 * 1000;
export const GC_TTL_P2_MS = 12 * 60 * 60 * 1000;
export const GC_TTL_P3_MS = 24 * 60 * 60 * 1000;

/**
* Pressure-adaptive workdir TTL (owner-approved 2026-11-14, incident
* 2026-11-13 ENOSPC): a flood of brand-new workdirs (e.g. a root-cause
* batch) leaves a fixed TTL with nothing eligible to delete while the disk
* drains. As free space shrinks the effective TTL shortens; the configured
* TTL remains the ceiling. null free space (df unavailable) keeps the
* configured TTL — never tighten blindly.
*/
export function pressureTtlMs(freeMb: number | null, configuredTtlMs: number): number {
if (freeMb === null || configuredTtlMs <= 0) return configuredTtlMs;
const floor =
freeMb < GC_TTL_P1_FREE_MB ? GC_TTL_P1_MS
: freeMb < GC_TTL_P2_FREE_MB ? GC_TTL_P2_MS
: freeMb < GC_TTL_P3_FREE_MB ? GC_TTL_P3_MS
: Number.MAX_SAFE_INTEGER;
return Math.min(configuredTtlMs, floor);
}

export async function purgeStaleNodeModules(
root: string,
ttlMs: number,
Expand Down
43 changes: 42 additions & 1 deletion packages/daemon/tests/workdir-gc.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { describe, expect, test, beforeAll, afterAll, afterEach } from "bun:test
import { mkdirSync, writeFileSync, rmSync, existsSync } from "node:fs";
import { dirname, join } from "node:path";
import { tmpdir } from "node:os";
import { freeSpaceMb, gcIntervalMs, purgeStaleNodeModules, purgeStaleWorkdirs, shouldBlockSpawn } from "../src/workdir-gc";
import { freeSpaceMb, gcIntervalMs, listBusyOpencodeWorkdirs, pressureTtlMs, purgeStaleNodeModules, purgeStaleWorkdirs, shouldBlockSpawn } from "../src/workdir-gc";

const ROOT = join(tmpdir(), `ework-gc-test-${Date.now()}`);
const DAY = 24 * 60 * 60 * 1000;
Expand Down Expand Up @@ -153,3 +153,44 @@ describe("pressure-aware GC cadence (gcIntervalMs)", () => {
expect(gcIntervalMs(null)).toBe(24 * 60 * 60 * 1000);
});
});

// Literals on purpose: these pin the incident-approved pressure-TTL policy
// (5G→6h / 10G→12h / 20G→24h, configured TTL stays the ceiling).
describe("pressure-adaptive workdir TTL (pressureTtlMs)", () => {
const H = 3_600_000;
const configured2d = 2 * 24 * H;

test("free < 5G → 6h floor", () => {
expect(pressureTtlMs(5 * 1024 - 1, configured2d)).toBe(6 * H);
expect(pressureTtlMs(0, configured2d)).toBe(6 * H);
});

test("5G <= free < 10G → 12h floor", () => {
expect(pressureTtlMs(5 * 1024, configured2d)).toBe(12 * H);
expect(pressureTtlMs(10 * 1024 - 1, configured2d)).toBe(12 * H);
});

test("10G <= free < 20G → 24h floor", () => {
expect(pressureTtlMs(10 * 1024, configured2d)).toBe(24 * H);
expect(pressureTtlMs(20 * 1024 - 1, configured2d)).toBe(24 * H);
});

test("free >= 20G → configured TTL unchanged", () => {
expect(pressureTtlMs(20 * 1024, configured2d)).toBe(configured2d);
expect(pressureTtlMs(102_400, configured2d)).toBe(configured2d);
});

test("effective TTL is min(configured, pressure floor)", () => {
expect(pressureTtlMs(100, 24 * H)).toBe(6 * H); // 1d configured, disk < 5G → floor 6h wins
expect(pressureTtlMs(50_000, 6 * H)).toBe(6 * H); // 6h configured, healthy disk → configured wins
});

test("null free space → configured TTL (never tighten blindly)", () => {
expect(pressureTtlMs(null, configured2d)).toBe(configured2d);
});

test("disabled (ttl <= 0) passes through unchanged", () => {
expect(pressureTtlMs(100, 0)).toBe(0);
expect(pressureTtlMs(100, -5)).toBe(-5);
});
});
Loading