From 472c17b70318e5b594315e53ac201f411bf472c7 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sat, 14 Nov 2026 20:14:21 +0800 Subject: [PATCH] =?UTF-8?q?fix(daemon):=20stuck-kill=20congestion=20backof?= =?UTF-8?q?f=20=E2=80=94=20hold=20the=20session=20out=2030min=E2=86=9260mi?= =?UTF-8?q?n=E2=86=926h=20instead=20of=20instantly=20re-dispatching?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2026-11-14 incident: under high concurrency the watchdog killed hung sessions and the same message was re-dispatched seconds later (kill → park interrupted → stranded sweep → pending → spawn), looping forever while the LLM fleet stayed saturated. Now a stuck kill parks the message on retry_after with exponential backoff (30min * 2^(n-1), capped 6h; give-up = 2h). The hold write is ordered after the park UPDATE (which clears retry_after), and the sweep guard treats the row field as authoritative, with the in-memory map only bridging the pre-park race window. No nudge message is created anymore. --- packages/daemon/package.json | 2 +- packages/daemon/src/opencode.ts | 110 +++++-- packages/daemon/tests/stuck-backoff.test.ts | 324 ++++++++++++++++++++ 3 files changed, 407 insertions(+), 29 deletions(-) create mode 100644 packages/daemon/tests/stuck-backoff.test.ts diff --git a/packages/daemon/package.json b/packages/daemon/package.json index 5e44010..f04f7ce 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -1,6 +1,6 @@ { "name": "ework-daemon", - "version": "0.4.217", + "version": "0.4.218", "description": "Issue-driven AI development daemon. Spawns opencode subprocesses to resolve Gitea issues.", "module": "src/index.ts", "type": "module", diff --git a/packages/daemon/src/opencode.ts b/packages/daemon/src/opencode.ts index 763f06d..3c5145b 100644 --- a/packages/daemon/src/opencode.ts +++ b/packages/daemon/src/opencode.ts @@ -937,6 +937,12 @@ export class Engine { // on a corrupted image part the model 500s on; every replay re-sent it). // Healthy completion resets it; ≥3 arms the fresh-session reset at spawn. private stuckKillCount = new Map(); + // Stuck-backoff holds keyed by MESSAGE id (2026-11-14): msg id → retry_after + // ISO. The sweep reads this in addition to the row's retry_after because the + // execProcess park ('interrupted') lands a tick after our setRetryAfter and + // its status flip CLEARS retry_after (op.ts updateMessageStatus) — the + // in-memory hold bridges that gap. Entries are pruned lazily (6h ceiling). + private stuckBackoffHolds = new Map(); /** Degrade-notice throttling for capped auto lanes (owner directive * 2026-10-24): `${issue.id}:${lane}` → last notify timestamp. In-memory * only — a restart may re-notify once, which is acceptable. */ @@ -1009,6 +1015,16 @@ export class Engine { private static MAX_PENDING_AGE_MS = 128 * 60 * 60_000; private static MAX_STUCK_NUDGE_ROUNDS = 1; private static MAX_RUNTIME_MS = 3 * 60 * 60 * 1000; + // Congestion backoff for stuck-kills (2026-11-14, owner): under high load a + // stalled session was force-killed and re-dispatched in seconds — stall → + // kill → immediate nudge → stall again, an infinite loop burning a full + // 30-40min model cycle per pass. The parked message now rides the + // messages.retry_after hold instead: 30min after the first stuck-kill, + // doubling per consecutive kill, capped at 6h; giving up after the nudge + // budget waits a flat 2h. + static readonly STUCK_BACKOFF_BASE_MS = 30 * 60 * 1000; + static readonly STUCK_BACKOFF_MAX_MS = 6 * 60 * 60 * 1000; + static readonly STUCK_GIVEUP_HOLD_MS = 2 * 60 * 60 * 1000; private static OBSERVER_INTERVAL_MS = 5 * 60 * 1000; private static WEB_RECONCILE_INTERVAL_MS = 60 * 60_000; // CI-lane post-fix verification (audit RC3). @@ -4424,25 +4440,6 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st ].join("\n"); } - private buildStuckNudgePrompt( - session: OpSession, - issue: Issue, - instructions: { issueRef: string }, - stuckMinutes: number - ): string { - return [ - `[SYSTEM] 检测到您的进程已经卡住 ${stuckMinutes} 分钟没有输出了,已经被强制重启。`, - ``, - `可能的原因:等待输入、死循环、网络请求挂起、长时间无响应的工具调用等。`, - ``, - `- 如果您之前的工作有阶段性成果,请立即向用户汇报当前进度和遇到的问题。`, - `- 如果您遇到了阻塞(权限不足、依赖缺失、不确定的方向等),请向用户说明并请求指导。`, - `- 如果您可以继续,请避开导致卡住的操作,换一种方式继续工作。`, - ``, - `使用 \`reply\` 工具向 ${instructions.issueRef} 给出回复(以 \`[bot]\` 开头)。其他方式的回复会被忽略。`, - ].join("\n"); - } - private handleLargeContent(workdir: string, content: string, filename: string): string { if (content.length <= Engine.MAX_INLINE_SIZE) return content; @@ -4822,6 +4819,19 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st } for (const msg of stranded) { if (this.destroyed) return; + // Stuck-backoff hold (2026-11-14): a just-killed session's message rides + // retry_after — waiting it out is the whole point of the backoff. The + // row field is authoritative once set; the in-memory map only bridges + // the (rare) window where the hold write raced the park's + // retry_after=NULL clear, so a row that carries its own retry_after + // must never be shadowed by a stale map entry. + const memHold = msg.retryAfter ? undefined : this.stuckBackoffHolds.get(msg.id); + if (memHold) { + if (new Date(memHold).getTime() > Date.now()) continue; + this.stuckBackoffHolds.delete(msg.id); // pruned once lapsed + await this.store.setRetryAfter(msg.id, null).catch(() => {}); + } + if (msg.retryAfter && msg.retryAfter.getTime() > Date.now()) continue; const session = await this.store.getSession(msg.sessionId); if (!session) continue; const issue = await this.store.getIssue(session.issueId); @@ -5119,8 +5129,17 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st const stuckNudgeRound = this.stuckNudgeRounds.get(k) ?? 0; if (stuckNudgeRound < this.maxStuckNudges) { - log.warn(`engine: stuck — no output for ${minutes}min on ${k}, killing + sending stuck nudge (round ${stuckNudgeRound + 1}/${this.maxStuckNudges})`); - await tracker.createComment(ref, `[system] ⏰ **${session.name}** no output for ${minutes} min, restarting...`).catch( + // Congestion backoff (2026-11-14): compute the hold BEFORE killing + // so the comment can state it; the parked message (finishRun flips + // it to 'interrupted') rides retry_after and the stranded sweep + // waits it out — no immediate re-dispatch, no nudge message. + const stuckKills = (this.stuckKillCount.get(k) ?? 0) + 1; + const backoffMs = Math.min(Engine.STUCK_BACKOFF_BASE_MS * 2 ** (stuckKills - 1), Engine.STUCK_BACKOFF_MAX_MS); + const backoffMin = Math.round(backoffMs / 60000); + + log.warn(`engine: stuck — no output for ${minutes}min on ${k}, killing + ~${backoffMin}min backoff (kill #${stuckKills}/${this.maxStuckNudges})`); + const heldMsgId = this.currentMessage.get(k); + await tracker.createComment(ref, `[system] ⏰ **${session.name}** no output for ${minutes} min — restarting in ~${backoffMin} min (congestion backoff, 第 ${stuckKills} 次无输出重启)`).catch( err => log.error(`engine: stuck-nudge comment failed for ${k}:`, (err as Error).message) ); @@ -5133,27 +5152,62 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st } // A stuck hang is the dead-provider signature: open the circuit so - // the pool stops picking it, and leave the nudge unpinned so the + // the pool stops picking it, and leave the model unpinned so the // re-spawn re-picks from healthy models instead of re-pinning the // one that just hung (dsh#133 flash-zombie family). const stuckModel = this.currentModel.get(k); if (stuckModel) this.modelCircuits.set(stuckModel, Date.now() + this.cfg.opencode.modelCooldownMs); this.currentModel.delete(k); this.stuckNudgeRounds.set(k, stuckNudgeRound + 1); - this.stuckKillCount.set(k, (this.stuckKillCount.get(k) ?? 0) + 1); - - const instructions = tracker.getTrackerInstructions(ref); - const nudgePrompt = this.buildStuckNudgePrompt(session, issue, instructions, minutes); - const nudgeMsg = await this.store.createMessage(session.id, nudgePrompt, undefined, undefined, this.currentModel.get(k)); - await this.dequeueOrIdle(k, session, issue, nudgeMsg); + this.stuckKillCount.set(k, stuckKills); + + // Arm the backoff hold on the parked message (execProcess's + // process-replaced path settles it 'interrupted' a tick later; a + // daemon death in that gap costs one fast retry at worst — the + // next kill re-arms with a larger hold). The stranded sweep + // resurrects it once the hold expires: no nudge message, no + // immediate re-dispatch. + if (heldMsgId) { + const holdIso = new Date(Date.now() + backoffMs).toISOString(); + this.stuckBackoffHolds.set(heldMsgId, holdIso); + // Arm the hold ONLY after the park lands: forceStop's execProcess + // continuation settles the row 'interrupted' a tick later, and + // that status flip (updateMessageStatus) CLEARS retry_after — + // breaking on "not running/pending" is too early (a 'failed' + // row from forceStop's own message sweep passes it while the + // park is still in flight, and the park then wipes the hold). + // status === 'interrupted' is the terminal state of that same + // UPDATE, so it is safe to write immediately after observing it. + for (let i = 0; i < 80; i++) { + const row = await this.store.getMessage(heldMsgId).catch(() => undefined); + if (row && row.status === "interrupted") break; + await new Promise((r) => setTimeout(r, 150)); + } + await this.store.setRetryAfter(heldMsgId, holdIso).catch(() => {}); + log.info(`engine: stuck-backoff hold on msg ${heldMsgId.slice(0, 8)} until ${holdIso}`); + } } else { log.warn(`engine: stuck for ${minutes}min on ${k}, stuck nudge exhausted (${stuckNudgeRound}/${this.maxStuckNudges}), giving up`); + const heldMsgId = this.currentMessage.get(k); await tracker.createComment(ref, `[system] ⛔ **${session.name}** stuck for ${minutes} min, gave up after ${this.maxStuckNudges} restart(s).`).catch( err => log.error(`engine: stuck-giveup comment failed for ${k}:`, (err as Error).message) ); if (this.processes.get(k)) { await this.forceStop(k); } + // Flat 2h cool-off before the stranded sweep may try again — + // give-up must mean "stop hammering", not "retry in seconds". + if (heldMsgId) { + const holdIso = new Date(Date.now() + Engine.STUCK_GIVEUP_HOLD_MS).toISOString(); + this.stuckBackoffHolds.set(heldMsgId, holdIso); + for (let i = 0; i < 80; i++) { + const row = await this.store.getMessage(heldMsgId).catch(() => undefined); + if (row && row.status === "interrupted") break; + await new Promise((r) => setTimeout(r, 150)); + } + await this.store.setRetryAfter(heldMsgId, holdIso).catch(() => {}); + log.info(`engine: stuck-giveup hold on msg ${heldMsgId.slice(0, 8)} until ${holdIso}`); + } this.stuckNudgeRounds.delete(k); this.stuckKillCount.set(k, (this.stuckKillCount.get(k) ?? 0) + 1); } diff --git a/packages/daemon/tests/stuck-backoff.test.ts b/packages/daemon/tests/stuck-backoff.test.ts new file mode 100644 index 0000000..f9829a7 --- /dev/null +++ b/packages/daemon/tests/stuck-backoff.test.ts @@ -0,0 +1,324 @@ +import { beforeAll, beforeEach, afterEach, describe, it, expect } from "bun:test"; + +// Real-stuck tests sleep through threshold + poll cycles; per-test 120s timeout (bun default 5s). +const TMO = 120_000; +import { writeFileSync, chmodSync, mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { Database } from "bun:sqlite"; +import { Store } from "../src/op"; +import { Engine, type TakeoverStrategy } from "../src/opencode"; +import { initDB, getDB } from "../src/db"; +import { loadConfig, type Config } from "../src/config"; +import type { + IssueTracker, + TrackerRef, + TrackerEvent, + TrackerComment, + TrackerInstructions, + Issue, + OpSession, +} from "../src/trackers/types"; + +// Stuck-kill congestion backoff (2026-11-14 owner decision). The old behavior +// re-dispatched instantly on every stuck kill (nudge message + immediate +// dequeue): under a congested upstream the pool killed the same session every +// ~30min, resurrected it, killed it again — an infinite loop that burned the +// release window (billion-context release v0.1.214 starvation, 11-13/11-14). +// New contract: each stuck kill parks the ORIGINAL message interrupted with an +// exponential retry_after hold (30min * 2^(n-1), cap 6h; give-up flat 2h), no +// nudge message, no immediate re-dispatch; sweepStrandedInterrupted skips rows +// whose hold has not expired and resurrects them once it has. + +const FAKE_BIN = join(tmpdir(), "fake-opencode-stuck-backoff.sh"); +const FAKE_SESSION_ID = "ses-test-stuck-bo-aaaa-bbbb-cccc-dddddddddddd"; +const LEASE_TTL_MS = 500; +const HEARTBEAT_MS = 100; + +let workdirBase: string; +let opencodeDbPath: string; +let fakeTracker: FakeTracker; +let trackerRegistry: Map; +let gateAllowed = true; +const liveEngines: Engine[] = []; + +beforeAll(async () => { + // Emits a sessionID (so spawn is "healthy") and then stays silent: the + // stuck detector's no-output threshold fires while the process is alive. + writeFileSync( + FAKE_BIN, + `#!/bin/sh\necho "{\\"sessionID\\":\\"${FAKE_SESSION_ID}\\"}"\nsleep 300\n`, + ); + chmodSync(FAKE_BIN, 0o755); + await initDB(); +}); + +beforeEach(async () => { + const db = getDB(); + const mysql = db.dialect === "mysql"; + await db.exec(mysql ? "SET FOREIGN_KEY_CHECKS = 0" : "PRAGMA foreign_keys = OFF"); + for (const t of ["messages", "op_sessions", "issues", "daemons"]) { + await db.exec(`DELETE FROM {{${t}}}`); + } + await db.exec(mysql ? "SET FOREIGN_KEY_CHECKS = 1" : "PRAGMA foreign_keys = ON"); + + workdirBase = `${tmpdir()}/ework-daemon-stuckbo-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + mkdirSync(workdirBase, { recursive: true }); + + opencodeDbPath = join(workdirBase, "opencode.db"); + const ocdb = new Database(opencodeDbPath); + ocdb.exec("CREATE TABLE session (id TEXT PRIMARY KEY)"); + ocdb.query("INSERT INTO session (id) VALUES (?)").run(FAKE_SESSION_ID); + ocdb.close(); + + fakeTracker = new FakeTracker(); + trackerRegistry = new Map(); + trackerRegistry.set(fakeTracker.type, fakeTracker); + gateAllowed = true; +}); + +afterEach(async () => { + for (const e of liveEngines) { + try { e.destroy(); } catch { /* already destroyed */ } + } + liveEngines.length = 0; + try { rmSync(workdirBase, { recursive: true, force: true }); } catch { /* gone */ } +}); + +class FakeTracker implements IssueTracker { + readonly type = "gitea"; + comments: TrackerComment[] = []; + private nextId = 1; + isBot = false; + + formatScopeKey(scope: Record): string { + return `${scope.owner}/${scope.repo}`; + } + + async createComment(_ref: TrackerRef, body: string): Promise<{ id: string }> { + const id = `c${this.nextId++}`; + this.comments.push({ id, body, author: this.isBot ? "ework-daemon" : "tester", createdAt: new Date().toISOString() }); + return { id }; + } + + async editComment(): Promise {} + async deleteComment(): Promise {} + async listComments(): Promise { return [...this.comments]; } + async closeIssue(): Promise {} + async updateStatus(): Promise {} + async setCommentModel(): Promise {} + async setReaction(): Promise {} + async setIssueReaction(): Promise {} + + getTrackerInstructions(_ref: TrackerRef): TrackerInstructions { + return { clone: "git clone fake", issueRef: "fake/ref" }; + } + + verifyWebhookSignature(): boolean { return true; } + parseWebhookEvent(): TrackerEvent | null { return null; } + isBotUser(author: string): boolean { return author === "ework-daemon"; } +} + +class TestTakeoverStrategy implements TakeoverStrategy { + constructor(private baseWorkdir: string) {} + async acquireWorkdir(session: OpSession, issue: Issue): Promise { + const dir = join(this.baseWorkdir, String(issue.trackerIssueId), session.name); + mkdirSync(dir, { recursive: true }); + return dir; + } + async resumeOpenCodeSession(): Promise { return null; } +} + +function makeConfig(stuck?: { thresholdMs: number; maxNudges: number }): Config { + const cfg = loadConfig(); + return { + ...cfg, + stuck: { + thresholdMs: stuck?.thresholdMs ?? 100, + maxNudges: stuck?.maxNudges ?? 2, + maxRuntimeMs: 3_600_000, + }, + opencode: { ...cfg.opencode, binary: FAKE_BIN, baseWorkdir: workdirBase, dbPath: opencodeDbPath }, + work: { ...cfg.work, capacity: 4, maxConcurrent: 4, maxConcurrentExplicit: false, heartbeatMs: HEARTBEAT_MS, leaseTtlMs: LEASE_TTL_MS, reconcileScopes: [] }, + }; +} + +async function bootEngine(name: string, store: Store, cfg: Config, port: number): Promise { + await store.releaseDeadOwners(cfg.work.leaseTtlMs); + const daemonId = await store.registerDaemon( + `host-${name}`, + `127.0.0.1:${port}`, + cfg.work.capacity, + cfg.work.leaseTtlMs, + ); + await store.claimAllOwnerless(daemonId); + const engine = new Engine(cfg, store, trackerRegistry, { + daemonId, + gateChecker: async () => (gateAllowed ? { allowed: true, reason: "test" } : { allowed: false, reason: "dispatch off" }), + takeover: new TestTakeoverStrategy(workdirBase), + }); + engine.startHeartbeat(cfg.work.heartbeatMs); + liveEngines.push(engine); + return engine; +} + +function commentEvent(issueId: string, body: string): TrackerEvent { + return { + type: "comment_created", + ref: { trackerType: "gitea", scope: { owner: "dog", repo: "repo" }, issueId }, + issue: { title: "stuck backoff test", body: "b", state: "open", author: "tester" }, + comment: { id: `wc-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`, body, author: "tester" }, + }; +} + +async function waitFor(fn: () => T | undefined | Promise, timeoutMs = 8000): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const result = await fn(); + if (result !== undefined) return result; + await new Promise((r) => setTimeout(r, 50)); + } + return fn(); +} + +async function messageRow(): Promise<{ uid: string; status: string; retry_after: string | null } | undefined> { + const db = getDB(); + const row = await db.get<{ uid: string; status: string; retry_after: string | null }>( + `SELECT uid, status, retry_after FROM {{messages}} LIMIT 1`, + ); + return row ?? undefined; +} + +// Drive the observer by hand: the engine's real cycle is 5min apart, which +// would make each test take minutes. Poll the DB, and once the row exists run +// an observer cycle every 2s (well past the 100ms no-output threshold) until +// the stuck kill settles the row, or the timeout elapses. +async function waitForStuckSettle(engine: Engine, timeoutMs = 60_000) { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const row = await messageRow(); + if (row && row.status === "interrupted" && row.retry_after) return row; + await Bun.sleep(2000); + await (engine as unknown as { runObserverCycle(): Promise }).runObserverCycle().catch(() => {}); + } + const row = await messageRow(); + return row && row.status === "interrupted" ? row : undefined; +} + +// Flip a parked interrupted row back to pending (hold cleared) and let the +// engine re-dispatch it — simulates the hold expiring plus the next observer +// cycle, without waiting the real backoff window. +async function fastForwardResurrect(engine: Engine, msgUid: string): Promise { + const db = getDB(); + await db.exec(`UPDATE {{messages}} SET status = 'pending', retry_after = NULL WHERE uid = '${msgUid}'`); + await (engine as unknown as { runObserverCycle(): Promise }).runObserverCycle(); +} + +function heldMinutes(row: { retry_after: string | null }): number { + expect(row.retry_after).toBeTruthy(); + return (new Date(row.retry_after as string).getTime() - Date.now()) / 60_000; +} + +describe("stuck-kill congestion backoff (2026-11-14)", () => { + it("first stuck kill: parks original row interrupted with a ~30min hold, no nudge message, no re-dispatch", TMO, async () => { + const store = new Store(); + const engine = await bootEngine("A", store, makeConfig(), 7401); + + await engine.handleEvent(commentEvent("701", "hang on")); + const row = await waitForStuckSettle(engine); + expect(row).toBeTruthy(); + expect(row!.status).toBe("interrupted"); + + // Hold ≈ 30min (first kill: 30 * 2^0). Tolerate test-run drift. + const mins = heldMinutes(row!); + expect(mins).toBeGreaterThanOrEqual(25); + expect(mins).toBeLessThanOrEqual(40); + + // Exactly ONE message row — the old behavior minted a second nudge row + // that raced the original back into the pool immediately. + const db = getDB(); + const count = (await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM {{messages}}`))!.n; + expect(count).toBe(1); + + // The visible notice carries the backoff, not a bare "restarting". + const notice = fakeTracker.comments.map((c) => c.body).find((b) => b.includes("congestion backoff")); + expect(notice).toBeTruthy(); + expect(notice).toContain("~30 min"); + + // And nothing came back up: the session must be idle, not running. + await Bun.sleep(300); + const after = await messageRow(); + expect(after?.status).toBe("interrupted"); + }); + + it("second stuck kill doubles the hold to ~60min (exponential, no loop)", TMO, async () => { + const store = new Store(); + const engine = await bootEngine("B", store, makeConfig(), 7402); + + await engine.handleEvent(commentEvent("702", "hang one")); + const first = await waitForStuckSettle(engine); + expect(first).toBeTruthy(); + + // Fast-forward the 30min hold, re-dispatch, and let it stick again. + await fastForwardResurrect(engine, first!.uid); + const second = await waitForStuckSettle(engine); + expect(second).toBeTruthy(); + expect(second!.status).toBe("interrupted"); + + const mins = heldMinutes(second!); + expect(mins).toBeGreaterThanOrEqual(50); + expect(mins).toBeLessThanOrEqual(80); + + const notices = fakeTracker.comments.map((c) => c.body).filter((b) => b.includes("congestion backoff")); + expect(notices.length).toBe(2); + expect(notices[1]).toContain("~60 min"); + }); + + it("sweep skips a parked row whose hold has not expired, and resurrects it once the hold lapses", TMO, async () => { + const store = new Store(); + const engine = await bootEngine("C", store, makeConfig(), 7403); + + await engine.handleEvent(commentEvent("703", "hang two")); + const row = await waitForStuckSettle(engine); + expect(row).toBeTruthy(); + + // Hold still in the future: the restart-stranding repair must NOT yank + // this row back early (that was the infinite kill/resurrect loop). + await engine.sweepStrandedInterrupted(); + expect((await messageRow())?.status).toBe("interrupted"); + + // Hold lapses: same sweep now resurrects it to pending and drains. + const db = getDB(); + await db.exec(`UPDATE {{messages}} SET retry_after = datetime('now', '-1 minute')`); + await engine.sweepStrandedInterrupted(); + const revived = await waitFor(async () => { + const r = await messageRow(); + return r && r.status !== "interrupted" ? r : undefined; + }); + expect(revived).toBeTruthy(); + expect(["pending", "running", "done"]).toContain(revived!.status); + }); + + it("give-up (nudge budget spent) applies a flat 2h hold, not an instant retry", TMO, async () => { + const store = new Store(); + // maxNudges=1 → the FIRST kill is kill #1 (backoff), the SECOND hits + // stuckNudgeRound == maxNudges and takes the give-up branch. + const engine = await bootEngine("D", store, makeConfig({ thresholdMs: 100, maxNudges: 1 }), 7404); + + await engine.handleEvent(commentEvent("704", "hang three")); + const first = await waitForStuckSettle(engine); + expect(first).toBeTruthy(); + + await fastForwardResurrect(engine, first!.uid); + const second = await waitForStuckSettle(engine); + expect(second).toBeTruthy(); + expect(second!.status).toBe("interrupted"); + + const mins = heldMinutes(second!); + expect(mins).toBeGreaterThanOrEqual(110); + expect(mins).toBeLessThanOrEqual(130); + + const giveup = fakeTracker.comments.map((c) => c.body).find((b) => b.includes("gave up after")); + expect(giveup).toBeTruthy(); + }); +});