Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/daemon/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "ework-daemon",
"version": "0.4.217",
"version": "0.4.218",
"description": "Issue-driven AI development daemon. Spawns opencode subprocesses to resolve Gitea issues.",
"module": "src/index.ts",
"type": "module",
Expand Down
120 changes: 90 additions & 30 deletions packages/daemon/src/opencode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -937,6 +937,12 @@ export class Engine {
// on a corrupted image part the model 500s on; every replay re-sent it).
// Healthy completion resets it; ≥3 arms the fresh-session reset at spawn.
private stuckKillCount = new Map<string, number>();
// Stuck-backoff holds keyed by MESSAGE id (2026-11-14): msg id → retry_after
// ISO. The sweep reads this in addition to the row's retry_after because the
// execProcess park ('interrupted') lands a tick after our setRetryAfter and
// its status flip CLEARS retry_after (op.ts updateMessageStatus) — the
// in-memory hold bridges that gap. Entries are pruned lazily (6h ceiling).
private stuckBackoffHolds = new Map<string, string>();
/** Degrade-notice throttling for capped auto lanes (owner directive
* 2026-10-24): `${issue.id}:${lane}` → last notify timestamp. In-memory
* only — a restart may re-notify once, which is acceptable. */
Expand Down Expand Up @@ -1009,6 +1015,16 @@ export class Engine {
private static MAX_PENDING_AGE_MS = 128 * 60 * 60_000;
private static MAX_STUCK_NUDGE_ROUNDS = 1;
private static MAX_RUNTIME_MS = 3 * 60 * 60 * 1000;
// Congestion backoff for stuck-kills (2026-11-14, owner): under high load a
// stalled session was force-killed and re-dispatched in seconds — stall →
// kill → immediate nudge → stall again, an infinite loop burning a full
// 30-40min model cycle per pass. The parked message now rides the
// messages.retry_after hold instead: 30min after the first stuck-kill,
// doubling per consecutive kill, capped at 6h; giving up after the nudge
// budget waits a flat 2h.
static readonly STUCK_BACKOFF_BASE_MS = 30 * 60 * 1000;
static readonly STUCK_BACKOFF_MAX_MS = 6 * 60 * 60 * 1000;
static readonly STUCK_GIVEUP_HOLD_MS = 2 * 60 * 60 * 1000;
private static OBSERVER_INTERVAL_MS = 5 * 60 * 1000;
private static WEB_RECONCILE_INTERVAL_MS = 60 * 60_000;
// CI-lane post-fix verification (audit RC3).
Expand Down Expand Up @@ -2062,6 +2078,12 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st
`Push discipline (owner policy 2026-11-07): other agents may be working on this same PR branch concurrently — before EVERY push run \`git fetch origin\` and integrate the remote head (\`git pull --rebase origin/<branch>\`); NEVER force-push except when completing a conflict-rebase this task explicitly told you to perform; if a push is rejected twice or your rebase hits conflicts you did not cause, STOP and report it plainly in your reply — do not fight for the branch.`;
const statusFreshness =
`Status freshness (owner policy 2026-11-07): never assert CI green/red, merge state, labels, or approvals from memory or from earlier comments — verify against the LIVE sources available to you IN THIS RUN first; if you cannot verify it live, say so instead of guessing.`;
// CI-hold marker convention (owner policy 2026-11-15, billion-context #6783/PR #6784):
// every intermediate push carries [ework:ci-held] in its commit message (repo
// workflows skip all PR jobs on it); exactly ONE unmarked final head push is the
// branch's single full CI pass. No more one-push-one-fleet queue floods.
const ciHoldDiscipline =
`CI-hold push discipline (owner policy 2026-11-15): every push that is NOT the final head of this PR MUST carry the line [ework:ci-held] in its commit message — put it in the commit BODY (git commit -m "<subject>" -m "[ework:ci-held]"). Those pushes trigger NO CI. Only the FINAL head push — the exact commit you want reviewed, pushed after you verified the work locally — must NOT carry the marker; it is this branch's one full CI pass. At wake, check the branch head first: if its commit message still carries [ework:ci-held], a previous run died mid-session — finish the work and push the unmarked final commit.`;
const windowMs = this.cfg.work.autoLaneWindowMs;
const cap = this.engineerCap;
const attempts = await this.store.countAutoLaneEngineerMessages(
Expand Down Expand Up @@ -2095,7 +2117,7 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st
}
if (parts.ci) {
const checkList = parts.ci.checks.map((c) => `- ${c.name}${c.html_url ? ` (${c.html_url})` : ""}`).join("\n");
steps.push(`${stepN++}. Failing checks:\n${checkList}\nFix them directly on the PR branch (fetch check logs via the API as needed), push incrementally. RE-VERIFY the live check state FIRST — an earlier wake may already have fixed what this event reports; if everything is green already, confirm that in one short comment and stop without further work.`);
steps.push(`${stepN++}. Failing checks:\n${checkList}\nFix them directly on the PR branch (fetch check logs via the API as needed), push incrementally WITH the CI-hold marker on every intermediate commit (see CI-hold push discipline below). RE-VERIFY the live check state FIRST — an earlier wake may already have fixed what this event reports; if everything is green already, confirm that in one short comment and stop without further work.`);
}
const body =
// Push/status rules are appended below the body (owner policy 2026-11-07).
Expand All @@ -2105,7 +2127,7 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st
`\nWhen done, reply with a 七问 summary covering everything you did this run.\n` +
`This is an auto-fix lane: attempt ${attempts + 1}/${cap}. Rules: small mechanical fixes only; when something needs a design decision, NEVER open a NEW issue — post a comment here whose first line is "🔶 需要决策" listing options A/B/C + recommendation + default-if-undecided, apply the needs-decision label via add-label.sh (workdir-root tools/ dir, see AGENTS.md §需要人工决策时), and stop (owner policy 2026-11-01). ` +
`Authorization checkpoints are NOT yours to satisfy: if a gate blocks you (kernel-guard on kernel/ diffs or any owner-approval guard), NEVER edit the PR body to add a declaration UNLESS the gated diff is a mechanical kernel bug YOU fixed yourself (owner policy 2026-11-06 exemption — then adding ONLY the \`kernel-change:\` line plus your in-thread report is permitted; no other body edits, ever) — otherwise post a "🔶 需要决策" comment requesting the owner's explicit authorization citing the PR number and the exact gated diff, apply the needs-decision label, then stop touching that PR (owner policy 2026-11-01, superseding 2026-10-26). ` +
`Check states come from the mirror cache: curl -s -H "Authorization: token $(grep ^GITEA_TOKEN= /var/lib/ework-daemon/share/ework-daemon/.env | cut -d= -f2)" "http://127.0.0.1:3002/api/v1/upstream-checks?owner=<owner>&repo=<repo>&number=<PR>" (refreshed by the mirror poller ~60s). NEVER gh-poll check states — the shared GitHub PAT budget exhausts (owner policy 2026-10-27); gh is only for git push/PR operations.\n\n${pushDiscipline}\n\n${statusFreshness}`;
`Check states come from the mirror cache: curl -s -H "Authorization: token $(grep ^GITEA_TOKEN= /var/lib/ework-daemon/share/ework-daemon/.env | cut -d= -f2)" "http://127.0.0.1:3002/api/v1/upstream-checks?owner=<owner>&repo=<repo>&number=<PR>" (refreshed by the mirror poller ~60s). NEVER gh-poll check states — the shared GitHub PAT budget exhausts (owner policy 2026-10-27); gh is only for git push/PR operations.\n\n${pushDiscipline}\n\n${statusFreshness}\n\n${ciHoldDiscipline}`;
const prompt = this.buildForwardPrompt(
session.name,
this.handleLargeContent(workdir, body, `ci-engineer-${ref.issueId}.txt`),
Expand Down Expand Up @@ -4424,25 +4446,6 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st
].join("\n");
}

private buildStuckNudgePrompt(
session: OpSession,
issue: Issue,
instructions: { issueRef: string },
stuckMinutes: number
): string {
return [
`[SYSTEM] 检测到您的进程已经卡住 ${stuckMinutes} 分钟没有输出了,已经被强制重启。`,
``,
`可能的原因:等待输入、死循环、网络请求挂起、长时间无响应的工具调用等。`,
``,
`- 如果您之前的工作有阶段性成果,请立即向用户汇报当前进度和遇到的问题。`,
`- 如果您遇到了阻塞(权限不足、依赖缺失、不确定的方向等),请向用户说明并请求指导。`,
`- 如果您可以继续,请避开导致卡住的操作,换一种方式继续工作。`,
``,
`使用 \`reply\` 工具向 ${instructions.issueRef} 给出回复(以 \`[bot]\` 开头)。其他方式的回复会被忽略。`,
].join("\n");
}

private handleLargeContent(workdir: string, content: string, filename: string): string {
if (content.length <= Engine.MAX_INLINE_SIZE) return content;

Expand Down Expand Up @@ -4822,6 +4825,19 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st
}
for (const msg of stranded) {
if (this.destroyed) return;
// Stuck-backoff hold (2026-11-14): a just-killed session's message rides
// retry_after — waiting it out is the whole point of the backoff. The
// row field is authoritative once set; the in-memory map only bridges
// the (rare) window where the hold write raced the park's
// retry_after=NULL clear, so a row that carries its own retry_after
// must never be shadowed by a stale map entry.
const memHold = msg.retryAfter ? undefined : this.stuckBackoffHolds.get(msg.id);
if (memHold) {
if (new Date(memHold).getTime() > Date.now()) continue;
this.stuckBackoffHolds.delete(msg.id); // pruned once lapsed
await this.store.setRetryAfter(msg.id, null).catch(() => {});
}
if (msg.retryAfter && msg.retryAfter.getTime() > Date.now()) continue;
const session = await this.store.getSession(msg.sessionId);
if (!session) continue;
const issue = await this.store.getIssue(session.issueId);
Expand Down Expand Up @@ -5119,8 +5135,17 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st

const stuckNudgeRound = this.stuckNudgeRounds.get(k) ?? 0;
if (stuckNudgeRound < this.maxStuckNudges) {
log.warn(`engine: stuck — no output for ${minutes}min on ${k}, killing + sending stuck nudge (round ${stuckNudgeRound + 1}/${this.maxStuckNudges})`);
await tracker.createComment(ref, `[system] ⏰ **${session.name}** no output for ${minutes} min, restarting...`).catch(
// Congestion backoff (2026-11-14): compute the hold BEFORE killing
// so the comment can state it; the parked message (finishRun flips
// it to 'interrupted') rides retry_after and the stranded sweep
// waits it out — no immediate re-dispatch, no nudge message.
const stuckKills = (this.stuckKillCount.get(k) ?? 0) + 1;
const backoffMs = Math.min(Engine.STUCK_BACKOFF_BASE_MS * 2 ** (stuckKills - 1), Engine.STUCK_BACKOFF_MAX_MS);
const backoffMin = Math.round(backoffMs / 60000);

log.warn(`engine: stuck — no output for ${minutes}min on ${k}, killing + ~${backoffMin}min backoff (kill #${stuckKills}/${this.maxStuckNudges})`);
const heldMsgId = this.currentMessage.get(k);
await tracker.createComment(ref, `[system] ⏰ **${session.name}** no output for ${minutes} min — restarting in ~${backoffMin} min (congestion backoff, 第 ${stuckKills} 次无输出重启)`).catch(
err => log.error(`engine: stuck-nudge comment failed for ${k}:`, (err as Error).message)
);

Expand All @@ -5133,27 +5158,62 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st
}

// A stuck hang is the dead-provider signature: open the circuit so
// the pool stops picking it, and leave the nudge unpinned so the
// the pool stops picking it, and leave the model unpinned so the
// re-spawn re-picks from healthy models instead of re-pinning the
// one that just hung (dsh#133 flash-zombie family).
const stuckModel = this.currentModel.get(k);
if (stuckModel) this.modelCircuits.set(stuckModel, Date.now() + this.cfg.opencode.modelCooldownMs);
this.currentModel.delete(k);
this.stuckNudgeRounds.set(k, stuckNudgeRound + 1);
this.stuckKillCount.set(k, (this.stuckKillCount.get(k) ?? 0) + 1);

const instructions = tracker.getTrackerInstructions(ref);
const nudgePrompt = this.buildStuckNudgePrompt(session, issue, instructions, minutes);
const nudgeMsg = await this.store.createMessage(session.id, nudgePrompt, undefined, undefined, this.currentModel.get(k));
await this.dequeueOrIdle(k, session, issue, nudgeMsg);
this.stuckKillCount.set(k, stuckKills);

// Arm the backoff hold on the parked message (execProcess's
// process-replaced path settles it 'interrupted' a tick later; a
// daemon death in that gap costs one fast retry at worst — the
// next kill re-arms with a larger hold). The stranded sweep
// resurrects it once the hold expires: no nudge message, no
// immediate re-dispatch.
if (heldMsgId) {
const holdIso = new Date(Date.now() + backoffMs).toISOString();
this.stuckBackoffHolds.set(heldMsgId, holdIso);
// Arm the hold ONLY after the park lands: forceStop's execProcess
// continuation settles the row 'interrupted' a tick later, and
// that status flip (updateMessageStatus) CLEARS retry_after —
// breaking on "not running/pending" is too early (a 'failed'
// row from forceStop's own message sweep passes it while the
// park is still in flight, and the park then wipes the hold).
// status === 'interrupted' is the terminal state of that same
// UPDATE, so it is safe to write immediately after observing it.
for (let i = 0; i < 80; i++) {
const row = await this.store.getMessage(heldMsgId).catch(() => undefined);
if (row && row.status === "interrupted") break;
await new Promise((r) => setTimeout(r, 150));
}
await this.store.setRetryAfter(heldMsgId, holdIso).catch(() => {});
log.info(`engine: stuck-backoff hold on msg ${heldMsgId.slice(0, 8)} until ${holdIso}`);
}
} else {
log.warn(`engine: stuck for ${minutes}min on ${k}, stuck nudge exhausted (${stuckNudgeRound}/${this.maxStuckNudges}), giving up`);
const heldMsgId = this.currentMessage.get(k);
await tracker.createComment(ref, `[system] ⛔ **${session.name}** stuck for ${minutes} min, gave up after ${this.maxStuckNudges} restart(s).`).catch(
err => log.error(`engine: stuck-giveup comment failed for ${k}:`, (err as Error).message)
);
if (this.processes.get(k)) {
await this.forceStop(k);
}
// Flat 2h cool-off before the stranded sweep may try again —
// give-up must mean "stop hammering", not "retry in seconds".
if (heldMsgId) {
const holdIso = new Date(Date.now() + Engine.STUCK_GIVEUP_HOLD_MS).toISOString();
this.stuckBackoffHolds.set(heldMsgId, holdIso);
for (let i = 0; i < 80; i++) {
const row = await this.store.getMessage(heldMsgId).catch(() => undefined);
if (row && row.status === "interrupted") break;
await new Promise((r) => setTimeout(r, 150));
}
await this.store.setRetryAfter(heldMsgId, holdIso).catch(() => {});
log.info(`engine: stuck-giveup hold on msg ${heldMsgId.slice(0, 8)} until ${holdIso}`);
}
this.stuckNudgeRounds.delete(k);
this.stuckKillCount.set(k, (this.stuckKillCount.get(k) ?? 0) + 1);
}
Expand Down
11 changes: 11 additions & 0 deletions packages/daemon/tests/ci-engineer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,17 @@ describe("unified CI-engineer routing", () => {
expect(prompt).toContain("update-branch");
});


test("engineer prompt enforces the [ework:ci-held] intermediate-push discipline (owner policy 2026-11-15)", async () => {
const { self, enqueueOrRun, tracker } = makeSelf(3);
await callCi(self, tracker);
const [prompt] = enqueueArgs(enqueueOrRun);
expect(prompt).toContain("[ework:ci-held]");
expect(prompt).toContain("NOT the final head of this PR MUST carry the line [ework:ci-held]");
expect(prompt).toContain("commit BODY");
expect(prompt).toContain("must NOT carry the marker");
expect(prompt).toContain("one full CI pass");
});
});

describe("execProcess engineer kill-switch", () => {
Expand Down
Loading
Loading