Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/daemon/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "ework-daemon",
"version": "0.4.216",
"version": "0.4.217",
"description": "Issue-driven AI development daemon. Spawns opencode subprocesses to resolve Gitea issues.",
"module": "src/index.ts",
"type": "module",
Expand Down
14 changes: 11 additions & 3 deletions packages/daemon/src/opencode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3167,21 +3167,29 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st
// Owner policy 2026-10-14: infra fix work must not fan out across the
// whole fleet — deferred messages stay pending; the next completion event
// drains them. Engineer mode (2026-11-03) counts BOTH prefixes in one pool.
//
// Owner decision 2026-11-14 (billion-context release flood): the CI lane
// must NOT borrow idle fleet capacity — every rebase push onto an old PR
// branch re-triggers the full upstream suite (~13 workflow runs), so a
// pool that idles up to the global cap (infraIdleExpandedCap) turned an
// idle fleet into a 10-wide rebase storm that starved release runs. The
// CI pool is now hard-capped at its configured value (×2 at night).
if (msg.sourceCommentId?.startsWith("merge-conflict-") || msg.sourceCommentId?.startsWith("ci-fail-")) {
let poolCap: number;
let runningPool: number;
let poolName: string;
if (this.engineerMode) {
// One shared budget for the unified CI engineer.
poolCap = this.infraIdleExpandedCap(this.infraNightActive() ? this.ciMaxConcurrent * 2 : this.ciMaxConcurrent);
// One shared budget for the unified CI engineer — fixed, no idle
// expansion (2026-11-14 flood fix, see block comment above).
poolCap = this.infraNightActive() ? this.ciMaxConcurrent * 2 : this.ciMaxConcurrent;
runningPool = await this.store.countRunningEngineerMessages(msg.id);
poolName = "ci-engineer";
} else if (msg.sourceCommentId.startsWith("merge-conflict-")) {
poolCap = this.infraIdleExpandedCap(this.infraNightActive() ? this.conflictMaxConcurrent * 2 : this.conflictMaxConcurrent);
runningPool = await this.store.countRunningConflictMessages(msg.id);
poolName = "merge-conflict";
} else {
poolCap = this.infraIdleExpandedCap(this.infraNightActive() ? this.ciMaxConcurrent * 2 : this.ciMaxConcurrent);
poolCap = this.infraNightActive() ? this.ciMaxConcurrent * 2 : this.ciMaxConcurrent;
runningPool = await this.store.countRunningCiMessages(msg.id);
poolName = "ci-fix";
}
Expand Down
79 changes: 79 additions & 0 deletions packages/daemon/tests/ci-lane-cap.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
// 2026-11-14 owner decision (billion-context release flood): the CI lane pool
// must stay at its configured cap (WORK_CI_MAX_CONCURRENT, ×2 at night) and
// must NOT borrow idle fleet capacity via infraIdleExpandedCap. One rebase
// push onto an old PR branch re-triggers the full upstream suite (~13
// workflow runs), so an idle-expanded pool of ~10 turned a quiet fleet into
// a 10-wide rebase storm that starved release runs.
import { describe, expect, test, vi } from "bun:test";

// eslint-disable-next-line @typescript-eslint/no-explicit-any
const { Engine } = require("../src/opencode") as { Engine: any };

function makeSelf(opts: { cap: number; runningEngineer: number; engineerMode: boolean }) {
const updateMessageStatus = vi.fn(async () => {});
const gateChecker = vi.fn(async () => ({ allowed: false, unreachable: false, reason: "test-exit" }));
const self = Object.create(Engine.prototype) as Record<string, unknown>;
self.store = {
getSession: vi.fn(async () => ({ id: "s1", name: "ework-daemon" })),
getIssue: vi.fn(async () => ({ id: 7, state: "open" })),
countRunningEngineerMessages: vi.fn(async () => opts.runningEngineer),
countRunningCiMessages: vi.fn(async () => opts.runningEngineer),
countRunningConflictMessages: vi.fn(async () => 0),
updateMessageStatus,
};
self.gateChecker = gateChecker;
// engineerMode is a getter over cfg.work.ciEngineerMaxAttempts — drive it
// through the config, not by assigning the readonly accessors.
self.cfg = { work: { ciFixMaxAttempts: 3, conflictFixMaxAttempts: 3, ciEngineerMaxAttempts: opts.engineerMode ? 3 : undefined } };
self.ciMaxConcurrent = opts.cap;
self.conflictMaxConcurrent = 1;
// infraNightBoost stays unset (falsy) => infraNightActive() is false.
// IDLE fleet: freeGlobal = maxConcurrent - 0, so infraIdleExpandedCap WOULD
// expand a pool of `cap` up to `cap + 8` if it were still applied here.
self.maxConcurrent = opts.cap + 8;
self.running = new Map<string, unknown>();
self.lastOutputAt = new Map();
self.apiIdleGraceMs = 180_000;
self.startedAt = new Map();
self.currentMessage = new Map();
self.currentModel = new Map();
self.generation = new Map();
self.projectConcurrencyCache = new Map();
self.dequeueAfterGate = vi.fn(async () => {});
return { self, updateMessageStatus, gateChecker };
}

// eslint-disable-next-line @typescript-eslint/no-explicit-any
const session: any = { id: "s1", name: "ework-daemon" };
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const issue: any = { id: 7, trackerScopeKey: "acme/widgets" };

describe("CI lane pool: hard cap, no idle-fleet expansion (2026-11-14 flood fix)", () => {
test("engineer mode: pool stays at configured cap while the fleet is idle", async () => {
// 2 running engineer messages, configured cap 2, fleet wide open.
// Pre-fix this admitted a 10th (freeGlobal) — the flood.
const msg = { id: "m1", uid: "m1", sourceCommentId: "ci-fail-42-1" };
const { self, updateMessageStatus, gateChecker } = makeSelf({ cap: 2, runningEngineer: 2, engineerMode: true });
await Engine.prototype.execProcess.call(self, "k1", session, issue, msg);
expect(updateMessageStatus).toHaveBeenCalledWith("m1", "pending");
expect(gateChecker).not.toHaveBeenCalled();
expect((self.running as Map<string, unknown>).has("k1")).toBe(false);
});

test("engineer mode: cap 2 admits exactly the second slot, not the idle-expanded ones", async () => {
const msg = { id: "m2", uid: "m2", sourceCommentId: "ci-fail-43-1" };
const { self, updateMessageStatus, gateChecker } = makeSelf({ cap: 2, runningEngineer: 1, engineerMode: true });
await Engine.prototype.execProcess.call(self, "k2", session, issue, msg);
expect(gateChecker).toHaveBeenCalledTimes(1);
// gate fails in the fixture — proves the pool check admitted it.
expect(updateMessageStatus).toHaveBeenCalledWith("m2", "failed", "gate: test-exit");
});

test("legacy ci-fix mode (no engineer lane): same hard cap", async () => {
const msg = { id: "m3", uid: "m3", sourceCommentId: "ci-fail-44-1" };
const { self, updateMessageStatus, gateChecker } = makeSelf({ cap: 1, runningEngineer: 1, engineerMode: false });
await Engine.prototype.execProcess.call(self, "k3", session, issue, msg);
expect(updateMessageStatus).toHaveBeenCalledWith("m3", "pending");
expect(gateChecker).not.toHaveBeenCalled();
});
});
Loading