From 48376ab9345a245a34b7ba8b9679341453043d77 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sat, 14 Nov 2026 13:05:58 +0800 Subject: [PATCH] =?UTF-8?q?fix(daemon):=20hard-cap=20the=20CI=20lane=20poo?= =?UTF-8?q?l=20=E2=80=94=20no=20idle-fleet=20expansion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every rebase push onto an old PR branch re-triggers the full upstream CI suite (~13 workflow runs on billion-context). The ci-engineer pool used infraIdleExpandedCap(), which borrows idle fleet slots: with an idle fleet the pool grew to ~10, and 10 parallel rebases flooded the Actions queue, starving the 2026-11-13 release runs (owner report 2026-11-14). The CI lane pool (engineer mode + legacy ci-fix) now stays at its configured cap (WORK_CI_MAX_CONCURRENT, x2 at night). The merge-conflict legacy branch keeps the old expansion. New tests pin the no-expansion behavior on an idle fleet. --- packages/daemon/package.json | 2 +- packages/daemon/src/opencode.ts | 14 +++- packages/daemon/tests/ci-lane-cap.test.ts | 79 +++++++++++++++++++++++ 3 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 packages/daemon/tests/ci-lane-cap.test.ts diff --git a/packages/daemon/package.json b/packages/daemon/package.json index 4fbc78e..5e44010 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -1,6 +1,6 @@ { "name": "ework-daemon", - "version": "0.4.216", + "version": "0.4.217", "description": "Issue-driven AI development daemon. Spawns opencode subprocesses to resolve Gitea issues.", "module": "src/index.ts", "type": "module", diff --git a/packages/daemon/src/opencode.ts b/packages/daemon/src/opencode.ts index 1d1f0f7..763f06d 100644 --- a/packages/daemon/src/opencode.ts +++ b/packages/daemon/src/opencode.ts @@ -3167,13 +3167,21 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st // Owner policy 2026-10-14: infra fix work must not fan out across the // whole fleet — deferred messages stay pending; the next completion event // drains them. Engineer mode (2026-11-03) counts BOTH prefixes in one pool. + // + // Owner decision 2026-11-14 (billion-context release flood): the CI lane + // must NOT borrow idle fleet capacity — every rebase push onto an old PR + // branch re-triggers the full upstream suite (~13 workflow runs), so a + // pool that idles up to the global cap (infraIdleExpandedCap) turned an + // idle fleet into a 10-wide rebase storm that starved release runs. The + // CI pool is now hard-capped at its configured value (×2 at night). if (msg.sourceCommentId?.startsWith("merge-conflict-") || msg.sourceCommentId?.startsWith("ci-fail-")) { let poolCap: number; let runningPool: number; let poolName: string; if (this.engineerMode) { - // One shared budget for the unified CI engineer. - poolCap = this.infraIdleExpandedCap(this.infraNightActive() ? this.ciMaxConcurrent * 2 : this.ciMaxConcurrent); + // One shared budget for the unified CI engineer — fixed, no idle + // expansion (2026-11-14 flood fix, see block comment above). + poolCap = this.infraNightActive() ? this.ciMaxConcurrent * 2 : this.ciMaxConcurrent; runningPool = await this.store.countRunningEngineerMessages(msg.id); poolName = "ci-engineer"; } else if (msg.sourceCommentId.startsWith("merge-conflict-")) { @@ -3181,7 +3189,7 @@ Check states: read the mirror cache (see AGENTS.md), do not poll gh for check st runningPool = await this.store.countRunningConflictMessages(msg.id); poolName = "merge-conflict"; } else { - poolCap = this.infraIdleExpandedCap(this.infraNightActive() ? this.ciMaxConcurrent * 2 : this.ciMaxConcurrent); + poolCap = this.infraNightActive() ? this.ciMaxConcurrent * 2 : this.ciMaxConcurrent; runningPool = await this.store.countRunningCiMessages(msg.id); poolName = "ci-fix"; } diff --git a/packages/daemon/tests/ci-lane-cap.test.ts b/packages/daemon/tests/ci-lane-cap.test.ts new file mode 100644 index 0000000..afb2f1d --- /dev/null +++ b/packages/daemon/tests/ci-lane-cap.test.ts @@ -0,0 +1,79 @@ +// 2026-11-14 owner decision (billion-context release flood): the CI lane pool +// must stay at its configured cap (WORK_CI_MAX_CONCURRENT, ×2 at night) and +// must NOT borrow idle fleet capacity via infraIdleExpandedCap. One rebase +// push onto an old PR branch re-triggers the full upstream suite (~13 +// workflow runs), so an idle-expanded pool of ~10 turned a quiet fleet into +// a 10-wide rebase storm that starved release runs. +import { describe, expect, test, vi } from "bun:test"; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const { Engine } = require("../src/opencode") as { Engine: any }; + +function makeSelf(opts: { cap: number; runningEngineer: number; engineerMode: boolean }) { + const updateMessageStatus = vi.fn(async () => {}); + const gateChecker = vi.fn(async () => ({ allowed: false, unreachable: false, reason: "test-exit" })); + const self = Object.create(Engine.prototype) as Record; + self.store = { + getSession: vi.fn(async () => ({ id: "s1", name: "ework-daemon" })), + getIssue: vi.fn(async () => ({ id: 7, state: "open" })), + countRunningEngineerMessages: vi.fn(async () => opts.runningEngineer), + countRunningCiMessages: vi.fn(async () => opts.runningEngineer), + countRunningConflictMessages: vi.fn(async () => 0), + updateMessageStatus, + }; + self.gateChecker = gateChecker; + // engineerMode is a getter over cfg.work.ciEngineerMaxAttempts — drive it + // through the config, not by assigning the readonly accessors. + self.cfg = { work: { ciFixMaxAttempts: 3, conflictFixMaxAttempts: 3, ciEngineerMaxAttempts: opts.engineerMode ? 3 : undefined } }; + self.ciMaxConcurrent = opts.cap; + self.conflictMaxConcurrent = 1; + // infraNightBoost stays unset (falsy) => infraNightActive() is false. + // IDLE fleet: freeGlobal = maxConcurrent - 0, so infraIdleExpandedCap WOULD + // expand a pool of `cap` up to `cap + 8` if it were still applied here. + self.maxConcurrent = opts.cap + 8; + self.running = new Map(); + self.lastOutputAt = new Map(); + self.apiIdleGraceMs = 180_000; + self.startedAt = new Map(); + self.currentMessage = new Map(); + self.currentModel = new Map(); + self.generation = new Map(); + self.projectConcurrencyCache = new Map(); + self.dequeueAfterGate = vi.fn(async () => {}); + return { self, updateMessageStatus, gateChecker }; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const session: any = { id: "s1", name: "ework-daemon" }; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const issue: any = { id: 7, trackerScopeKey: "acme/widgets" }; + +describe("CI lane pool: hard cap, no idle-fleet expansion (2026-11-14 flood fix)", () => { + test("engineer mode: pool stays at configured cap while the fleet is idle", async () => { + // 2 running engineer messages, configured cap 2, fleet wide open. + // Pre-fix this admitted a 10th (freeGlobal) — the flood. + const msg = { id: "m1", uid: "m1", sourceCommentId: "ci-fail-42-1" }; + const { self, updateMessageStatus, gateChecker } = makeSelf({ cap: 2, runningEngineer: 2, engineerMode: true }); + await Engine.prototype.execProcess.call(self, "k1", session, issue, msg); + expect(updateMessageStatus).toHaveBeenCalledWith("m1", "pending"); + expect(gateChecker).not.toHaveBeenCalled(); + expect((self.running as Map).has("k1")).toBe(false); + }); + + test("engineer mode: cap 2 admits exactly the second slot, not the idle-expanded ones", async () => { + const msg = { id: "m2", uid: "m2", sourceCommentId: "ci-fail-43-1" }; + const { self, updateMessageStatus, gateChecker } = makeSelf({ cap: 2, runningEngineer: 1, engineerMode: true }); + await Engine.prototype.execProcess.call(self, "k2", session, issue, msg); + expect(gateChecker).toHaveBeenCalledTimes(1); + // gate fails in the fixture — proves the pool check admitted it. + expect(updateMessageStatus).toHaveBeenCalledWith("m2", "failed", "gate: test-exit"); + }); + + test("legacy ci-fix mode (no engineer lane): same hard cap", async () => { + const msg = { id: "m3", uid: "m3", sourceCommentId: "ci-fail-44-1" }; + const { self, updateMessageStatus, gateChecker } = makeSelf({ cap: 1, runningEngineer: 1, engineerMode: false }); + await Engine.prototype.execProcess.call(self, "k3", session, issue, msg); + expect(updateMessageStatus).toHaveBeenCalledWith("m3", "pending"); + expect(gateChecker).not.toHaveBeenCalled(); + }); +});