From b8cad881be1e610058351f86f442d8774c4912b5 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Wed, 14 Oct 2026 14:57:38 +0800 Subject: [PATCH] ci: shard the test suite across the matrix to cut wall time ~3x (#3143) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suite outgrew one runner: ~656 top-level files (~190k LOC) serialized through --test-concurrency=4 on a single 4-core leg took ~17-20 min (windows slowest; measured 1225s on the last full green run). Add a `shard` matrix dimension to the test job so each leg runs a balanced slice; run-ci-tests.mjs now partitions deterministically + totally (LPT bin-packing over file size) so the union of all shards == allFiles and no test is skipped. Same tests, assertions, env, OS and node versions — zero coverage loss. Wall drops ~3x (~20 -> ~7 min). --- .github/workflows/ci.yml | 10 +++++++ AGENTS.md | 2 +- reference/testing.md | 10 +++++++ scripts/run-ci-tests.mjs | 57 ++++++++++++++++++++++++++++++++++++++-- 4 files changed, 76 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 418d2d9802..663ab42562 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,14 @@ jobs: # paying 4x(install+typecheck+suites+build). Guards with `if: pull_request` # skip on push/dispatch (skipped needs count as satisfied). Behavior change: # on guard failure the required legs show pending (not red); merge blocked. + # #3143: the suite outgrew one runner — ~656 top-level files (~190k LOC) now + # serialize through c=4 on a single 4-core leg for ~17-20 min (windows slowest; + # measured 1225s on the last full green run). Sharding spreads the SAME files + # across more legs: each runs a balanced slice (scripts/run-ci-tests.mjs partitions + # deterministically + totally, so union of shards == allFiles, nothing skipped, + # same tests/assertions/env, c=4 within each shard). Wall drops ~3x (~20 -> ~7min) + # with zero coverage loss. To go faster, add entries to the list AND bump the "/3" + # below together (they must stay equal). test: needs: [version-guard, kernel-guard, pi-subagents-guard, lane-guard, docs-sync, docs-coverage, unused-exports, lock-sync, test-import-graph] strategy: @@ -44,6 +52,7 @@ jobs: matrix: os: [ubuntu-latest, windows-latest] node-version: [22, 24] + shard: [1, 2, 3] # keep in sync with CI_TEST_SHARD "/3" below runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -67,6 +76,7 @@ jobs: - run: node scripts/run-ci-tests.mjs env: ACP_TEST_CLAUDE_NATIVE: "1" + CI_TEST_SHARD: "${{ matrix.shard }}/3" # denominator MUST equal matrix.shard length # #2816: typecheck + the in-repo component suites + the bundle build used # to run sequentially in front of the test job (~90s of critical path per diff --git a/AGENTS.md b/AGENTS.md index 247bc14f92..d36f0a2346 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -211,7 +211,7 @@ Commit convention: `feat:` / `fix:` / `refactor:` / `test:` / `docs:` / `release ### 7.2 Review Discipline - **Rebase to CURRENT master before claiming mergeable.** Verify against live master, not the PR's original base; after rebase run typecheck + build locally — the FULL-suite verdict comes from remote CI on the pushed head (next bullet). Stale base is the #1 cause of second-round rework (#425 Aug-31 base, #467 43 commits behind, #517). -- **Full regression = remote GitHub CI, not a local full-suite run.** The authoritative green/red for the full suite is the GitHub Actions `test` lane on the pushed head (`scripts/run-ci-tests.mjs`, #2816: hermetic runner, pinned concurrency, ubuntu/windows × node 22/24). Workflow: run the TARGETED suites locally for the area you touched (`node --import tsx --test tests/.test.ts`), push the branch / update the PR, then read the check runs. A local `npm test` full run is environment noise, not evidence — live lanes probing firewalled networks, port collisions with a concurrently running bili instance (e.g. claude-native binding 18787, #394), and cross-file interference all produce failures that CI proves green. Never cite a local full-suite result as the merge verdict, and never chase a local full-suite failure before confirming it reproduces on CI (or in that single file run standalone). +- **Full regression = remote GitHub CI, not a local full-suite run.** The authoritative green/red for the full suite is the GitHub Actions `test` lane on the pushed head (`scripts/run-ci-tests.mjs`, #2816: hermetic runner, pinned concurrency, ubuntu/windows × node 22/24 × 3 balanced shards (#3143)). Workflow: run the TARGETED suites locally for the area you touched (`node --import tsx --test tests/.test.ts`), push the branch / update the PR, then read the check runs. A local `npm test` full run is environment noise, not evidence — live lanes probing firewalled networks, port collisions with a concurrently running bili instance (e.g. claude-native binding 18787, #394), and cross-file interference all produce failures that CI proves green. Never cite a local full-suite result as the merge verdict, and never chase a local full-suite failure before confirming it reproduces on CI (or in that single file run standalone). - **Watch hot-file contention.** `src/server.ts`, the preflight paths, `src/persist.ts`, and the `src/agent/*` extension types are rewritten by many concurrent PRs — expect semantic (not textual) conflicts; coordinate/sequence, resolve by union of intent, then prove by running tests (#517↔#587, #571↔#558). - **One linear commit, clean diff.** No diff-exploding rebases, no incidental whitespace re-alignment; every line relates to the PR's purpose (#571 "diff-爆炸", #467). - **Minor flaws found during review get fixed, not filed.** Per the owner rule "小问题直接修" (§4 Issue & Problem Tracking): when your own review of a PR turns up a small, non-semantic flaw (typo, misaligned indent, log wording), push the one-line fix to the PR branch immediately instead of listing it as a caveat — on ANY PR branch, external contributors included. diff --git a/reference/testing.md b/reference/testing.md index 78659363b0..872f7c4d1b 100644 --- a/reference/testing.md +++ b/reference/testing.md @@ -37,6 +37,16 @@ rides c=4 unreviewed. Scheduling only — same tests, same assertions, same env vars; local `npm test` is untouched. +Since #3143 the `test` job also SHARDS across the matrix: each leg runs one of +three balanced slices (`CI_TEST_SHARD="i/3"`), partitioned by +`scripts/run-ci-tests.mjs` with deterministic LPT bin-packing over file size — +total (union of shards == every file) and disjoint (no file in two shards), so no +test is ever skipped and the LIVE_FILES tripwire still scans the full set in every +shard. The suite had grown past one runner's ~c=4 ceiling (~656 files / ~190k LOC → +~20 min wall on windows), so sharding cut the wall ~3x (~7 min) at zero coverage +cost. Tune by editing the `shard:` list in `.github/workflows/ci.yml` AND its `/3` +denominator together (they must stay equal). + The `static-gates` job (same os×node matrix) runs typecheck + kernel suite + pi-subagents suite + build IN PARALLEL with the `test` job; they used to be sequential steps ahead of the tests (~90s of critical path per leg). diff --git a/scripts/run-ci-tests.mjs b/scripts/run-ci-tests.mjs index 95f7ae0d7f..b2bb0d4259 100644 --- a/scripts/run-ci-tests.mjs +++ b/scripts/run-ci-tests.mjs @@ -100,6 +100,46 @@ function die(msg) { process.exit(1); } +// #3143: shard parsing + balanced partitioning. CI_TEST_SHARD="i/N" selects this +// leg's slice; absent means "run everything" (local / back-compat, unchanged). +// The partition is DETERMINISTIC and TOTAL: LPT bin-packing over a line-count +// weight (cheap, runtime-derived, self-maintaining for new files) puts every file +// in exactly ONE shard, so the union of all shards == allFiles and NOTHING is ever +// skipped. Same input => same partition; a newly-added file only re-partitions the +// bins, it never drops another. Balance targets the HEAVIEST shard (the wall clock), +// not equal file counts — the one 7k-line monster must not pile onto one leg. +function parseShard() { + const raw = (process.env.CI_TEST_SHARD ?? "").trim(); + if (raw === "") return null; + const m = raw.match(/^(\d+)\s*\/\s*(\d+)$/); + if (!m) die(`CI_TEST_SHARD must look like "/" (got "${raw}")`); + const i = Number(m[1]); + const n = Number(m[2]); + if (n < 1 || i < 1 || i > n) die(`CI_TEST_SHARD "${raw}" out of range (need 1 <= i <= N)`); + return { i, n }; +} + +function partitionAll(files, n) { + const weight = new Map(); + for (const f of files) { + let txt = ""; + try { txt = readFileSync(join(root, f), "utf8"); } catch { /* unreadable => weight floor 1 */ } + weight.set(f, Math.max(1, txt.split("\n").length)); + } + const order = [...files].sort( + (a, b) => weight.get(b) - weight.get(a) || (a < b ? -1 : a > b ? 1 : 0), + ); + const bins = Array.from({ length: n }, () => []); + const load = new Array(n).fill(0); + for (const f of order) { + let best = 0; + for (let k = 1; k < n; k++) if (load[k] < load[best]) best = k; // lowest index breaks ties + bins[best].push(f); + load[best] += weight.get(f); + } + return bins; +} + for (const f of LIVE_FILES) { if (!allFiles.includes(f)) die(`LIVE_FILES entry ${f} does not exist`); } @@ -114,6 +154,19 @@ if (unregistered.length > 0) { ); } +// #3143: select this leg's slice AFTER the global tripwire above (which still scans +// the FULL set in every shard, so an unregistered spawner fails whichever shard you +// inspect). Unsharded => the whole suite, exactly as before. +const shard = parseShard(); +let selectedFiles = allFiles; +if (shard) { + selectedFiles = partitionAll(allFiles, shard.n)[shard.i - 1]; + console.log( + `[ci-tests] shard ${shard.i}/${shard.n}: ${selectedFiles.length}/${allFiles.length} files ` + + `(union of all shards == allFiles — nothing skipped)`, + ); +} + // Parity with the old `npm test` step: its pretest hook builds kernel/ and // pi-subagents/ dist bundles that the suites import. On Windows npm is a .cmd // batch file, which CreateProcess cannot exec directly (EINVAL) — route it @@ -218,7 +271,7 @@ function runIsolated(fileRel) { } (async () => { - const first = await runNodeTest(allFiles, 4, "all"); + const first = await runNodeTest(selectedFiles, 4, "all"); if (first.code === 0) { console.log(`[ci-tests] all ok in ${(first.ms / 1000).toFixed(1)}s`); return; @@ -258,7 +311,7 @@ function runIsolated(fileRel) { ); } console.error("[ci-tests] retrying full phase once to confirm flake (#3029)"); - const retry = await runNodeTest(allFiles, 4, "all (retry)"); + const retry = await runNodeTest(selectedFiles, 4, "all (retry)"); if (retry.code === 0) { console.log( `[ci-tests] FLAKE CONFIRMED: first run failed on ${swallowed.join(", ")}, full retry green in ` +