diff --git a/.github/scripts/flake-issues.test.ts b/.github/scripts/flake-issues.test.ts new file mode 100644 index 0000000..3ce9e1d --- /dev/null +++ b/.github/scripts/flake-issues.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, test } from "bun:test"; + +import { collectFailures, missingReports } from "./flake-issues"; + +const suite = (tests: { status: string; results: { status: string }[] }[]) => [ + { + file: "e2e/a.spec.ts", + specs: [ + { + file: "e2e/a.spec.ts", + tests: tests.map((entry) => ({ + results: entry.results.map((result) => ({ + error: { message: `boom (${result.status})` }, + status: result.status, + })), + status: entry.status, + })), + title: "does x", + }, + ], + title: "e2e/a.spec.ts", + }, +]; + +describe("collectFailures", () => { + test("a test.fail() that failed as declared is green, never filed", () => { + expect( + collectFailures( + suite([{ results: [{ status: "failed" }], status: "expected" }]), + [], + undefined, + [] + ) + ).toEqual([]); + }); + + test("a test.fail() that passed is filed as an unexpected pass", () => { + const [failure] = collectFailures( + suite([{ results: [{ status: "passed" }], status: "unexpected" }]), + [], + undefined, + [] + ); + + expect(failure?.status).toBe("unexpected pass"); + expect(failure?.error).toContain("declared to fail"); + }); + + test("an unexpected failure carries its failing result's error", () => { + const [failure] = collectFailures( + suite([{ results: [{ status: "timedOut" }], status: "unexpected" }]), + [], + undefined, + [] + ); + + expect(failure?.status).toBe("timedOut"); + expect(failure?.error).toBe("boom (timedOut)"); + expect(failure?.title).toBe("e2e/a.spec.ts › does x"); + }); + + test("skipped and passing tests file nothing", () => { + expect( + collectFailures( + suite([ + { results: [], status: "skipped" }, + { results: [{ status: "passed" }], status: "expected" }, + ]), + [], + undefined, + [] + ) + ).toEqual([]); + }); +}); + +describe("missingReports", () => { + test("every planned lane, round and shard must have reported", () => { + const present = new Set(["single-r1-s1", "single-r1-s2", "multi-r1-s1"]); + + expect(missingReports(present, ["single", "multi"], [1], 2)).toEqual([ + "multi-r1-s2", + ]); + }); + + test("a full matrix is complete", () => { + const present = new Set([ + "single-r1-s1", + "single-r1-s2", + "single-r2-s1", + "single-r2-s2", + ]); + + expect(missingReports(present, ["single"], [1, 2], 2)).toEqual([]); + }); +}); diff --git a/.github/scripts/flake-issues.ts b/.github/scripts/flake-issues.ts new file mode 100755 index 0000000..2a1ba57 --- /dev/null +++ b/.github/scripts/flake-issues.ts @@ -0,0 +1,439 @@ +#!/usr/bin/env bun +// Nightly flake hunt: read every lane's Playwright JSON results and file one +// `flake`-labelled issue per failing spec title, or comment on the +// already-open one. Green runs file nothing and exit 0. +// +// bun .github/scripts/flake-issues.ts --reports /tmp/e2e-reports \ +// --run-url https://github.com/OWNER/REPO/actions/runs/ID +// +// `--reports` holds one directory per uploaded artifact, named +// `playwright-report--r-s`, each with `e2e-results.json` +// (the `json` reporter in apps/noite/playwright.config.ts). Auth through +// GH_TOKEN, like every other `gh` call on the runners. +import { execFileSync } from "node:child_process"; +import { + existsSync, + mkdtempSync, + readdirSync, + readFileSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +const MAX_TITLE = 200; +const MAX_ERROR_LINES = 30; +const MAX_ERROR_CHARS = 3000; + +const REPORT_DIR = + /^playwright-report-(?.+)-r(?\d+)-s(?\d+)$/u; + +const BAD_STATUSES = { + failed: true, + interrupted: true, + timedOut: true, +} satisfies Record; + +// The Playwright `json` reporter shape, narrowed to the fields this script +// reads. Unknown report internals stay out: every access below defaults. +interface ReportResult { + error?: { message?: string; stack?: string } | null; + status?: string; +} + +// `status` is Playwright's per-test outcome: `expected` (including a +// `test.fail()` that failed as declared), `unexpected`, `flaky` or `skipped`. +interface ReportTest { + results?: ReportResult[]; + status?: string; +} + +interface ReportSpec { + file?: string; + tests?: ReportTest[]; + title?: string; +} + +interface ReportSuite { + file?: string; + specs?: ReportSpec[]; + suites?: ReportSuite[]; + title?: string; +} + +interface PlaywrightReport { + suites?: ReportSuite[]; +} + +interface Failure { + error: string; + file: string; + status: string; + title: string; +} + +type Occurrence = Failure & { + lane: string; + round: string; + shard: string; +}; + +interface OpenIssue { + number: number; + title: string; +} + +const usage = (): never => { + console.error( + "usage: flake-issues.ts --reports DIR --run-url URL --lanes a,b --rounds '[1,2]' --shards N" + ); + + process.exit(2); +}; + +const FLAGS = { + "--lanes": "lanes", + "--reports": "reports", + "--rounds": "rounds", + "--run-url": "runUrl", + "--shards": "shards", +} satisfies Record; + +const parseArgs = (argv: string[]) => { + const out: Record = {}; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const value = argv[index + 1]; + + if (arg !== undefined && Object.hasOwn(FLAGS, arg) && value !== undefined) { + // SAFETY: `Object.hasOwn` just proved `arg` is a FLAGS key. + out[FLAGS[arg as keyof typeof FLAGS]] = value; + index += 1; + } else { + usage(); + } + } + + const { lanes, reports, rounds, runUrl, shards } = out; + + if ( + lanes === undefined || + reports === undefined || + rounds === undefined || + runUrl === undefined || + shards === undefined + ) { + return usage(); + } + + // SAFETY: the plan job emits `rounds` as a JSON array of round numbers; + // anything else fails the check below instead of shrinking the matrix. + const roundList = JSON.parse(rounds) as unknown; + const shardCount = Number(shards); + + if ( + !Array.isArray(roundList) || + !roundList.every((round) => Number.isInteger(round)) || + !Number.isInteger(shardCount) || + shardCount < 1 + ) { + return usage(); + } + + return { + lanes: lanes.split(",").filter((lane) => lane !== ""), + reports, + // SAFETY: the check above proved an array of integers. + rounds: roundList as number[], + runUrl, + shards: shardCount, + }; +}; + +const gh = (args: string[]): string => + execFileSync("gh", args, { encoding: "utf-8" }).trim(); + +// What a filed test reports: its first failing result's error and status, +// `flaky` when a retry passed, or an unexpected pass when a `test.fail()` +// passed (no failing result at all). +const outcomeOf = (failing: ReportTest): Pick => { + const bad = (failing.results ?? []).find( + (result) => + result.status !== undefined && Object.hasOwn(BAD_STATUSES, result.status) + ); + + if (bad === undefined) { + return { + error: "(passed, but the test is declared to fail)", + status: failing.status === "flaky" ? "flaky" : "unexpected pass", + }; + } + + return { + error: bad.error?.message ?? bad.error?.stack ?? "(no error text)", + status: failing.status === "flaky" ? "flaky" : (bad.status ?? "failed"), + }; +}; + +// Walk the JSON reporter's suite tree, carrying ancestor titles, and collect +// every spec whose test outcome is `unexpected` or `flaky`. The outcome, not +// a result status, decides: a `test.fail()` that failed as declared is +// `expected` (green, never filed), and one that passed is `unexpected` with +// no failing result (filed as an unexpected pass). +export const collectFailures = ( + suites: ReportSuite[] | undefined, + ancestors: string[], + file: string | undefined, + out: Failure[] +): Failure[] => { + for (const suite of suites ?? []) { + const titles = [...ancestors, suite.title ?? ""].filter( + (part) => part !== "" + ); + + const suiteFile = suite.file ?? file ?? "unknown"; + + for (const spec of suite.specs ?? []) { + const failing = (spec.tests ?? []).find( + (test) => test.status === "unexpected" || test.status === "flaky" + ); + + if (failing !== undefined) { + out.push({ + ...outcomeOf(failing), + file: spec.file ?? suiteFile, + title: [...titles, spec.title ?? ""] + .filter((part) => part !== "") + .join(" › "), + }); + } + } + + collectFailures(suite.suites, titles, suiteFile, out); + } + + return out; +}; + +// Every planned (lane, round, shard) cell must have uploaded its results: a +// shard that died before Playwright wrote them (a lane that never booted, a +// cancelled job) would otherwise shrink the hunt silently. +export const missingReports = ( + present: ReadonlySet, + lanes: readonly string[], + rounds: readonly number[], + shards: number +): string[] => + lanes.flatMap((lane) => + rounds.flatMap((round) => + Array.from({ length: shards }, (_, index) => { + const cell = `${lane}-r${round}-s${index + 1}`; + + return present.has(cell) ? [] : [cell]; + }).flat() + ) + ); + +const errorSnippet = (error: string): string => { + const lines = error.split("\n").slice(0, MAX_ERROR_LINES).join("\n"); + + return lines.length > MAX_ERROR_CHARS + ? `${lines.slice(0, MAX_ERROR_CHARS)}\n…(truncated)` + : lines; +}; + +const issueBody = ( + key: string, + first: Occurrence, + occurrences: Occurrence[], + runUrl: string +): string => { + const rows = occurrences + .map((o) => `| ${o.lane} | ${o.round} | ${o.shard} | ${o.status} |`) + .join("\n"); + + return [ + "A scheduled e2e round failed this spec. No retries anywhere: a failure is a finding.", + "", + `Run: ${runUrl}`, + "", + "| lane | round | shard | status |", + "| --- | --- | --- | --- |", + rows, + "", + `
First error (${first.file}, lane ${first.lane} round ${first.round} shard ${first.shard})`, + "", + "```", + errorSnippet(first.error), + "```", + "
", + "", + `Spec key: \`${key}\``, + ].join("\n"); +}; + +const fileIssues = ( + bySpec: ReadonlyMap, + runUrl: string +): void => { + try { + gh([ + "label", + "create", + "flake", + "--description", + "Found by the nightly e2e flake hunt", + "--color", + "F4667C", + ]); + } catch { + // Already exists (or otherwise uncreateable — the create/comment below + // will fail loudly if the label is really missing). + } + + // Every open `flake` issue, paginated (a capped list could miss one and + // file a duplicate); titles are matched exactly in memory below. The + // issues endpoint also lists pull requests, which never match a title. + // SAFETY: `gh api --paginate --slurp` answers an array of pages, each an + // array of issue objects; a CLI or API change fails this job loudly. + const open = ( + JSON.parse( + gh([ + "api", + "--paginate", + "--slurp", + "repos/{owner}/{repo}/issues?state=open&labels=flake&per_page=100", + ]) + ) as OpenIssue[][] + ).flat(); + + const tmp = mkdtempSync(path.join(tmpdir(), "flake-")); + + for (const [key, occurrences] of bySpec) { + // Keys are only set beside a push above, so a first occurrence exists. + const [first] = occurrences; + + if (first === undefined) { + continue; + } + + const title = `[flake] ${key}`; + + const bodyFile = path.join( + tmp, + `body-${occurrences.length}-${Date.now()}.md` + ); + + writeFileSync(bodyFile, issueBody(key, first, occurrences, runUrl)); + + const existing = open.find((issue) => issue.title === title); + + if (existing === undefined) { + const url = gh([ + "issue", + "create", + "--label", + "flake", + "--title", + title, + "--body-file", + bodyFile, + ]); + + console.log(`opened ${url} for ${key}`); + } else { + gh([ + "issue", + "comment", + String(existing.number), + "--body-file", + bodyFile, + ]); + + console.log(`commented on #${existing.number} for ${key}`); + } + } +}; + +const main = (): void => { + const { lanes, reports, rounds, runUrl, shards } = parseArgs( + process.argv.slice(2) + ); + + const dirs = readdirSync(reports, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .toSorted(); + + const bySpec = new Map(); + const present = new Set(); + + for (const dir of dirs) { + const match = REPORT_DIR.exec(dir); + + if (match === null || match.groups === undefined) { + continue; + } + + const { lane, round, shard } = match.groups; + const resultsPath = path.join(reports, dir, "e2e-results.json"); + + if (!existsSync(resultsPath)) { + continue; + } + + present.add(`${lane}-r${round}-s${shard}`); + + // SAFETY: the file is the Playwright `json` reporter output the lane just + // wrote; every field this script reads is optional and defaulted above, + // so an older or newer reporter shape collects fewer failures instead of + // crashing (and zero files still fails the job below). + const report = JSON.parse( + readFileSync(resultsPath, "utf-8") + ) as PlaywrightReport; + + for (const failure of collectFailures(report.suites, [], undefined, [])) { + // The file suite's own title is the file path, which the key already + // carries: `e2e/a.spec.ts › admin › does x`, not the doubled form. + const shortTitle = failure.title.startsWith(`${failure.file} › `) + ? failure.title.slice(failure.file.length + 3) + : failure.title; + + const key = `${failure.file} › ${shortTitle}`.slice(0, MAX_TITLE); + + if (!bySpec.has(key)) { + bySpec.set(key, []); + } + + const occurrences = bySpec.get(key); + + occurrences?.push({ ...failure, lane, round, shard }); + } + } + + const missing = missingReports(present, lanes, rounds, shards); + + console.log( + `read ${present.size} result file(s), ${bySpec.size} failing spec(s), ${missing.length} missing` + ); + + if (bySpec.size > 0) { + fileIssues(bySpec, runUrl); + } + + // Checked after filing, so the failures that did report still land: a + // missing cell (a lane that never booted, a shard that died before its + // report) fails the hunt loudly instead of passing on partial coverage. + if (missing.length > 0) { + console.error( + `::error::no e2e-results.json for ${missing.join(", ")}: the hunt did not cover the planned matrix` + ); + + process.exit(1); + } +}; + +if (import.meta.main) { + main(); +} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 926547e..c0185c2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -33,8 +33,10 @@ jobs: run: bun run check - name: version pins run: bun docker/check-versions.ts - - name: docker script tests - run: bun test docker/ + # Script unit tests: docker/ and the CI scripts (bun skips dot + # directories unless they're named explicitly). + - name: docker and CI script tests + run: bun test docker/ ./.github/scripts/ # `config` needs no daemon, so a broken compose file fails here instead # of on someone's server. Every variant, including the test overlays. - name: render compose (prod, build, dev, e2e, multi) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b193706..009ab62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,7 +110,9 @@ jobs: # replicates — every past image-only breakage (trustedDependencies, # .assetsignore handling, SSR snapshot atoms) was green here. Single # platform plus the shared GHA cache (same scope as the release build, so - # main's layers warm the PR build) keeps it incremental. + # main's layers warm the PR build) keeps it incremental. Built once per run + # and handed to the e2e lanes as an artifact, so no lane rebuilds it + # (every lane paid ~280 s for its own build before). image: runs-on: ubuntu-latest permissions: @@ -124,8 +126,23 @@ jobs: file: docker/Dockerfile target: noite push: false + load: true + tags: noite:ci cache-from: type=gha,scope=noite-linux/amd64 cache-to: type=gha,mode=max,scope=noite-linux/amd64 + - name: save lane image + run: | + mkdir -p /tmp/noite-image + docker save noite:ci | zstd -T0 -3 -o /tmp/noite-image/noite-ci.tar.zst + ls -la /tmp/noite-image + - name: upload lane image + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: noite-ci-image + path: /tmp/noite-image/noite-ci.tar.zst + retention-days: 1 + compression-level: 0 + if-no-files-found: error # Which runtime-code filter decides the automatic fast run. The `e2e` label # upgrades a pull request to the release suite; docs-only PRs run no lane. @@ -162,11 +179,20 @@ jobs: # Actions tab, always release on merge_group. The lanes live in e2e.yml, # which `images.yml` also runs as the release gate. e2e: - needs: changes - if: always() && (github.event_name == 'workflow_dispatch' || github.event_name == 'merge_group' || contains(github.event.pull_request.labels.*.name, 'e2e') || needs.changes.outputs.code == 'true') + needs: [changes, image] + if: always() && needs.image.result == 'success' && (github.event_name == 'workflow_dispatch' || github.event_name == 'merge_group' || contains(github.event.pull_request.labels.*.name, 'e2e') || needs.changes.outputs.code == 'true') + # Permission envelope for the called workflow: GitHub validates the + # callee's jobs against this. The lanes download the shared image + # artifact (`actions: read`) and the nightly follow-up files `flake` + # issues (`issues: write`). + permissions: + contents: read + actions: read + issues: write uses: ./.github/workflows/e2e.yml with: suite: ${{ (github.event_name == 'merge_group' || contains(github.event.pull_request.labels.*.name, 'e2e')) && 'release' || inputs.suite || 'fast' }} + image_artifact: noite-ci-image # The merge-queue gate (required check, with `security-gate`): fails if any # needed job failed or was cancelled. `skipped` is accepted only where the diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5d499fe..31bb69f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -10,13 +10,13 @@ # schedule runs `release` on main. Locally: `make e2e`, # `E2E_SUITE=fast make e2e`, and `make e2e-isolation`. # -# Speed shape: each lane builds the image once with buildx into the local -# daemon (no per-lane from-scratch compose build) and tests it via -# E2E_IMAGE. Unchanged stages come from the GHA layer cache shared with -# `ci.yml`'s `image` job; the build's cache mounts (cargo registry and -# target/, bun's package cache) come from the Actions cache, so a source -# change recompiles the runner crate alone. Playwright then runs spec files -# in parallel (E2E_WORKERS) against the one stack. +# Speed shape: `ci.yml` builds the image once per run (its `image` job) and +# hands it over as an artifact; each lane below downloads and `docker load`s +# it instead of rebuilding (every lane paid ~280 s for its own build before). +# Playwright then runs each lane's spec files sharded across 2 jobs +# (E2E_SHARD) against their own stacks, 3 workers per shard as before. +# Without the artifact input (the tag gate, the nightly schedule, dispatch), +# each lane builds exactly as before. name: e2e on: @@ -27,6 +27,16 @@ on: description: Which spec set to run (fast = the tagged PR gate, release = every spec). required: false type: string + image_artifact: + default: "" + description: Run-local artifact holding the lane image (docker save piped to zstd). When set, each lane downloads and loads it instead of building. + required: false + type: string + rounds: + default: "1" + description: How many back-to-back rounds to run per lane (the nightly hunt runs 3). + required: false + type: string workflow_dispatch: inputs: suite: @@ -35,10 +45,17 @@ on: options: [release, fast] required: false type: choice + rounds: + default: "1" + description: How many back-to-back rounds to run per lane (the nightly hunt runs 3). + options: ["1", "2", "3"] + required: false + type: choice # Nightly release-suite run on main: the tag gate only runs on `v*`, so this # is what catches a slow main-branch regression in a spec outside the fast # set. `inputs.suite` is empty on a schedule trigger; the lane step below - # falls back to `release`. + # falls back to `release`. Three rounds per lane, so flakes surface; the + # flakes job below files them as `flake` issues. schedule: - cron: "0 4 * * *" @@ -46,17 +63,42 @@ permissions: contents: read jobs: + # Which rounds to run: 3 on the nightly schedule, the `rounds` input on + # dispatch, 1 for every reusable caller. A job (not an inline expression) + # so the lane matrix below stays clean instead of scattering skipped cells. + plan: + runs-on: ubuntu-latest + outputs: + rounds: ${{ steps.rounds.outputs.json }} + steps: + - id: rounds + run: | + n='${{ inputs.rounds }}' + if [ '${{ github.event_name }}' = 'schedule' ]; then n=3; fi + case "$n" in + 1) echo 'json=[1]' >> "$GITHUB_OUTPUT" ;; + 2) echo 'json=[1,2]' >> "$GITHUB_OUTPUT" ;; + *) echo 'json=[1,2,3]' >> "$GITHUB_OUTPUT" ;; + esac + e2e: - name: e2e (${{ matrix.lane }}) + name: e2e (${{ matrix.lane }} round ${{ matrix.round }} shard ${{ matrix.shard }}) + needs: plan runs-on: ubuntu-latest timeout-minutes: 40 - # The two lanes boot their own stack from the same tree and share nothing, - # so they run side by side: the gate takes as long as the slower lane - # instead of both in a row. `fail-fast: false` keeps the other lane's - # verdict (and its report) when one fails. + permissions: + contents: read + actions: read + # The lanes boot their own stack from the same image and share nothing, + # so shards and lanes run side by side: the gate takes as long as the + # slowest shard instead of everything in a row. `fail-fast: false` keeps + # the other cells' verdicts (and their reports) when one fails. strategy: fail-fast: false matrix: + lane: [single, multi] + round: ${{ fromJSON(needs.plan.outputs.rounds) }} + shard: [1, 2] include: - lane: single target: e2e @@ -73,8 +115,9 @@ jobs: working-directory: apps/noite run: bun install --frozen-lockfile # Browser binaries (~170 MB) survive across runs; the install below - # stays `--with-deps` so the OS libraries are still ensured, and it - # skips binaries already on disk. + # skips binaries already on disk. No `--with-deps`: the ubuntu runner + # image already ships Chromium's shared libraries, and the apt step swung + # from 21 to 260 s run to run. - name: cache playwright browsers uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 with: @@ -82,14 +125,16 @@ jobs: path: ~/.cache/ms-playwright - name: playwright chromium working-directory: apps/noite - run: bunx playwright install --with-deps chromium + run: bunx playwright install chromium - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 # The runner and UI stages compile into cache mounts, which a hosted # runner's throwaway builder never keeps: every lane recompiled every # Rust dependency (~4 min) even with the layer cache warm. Carry the # mounts in the Actions cache, keyed on both lockfiles, and inject them - # into the builder; a lockfile change saves a fresh copy. + # into the builder; a lockfile change saves a fresh copy. Skipped when + # the lane loads the shared image instead of building. - name: cache build mounts + if: inputs.image_artifact == '' id: mounts uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 with: @@ -100,6 +145,7 @@ jobs: runner-target bun-cache - name: inject build mounts + if: inputs.image_artifact == '' uses: reproducible-containers/buildkit-cache-dance@5422eac04292c961a382e0f584ea0f03ad9da723 # v3.4.0 with: cache-map: | @@ -113,8 +159,10 @@ jobs: # The lane's image, built once per lane straight into the daemon the # compose stack below uses — no second from-scratch build inside # `make e2e`. Same Dockerfile target and GHA cache scope as `ci.yml`'s - # `image` job, so PR layers warm this build. + # `image` job, so PR layers warm this build. Skipped when the caller + # hands over the shared image (see below). - name: build lane image + if: inputs.image_artifact == '' uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 with: cache-from: type=gha,scope=noite-linux/amd64 @@ -124,12 +172,54 @@ jobs: load: true tags: noite:e2e target: noite + # The shared image `ci.yml` built once for this run: download and load + # it instead of the build above. Same bytes every lane tests. + - name: download lane image + if: inputs.image_artifact != '' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: ${{ inputs.image_artifact }} + path: /tmp/noite-image + - name: load lane image + if: inputs.image_artifact != '' + run: zstd -d -c /tmp/noite-image/noite-ci.tar.zst | docker load - name: e2e lane - run: E2E_IMAGE=noite:e2e E2E_SUITE=${{ inputs.suite || 'release' }} make ${{ matrix.target }} + run: E2E_IMAGE=${{ inputs.image_artifact == '' && 'noite:e2e' || 'noite:ci' }} E2E_SUITE=${{ inputs.suite || 'release' }} E2E_SHARD=${{ matrix.shard }}/2 make ${{ matrix.target }} - name: upload playwright report if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: - name: playwright-report-${{ matrix.lane }} + name: playwright-report-${{ matrix.lane }}-r${{ matrix.round }}-s${{ matrix.shard }} path: apps/noite/playwright-report/ retention-days: 7 + + # The nightly flake hunt's paper trail: after every scheduled (and every + # hand-dispatched multi-round) run, read each lane's JSON results and file + # one `flake`-labelled issue per failing spec title — or comment on the + # already-open one. No retries anywhere: a failure is a finding. Green runs + # file nothing and pass. + flakes: + name: flake issues + needs: [plan, e2e] + if: ${{ !cancelled() && needs.e2e.result != 'skipped' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + - name: download result reports + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + pattern: playwright-report-* + path: /tmp/e2e-reports + # `--lanes`/`--shards` mirror the e2e matrix above and `--rounds` is the + # plan's output: a planned cell without a result file fails this job. + - name: open or update flake issues + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ROUNDS: ${{ needs.plan.outputs.rounds }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: bun .github/scripts/flake-issues.ts --reports /tmp/e2e-reports --run-url "$RUN_URL" --lanes single,multi --rounds "$ROUNDS" --shards 2 diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 595fe6e..f8799a5 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -56,6 +56,13 @@ jobs: e2e: if: startsWith(github.ref, 'refs/tags/v') needs: check + # Permission envelope for the called workflow (see ci.yml): the lanes + # download nothing here (no artifact input) but still declare + # `actions: read`, and the envelope must cover the callee's jobs. + permissions: + contents: read + actions: read + issues: write uses: ./.github/workflows/e2e.yml with: suite: release diff --git a/AGENTS.md b/AGENTS.md index 1957ac1..9b8ee97 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ Ground truth for working on Noite. Read this before touching code — the rules - `make up` — the `noite` service built from the tree (`compose.build.yaml`) + `rustfs`; `make up-prod` — same file, pulls the release image; `make dev` — the same service from the Dockerfile's `dev` target with the sources bind-mounted (`docker/dev.sh`: cargo-watch runner + `vite dev` on `127.0.0.1:8090`, where the control fleet would be) - `make logs`, `make doctor` (`/ready` and its failing gates, API auth, edge, control auth, celld control node), `make down` (keeps volumes), `make nuke` (every local database: the project's volumes + local `.wrangler`/`.void` D1/SQLite). `nuke` removes containers by project label and volumes by name as well as via `down -v` because podman-compose's `down -v` aborts on the first missing container and then never removes named volumes — which left the runner's SQLite (every app row) in place and the next `apps.create` answered `409 slug already taken`. BYO S3 installs (`--scale rustfs=0`) keep their objects outside compose: delete them at the provider. - `make up`/`make dev` pass `--force-recreate` on purpose: podman-compose does **not** recreate a container whose image changed (`docker compose` does), so an image rebuild would otherwise silently keep the old bytes running. - - E2E: `make e2e` — boots the prod image in its own compose project (`-p noite-e2e`): control UI through Caddy on :8090, runner API/git :8080, tenants 20000+, rustfs :19000 (`E2E_RUSTFS_PORT`); then runs doctor + Playwright (passkey signup, git push, deploy, tenant serve). Builds the image from the tree by default; `TAG= make e2e` pulls that release from GHCR instead. `make e2e-isolation` is the same lane in `NOITE_TENANCY=multi` plus the hostile-tenant spec (`E2E_HOSTILE=1`). Two suites over the same specs: `E2E_SUITE=fast` runs the tagged PR gate (`@fast` on single, `@isolation` on multi, plus the setup/policy projects), and the default `release` runs every spec. Tagging is opt-in: a new test joins `release`, and joining the `fast` gate is a decision. On PRs, `ci.yml` runs `fast` automatically when runtime code changes (`apps/**`, `packages/**`, `docker/**`, `Makefile`, `bun.lock`, `.env.example`, `e2e.yml`) and `release` on the `e2e` label; `E2E_SPECS="e2e/a.spec.ts …"` narrows either suite to files. `E2E_KEEP=1` leaves the stack up for debugging. Always fresh: the lane removes its own project's containers and volumes (by label and name) and uses its own `noite-e2e` bucket. Stop the prod/dev stack first (same host ports). The lane lock is shared (`/tmp/noite-e2e-lane.lock`): wrap runs in `flock`, never hold it more than 30 min, and prefer CI for full runs. + - E2E: `make e2e` — boots the prod image in its own compose project (`-p noite-e2e`): control UI through Caddy on :8090, runner API/git :8080, tenants 20000+, rustfs :19000 (`E2E_RUSTFS_PORT`); then runs doctor + Playwright (passkey signup, git push, deploy, tenant serve). Builds the image from the tree by default; `TAG= make e2e` pulls that release from GHCR instead. `make e2e-isolation` is the same lane in `NOITE_TENANCY=multi` plus the hostile-tenant spec (`E2E_HOSTILE=1`). Two suites over the same specs: `E2E_SUITE=fast` runs the tagged PR gate (`@fast` on single, `@isolation` on multi, plus the setup/policy projects), and the default `release` runs every spec. Tagging is opt-in: a new test joins `release`, and joining the `fast` gate is a decision. On PRs, `ci.yml` runs `fast` automatically when runtime code changes (`apps/**`, `packages/**`, `docker/**`, `Makefile`, `bun.lock`, `.env.example`, `e2e.yml`) and `release` on the `e2e` label; `E2E_SPECS="e2e/a.spec.ts …"` narrows either suite to files. `E2E_KEEP=1` leaves the stack up for debugging. Always fresh: the lane removes its own project's containers and volumes (by label and name) and uses its own `noite-e2e` bucket. Stop the prod/dev stack first (same host ports). The lane lock is shared (`/tmp/noite-e2e-lane.lock`): wrap runs in `flock`, never hold it more than 30 min, and prefer CI for full runs. CI builds the image once per run (`ci.yml` `image` job) and each lane loads it; each lane shards its suite across 2 jobs (`E2E_SHARD`, reports `playwright-report-{lane}-r{round}-s{shard}`); the nightly schedule runs 3 rounds per lane and files `flake` issues. - `up`/`dev` build with cache every run, so no separate build targets ## Merging @@ -40,7 +40,7 @@ Ground truth for working on Noite. Read this before touching code — the rules - `packages/cli` — `@noitenow/cli` (Effect CLI `noite deploy`: CI-built dist over Git smart-HTTP). - `apps/noite/test` — sample app + `deploy.sh`; also a nested git repo. - `docker/` — **every Docker artifact lives here**: the one `Dockerfile` (targets `noite` = the release image, the default, and `dev`), its context guard `Dockerfile.dockerignore` (`.dockerignore` wins over a context-root `.dockerignore`, and patterns are relative to the repo-root context, so nested paths need `**/`), `install-sidecars.sh`, `dev.sh` (the dev target's command), Compose files (`compose.yaml` = the whole install, pull-only, every variable defaulted, Compose + Coolify via env; `compose.build.yaml` builds it from the tree; `compose.dev.yaml` dev overlay; `compose.e2e.yaml` raw-port test lane), `doctor.sh`, `e2e-local.sh`, `backup.sh`, `restore.sh`, `check-versions.ts` (pin drift guard), and `lib.sh` (the shared helper sourced by the other `docker/*.sh` scripts; `install.sh`/`uninstall.sh` stay self-contained because `run.sh` curls each of them standalone). No `Dockerfile*`/`.dockerignore`/`compose*` may sit outside this directory. `Makefile` at the root. - - `.github/workflows/` — `check.yml` is the reusable gate (`workflow_call`) consumed by `ci.yml` (pull requests and the merge queue) and `images.yml` (before a release image): lint/format, version-pin drift, every compose variant, script parsing, schema idempotency, the CLI bundle. `ci.yml` adds the Rust crate, the worker bundle and the `ci-ok` aggregate (the required check); `security-gate.yml` holds the sensitive-change review (required check, with `ci-ok`). `e2e.yml` holds the container lanes (callable with a `suite` input, dispatched by hand, nightly on a schedule). The `types` job (in `check.yml`, so both callers get it) regenerates the runner→UI bindings and fails on any diff. + - `.github/workflows/` — `check.yml` is the reusable gate (`workflow_call`) consumed by `ci.yml` (pull requests and the merge queue) and `images.yml` (before a release image): lint/format, version-pin drift, every compose variant, script parsing, schema idempotency, the CLI bundle. `ci.yml` adds the Rust crate, the worker bundle and the `ci-ok` aggregate (the required check), and builds the release image once and hands it to the lanes as an artifact; `security-gate.yml` holds the sensitive-change review (required check, with `ci-ok`). `e2e.yml` holds the container lanes (callable with `suite`, `image_artifact` and `rounds` inputs, dispatched by hand, nightly on a schedule with 3 rounds per lane and a `flake`-issue follow-up). The `types` job (in `check.yml`, so both callers get it) regenerates the runner→UI bindings and fails on any diff. - **Runner → UI types are generated** ([ts-rs](https://github.com/Aleph-Alpha/ts-rs)): every Rust struct/enum on the REST boundary carries `#[derive(ts_rs::TS)] #[ts(export)]`, and `cd apps/runner && ~/.cargo/bin/cargo test --locked export_bindings` writes them to `apps/noite/src/lib/runner-types`. `apps/runner/.cargo/config.toml` sets `TS_RS_EXPORT_DIR` (that directory, relative) and `TS_RS_LARGE_INT = "number"` (i64/u64 stay TypeScript `number`), and `apps/noite/src/lib/runner.ts` imports and re-exports the files under the UI's long-standing `Runner*` aliases — UI-only derived shapes (the D1 caps extensions, `D1Cell`/`D1Key`/`D1RowsQuery`, error status, admin stats) stay hand-written there. The directory is generated (never edit it by hand; oxlint/oxfmt ignore it), and CI's `types` job (in `check.yml`) regenerates it and fails on any diff. - URLs: control UI `http://noite.localhost:9080` (prod `https://app.noite.now` via `CONTROL_SUBDOMAIN=app`; the dev base is a `*.localhost` name — a browser secure context, and the only hostname family Bitwarden's matcher accepts without https), runner REST `http://api.noite.localhost:9080`, Git HTTP `http://git.noite.localhost:9080/{slug}`, rustfs S3 `:9000`, console `:9001`, deployed apps `http://{slug}.noite.localhost:9080` (prod `https://{slug}.noite.now`; `app`/`api`/`git` slugs reserved). diff --git a/apps/noite/playwright.config.ts b/apps/noite/playwright.config.ts index e48a25d..56fc362 100644 --- a/apps/noite/playwright.config.ts +++ b/apps/noite/playwright.config.ts @@ -19,6 +19,26 @@ const baseURL = process.env.E2E_BASE_URL ?? "http://noite.localhost:8090"; // (per-app locks only), so concurrent deploys contend on CPU but proceed; // each deploy keeps its 8-minute budget, and 3 workers leave headroom on a // 4-vCPU GitHub runner. +// Sharding across CI jobs (E2E_SHARD="/", set by e2e.yml): +// each shard still runs the setup and policy dependency projects in full +// (verified: Playwright repeats project dependencies on every shard), so +// every shard signs up and probes policy beside its half of the bulk. Unset +// locally means today's behaviour: one unsharded run. +const shardValue = process.env.E2E_SHARD; +let shard: { current: number; total: number } | undefined; + +if (shardValue !== undefined && shardValue !== "") { + const match = /^(?\d+)\/(?\d+)$/u.exec(shardValue); + const current = Number(match?.groups?.current); + const total = Number(match?.groups?.total); + + if (match === null || total < 1 || current < 1 || current > total) { + throw new Error(`E2E_SHARD must be "/", not ${shardValue}`); + } + + shard = { current, total }; +} + const workers = Number(process.env.E2E_WORKERS ?? 3); // Two suites over the same specs (E2E_SUITE, set by docker/e2e-local.sh): @@ -77,7 +97,15 @@ export default defineConfig({ use: { storageState: "e2e/.auth/user.json" }, }, ], - reporter: [["list"], ["html", { open: "never" }]], + reporter: [ + ["list"], + ["html", { open: "never" }], + // Machine-readable results for the nightly flake hunt (e2e.yml's flakes + // job reads this from the uploaded report). Local runs write it too, + // next to the html report; both are gitignored. + ["json", { outputFile: "playwright-report/e2e-results.json" }], + ], + shard, testDir: "./e2e", // Deploys (bun install + celld deploy + fleet boot) take minutes. timeout: 10 * 60_000, diff --git a/changelog.d/ci-e2e-speed.md b/changelog.d/ci-e2e-speed.md new file mode 100644 index 0000000..fe6dc7a --- /dev/null +++ b/changelog.d/ci-e2e-speed.md @@ -0,0 +1,3 @@ +### Changed + +- **CI builds the lane image once and shards each e2e lane.** The `ci.yml` image job hands its build to the lanes as a short-lived artifact, so neither lane rebuilds it; each lane's suite then splits across 2 jobs, and the Playwright browser install skips the apt step. The nightly schedule runs 3 rounds per lane and files a `flake` issue per failing spec (or comments on the open one); `workflow_dispatch` on `e2e.yml` takes a `rounds` input for the same. No suite lost a spec and nothing retries.