From 350a1d36834e2644c22236ea7f36093162e8991c Mon Sep 17 00:00:00 2001 From: sarahxsanders Date: Wed, 17 Jun 2026 17:38:01 -0400 Subject: [PATCH 1/6] =?UTF-8?q?test(cli):=20ANSI=20screenshot=20harness=20?= =?UTF-8?q?for=20command=20=E2=86=92=20intro-screen=20wiring?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds scripts/cli-screenshots.mjs (pnpm screens:cli): for each command it runs the real binary in a pseudo-terminal (via `script`, since Ink needs a TTY), captures the raw ANSI output, and diffs it against a committed golden dump under scripts/__screenshots__/. Catches regressions where a command falls through to the default flow, renders the wrong intro screen, or doesn't render at all — the class of bug we hit this cycle. Goldens are raw ANSI ("jank screenshots"); normalize() strips volatile cursor/spinner sequences before comparing so a live TUI doesn't flake. Complements the component-level scripts/check-screens.tsx. Includes a workflow_dispatch CI job. NOT yet wired to gate PRs: the goldens need seeding once in a real terminal/CI (`pnpm build && pnpm screens:cli --update`, commit scripts/__screenshots__/), then enable the pull_request/push triggers. (Couldn't seed here — the sandbox has no pty.) Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/cli-screenshots.yml | 56 ++++++++ package.json | 3 +- scripts/cli-screenshots.mjs | 182 ++++++++++++++++++++++++++ 3 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/cli-screenshots.yml create mode 100644 scripts/cli-screenshots.mjs diff --git a/.github/workflows/cli-screenshots.yml b/.github/workflows/cli-screenshots.yml new file mode 100644 index 000000000..d619ee621 --- /dev/null +++ b/.github/workflows/cli-screenshots.yml @@ -0,0 +1,56 @@ +name: CLI Screenshot Tests + +# Verifies every command renders its expected intro screen by capturing the +# real binary's ANSI output in a pseudo-terminal and diffing against committed +# goldens under scripts/__screenshots__/. See scripts/cli-screenshots.mjs. +# +# Trigger is workflow_dispatch (manual) until the goldens are seeded: run +# pnpm build && pnpm screens:cli --update +# in a real terminal, commit scripts/__screenshots__/, then add the triggers +# below so it gates PRs: +# pull_request: +# push: +# branches: [main] +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + cli-screenshots: + name: CLI screenshot tests + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Install pnpm + uses: pnpm/action-setup@eae0cfeb286e66ffb5155f1a79b90583a127a68b # v2.4.1 + with: + version: 10.23.0 + run_install: false + + - name: Set up Node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: 'package.json' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build wizard + run: pnpm build + + - name: Check command screens + run: node scripts/cli-screenshots.mjs + + - name: Upload captured screenshots + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cli-screenshots + path: scripts/__screenshots__/** + if-no-files-found: ignore diff --git a/package.json b/package.json index 63f36479e..2002d741b 100644 --- a/package.json +++ b/package.json @@ -120,7 +120,8 @@ "dev": "pnpm build && pnpm link --global && pnpm build:watch", "test:watch": "jest --watch", "prepare": "husky", - "screens:check": "tsx scripts/check-screens.tsx" + "screens:check": "tsx scripts/check-screens.tsx", + "screens:cli": "node scripts/cli-screenshots.mjs" }, "jest": { "collectCoverage": true, diff --git a/scripts/cli-screenshots.mjs b/scripts/cli-screenshots.mjs new file mode 100644 index 000000000..b1f0a2760 --- /dev/null +++ b/scripts/cli-screenshots.mjs @@ -0,0 +1,182 @@ +#!/usr/bin/env node +/** + * ANSI "screenshot" tests for the wizard CLI command surface. + * + * For each command below, this launches the REAL built binary in a + * pseudo-terminal (Ink won't render without a TTY) using the system `script` + * command, captures the raw ANSI output, and compares it to a committed golden + * dump under `scripts/__screenshots__/`. It catches regressions in the + * command → intro-screen wiring: a command falling through to the default + * flow, an intro screen not rendering, or the wrong screen showing. + * + * The goldens are the "jank screenshots" — raw ANSI bytes, so they capture + * colour + layout, not just stripped text. + * + * A live TUI animates (spinners, cursor moves), so a byte-exact compare would + * flake. `normalize()` strips the volatile escape sequences before comparing + * (it keeps SGR colour codes so the screenshot still shows styling). Tune it if + * a screen still flakes. + * + * Usage: + * pnpm build && node scripts/cli-screenshots.mjs # check vs goldens + * pnpm build && node scripts/cli-screenshots.mjs --update # (re)capture goldens + * + * NOTE: a pseudo-terminal is required, so goldens must be seeded once with + * `--update` in a real terminal or CI runner — they are not generated in + * environments without a TTY. + */ + +import { spawn } from 'node:child_process'; +import { + existsSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const REPO = path.resolve(HERE, '..'); +const BIN = path.join(REPO, 'dist', 'bin.js'); +const GOLDEN_DIR = path.join(HERE, '__screenshots__'); + +/** How long to let a screen render before killing it (intro screens wait for input). */ +const CAPTURE_MS = 4000; + +/** + * Commands to snapshot. `slug` is the golden filename; `args` is the wizard + * argv. Keep this in sync with the command surface (`bin.ts` + the audit + * family). `unknown-command` is a negative case — it must error, not run a flow. + */ +const COMMANDS = [ + { slug: 'default', args: [] }, + { slug: 'audit', args: ['audit'] }, + { slug: 'audit-events', args: ['audit', 'events'] }, + { slug: 'audit-all', args: ['audit', 'all'] }, + { slug: 'revenue-analytics', args: ['revenue-analytics'] }, + { slug: 'migrate', args: ['migrate'] }, + { slug: 'upload-source-maps', args: ['upload-source-maps'] }, + { slug: 'mcp-add', args: ['mcp', 'add'] }, + { slug: 'slack-add', args: ['slack', 'add'] }, + { slug: 'skill-list', args: ['skill', 'list'] }, + { slug: 'doctor', args: ['doctor'] }, + { slug: 'unknown-command', args: ['asdf'] }, +]; + +const UPDATE = process.argv.includes('--update'); + +/** Build the platform-specific `script` invocation that runs the wizard in a pty. */ +function scriptInvocation(outFile, args) { + const inner = ['node', BIN, ...args, '--no-telemetry']; + if (process.platform === 'darwin') { + // BSD script: `script -q ` + return ['-q', outFile, ...inner]; + } + // util-linux: `script -q -e -c "" ` + const command = inner.map((a) => (/\s/.test(a) ? JSON.stringify(a) : a)).join(' '); + return ['-q', '-e', '-c', command, outFile]; +} + +/** Run one command in a pty, kill it after CAPTURE_MS, return the captured bytes. */ +function capture(args) { + return new Promise((resolve, reject) => { + const outFile = path.join( + tmpdir(), + `wizard-shot-${process.pid}-${Math.random().toString(36).slice(2)}.ans`, + ); + const child = spawn('script', scriptInvocation(outFile, args), { + stdio: 'ignore', + }); + const timer = setTimeout(() => child.kill('SIGTERM'), CAPTURE_MS); + child.on('error', (err) => { + clearTimeout(timer); + reject(err); + }); + child.on('close', () => { + clearTimeout(timer); + try { + const buf = existsSync(outFile) ? readFileSync(outFile) : Buffer.alloc(0); + rmSync(outFile, { force: true }); + resolve(buf); + } catch (err) { + reject(err); + } + }); + }); +} + +/** Strip volatile terminal noise so comparisons don't flake on animation/timing. */ +function normalize(buf) { + return ( + buf + .toString('utf8') + // `script` wrapper lines + .replace(/^Script (started|done).*$/gm, '') + // cursor moves / clear-line / clear-screen — i.e. how spinners repaint. + // Keep SGR (`\x1b[...m`) so the screenshot still carries colour. + .replace(/\x1b\[[0-9;]*[ABCDEFGHJKnsu]/g, '') + .replace(/\r/g, '') + .replace(/\n{3,}/g, '\n\n') + .trim() + ); +} + +async function main() { + if (!existsSync(BIN)) { + console.error(`✖ ${BIN} not found — run \`pnpm build\` first.`); + process.exit(1); + } + mkdirSync(GOLDEN_DIR, { recursive: true }); + + let failures = 0; + for (const { slug, args } of COMMANDS) { + const goldenFile = path.join(GOLDEN_DIR, `${slug}.ans`); + const label = `wizard ${args.join(' ') || '(default)'}`; + + let captured; + try { + captured = await capture(args); + } catch (err) { + console.error(`✖ ${slug} (${label}): capture failed — ${err.message}`); + failures++; + continue; + } + if (captured.length === 0) { + // No output means no pty (e.g. `script` couldn't allocate one) — fail + // loudly rather than write/compare an empty golden. + console.error( + `✖ ${slug} (${label}): empty capture — needs a real terminal/CI (is \`script\` available?)`, + ); + failures++; + continue; + } + + if (UPDATE) { + writeFileSync(goldenFile, captured); + console.log(`updated ${slug}`); + continue; + } + if (!existsSync(goldenFile)) { + console.error(`✖ ${slug} (${label}): no golden yet — run with --update to seed it.`); + failures++; + continue; + } + if (normalize(captured) === normalize(readFileSync(goldenFile))) { + console.log(`ok ${slug}`); + } else { + console.error(`✖ ${slug} (${label}): output changed vs golden`); + failures++; + } + } + + if (failures > 0) { + console.error(`\n${failures} screenshot check(s) failed`); + process.exit(1); + } + console.log(`\nAll ${COMMANDS.length} screenshot checks passed`); +} + +main(); From 5862ec87054d445f92a4c276e156503c3c01e66b Mon Sep 17 00:00:00 2001 From: sarahxsanders Date: Wed, 17 Jun 2026 17:54:48 -0400 Subject: [PATCH 2/6] fix(screenshots): kill the capture's whole process group (no orphans) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SIGTERM to `script` left the inner wizard running (script doesn't forward signals), and a stray wizard holds resources like the OAuth-callback port — so the first capture worked and every one after it came back empty. Spawn detached and SIGINT the process group (Ink then restores the terminal; the inner node dies), with a SIGKILL fallback. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/cli-screenshots.mjs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/cli-screenshots.mjs b/scripts/cli-screenshots.mjs index b1f0a2760..34aacbf7a 100644 --- a/scripts/cli-screenshots.mjs +++ b/scripts/cli-screenshots.mjs @@ -89,14 +89,30 @@ function capture(args) { ); const child = spawn('script', scriptInvocation(outFile, args), { stdio: 'ignore', + // Own process group, so we can signal the inner wizard too — `script` + // doesn't forward signals to its child. + detached: true, }); - const timer = setTimeout(() => child.kill('SIGTERM'), CAPTURE_MS); + // SIGINT (not SIGTERM) so Ink restores the terminal; signal the whole group + // so the inner `node` actually dies. A stray wizard left running holds + // resources (e.g. the OAuth-callback port) and empties the next capture. + const stop = (signal) => { + try { + if (child.pid) process.kill(-child.pid, signal); + } catch { + /* already exited */ + } + }; + const timer = setTimeout(() => stop('SIGINT'), CAPTURE_MS); + const hardTimer = setTimeout(() => stop('SIGKILL'), CAPTURE_MS + 2000); child.on('error', (err) => { clearTimeout(timer); + clearTimeout(hardTimer); reject(err); }); child.on('close', () => { clearTimeout(timer); + clearTimeout(hardTimer); try { const buf = existsSync(outFile) ? readFileSync(outFile) : Buffer.alloc(0); rmSync(outFile, { force: true }); From 739119cf7feb219ef5d00815802e18de3e77e5f5 Mon Sep 17 00:00:00 2001 From: sarahxsanders Date: Wed, 17 Jun 2026 18:14:29 -0400 Subject: [PATCH 3/6] test(cli): capture screenshots with node-pty (sized pty, raw binary, byte-exact) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the `script`-based capture, which painted blank frames because the pty had no window size when detached. node-pty gives a real pty with a fixed size (COLS×ROWS), forced colour (stable local↔CI), raw-byte capture (encoding: null), and a clean kill. Capture waits for the screen to settle, then snapshots; goldens are raw binary ANSI dumps compared byte-for-byte. Adds node-pty as a devDependency (not bundled into the shipped wizard) and to pnpm.onlyBuiltDependencies so its native build runs on install. Co-Authored-By: Claude Opus 4.8 (1M context) --- package.json | 6 ++ pnpm-lock.yaml | 15 ++++ scripts/cli-screenshots.mjs | 167 +++++++++++++++--------------------- 3 files changed, 90 insertions(+), 98 deletions(-) diff --git a/package.json b/package.json index 2002d741b..7d990b3f6 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "jest": "^29.5.0", "lint-staged": "^15.5.1", "msw": "^2.10.4", + "node-pty": "^1.1.0", "prettier": "^2.8.7", "rimraf": "^3.0.2", "ts-jest": "^29.1.0", @@ -98,6 +99,11 @@ "npm": ">=3.10.7" }, "packageManager": "pnpm@10.23.0+sha512.21c4e5698002ade97e4efe8b8b4a89a8de3c85a37919f957e7a0f30f38fbc5bbdd05980ffe29179b2fb6e6e691242e098d945d1601772cad0fef5fb6411e2a4b", + "pnpm": { + "onlyBuiltDependencies": [ + "node-pty" + ] + }, "scripts": { "clean": "rm -rf ./dist", "prebuild": "pnpm clean && node scripts/generate-version.cjs", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 336708a77..b2f02ffc7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -162,6 +162,9 @@ importers: msw: specifier: ^2.10.4 version: 2.10.4(@types/node@18.19.76)(typescript@5.7.3) + node-pty: + specifier: ^1.1.0 + version: 1.1.0 prettier: specifier: ^2.8.7 version: 2.8.8 @@ -3134,9 +3137,15 @@ packages: resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} engines: {node: '>= 0.6'} + node-addon-api@7.1.1: + resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} + node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} + node-pty@1.1.0: + resolution: {integrity: sha512-20JqtutY6JPXTUnL0ij1uad7Qe1baT46lyolh2sSENDd4sTzKZ4nmAFkeAARDKwmlLjPx6XKRlwRUxwjOy+lUg==} + node-releases@2.0.19: resolution: {integrity: sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==} @@ -7471,8 +7480,14 @@ snapshots: negotiator@1.0.0: {} + node-addon-api@7.1.1: {} + node-int64@0.4.0: {} + node-pty@1.1.0: + dependencies: + node-addon-api: 7.1.1 + node-releases@2.0.19: {} node-releases@2.0.27: {} diff --git a/scripts/cli-screenshots.mjs b/scripts/cli-screenshots.mjs index 34aacbf7a..f7de9a507 100644 --- a/scripts/cli-screenshots.mjs +++ b/scripts/cli-screenshots.mjs @@ -2,54 +2,54 @@ /** * ANSI "screenshot" tests for the wizard CLI command surface. * - * For each command below, this launches the REAL built binary in a - * pseudo-terminal (Ink won't render without a TTY) using the system `script` - * command, captures the raw ANSI output, and compares it to a committed golden - * dump under `scripts/__screenshots__/`. It catches regressions in the - * command → intro-screen wiring: a command falling through to the default - * flow, an intro screen not rendering, or the wrong screen showing. + * For each command below, this launches the REAL built binary in a real + * pseudo-terminal (via node-pty — Ink won't render without a TTY, and the pty + * needs a fixed size or screens paint blank), captures the **raw bytes** it + * writes to the terminal, and compares them **byte-for-byte** against a + * committed golden dump under scripts/__screenshots__/. It catches regressions + * in the command → intro-screen wiring: a command falling through to the + * default flow, the wrong screen, or nothing rendering. * - * The goldens are the "jank screenshots" — raw ANSI bytes, so they capture - * colour + layout, not just stripped text. - * - * A live TUI animates (spinners, cursor moves), so a byte-exact compare would - * flake. `normalize()` strips the volatile escape sequences before comparing - * (it keeps SGR colour codes so the screenshot still shows styling). Tune it if - * a screen still flakes. + * The goldens are raw binary ANSI dumps — the actual "screenshot" (colour + + * layout), not stripped text. For byte-exact comparison to be stable: + * - the pty size is fixed (COLS×ROWS), + * - colour output is forced (FORCE_COLOR) so it's identical local ↔ CI, + * - capture waits for the screen to SETTLE (output stops) before snapshotting. + * If a screen animates (e.g. a spinner) it won't settle to a stable frame — + * handle those case-by-case (pin the screen, or exclude it) rather than + * loosening the whole comparison. * * Usage: * pnpm build && node scripts/cli-screenshots.mjs # check vs goldens * pnpm build && node scripts/cli-screenshots.mjs --update # (re)capture goldens * - * NOTE: a pseudo-terminal is required, so goldens must be seeded once with - * `--update` in a real terminal or CI runner — they are not generated in - * environments without a TTY. + * Requires node-pty (devDependency) built — `pnpm install` with node-pty in + * pnpm.onlyBuiltDependencies. */ -import { spawn } from 'node:child_process'; -import { - existsSync, - mkdirSync, - readFileSync, - rmSync, - writeFileSync, -} from 'node:fs'; -import { tmpdir } from 'node:os'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import pty from 'node-pty'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const REPO = path.resolve(HERE, '..'); const BIN = path.join(REPO, 'dist', 'bin.js'); const GOLDEN_DIR = path.join(HERE, '__screenshots__'); -/** How long to let a screen render before killing it (intro screens wait for input). */ -const CAPTURE_MS = 4000; +// Fixed terminal geometry — layout (and therefore the bytes) must be identical +// when seeding and when checking, on macOS and in CI alike. +const COLS = 100; +const ROWS = 40; +/** Snapshot once output has been quiet for this long (screen has painted). */ +const SETTLE_MS = 1200; +/** Hard cap, in case a screen never goes quiet. */ +const MAX_CAPTURE_MS = 12000; /** * Commands to snapshot. `slug` is the golden filename; `args` is the wizard - * argv. Keep this in sync with the command surface (`bin.ts` + the audit - * family). `unknown-command` is a negative case — it must error, not run a flow. + * argv. Keep in sync with the command surface (`bin.ts` + the audit family). + * `unknown-command` is a negative case — it must error, not run a flow. */ const COMMANDS = [ { slug: 'default', args: [] }, @@ -68,78 +68,53 @@ const COMMANDS = [ const UPDATE = process.argv.includes('--update'); -/** Build the platform-specific `script` invocation that runs the wizard in a pty. */ -function scriptInvocation(outFile, args) { - const inner = ['node', BIN, ...args, '--no-telemetry']; - if (process.platform === 'darwin') { - // BSD script: `script -q ` - return ['-q', outFile, ...inner]; - } - // util-linux: `script -q -e -c "" ` - const command = inner.map((a) => (/\s/.test(a) ? JSON.stringify(a) : a)).join(' '); - return ['-q', '-e', '-c', command, outFile]; -} - -/** Run one command in a pty, kill it after CAPTURE_MS, return the captured bytes. */ +/** Run one command in a sized pty, snapshot once it settles, return raw bytes. */ function capture(args) { return new Promise((resolve, reject) => { - const outFile = path.join( - tmpdir(), - `wizard-shot-${process.pid}-${Math.random().toString(36).slice(2)}.ans`, - ); - const child = spawn('script', scriptInvocation(outFile, args), { - stdio: 'ignore', - // Own process group, so we can signal the inner wizard too — `script` - // doesn't forward signals to its child. - detached: true, - }); - // SIGINT (not SIGTERM) so Ink restores the terminal; signal the whole group - // so the inner `node` actually dies. A stray wizard left running holds - // resources (e.g. the OAuth-callback port) and empties the next capture. - const stop = (signal) => { + let proc; + try { + proc = pty.spawn('node', [BIN, ...args, '--no-telemetry'], { + name: 'xterm-256color', + cols: COLS, + rows: ROWS, + cwd: REPO, + // Force a stable colour level so the bytes match across environments. + env: { ...process.env, FORCE_COLOR: '3', TERM: 'xterm-256color' }, + encoding: null, // hand back Buffers, not decoded strings + }); + } catch (err) { + reject(err); + return; + } + + const chunks = []; + let settleTimer; + let done = false; + const finish = () => { + if (done) return; + done = true; + clearTimeout(settleTimer); + clearTimeout(maxTimer); try { - if (child.pid) process.kill(-child.pid, signal); + proc.kill(); } catch { - /* already exited */ + /* already gone */ } }; - const timer = setTimeout(() => stop('SIGINT'), CAPTURE_MS); - const hardTimer = setTimeout(() => stop('SIGKILL'), CAPTURE_MS + 2000); - child.on('error', (err) => { - clearTimeout(timer); - clearTimeout(hardTimer); - reject(err); + proc.onData((data) => { + chunks.push(Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf8')); + clearTimeout(settleTimer); + settleTimer = setTimeout(finish, SETTLE_MS); }); - child.on('close', () => { - clearTimeout(timer); - clearTimeout(hardTimer); - try { - const buf = existsSync(outFile) ? readFileSync(outFile) : Buffer.alloc(0); - rmSync(outFile, { force: true }); - resolve(buf); - } catch (err) { - reject(err); - } + const maxTimer = setTimeout(finish, MAX_CAPTURE_MS); + proc.onExit(() => { + clearTimeout(settleTimer); + clearTimeout(maxTimer); + resolve(Buffer.concat(chunks)); }); }); } -/** Strip volatile terminal noise so comparisons don't flake on animation/timing. */ -function normalize(buf) { - return ( - buf - .toString('utf8') - // `script` wrapper lines - .replace(/^Script (started|done).*$/gm, '') - // cursor moves / clear-line / clear-screen — i.e. how spinners repaint. - // Keep SGR (`\x1b[...m`) so the screenshot still carries colour. - .replace(/\x1b\[[0-9;]*[ABCDEFGHJKnsu]/g, '') - .replace(/\r/g, '') - .replace(/\n{3,}/g, '\n\n') - .trim() - ); -} - async function main() { if (!existsSync(BIN)) { console.error(`✖ ${BIN} not found — run \`pnpm build\` first.`); @@ -161,18 +136,14 @@ async function main() { continue; } if (captured.length === 0) { - // No output means no pty (e.g. `script` couldn't allocate one) — fail - // loudly rather than write/compare an empty golden. - console.error( - `✖ ${slug} (${label}): empty capture — needs a real terminal/CI (is \`script\` available?)`, - ); + console.error(`✖ ${slug} (${label}): empty capture — the screen rendered nothing`); failures++; continue; } if (UPDATE) { writeFileSync(goldenFile, captured); - console.log(`updated ${slug}`); + console.log(`updated ${slug} (${captured.length} bytes)`); continue; } if (!existsSync(goldenFile)) { @@ -180,10 +151,10 @@ async function main() { failures++; continue; } - if (normalize(captured) === normalize(readFileSync(goldenFile))) { + if (captured.equals(readFileSync(goldenFile))) { console.log(`ok ${slug}`); } else { - console.error(`✖ ${slug} (${label}): output changed vs golden`); + console.error(`✖ ${slug} (${label}): bytes differ from golden`); failures++; } } From de40c0114b2d048d1e91afc0fc8bda84dfc6a28c Mon Sep 17 00:00:00 2001 From: sarahxsanders Date: Thu, 18 Jun 2026 10:29:10 -0400 Subject: [PATCH 4/6] test(cli): assert each command loads the right skill via rendered screenshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run each command in a sized pty (node-pty), render the settled screen through a headless emulator (@xterm/headless), and assert it contains a source-grounded marker — the program/skill id the command wires into its intro screen. Catches command -> screen wiring regressions. - marker match, not byte-for-byte golden: survives spinners, async fetches, and project-dependent intros that make whole-frame matching flake - self-heal node-pty's spawn-helper +x bit (pnpm drops it on install) - CI: path-filtered PR gate + daily drift monitor, Slack on scheduled fail Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/cli-screenshots.yml | 37 +++-- AGENTS.md | 1 + package.json | 2 + pnpm-lock.yaml | 16 ++ scripts/__screenshots__/.gitignore | 2 + scripts/cli-screenshots.mjs | 228 +++++++++++++++++++------- 6 files changed, 214 insertions(+), 72 deletions(-) create mode 100644 scripts/__screenshots__/.gitignore diff --git a/.github/workflows/cli-screenshots.yml b/.github/workflows/cli-screenshots.yml index d619ee621..a9da71ad6 100644 --- a/.github/workflows/cli-screenshots.yml +++ b/.github/workflows/cli-screenshots.yml @@ -1,18 +1,21 @@ name: CLI Screenshot Tests -# Verifies every command renders its expected intro screen by capturing the -# real binary's ANSI output in a pseudo-terminal and diffing against committed -# goldens under scripts/__screenshots__/. See scripts/cli-screenshots.mjs. -# -# Trigger is workflow_dispatch (manual) until the goldens are seeded: run -# pnpm build && pnpm screens:cli --update -# in a real terminal, commit scripts/__screenshots__/, then add the triggers -# below so it gates PRs: -# pull_request: -# push: -# branches: [main] +# Asserts each command loads the right skill/program. See scripts/cli-screenshots.mjs. on: workflow_dispatch: + # Drift monitor: catches a context-mill change breaking a command, no PR needed. + # Runs from `main` only (GitHub limitation). + schedule: + - cron: '37 13 * * *' # daily ~06:37 PT + # Pre-merge gate, scoped to files that decide command → screen wiring. + pull_request: + paths: + - 'bin.ts' + - 'src/commands/**' + - 'src/ui/tui/**' + - 'src/lib/programs/**' + - 'scripts/cli-screenshots.mjs' + - '.github/workflows/cli-screenshots.yml' permissions: contents: read @@ -54,3 +57,15 @@ jobs: name: cli-screenshots path: scripts/__screenshots__/** if-no-files-found: ignore + + # Scheduled-only: a PR failure shows a red ❌ already; a scheduled one pings nobody. + - name: Notify Slack on scheduled failure + if: failure() && github.event_name == 'schedule' + uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 + with: + webhook: ${{ secrets.SLACK_WEBHOOK_WIZARD_CHANNEL }} + webhook-type: incoming-webhook + payload: | + { + "text": "🖼️ CLI screenshot check failed on the scheduled run — a command may be loading the wrong screen, or context-mill drifted. <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" + } diff --git a/AGENTS.md b/AGENTS.md index 68189191a..4fb547d60 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -121,6 +121,7 @@ pnpm build # Compile TypeScript pnpm test # Unit tests (builds first) pnpm test:watch # Unit tests in watch mode pnpm test:e2e # End-to-end tests +pnpm screens:cli # Assert each command loads the right skill (renders + checks its intro screen) pnpm lint # Prettier + ESLint checks pnpm fix # Auto-fix lint issues pnpm dev # Build, link globally, watch for changes diff --git a/package.json b/package.json index 7d990b3f6..b0abc85ad 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,8 @@ "@types/yargs": "^16.0.9", "@typescript-eslint/eslint-plugin": "^5.13.0", "@typescript-eslint/parser": "^5.13.0", + "@xterm/addon-serialize": "0.14.0", + "@xterm/headless": "6.0.0", "babel-jest": "^29.7.0", "dotenv": "^16.4.7", "eslint": "^8.18.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b2f02ffc7..58c478a11 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -132,6 +132,12 @@ importers: '@typescript-eslint/parser': specifier: ^5.13.0 version: 5.62.0(eslint@8.57.1)(typescript@5.7.3) + '@xterm/addon-serialize': + specifier: 0.14.0 + version: 0.14.0 + '@xterm/headless': + specifier: 6.0.0 + version: 6.0.0 babel-jest: specifier: ^29.7.0 version: 29.7.0(@babel/core@7.29.0) @@ -1642,6 +1648,12 @@ packages: resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==} engines: {node: '>=10.0.0'} + '@xterm/addon-serialize@0.14.0': + resolution: {integrity: sha512-uteyTU1EkrQa2Ux6P/uFl2fzmXI46jy5uoQMKEOM0fKTyiW7cSn0WrFenHm5vO5uEXX/GpwW/FgILvv3r0WbkA==} + + '@xterm/headless@6.0.0': + resolution: {integrity: sha512-5Yj1QINYCyzrZtf8OFIHi47iQtI+0qYFPHmouEfG8dHNxbZ9Tb9YGSuLcsEwj9Z+OL75GJqPyJbyoFer80a2Hw==} + accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} @@ -5789,6 +5801,10 @@ snapshots: '@xmldom/xmldom@0.8.10': {} + '@xterm/addon-serialize@0.14.0': {} + + '@xterm/headless@6.0.0': {} + accepts@2.0.0: dependencies: mime-types: 3.0.2 diff --git a/scripts/__screenshots__/.gitignore b/scripts/__screenshots__/.gitignore new file mode 100644 index 000000000..b24039a1c --- /dev/null +++ b/scripts/__screenshots__/.gitignore @@ -0,0 +1,2 @@ +# Rendered screenshots — regenerated every run, not committed. +*.ans diff --git a/scripts/cli-screenshots.mjs b/scripts/cli-screenshots.mjs index f7de9a507..c57a644d7 100644 --- a/scripts/cli-screenshots.mjs +++ b/scripts/cli-screenshots.mjs @@ -1,72 +1,157 @@ #!/usr/bin/env node /** - * ANSI "screenshot" tests for the wizard CLI command surface. + * "Screenshot" tests for the wizard CLI command surface. * - * For each command below, this launches the REAL built binary in a real - * pseudo-terminal (via node-pty — Ink won't render without a TTY, and the pty - * needs a fixed size or screens paint blank), captures the **raw bytes** it - * writes to the terminal, and compares them **byte-for-byte** against a - * committed golden dump under scripts/__screenshots__/. It catches regressions - * in the command → intro-screen wiring: a command falling through to the - * default flow, the wrong screen, or nothing rendering. + * Per command: run the built binary in a sized pty, render the settled screen + * through a headless emulator, and assert it contains a `marker` — the + * program/skill id the command's source wires into the intro screen. Catches a + * command routing to the wrong screen, the default flow, or nothing. * - * The goldens are raw binary ANSI dumps — the actual "screenshot" (colour + - * layout), not stripped text. For byte-exact comparison to be stable: - * - the pty size is fixed (COLS×ROWS), - * - colour output is forced (FORCE_COLOR) so it's identical local ↔ CI, - * - capture waits for the screen to SETTLE (output stops) before snapshotting. - * If a screen animates (e.g. a spinner) it won't settle to a stable frame — - * handle those case-by-case (pin the screen, or exclude it) rather than - * loosening the whole comparison. + * Marker, not byte-for-byte golden: spinners, async fetches, and + * project-dependent intros make whole-frame matching flake. We read the same + * rendered screenshot, just assert the one line that proves the routing. * - * Usage: - * pnpm build && node scripts/cli-screenshots.mjs # check vs goldens - * pnpm build && node scripts/cli-screenshots.mjs --update # (re)capture goldens - * - * Requires node-pty (devDependency) built — `pnpm install` with node-pty in - * pnpm.onlyBuiltDependencies. + * Usage: pnpm build && node scripts/cli-screenshots.mjs */ -import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { + chmodSync, + existsSync, + mkdirSync, + readdirSync, + writeFileSync, +} from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import pty from 'node-pty'; +// CommonJS — default-import then destructure (named ESM imports fail). +import xtermHeadless from '@xterm/headless'; +import addonSerialize from '@xterm/addon-serialize'; + +const { Terminal } = xtermHeadless; +const { SerializeAddon } = addonSerialize; const HERE = path.dirname(fileURLToPath(import.meta.url)); const REPO = path.resolve(HERE, '..'); const BIN = path.join(REPO, 'dist', 'bin.js'); -const GOLDEN_DIR = path.join(HERE, '__screenshots__'); +// Rendered screenshots for inspection / CI artifacts — gitignored, not goldens. +const SHOTS_DIR = path.join(HERE, '__screenshots__'); -// Fixed terminal geometry — layout (and therefore the bytes) must be identical -// when seeding and when checking, on macOS and in CI alike. +// Fixed geometry: capture and replay must agree, on macOS and CI alike. const COLS = 100; const ROWS = 40; /** Snapshot once output has been quiet for this long (screen has painted). */ const SETTLE_MS = 1200; -/** Hard cap, in case a screen never goes quiet. */ +/** Hard cap, in case a screen never goes quiet (e.g. a live spinner). */ const MAX_CAPTURE_MS = 12000; /** - * Commands to snapshot. `slug` is the golden filename; `args` is the wizard - * argv. Keep in sync with the command surface (`bin.ts` + the audit family). - * `unknown-command` is a negative case — it must error, not run a flow. + * Each command and the marker its intro screen must contain. `marker` is in + * normalized form (see normalize(): ANSI stripped, hyphens → spaces, collapsed, + * lowercased). `src` ties the marker to wizard source, so it's a deliberate + * "command → skill/program" assertion, not an eyeballed string. */ const COMMANDS = [ - { slug: 'default', args: [] }, - { slug: 'audit', args: ['audit'] }, - { slug: 'audit-events', args: ['audit', 'events'] }, - { slug: 'audit-all', args: ['audit', 'all'] }, - { slug: 'revenue-analytics', args: ['revenue-analytics'] }, - { slug: 'migrate', args: ['migrate'] }, - { slug: 'upload-source-maps', args: ['upload-source-maps'] }, - { slug: 'mcp-add', args: ['mcp', 'add'] }, - { slug: 'slack-add', args: ['slack', 'add'] }, - { slug: 'skill-list', args: ['skill', 'list'] }, - { slug: 'doctor', args: ['doctor'] }, - { slug: 'unknown-command', args: ['asdf'] }, + // Strong: marker is the program/skill id the source wires into the intro row. + { + slug: 'default', + args: [], + marker: 'program ✔ posthog integration', + src: "posthog-integration/index.ts id:'posthog-integration'", + }, + { + slug: 'audit', + args: ['audit'], + marker: 'program ✔ audit', + src: "audit/index.ts id:'audit' (default leaf, context-mill #187)", + }, + { + slug: 'audit-events', + args: ['audit', 'events'], + marker: 'skill ✔ audit events', + src: "agent-skill skillId 'audit-events' (AgentSkillIntroScreen)", + }, + { + slug: 'audit-all', + args: ['audit', 'all'], + marker: 'program ✔ audit', + src: "audit/index.ts id:'audit'", + }, + { + slug: 'migrate', + args: ['migrate'], + marker: 'program ✔ migration', + src: "migration/index.ts id:'migration'", + }, + // These preflight-block in the wizard's OWN repo (no Stripe SDK / RN), so they + // don't reach the skill-id intro. Marker proves routing, not the skill id. + { + slug: 'revenue-analytics', + args: ['revenue-analytics'], + marker: 'revenue analytics', + src: "revenue-analytics-setup; preflight block in-repo (routing only)", + }, + { + slug: 'upload-source-maps', + args: ['upload-source-maps'], + marker: 'source map', + src: 'error-tracking-upload-source-maps; preflight block in-repo (routing only)', + }, + // Native / utility screens — marker is a stable, screen-specific phrase. + { + slug: 'doctor', + args: ['doctor'], + marker: 'posthog doctor', + src: 'posthog-doctor intro title', + }, + { + slug: 'mcp-add', + args: ['mcp', 'add'], + marker: 'posthog mcp', + src: 'McpScreen.tsx', + }, + // Slack starts OAuth immediately, so the settled screen is the auth wait. + // Weak: proves slack didn't fall through to the default flow, not a skill id. + { + slug: 'slack-add', + args: ['slack', 'add'], + marker: 'waiting for authentication', + src: 'SlackConnectScreen.tsx (OAuth wait; routing smoke check)', + }, + { + slug: 'skill-list', + args: ['skill', 'list'], + marker: 'wizard audit events', + src: 'skill catalog listing (skill.ts)', + }, + // Negative case — a bogus command must error, not run a flow. + { + slug: 'unknown-command', + args: ['asdf'], + marker: 'unknown command', + src: 'wizard.ts strictCommands() .fail()', + }, ]; -const UPDATE = process.argv.includes('--update'); +/** + * pnpm's extraction drops the +x bit on node-pty's prebuilt `spawn-helper`, so + * `pty.spawn` dies with "posix_spawnp failed" on a fresh install. Re-add it. + * No-op once node-pty fixes the packaging upstream. + */ +function ensureSpawnHelperExecutable() { + const root = path.join(REPO, 'node_modules', 'node-pty'); + if (!existsSync(root)) return; + // spawn-helper lives in prebuilds// or build/Release/. + const stack = [root]; + while (stack.length) { + const dir = stack.pop(); + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) stack.push(full); + else if (entry.name === 'spawn-helper') chmodSync(full, 0o755); + } + } +} /** Run one command in a sized pty, snapshot once it settles, return raw bytes. */ function capture(args) { @@ -78,7 +163,7 @@ function capture(args) { cols: COLS, rows: ROWS, cwd: REPO, - // Force a stable colour level so the bytes match across environments. + // Force a stable colour level so the render matches across environments. env: { ...process.env, FORCE_COLOR: '3', TERM: 'xterm-256color' }, encoding: null, // hand back Buffers, not decoded strings }); @@ -115,16 +200,40 @@ function capture(args) { }); } +/** + * Replay raw bytes into a headless emulator and serialize the screen to ANSI. + * Spinner ticks and wait-frames collapse into the single settled frame. + */ +function renderFinalFrame(bytes) { + return new Promise((resolve) => { + const term = new Terminal({ cols: COLS, rows: ROWS, allowProposedApi: true }); + const serializer = new SerializeAddon(); + term.loadAddon(serializer); + // write() is async — serialize only once the bytes have been parsed. + term.write(bytes, () => resolve(serializer.serialize())); + }); +} + +/** Normalize a frame for marker matching: drop ANSI, hyphens → spaces, lower. */ +function normalize(frame) { + return frame + .replace(/\x1b\[[0-9;?]*[a-zA-Z]/g, '') // strip ANSI escapes + .replace(/-/g, ' ') // 'audit-events' ↔ 'audit events' + .replace(/\s+/g, ' ') // collapse whitespace + .trim() + .toLowerCase(); +} + async function main() { if (!existsSync(BIN)) { console.error(`✖ ${BIN} not found — run \`pnpm build\` first.`); process.exit(1); } - mkdirSync(GOLDEN_DIR, { recursive: true }); + mkdirSync(SHOTS_DIR, { recursive: true }); + ensureSpawnHelperExecutable(); let failures = 0; - for (const { slug, args } of COMMANDS) { - const goldenFile = path.join(GOLDEN_DIR, `${slug}.ans`); + for (const { slug, args, marker, src } of COMMANDS) { const label = `wizard ${args.join(' ') || '(default)'}`; let captured; @@ -136,25 +245,22 @@ async function main() { continue; } if (captured.length === 0) { - console.error(`✖ ${slug} (${label}): empty capture — the screen rendered nothing`); + console.error(`✖ ${slug} (${label}): empty capture — nothing rendered`); failures++; continue; } - if (UPDATE) { - writeFileSync(goldenFile, captured); - console.log(`updated ${slug} (${captured.length} bytes)`); - continue; - } - if (!existsSync(goldenFile)) { - console.error(`✖ ${slug} (${label}): no golden yet — run with --update to seed it.`); - failures++; - continue; - } - if (captured.equals(readFileSync(goldenFile))) { - console.log(`ok ${slug}`); + const frame = await renderFinalFrame(captured); + // Save the actual screenshot for inspection / CI artifacts (gitignored). + writeFileSync(path.join(SHOTS_DIR, `${slug}.ans`), frame); + + if (normalize(frame).includes(marker)) { + console.log(`ok ${slug} (found "${marker}")`); } else { - console.error(`✖ ${slug} (${label}): bytes differ from golden`); + console.error( + `✖ ${slug} (${label}): expected "${marker}" [${src}] — not on screen.\n` + + ` See scripts/__screenshots__/${slug}.ans for what rendered.`, + ); failures++; } } From 1f29121236477148bd0db09d7128f1aa3cb1e017 Mon Sep 17 00:00:00 2001 From: sarahxsanders Date: Thu, 18 Jun 2026 10:30:20 -0400 Subject: [PATCH 5/6] test(cli): trim workflow comments Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/cli-screenshots.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cli-screenshots.yml b/.github/workflows/cli-screenshots.yml index a9da71ad6..b796977c3 100644 --- a/.github/workflows/cli-screenshots.yml +++ b/.github/workflows/cli-screenshots.yml @@ -3,11 +3,10 @@ name: CLI Screenshot Tests # Asserts each command loads the right skill/program. See scripts/cli-screenshots.mjs. on: workflow_dispatch: - # Drift monitor: catches a context-mill change breaking a command, no PR needed. - # Runs from `main` only (GitHub limitation). + # Drift monitor (context-mill can break a command with no PR). Runs from main only. schedule: - cron: '37 13 * * *' # daily ~06:37 PT - # Pre-merge gate, scoped to files that decide command → screen wiring. + # Pre-merge gate, scoped to files that affect command → screen wiring. pull_request: paths: - 'bin.ts' @@ -58,7 +57,7 @@ jobs: path: scripts/__screenshots__/** if-no-files-found: ignore - # Scheduled-only: a PR failure shows a red ❌ already; a scheduled one pings nobody. + # Scheduled-only — a PR failure already shows a red ❌. - name: Notify Slack on scheduled failure if: failure() && github.event_name == 'schedule' uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 From eded10f141837eb0911ab37864338e2665a0e831 Mon Sep 17 00:00:00 2001 From: sarahxsanders Date: Thu, 18 Jun 2026 10:41:17 -0400 Subject: [PATCH 6/6] test(cli): accept multiple entry screens for env-dependent flows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The integration (default) and slack flows render different screens depending on the directory, detection speed, and login state — cold CI catches `default` still detecting, and logged-out CI shows slack's connect intro rather than the login wait. A single static marker can't match all of them. Allow `marker` to be an array (pass if any appears); every accepted marker is still flow-specific, so a fall-through to the wrong flow fails. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/cli-screenshots.mjs | 38 ++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/scripts/cli-screenshots.mjs b/scripts/cli-screenshots.mjs index c57a644d7..5cf6bc71e 100644 --- a/scripts/cli-screenshots.mjs +++ b/scripts/cli-screenshots.mjs @@ -46,17 +46,24 @@ const SETTLE_MS = 1200; const MAX_CAPTURE_MS = 12000; /** - * Each command and the marker its intro screen must contain. `marker` is in - * normalized form (see normalize(): ANSI stripped, hyphens → spaces, collapsed, - * lowercased). `src` ties the marker to wizard source, so it's a deliberate - * "command → skill/program" assertion, not an eyeballed string. + * Each command and the marker(s) its screen must contain. `marker` is a string, + * or an array if the flow has several valid entry screens (the check passes if + * ANY appear) — in normalized form (see normalize(): ANSI stripped, hyphens → + * spaces, collapsed, lowercased). `src` ties markers to wizard source, so it's a + * deliberate "command → skill/program" assertion, not an eyeballed string. */ const COMMANDS = [ - // Strong: marker is the program/skill id the source wires into the intro row. + // The integration flow renders one of three screens depending on the dir and + // how fast detection finishes (cold CI may still be detecting at capture) — + // all framework-specific, none appear in other flows. { slug: 'default', args: [], - marker: 'program ✔ posthog integration', + marker: [ + 'program ✔ posthog integration', // intro (framework detected) + 'detecting project framework', // still detecting + 'select your framework', // picker (none detected) + ], src: "posthog-integration/index.ts id:'posthog-integration'", }, { @@ -110,13 +117,13 @@ const COMMANDS = [ marker: 'posthog mcp', src: 'McpScreen.tsx', }, - // Slack starts OAuth immediately, so the settled screen is the auth wait. - // Weak: proves slack didn't fall through to the default flow, not a skill id. + // Logged out (CI) shows the Slack connect intro; logged in jumps to the + // PostHog login wait. Either proves slack didn't fall through to another flow. { slug: 'slack-add', args: ['slack', 'add'], - marker: 'waiting for authentication', - src: 'SlackConnectScreen.tsx (OAuth wait; routing smoke check)', + marker: ['@posthog in slack', 'open slack setup', 'waiting for authentication'], + src: 'SlackConnectScreen.tsx (connect intro / login wait; routing check)', }, { slug: 'skill-list', @@ -254,11 +261,16 @@ async function main() { // Save the actual screenshot for inspection / CI artifacts (gitignored). writeFileSync(path.join(SHOTS_DIR, `${slug}.ans`), frame); - if (normalize(frame).includes(marker)) { - console.log(`ok ${slug} (found "${marker}")`); + const wanted = Array.isArray(marker) ? marker : [marker]; + const normalized = normalize(frame); + const hit = wanted.find((m) => normalized.includes(m)); + if (hit) { + console.log(`ok ${slug} (found "${hit}")`); } else { console.error( - `✖ ${slug} (${label}): expected "${marker}" [${src}] — not on screen.\n` + + `✖ ${slug} (${label}): expected ${wanted + .map((m) => `"${m}"`) + .join(' or ')} [${src}] — not on screen.\n` + ` See scripts/__screenshots__/${slug}.ans for what rendered.`, ); failures++;