diff --git a/.github/workflows/cli-screenshots.yml b/.github/workflows/cli-screenshots.yml new file mode 100644 index 000000000..b796977c3 --- /dev/null +++ b/.github/workflows/cli-screenshots.yml @@ -0,0 +1,70 @@ +name: CLI Screenshot Tests + +# Asserts each command loads the right skill/program. See scripts/cli-screenshots.mjs. +on: + workflow_dispatch: + # Drift monitor (context-mill can break a command with no PR). Runs from main only. + schedule: + - cron: '37 13 * * *' # daily ~06:37 PT + # Pre-merge gate, scoped to files that affect command → screen wiring. + pull_request: + paths: + - 'bin.ts' + - 'src/commands/**' + - 'src/ui/tui/**' + - 'src/lib/programs/**' + - 'scripts/cli-screenshots.mjs' + - '.github/workflows/cli-screenshots.yml' + +permissions: + contents: read + +jobs: + cli-screenshots: + name: CLI screenshot tests + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Install pnpm + uses: pnpm/action-setup@eae0cfeb286e66ffb5155f1a79b90583a127a68b # v2.4.1 + with: + version: 10.23.0 + run_install: false + + - name: Set up Node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: 'package.json' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build wizard + run: pnpm build + + - name: Check command screens + run: node scripts/cli-screenshots.mjs + + - name: Upload captured screenshots + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cli-screenshots + path: scripts/__screenshots__/** + if-no-files-found: ignore + + # Scheduled-only — a PR failure already shows a red ❌. + - name: Notify Slack on scheduled failure + if: failure() && github.event_name == 'schedule' + uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 + with: + webhook: ${{ secrets.SLACK_WEBHOOK_WIZARD_CHANNEL }} + webhook-type: incoming-webhook + payload: | + { + "text": "🖼️ CLI screenshot check failed on the scheduled run — a command may be loading the wrong screen, or context-mill drifted. <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" + } diff --git a/AGENTS.md b/AGENTS.md index 68189191a..4fb547d60 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -121,6 +121,7 @@ pnpm build # Compile TypeScript pnpm test # Unit tests (builds first) pnpm test:watch # Unit tests in watch mode pnpm test:e2e # End-to-end tests +pnpm screens:cli # Assert each command loads the right skill (renders + checks its intro screen) pnpm lint # Prettier + ESLint checks pnpm fix # Auto-fix lint issues pnpm dev # Build, link globally, watch for changes diff --git a/package.json b/package.json index 63f36479e..b0abc85ad 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,8 @@ "@types/yargs": "^16.0.9", "@typescript-eslint/eslint-plugin": "^5.13.0", "@typescript-eslint/parser": "^5.13.0", + "@xterm/addon-serialize": "0.14.0", + "@xterm/headless": "6.0.0", "babel-jest": "^29.7.0", "dotenv": "^16.4.7", "eslint": "^8.18.0", @@ -85,6 +87,7 @@ "jest": "^29.5.0", "lint-staged": "^15.5.1", "msw": "^2.10.4", + "node-pty": "^1.1.0", "prettier": "^2.8.7", "rimraf": "^3.0.2", "ts-jest": "^29.1.0", @@ -98,6 +101,11 @@ "npm": ">=3.10.7" }, "packageManager": "pnpm@10.23.0+sha512.21c4e5698002ade97e4efe8b8b4a89a8de3c85a37919f957e7a0f30f38fbc5bbdd05980ffe29179b2fb6e6e691242e098d945d1601772cad0fef5fb6411e2a4b", + "pnpm": { + "onlyBuiltDependencies": [ + "node-pty" + ] + }, "scripts": { "clean": "rm -rf ./dist", "prebuild": "pnpm clean && node scripts/generate-version.cjs", @@ -120,7 +128,8 @@ "dev": "pnpm build && pnpm link --global && pnpm build:watch", "test:watch": "jest --watch", "prepare": "husky", - "screens:check": "tsx scripts/check-screens.tsx" + "screens:check": "tsx scripts/check-screens.tsx", + "screens:cli": "node scripts/cli-screenshots.mjs" }, "jest": { "collectCoverage": true, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 336708a77..58c478a11 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -132,6 +132,12 @@ importers: '@typescript-eslint/parser': specifier: ^5.13.0 version: 5.62.0(eslint@8.57.1)(typescript@5.7.3) + '@xterm/addon-serialize': + specifier: 0.14.0 + version: 0.14.0 + '@xterm/headless': + specifier: 6.0.0 + version: 6.0.0 babel-jest: specifier: ^29.7.0 version: 29.7.0(@babel/core@7.29.0) @@ -162,6 +168,9 @@ importers: msw: specifier: ^2.10.4 version: 2.10.4(@types/node@18.19.76)(typescript@5.7.3) + node-pty: + specifier: ^1.1.0 + version: 1.1.0 prettier: specifier: ^2.8.7 version: 2.8.8 @@ -1639,6 +1648,12 @@ packages: resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==} engines: {node: '>=10.0.0'} + '@xterm/addon-serialize@0.14.0': + resolution: {integrity: sha512-uteyTU1EkrQa2Ux6P/uFl2fzmXI46jy5uoQMKEOM0fKTyiW7cSn0WrFenHm5vO5uEXX/GpwW/FgILvv3r0WbkA==} + + '@xterm/headless@6.0.0': + resolution: {integrity: sha512-5Yj1QINYCyzrZtf8OFIHi47iQtI+0qYFPHmouEfG8dHNxbZ9Tb9YGSuLcsEwj9Z+OL75GJqPyJbyoFer80a2Hw==} + accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} @@ -3134,9 +3149,15 @@ packages: resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} engines: {node: '>= 0.6'} + node-addon-api@7.1.1: + resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} + node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} + node-pty@1.1.0: + resolution: {integrity: sha512-20JqtutY6JPXTUnL0ij1uad7Qe1baT46lyolh2sSENDd4sTzKZ4nmAFkeAARDKwmlLjPx6XKRlwRUxwjOy+lUg==} + node-releases@2.0.19: resolution: {integrity: sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==} @@ -5780,6 +5801,10 @@ snapshots: '@xmldom/xmldom@0.8.10': {} + '@xterm/addon-serialize@0.14.0': {} + + '@xterm/headless@6.0.0': {} + accepts@2.0.0: dependencies: mime-types: 3.0.2 @@ -7471,8 +7496,14 @@ snapshots: negotiator@1.0.0: {} + node-addon-api@7.1.1: {} + node-int64@0.4.0: {} + node-pty@1.1.0: + dependencies: + node-addon-api: 7.1.1 + node-releases@2.0.19: {} node-releases@2.0.27: {} diff --git a/scripts/__screenshots__/.gitignore b/scripts/__screenshots__/.gitignore new file mode 100644 index 000000000..b24039a1c --- /dev/null +++ b/scripts/__screenshots__/.gitignore @@ -0,0 +1,2 @@ +# Rendered screenshots — regenerated every run, not committed. +*.ans diff --git a/scripts/cli-screenshots.mjs b/scripts/cli-screenshots.mjs new file mode 100644 index 000000000..5cf6bc71e --- /dev/null +++ b/scripts/cli-screenshots.mjs @@ -0,0 +1,287 @@ +#!/usr/bin/env node +/** + * "Screenshot" tests for the wizard CLI command surface. + * + * Per command: run the built binary in a sized pty, render the settled screen + * through a headless emulator, and assert it contains a `marker` — the + * program/skill id the command's source wires into the intro screen. Catches a + * command routing to the wrong screen, the default flow, or nothing. + * + * Marker, not byte-for-byte golden: spinners, async fetches, and + * project-dependent intros make whole-frame matching flake. We read the same + * rendered screenshot, just assert the one line that proves the routing. + * + * Usage: pnpm build && node scripts/cli-screenshots.mjs + */ + +import { + chmodSync, + existsSync, + mkdirSync, + readdirSync, + writeFileSync, +} from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import pty from 'node-pty'; +// CommonJS — default-import then destructure (named ESM imports fail). +import xtermHeadless from '@xterm/headless'; +import addonSerialize from '@xterm/addon-serialize'; + +const { Terminal } = xtermHeadless; +const { SerializeAddon } = addonSerialize; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const REPO = path.resolve(HERE, '..'); +const BIN = path.join(REPO, 'dist', 'bin.js'); +// Rendered screenshots for inspection / CI artifacts — gitignored, not goldens. +const SHOTS_DIR = path.join(HERE, '__screenshots__'); + +// Fixed geometry: capture and replay must agree, on macOS and CI alike. +const COLS = 100; +const ROWS = 40; +/** Snapshot once output has been quiet for this long (screen has painted). */ +const SETTLE_MS = 1200; +/** Hard cap, in case a screen never goes quiet (e.g. a live spinner). */ +const MAX_CAPTURE_MS = 12000; + +/** + * Each command and the marker(s) its screen must contain. `marker` is a string, + * or an array if the flow has several valid entry screens (the check passes if + * ANY appear) — in normalized form (see normalize(): ANSI stripped, hyphens → + * spaces, collapsed, lowercased). `src` ties markers to wizard source, so it's a + * deliberate "command → skill/program" assertion, not an eyeballed string. + */ +const COMMANDS = [ + // The integration flow renders one of three screens depending on the dir and + // how fast detection finishes (cold CI may still be detecting at capture) — + // all framework-specific, none appear in other flows. + { + slug: 'default', + args: [], + marker: [ + 'program ✔ posthog integration', // intro (framework detected) + 'detecting project framework', // still detecting + 'select your framework', // picker (none detected) + ], + src: "posthog-integration/index.ts id:'posthog-integration'", + }, + { + slug: 'audit', + args: ['audit'], + marker: 'program ✔ audit', + src: "audit/index.ts id:'audit' (default leaf, context-mill #187)", + }, + { + slug: 'audit-events', + args: ['audit', 'events'], + marker: 'skill ✔ audit events', + src: "agent-skill skillId 'audit-events' (AgentSkillIntroScreen)", + }, + { + slug: 'audit-all', + args: ['audit', 'all'], + marker: 'program ✔ audit', + src: "audit/index.ts id:'audit'", + }, + { + slug: 'migrate', + args: ['migrate'], + marker: 'program ✔ migration', + src: "migration/index.ts id:'migration'", + }, + // These preflight-block in the wizard's OWN repo (no Stripe SDK / RN), so they + // don't reach the skill-id intro. Marker proves routing, not the skill id. + { + slug: 'revenue-analytics', + args: ['revenue-analytics'], + marker: 'revenue analytics', + src: "revenue-analytics-setup; preflight block in-repo (routing only)", + }, + { + slug: 'upload-source-maps', + args: ['upload-source-maps'], + marker: 'source map', + src: 'error-tracking-upload-source-maps; preflight block in-repo (routing only)', + }, + // Native / utility screens — marker is a stable, screen-specific phrase. + { + slug: 'doctor', + args: ['doctor'], + marker: 'posthog doctor', + src: 'posthog-doctor intro title', + }, + { + slug: 'mcp-add', + args: ['mcp', 'add'], + marker: 'posthog mcp', + src: 'McpScreen.tsx', + }, + // Logged out (CI) shows the Slack connect intro; logged in jumps to the + // PostHog login wait. Either proves slack didn't fall through to another flow. + { + slug: 'slack-add', + args: ['slack', 'add'], + marker: ['@posthog in slack', 'open slack setup', 'waiting for authentication'], + src: 'SlackConnectScreen.tsx (connect intro / login wait; routing check)', + }, + { + slug: 'skill-list', + args: ['skill', 'list'], + marker: 'wizard audit events', + src: 'skill catalog listing (skill.ts)', + }, + // Negative case — a bogus command must error, not run a flow. + { + slug: 'unknown-command', + args: ['asdf'], + marker: 'unknown command', + src: 'wizard.ts strictCommands() .fail()', + }, +]; + +/** + * pnpm's extraction drops the +x bit on node-pty's prebuilt `spawn-helper`, so + * `pty.spawn` dies with "posix_spawnp failed" on a fresh install. Re-add it. + * No-op once node-pty fixes the packaging upstream. + */ +function ensureSpawnHelperExecutable() { + const root = path.join(REPO, 'node_modules', 'node-pty'); + if (!existsSync(root)) return; + // spawn-helper lives in prebuilds// or build/Release/. + const stack = [root]; + while (stack.length) { + const dir = stack.pop(); + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) stack.push(full); + else if (entry.name === 'spawn-helper') chmodSync(full, 0o755); + } + } +} + +/** Run one command in a sized pty, snapshot once it settles, return raw bytes. */ +function capture(args) { + return new Promise((resolve, reject) => { + let proc; + try { + proc = pty.spawn('node', [BIN, ...args, '--no-telemetry'], { + name: 'xterm-256color', + cols: COLS, + rows: ROWS, + cwd: REPO, + // Force a stable colour level so the render matches across environments. + env: { ...process.env, FORCE_COLOR: '3', TERM: 'xterm-256color' }, + encoding: null, // hand back Buffers, not decoded strings + }); + } catch (err) { + reject(err); + return; + } + + const chunks = []; + let settleTimer; + let done = false; + const finish = () => { + if (done) return; + done = true; + clearTimeout(settleTimer); + clearTimeout(maxTimer); + try { + proc.kill(); + } catch { + /* already gone */ + } + }; + proc.onData((data) => { + chunks.push(Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf8')); + clearTimeout(settleTimer); + settleTimer = setTimeout(finish, SETTLE_MS); + }); + const maxTimer = setTimeout(finish, MAX_CAPTURE_MS); + proc.onExit(() => { + clearTimeout(settleTimer); + clearTimeout(maxTimer); + resolve(Buffer.concat(chunks)); + }); + }); +} + +/** + * Replay raw bytes into a headless emulator and serialize the screen to ANSI. + * Spinner ticks and wait-frames collapse into the single settled frame. + */ +function renderFinalFrame(bytes) { + return new Promise((resolve) => { + const term = new Terminal({ cols: COLS, rows: ROWS, allowProposedApi: true }); + const serializer = new SerializeAddon(); + term.loadAddon(serializer); + // write() is async — serialize only once the bytes have been parsed. + term.write(bytes, () => resolve(serializer.serialize())); + }); +} + +/** Normalize a frame for marker matching: drop ANSI, hyphens → spaces, lower. */ +function normalize(frame) { + return frame + .replace(/\x1b\[[0-9;?]*[a-zA-Z]/g, '') // strip ANSI escapes + .replace(/-/g, ' ') // 'audit-events' ↔ 'audit events' + .replace(/\s+/g, ' ') // collapse whitespace + .trim() + .toLowerCase(); +} + +async function main() { + if (!existsSync(BIN)) { + console.error(`✖ ${BIN} not found — run \`pnpm build\` first.`); + process.exit(1); + } + mkdirSync(SHOTS_DIR, { recursive: true }); + ensureSpawnHelperExecutable(); + + let failures = 0; + for (const { slug, args, marker, src } of COMMANDS) { + const label = `wizard ${args.join(' ') || '(default)'}`; + + let captured; + try { + captured = await capture(args); + } catch (err) { + console.error(`✖ ${slug} (${label}): capture failed — ${err.message}`); + failures++; + continue; + } + if (captured.length === 0) { + console.error(`✖ ${slug} (${label}): empty capture — nothing rendered`); + failures++; + continue; + } + + const frame = await renderFinalFrame(captured); + // Save the actual screenshot for inspection / CI artifacts (gitignored). + writeFileSync(path.join(SHOTS_DIR, `${slug}.ans`), frame); + + const wanted = Array.isArray(marker) ? marker : [marker]; + const normalized = normalize(frame); + const hit = wanted.find((m) => normalized.includes(m)); + if (hit) { + console.log(`ok ${slug} (found "${hit}")`); + } else { + console.error( + `✖ ${slug} (${label}): expected ${wanted + .map((m) => `"${m}"`) + .join(' or ')} [${src}] — not on screen.\n` + + ` See scripts/__screenshots__/${slug}.ans for what rendered.`, + ); + failures++; + } + } + + if (failures > 0) { + console.error(`\n${failures} screenshot check(s) failed`); + process.exit(1); + } + console.log(`\nAll ${COMMANDS.length} screenshot checks passed`); +} + +main();