From a67db2ebc1604bd1484605d2a42be36be11ca677 Mon Sep 17 00:00:00 2001 From: Alexander Tenepere Date: Tue, 10 Nov 2026 00:10:52 +0800 Subject: [PATCH 1/8] feat: Track the commit a catalogue shows, visualice changes, and partial runs with crawl --only --- CHANGELOG.md | 27 ++ commands/visualice.md | 4 + docs/crawling.md | 4 + docs/viewer.md | 11 + manifest.schema.json | 18 + site/.vitepress/config.mts | 2 + skills/visualice/SKILL.md | 15 +- skills/visualice/references/cli.md | 41 +- skills/visualice/references/update-mode.md | 114 +++++ src/cli-args.ts | 4 + src/cli.ts | 187 ++++++++- src/crawl/carry.ts | 258 ++++++++++++ src/crawl/changes.ts | 464 +++++++++++++++++++++ src/crawl/orchestrator.ts | 346 ++++++++++++++- src/crawl/routes-file.ts | 29 +- src/crawl/source.ts | 192 +++++++++ src/shared/manifest-schema.ts | 8 + src/shared/types.ts | 54 +++ tests/carry.test.ts | 243 +++++++++++ tests/changes.test.ts | 216 ++++++++++ tests/integration/partial.test.ts | 79 ++++ tests/source.test.ts | 141 +++++++ viewer/src/App.tsx | 9 + viewer/src/Canvas.tsx | 16 +- viewer/src/Compare.tsx | 149 +++++++ viewer/src/Inspector.tsx | 49 ++- viewer/src/load.ts | 35 ++ 27 files changed, 2653 insertions(+), 62 deletions(-) create mode 100644 skills/visualice/references/update-mode.md create mode 100644 src/crawl/carry.ts create mode 100644 src/crawl/changes.ts create mode 100644 src/crawl/source.ts create mode 100644 tests/carry.test.ts create mode 100644 tests/changes.test.ts create mode 100644 tests/integration/partial.test.ts create mode 100644 tests/source.test.ts create mode 100644 viewer/src/Compare.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 6710d9c..656e587 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Changelog +## Unreleased + +### Updating after code changes + +- Each crawl records the git commit it shot, on a `visualice-source` line in + `visualice.routes.md` and as `source` in the run's `crawl-map.json` (branch, files with + uncommitted changes, whether the target was local, a hash of the config). +- `visualice changes` diffs the project against that commit: the commits and files since, + the changes that reach every page (dependencies, global styles, root layouts, header + and footer, the config), the pages file-based routing ties changed files to, and the + files left for an agent to trace. It gives a verdict (`none`, `partial`, `full`, + `unknown`) and the command to run; `--discover` also lists new and removed pages. +- `crawl --only=` shoots only those pages (paths or globs) and carries every + other page's shots over from the last finished run, as hard links where the disk + allows, so the new run is still the whole site. A page the last run can't cover (new, + or missing shots) is shot anyway. With `--replay` the named pages' states are + discovered again. `--dry-run` shows what would be shot and carried. +- The skill and `/visualice` follow it: run `changes`, trace the affected pages, ask + whether to rerun those or everything, then run it. + +### Viewer + +- Shots a partial run took carry an **updated** tag; a carried shot's details say which + run and commit it was really taken in. +- **Compare with another version** in the details panel shows the same shot from another + run beside this one. + ## 0.1.1 ### Fixes diff --git a/commands/visualice.md b/commands/visualice.md index 2890179..f304cf0 100644 --- a/commands/visualice.md +++ b/commands/visualice.md @@ -17,6 +17,10 @@ configured in `visualice.config.json` in the current directory. map with a dry run, show the user the page types, shot count and any writes and confirm once, dispatch one `visualice:visualice-explorer` agent per page type, then `visualice author` and `visualice crawl --replay`. + - **Already mapped, and the routes file records a commit** (a `visualice-source` + line) → run `visualice changes --json` first and follow the skill's + `references/update-mode.md`: trace the affected pages, ask the user whether to + rerun just those (`crawl --replay --only=…`, carrying the rest over) or everything. - **Already mapped** → `visualice crawl --replay --json --quiet`, then explore only what's new (pages added to the routes file, `wanted` lines), per "Later runs" in the same reference. `/visualice-explore` is for widening it on purpose. diff --git a/docs/crawling.md b/docs/crawling.md index a3580d4..4aa24cc 100644 --- a/docs/crawling.md +++ b/docs/crawling.md @@ -17,6 +17,10 @@ visualice crawl --resume # continue the newest interrupted run, keepin visualice crawl --dry-run # discover, then list what would be captured; capture nothing visualice crawl --no-states # pages only, no UI states (fast; cached states are kept) +# after code changes (the routes file records the commit each crawl shot): +visualice changes # what changed since, and whether some pages or all need reshooting +visualice crawl --replay --only=/pricing,/blog/** # reshoot those; carry every other page over + # other commands: visualice discover https://your.site # discovery only, no screenshots visualice init https://your.site # scaffold the config without crawling diff --git a/docs/viewer.md b/docs/viewer.md index 27a0035..fa39139 100644 --- a/docs/viewer.md +++ b/docs/viewer.md @@ -71,6 +71,17 @@ back to it. A card's trash button deletes the site and all its runs from disk af confirmation; the site `serve` was started on can't be deleted (409) while that server runs. +### Partial runs and comparing versions + +A partial run (`crawl --only`, see [updating after code changes](../skills/visualice/references/update-mode.md)) +shoots the pages that changed and carries the rest over. Its shots carry an **updated** +tag on the canvas; a carried shot's details panel says so, with the run and commit it +was really taken in. The details panel's footer shows the commit a shot came from. + +**Compare with another version**, in the details panel, puts the same shot from another +run of the site beside this one (the previous version by default). A shot that was +carried over unchanged says so instead of looking like a coincidence. + ## Share a run ```bash diff --git a/manifest.schema.json b/manifest.schema.json index fc961c0..247e987 100644 --- a/manifest.schema.json +++ b/manifest.schema.json @@ -740,6 +740,24 @@ }, "error": { "type": "string" + }, + "carried": { + "description": "Copied from an earlier run by a partial crawl (`--only`), not shot by this one.", + "type": "object", + "properties": { + "runId": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "capturedAt": { + "type": "string" + } + }, + "required": [ + "runId" + ] } } } diff --git a/site/.vitepress/config.mts b/site/.vitepress/config.mts index 8264647..70867dc 100644 --- a/site/.vitepress/config.mts +++ b/site/.vitepress/config.mts @@ -17,6 +17,7 @@ const PAGES: Record = { "site/guide/getting-started.md": "guide/getting-started.md", "skills/visualice/references/explore-mode.md": "guide/explore.md", "docs/crawling.md": "guide/crawling.md", + "skills/visualice/references/update-mode.md": "guide/updating.md", "docs/configuration.md": "guide/configuration.md", "skills/visualice/references/auth.md": "guide/auth.md", "docs/viewer.md": "guide/viewer.md", @@ -127,6 +128,7 @@ export default defineConfig({ text: "Guide", items: [ { text: "Crawling", link: "/guide/crawling" }, + { text: "Updating after code changes", link: "/guide/updating" }, { text: "Configuration", link: "/guide/configuration" }, { text: "Logging in", link: "/guide/auth" }, { text: "The viewer", link: "/guide/viewer" }, diff --git a/skills/visualice/SKILL.md b/skills/visualice/SKILL.md index d4ad53c..7bb8618 100644 --- a/skills/visualice/SKILL.md +++ b/skills/visualice/SKILL.md @@ -88,10 +88,18 @@ rediscovery and loses any Explore-mode states authored into it. hasn't been mapped: map it with a dry run, plan the page types with the user, explore each type, then capture with `--replay`. Read [`references/explore-mode.md`](references/explore-mode.md) and follow it. -2. **Replay or extend.** The site is mapped. To recapture, run +2. **Update after code changes.** The site is mapped and the project is a git repo + whose code changed since the last crawl. Run `visualice changes --json`: it diffs + against the commit the catalogue was shot from (recorded in the routes file) and + says whether nothing, some pages, or everything needs reshooting. Work out the + affected pages, ask the user whether to rerun just those or everything, then run + `visualice crawl --replay --only=`, which carries every other page over from + the last run. Read [`references/update-mode.md`](references/update-mode.md) and + follow it. +3. **Replay or extend.** The site is mapped. To recapture, run `visualice crawl --replay --json --quiet`. For new pages, `wanted` lines, or a focus the user names, explore just those page types first (explore-mode.md, "Later runs"). -3. **Quick.** Only when the user asks for it ("quick", "just screenshot it", pages only) +4. **Quick.** Only when the user asks for it ("quick", "just screenshot it", pages only) or you're scripting CI: `visualice crawl --json --quiet` runs the heuristics and captures in one go, with no exploring. Mention that exploring would find more. @@ -161,3 +169,6 @@ Full flag reference: [`references/cli.md`](references/cli.md). by default never publish, delete, pay, sign out or change account settings. - The cached model makes re-runs cheap — once a site is mapped, default to `--replay` and explore only what's new. +- After code changes, never reshoot part of a catalogue without asking: lay out what + `visualice changes` found and the pages you traced, and let the user choose between + those pages and a full run. When in doubt, recommend the full run. diff --git a/skills/visualice/references/cli.md b/skills/visualice/references/cli.md index 5b94813..9b514e0 100644 --- a/skills/visualice/references/cli.md +++ b/skills/visualice/references/cli.md @@ -7,25 +7,26 @@ the dedicated script where one exists, e.g. `npm run crawl -- `). ## Commands -| Command | Purpose | -| --------------------------- | ------------------------------------------------------------------------- | -| `init [url]` | Scaffold `visualice.config.json` (+ `.gitignore` entry) without crawling | -| `crawl [url]` | Discover routes + UI states, then capture across devices | -| `discover [url]` | Auto-discover routes + UI states only — no screenshots | -| `routes` | Regenerate `visualice.routes.md` from the cached site model | -| `validate` | Validate a crawl manifest / cached site model | -| `auth [--force]` | Run the configured login recipe, save storage state | -| `capture --manifest=` | Replay a given manifest's recipes and capture screenshots | -| `demo` | Capture the bundled fixture from a hand-written manifest | -| `demo-crawl` | Full discover → capture run against the bundled fixture | -| `fixture` | Serve the bundled fixture blog so you can browse it | -| `doctor [--no-fix]` | Idempotent preflight: check Node/Playwright/sharp/viewer, self-heal | -| `serve` | Launch the zoomable results viewer | -| `export ` | Write the viewer + a run as a static folder | -| `probe ` | JSON: the page's inventory, or (`--recipe`/`--state`) what a recipe does | -| `author [dir\|file]` | Merge explorer state files (default `.visualice/explore/`) into the cache | -| `--version` | Print the installed version | -| `help` | Print usage | +| Command | Purpose | +| --------------------------- | -------------------------------------------------------------------------- | +| `init [url]` | Scaffold `visualice.config.json` (+ `.gitignore` entry) without crawling | +| `crawl [url]` | Discover routes + UI states, then capture across devices | +| `discover [url]` | Auto-discover routes + UI states only — no screenshots | +| `routes` | Regenerate `visualice.routes.md` from the cached site model | +| `validate` | Validate a crawl manifest / cached site model | +| `auth [--force]` | Run the configured login recipe, save storage state | +| `capture --manifest=` | Replay a given manifest's recipes and capture screenshots | +| `demo` | Capture the bundled fixture from a hand-written manifest | +| `demo-crawl` | Full discover → capture run against the bundled fixture | +| `fixture` | Serve the bundled fixture blog so you can browse it | +| `doctor [--no-fix]` | Idempotent preflight: check Node/Playwright/sharp/viewer, self-heal | +| `serve` | Launch the zoomable results viewer | +| `export ` | Write the viewer + a run as a static folder | +| `probe ` | JSON: the page's inventory, or (`--recipe`/`--state`) what a recipe does | +| `author [dir\|file]` | Merge explorer state files (default `.visualice/explore/`) into the cache | +| `changes [--discover]` | JSON/text: what changed in git since the catalogue's commit, and a verdict | +| `--version` | Print the installed version | +| `help` | Print usage | ## Flags @@ -43,6 +44,8 @@ command are accepted by it. | `--manifest=` | `capture`/`validate` | Input manifest path | | `--replay` | `crawl`/`demo-crawl` | Reuse the cached site model, skip discovery | | `--resume` | `crawl` | Continue the newest interrupted run, keeping its shots | +| `--only=` | `crawl` | Shoot only these pages (paths or globs, comma-separated); every other page's shots are carried over from the last run. With `--replay`, rediscovers their states | +| `--discover` | `changes` | Also map the site's pages (no states, nothing written) to list new and removed ones | | `--json` | most | Print a machine-readable JSON summary to stdout | | `--quiet` | most | Suppress progress output on stderr | | `--fail-on-error` | capture cmds | Exit 3 if any capture failed | diff --git a/skills/visualice/references/update-mode.md b/skills/visualice/references/update-mode.md new file mode 100644 index 0000000..e32e6e6 --- /dev/null +++ b/skills/visualice/references/update-mode.md @@ -0,0 +1,114 @@ +# Updating a catalogue after code changes + +A crawl records the commit it shot in `visualice.routes.md`, on a line under the +title: + +```md + +``` + +Each run's `crawl-map.json` has the same record as `source`, with the list of files +that had uncommitted changes. When the project moves on (new pages, edited ones), you +don't have to reshoot everything: find out what changed, work out which pages it +touches, and reshoot only those. A partial run carries every other page's shots over +from the last run, so the new run is still the whole site. + +## 1. Ask what changed + +```bash +visualice changes --json # add --discover to also map the site for new/removed pages +``` + +The answer, in short: + +| Field | What it is | +| ------------ | ---------------------------------------------------------------------------------------------------------------- | +| `verdict` | `none` (nothing to do), `partial` (some pages), `full` (everything), `unknown` (no baseline: run a full crawl) | +| `reasons` | Why, most important first. Read these out to the user | +| `baseline` | The commit the catalogue shows, its branch, when it was shot, and whether the target was local | +| `commits` | Commits since (hash, subject), newest first; `commitCount` is the total | +| `files` | Every changed file that counts (committed since, uncommitted now, untracked, and files dirty at capture time) | +| `global` | Changed files that reach every page: dependencies, global styles, theme, root layouts, header/footer/nav, config | +| `candidates` | Changed files that file-based routing ties to pages: `{file, pattern, kind, pages}` | +| `unmapped` | Changed files that map to no page on their own: components, hooks, utilities, data | +| `routes` | With `--discover`: `added` and `removed` pages | +| `suggest` | The command to run, when the facts settle it (no unmapped files) | + +The visualice files themselves (`visualice.routes.md`, `.visualice/`) never count. +The config is compared by its hash: changing devices or capture settings means a full +run. + +## 2. Work out the affected pages + +`verdict: "full"` or `"unknown"` → skip to step 3 and recommend a full run. + +For `partial`: + +- **`candidates`** already list their pages. A `layout` candidate covers every page + under its pattern (`/docs/**`). A candidate with no `pages` is a page the catalogue + doesn't have yet: include its `pattern` as a new page. +- **`unmapped`** files are yours to trace. For each one, find who imports it (search + for its module name or path) and follow the chain up to page or layout files, then + map those the same way the candidates are mapped. Read the diff + (`git diff -- `) when the reach is unclear: a renamed CSS + class in a shared component reaches every page that renders it, a fixed typo in one + prop doesn't. + - A component used by most pages (a button, a card used everywhere, a design-system + primitive) is effectively global: say so and recommend a full run. + - Data files, API handlers and server code can change what a page shows without a + UI file changing. If a page reads that data, include it. + - Tests, docs, CI config, and server code no page renders don't affect screenshots: + leave them out, and say you did. +- **`routes.removed`** pages are gone: suggest a `- skip: /path` line in the routes + file so they stop being carried over. + +Map files to the page paths the routes file uses (`/pricing`, not `pricing.tsx`). Globs +work: `/blog/**` covers every post. + +## 3. Ask the user + +Lay it out in one short message, then ask: rerun the affected pages, or everything? +Recommend one. + +> Since 3f2a9c1 (6 commits): `/pricing` and `/compare` changed (PricingCard.tsx is used +> by both), and `/blog` is new. Nothing global changed. +> → **Rerun 3 pages (≈24 shots)**, carrying the other 38 pages over · or rerun +> everything (≈310 shots) + +Recommend everything when: + +- `verdict` is `full` or `unknown`; +- you couldn't trace an unmapped file with confidence; +- the baseline wasn't shot from a local server (`baseline.local: false`): it shows what + was deployed, which may not be that commit; +- the catalogue has gone through several partial runs since the last full one (check + the earlier runs' `partial` field): a full run now and then catches what tracing + missed. + +Add `--dry-run --json` to the command to show exact numbers before running it. With +`--only`, its `partial` field lists the pages to shoot and the pages to carry. + +## 4. Run it + +```bash +# Affected pages only: rediscovers their states (a changed page may have new ones), +# maps new pages, shoots them, carries every other page from the last run. +visualice crawl --replay --only=/pricing,/compare,/blog --json --quiet + +# Everything +visualice crawl --replay --json --quiet # or without --replay to rediscover the whole site +``` + +`--only` takes comma-separated paths or globs. A page outside it is still shot when the +last run can't cover it: a page new since, or one whose shots are missing or failed +there (a device added, a state added). The summary's `carried` count says how many +shots came from the earlier run. + +When the run finishes, the routes file records the new commit. Tell the user to commit +`visualice.routes.md` with their change, so the next `changes` starts from here. + +## In the viewer + +A partial run marks the shots it took with an **updated** tag. A carried shot says so +in its details panel, with the run and commit it was really taken in. **Compare with +another version** in the details panel puts the same shot from another run beside it. diff --git a/src/cli-args.ts b/src/cli-args.ts index c3b0190..e57e2fd 100644 --- a/src/cli-args.ts +++ b/src/cli-args.ts @@ -48,6 +48,8 @@ const FLAG_TYPES: Record = { inventory: "boolean", from: "string", "all-devices": "boolean", + only: "string", + discover: "boolean", }; /** @@ -93,6 +95,7 @@ export const COMMAND_FLAGS: Record = { "dry-run", "replay", "resume", + "only", "json", "quiet", "fail-on-error", @@ -112,6 +115,7 @@ export const COMMAND_FLAGS: Record = { doctor: ["json", "quiet", "no-fix"], export: ["config", "site", "run", "dest", "json", "quiet", "dir"], author: ["config", "target", "from", "json", "quiet", "dir"], + changes: ["config", "target", "discover", "site", "headed", "json", "quiet", "dir"], probe: [ "config", "target", diff --git a/src/cli.ts b/src/cli.ts index e3576ee..5efc4f8 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -4,7 +4,15 @@ import { promises as fs } from "node:fs"; import { spawn } from "node:child_process"; import { chromium } from "playwright"; import { ConfigLoadError, loadConfig, type Config } from "./shared/config"; -import { isCommand, UsageError, applyOverrides, parseFlags, str, type Flags } from "./cli-args"; +import { + isCommand, + UsageError, + applyOverrides, + csv, + parseFlags, + str, + type Flags, +} from "./cli-args"; import { countStates, loadInputManifest, modelCachePath } from "./crawl/manifest"; import { InputManifestSchema, formatManifestIssues } from "./shared/manifest-schema"; import { @@ -12,8 +20,11 @@ import { crawlPipeline, discoverOnlyPipeline, planCrawl, + previewRoutes, type CrawlPlan, } from "./crawl/orchestrator"; +import { computeChanges, type ChangesReport } from "./crawl/changes"; +import { CONFIG_FILES, findConfigFile, hashFile } from "./crawl/source"; import { setHeaded } from "./shared/launch"; import { computeWanted, @@ -29,6 +40,7 @@ import { serveViewer } from "./serve"; import { formatDoctor, runDoctor } from "./doctor"; import { latestRunId, + listRuns, listSites, migrateFlatRuns, newestSite, @@ -36,7 +48,7 @@ import { siteDir, siteKey, } from "./crawl/runs"; -import type { Manifest, RecipeStep } from "./shared/types"; +import type { Manifest, RecipeStep, SourceInfo } from "./shared/types"; import { exportRun } from "./export"; import { probePage } from "./crawl/probe"; import { listAuthoredFiles, mergeAuthored, readAuthoredFile } from "./crawl/author"; @@ -49,22 +61,9 @@ const HERE = path.dirname(fileURLToPath(import.meta.url)); const PROJECT_ROOT = path.resolve(HERE, ".."); const FIXTURE_DIR = path.join(PROJECT_ROOT, "tests", "fixture"); -/** Config files picked up automatically from the working directory. */ -const CONFIG_CANDIDATES = [ - "visualice.config.ts", - "visualice.config.mjs", - "visualice.config.js", - "visualice.config.json", -]; - -async function detectConfigFile(): Promise { - for (const c of CONFIG_CANDIDATES) if (await pathExists(path.resolve(c))) return c; - return undefined; -} - /** Explicit --config wins; otherwise a visualice.config.* in the cwd is picked up. */ async function resolveConfigPath(flags: Flags): Promise { - return str(flags, "config") ?? (await detectConfigFile()); + return findConfigFile(process.cwd(), str(flags, "config")); } async function loadConfigWithOverrides(flags: Flags): Promise { @@ -92,6 +91,7 @@ Usage: visualice export Write the viewer + a run as a static folder to share visualice probe Report a page's inventory, or what a recipe does on it (JSON) visualice author [dir|file] Merge explorer state files (.visualice/explore/) into the cache + visualice changes [opts] What changed in git since the catalogue was shot, and what to rerun visualice serve [opts] Launch the zoomable results viewer visualice --version Print the version @@ -109,6 +109,9 @@ Options (flags accept --name=value or --name value): --manifest= input manifest for 'capture' / 'validate' --replay for 'crawl'/'demo-crawl': reuse cached site model, skip discovery --resume for 'crawl': continue the newest interrupted run + --only= for 'crawl': shoot only these pages (paths or globs, comma-separated); + every other page's shots are carried over from the last run + --discover for 'changes': also map the site's pages, to list new and removed ones --json print a machine-readable JSON summary to stdout --quiet suppress progress output on stderr --fail-on-error exit 3 if any capture failed @@ -156,6 +159,7 @@ function reportDone(runDir: string, manifest: Manifest, flags: Flags, log: Log): routes: stats?.routes ?? 0, states: stats?.states ?? 0, shots: stats?.shots ?? 0, + ...(stats?.carried !== undefined ? { carried: stats.carried } : {}), failed: stats?.failed ?? 0, failures: stats?.failures ?? [], wanted: stats?.wanted ?? [], @@ -165,7 +169,9 @@ function reportDone(runDir: string, manifest: Manifest, flags: Flags, log: Log): } else { log(""); log( - `done — ${stats?.shots ?? 0} screenshots${stats?.failed ? `, ${stats.failed} FAILED` : ""}`, + `done — ${stats?.shots ?? 0} screenshots` + + (stats?.carried ? ` (+${stats.carried} carried over)` : "") + + (stats?.failed ? `, ${stats.failed} FAILED` : ""), ); console.log(runDir); log(` manifest : ${path.join(runDir, "crawl-map.json")}`); @@ -186,7 +192,14 @@ async function cmdCapture(flags: Flags): Promise { if (!baseUrl) throw new UsageError("no target URL: pass --target= or set target in config"); const inputManifest = await loadInputManifest(manifestPath); - const { runDir, manifest } = await capturePipeline({ config, inputManifest, baseUrl, log }); + const configFile = await resolveConfigPath(flags); + const { runDir, manifest } = await capturePipeline({ + config, + inputManifest, + baseUrl, + configFile, + log, + }); reportDone(runDir, manifest, flags, log); } @@ -502,6 +515,9 @@ async function cmdCrawl(flags: Flags): Promise { await scaffoldProject(baseUrl, log); } const pagesOnly = flags["no-states"] === true; + const onlyFlag = str(flags, "only"); + const only = onlyFlag === undefined ? undefined : csv(onlyFlag); + if (only && !only.length) throw new UsageError("--only needs at least one path or glob"); if (flags["dry-run"] === true) { if (flags.resume === true) throw new UsageError("pass --dry-run or --resume, not both"); const plan = await planCrawl({ @@ -509,6 +525,7 @@ async function cmdCrawl(flags: Flags): Promise { baseUrl, replay: flags.replay === true, pagesOnly, + only, log, }); reportPlan(plan, flags, log); @@ -520,6 +537,8 @@ async function cmdCrawl(flags: Flags): Promise { replay: flags.replay === true, resume: flags.resume === true, pagesOnly, + only, + configFile: configPath, log, }); reportDone(runDir, manifest, flags, log); @@ -540,9 +559,139 @@ function reportPlan(plan: CrawlPlan, flags: Flags, log: Log): void { `dry run — ${plan.routes.length} page(s), ${plan.states} state(s), ` + `${plan.shots} screenshot(s) across ${plan.devices.join(", ")}; nothing captured`, ); + const p = plan.partial; + if (p) { + if (p.carriedFrom) { + log( + ` --only: shoot ${p.capture.length} page(s), carry ${p.carry.length} page(s) ` + + `(${p.carriedShots} screenshot(s)) from run ${p.carriedFrom}`, + ); + for (const f of p.forced) + log(` + ${f.url} shot too (${f.reason === "new" ? "new" : "incomplete in that run"})`); + } else { + log(" --only: no finished run to carry from, so every page would be shot"); + } + } log(` model : ${plan.cachePath}`); } +/** The commit records in a site's runs, newest first. */ +async function runSources(config: Config, baseUrl: string | undefined): Promise { + const outRoot = path.resolve(config.output.dir); + const site = baseUrl ? siteKey(baseUrl, config.output.site) : config.output.site; + if (!site) return []; + const root = siteDir(outRoot, site); + const out: SourceInfo[] = []; + for (const id of [...(await listRuns(root))].reverse()) { + try { + const m = JSON.parse( + await fs.readFile(path.join(runDir(root, id), "crawl-map.json"), "utf8"), + ) as Manifest; + if (m.source?.commit) out.push(m.source); + } catch { + // An unfinished or unreadable run has no record to offer. + } + } + return out; +} + +async function cmdChanges(flags: Flags): Promise { + const log = makeLog(flags); + const config = await loadConfigWithOverrides(flags); + const intent = await loadRoutesIntent(config); + const baseUrl = str(flags, "target") ?? config.target ?? intent.target; + const recorded = intent.source; + // The run that matches the routes file's record holds its list of dirty files; with + // no record in the file (not committed, or an older catalogue), the newest run's. + const sources = await runSources(config, baseUrl); + const runSource = + (recorded && + (sources.find((s) => s.commit === recorded.commit && s.capturedAt === recorded.capturedAt) ?? + sources.find((s) => s.commit === recorded.commit))) || + (recorded ? undefined : sources[0]); + + const cachePath = modelCachePath(path.resolve(config.output.dir)); + const model = (await pathExists(cachePath)) ? await loadInputManifest(cachePath) : undefined; + const pages = model?.routes.map((r) => r.url) ?? intent.routes.map((r) => r.path); + + let routes: ChangesReport["routes"]; + if (flags.discover === true) { + if (!baseUrl) + throw new UsageError("--discover needs a target: pass --target= or set target"); + const preview = await previewRoutes({ config, baseUrl, log }); + routes = { added: preview.added, removed: preview.capped ? [] : preview.removed }; + if (preview.capped) + log( + `⚠ discovery stopped at scope.maxPages (${config.scope.maxPages}): removed pages aren't listed`, + ); + } + + const routesFile = routesFilePath(config); + const configFile = (await resolveConfigPath(flags)) ?? path.resolve("visualice.config.json"); + const report = await computeChanges({ + cwd: process.cwd(), + recorded, + runSource, + ignore: [ + path.resolve(config.output.dir), + ...(routesFile ? [routesFile] : []), + ...CONFIG_FILES.map((name) => path.resolve(name)), + configFile, + ], + config: { file: configFile, hash: await hashFile(configFile) }, + pages, + routes, + }); + if (flags.json === true) { + console.log(JSON.stringify(report, null, 2)); + return; + } + printChanges(report); +} + +function printChanges(r: ChangesReport): void { + const short = (c: string) => c.slice(0, 12); + if (r.baseline) { + const b = r.baseline; + console.log( + `catalogue: ${short(b.commit)}${b.branch ? ` (${b.branch})` : ""}` + + `${b.dirty ? `, ${b.dirty} uncommitted change(s)` : ""}` + + `${b.capturedAt ? `, shot ${b.capturedAt}` : ""}`, + ); + } + if (r.head) { + console.log( + `now: ${short(r.head.commit)}${r.head.branch ? ` (${r.head.branch})` : ""}` + + `${r.head.dirty ? `, ${r.head.dirty} uncommitted change(s)` : ""}` + + (r.baseline ? ` — ${r.commitCount} commit(s) later` : ""), + ); + } + console.log(`verdict: ${r.verdict}`); + for (const reason of r.reasons) console.log(` · ${reason}`); + if (r.global.length) { + console.log("\nchanges every page:"); + for (const g of r.global) console.log(` ${g.path} (${g.reason})`); + } + if (r.candidates.length) { + console.log("\npages by file:"); + for (const c of r.candidates) { + const pages = c.pages.length ? c.pages.join(", ") : "no page in the catalogue yet"; + console.log( + ` ${c.file} → ${c.pattern}${c.kind === "layout" ? " (layout)" : ""}: ${pages}`, + ); + } + } + if (r.unmapped.length) { + console.log("\nno page on their own (trace who imports them):"); + for (const f of r.unmapped) console.log(` ${f}`); + } + if (r.routes) { + if (r.routes.added.length) console.log(`\nnew pages: ${r.routes.added.join(", ")}`); + if (r.routes.removed.length) console.log(`gone pages: ${r.routes.removed.join(", ")}`); + } + if (r.suggest) console.log(`\nnext: ${r.suggest.command}`); +} + async function cmdRoutes(flags: Flags): Promise { const log = makeLog(flags); const config = await loadConfigWithOverrides(flags); @@ -896,6 +1045,8 @@ async function main(): Promise { return cmdProbe(flags); case "author": return cmdAuthor(flags); + case "changes": + return cmdChanges(flags); } } diff --git a/src/crawl/carry.ts b/src/crawl/carry.ts new file mode 100644 index 0000000..d35ef23 --- /dev/null +++ b/src/crawl/carry.ts @@ -0,0 +1,258 @@ +import path from "node:path"; +import { promises as fs } from "node:fs"; +import type { Device, Manifest, RouteNode, ShotRef, StateNode } from "../shared/types"; +import { matchesAny } from "../shared/glob"; +import { pathExists } from "../shared/fsx"; +import { stateSlug } from "../shared/slug"; +import { isRunComplete, listRuns, runDir as runDirFor } from "./runs"; +import { loadManifest } from "./manifest"; + +// Partial runs (`crawl --only`): shoot the pages a change touched, and carry every +// other page's shots over from the last finished run, so the new run is still the +// whole site. A carried shot is a hard link to the earlier file where the disk +// allows it (no extra space, and it outlives the earlier run being pruned), a copy +// where it doesn't, and records the run and commit it was really shot in. + +/** The finished run a partial run carries from. */ +export interface CarrySource { + runId: string; + runDir: string; + manifest: Manifest; +} + +/** + * The newest finished run of a site with a manifest to carry from, other than + * `exclude` (the run being written, which a `--resume` may have reopened). + */ +export async function findCarrySource( + site: string, + exclude?: string, +): Promise { + for (const id of [...(await listRuns(site))].reverse()) { + if (id === exclude) continue; + const dir = runDirFor(site, id); + if (!(await isRunComplete(dir))) continue; + const file = path.join(dir, "crawl-map.json"); + if (!(await pathExists(file))) continue; + try { + return { runId: id, runDir: dir, manifest: await loadManifest(file) }; + } catch { + // A map that doesn't parse can't be carried from; try the run before it. + } + } + return undefined; +} + +/** A route's path, for matching `--only` globs against: `/pricing`, never a full URL. */ +function routePath(url: string): string { + try { + return new URL(url, "http://_local_").pathname; + } catch { + return url; + } +} + +/** Does a route match one of the `--only` paths or globs? */ +export function matchesOnly(route: Pick, only: string[]): boolean { + return matchesAny(route.url, only) || matchesAny(routePath(route.url), only); +} + +/** The devices a state is captured on. */ +function devicesFor(state: StateNode, devices: Device[]): string[] { + const names = devices.map((d) => d.name); + return state.devices ? names.filter((n) => state.devices!.includes(n)) : names; +} + +/** + * The component states the earlier run looked for on a route, by state id → selector, + * from its config snapshot: a component it sought there and didn't find isn't on the + * route, but one added to the config (or pointed at the route) since was never sought. + */ +export function componentsSought(prev: Manifest, url: string): Map { + const sought = new Map(); + const snapshot = prev.configSnapshot as + | { components?: { name?: unknown; selector?: unknown; routes?: unknown }[] } + | undefined; + for (const c of snapshot?.components ?? []) { + if (typeof c.name !== "string" || typeof c.selector !== "string") continue; + const globs = Array.isArray(c.routes) ? c.routes.filter((g) => typeof g === "string") : []; + if (matchesAny(routePath(url), globs)) sought.set(`component-${stateSlug(c.name)}`, c.selector); + } + return sought; +} + +/** + * Can every shot this route's capture would take be carried from `prev`? Only if + * the earlier run shot the same state on each device it applies to, and the shot + * succeeded: a state new since then, a device added since, or a shot that failed + * then all mean the route is shot again instead. A component the earlier run sought + * on the route (`sought`) and didn't find was dropped from it, so that is no gap; a + * component it never looked for is. + */ +export function canCarry( + route: RouteNode, + prev: RouteNode | undefined, + devices: Device[], + sought: Map = new Map(), +): boolean { + if (!prev) return false; + const byId = new Map(prev.states.map((s) => [s.id, s])); + for (const state of route.states) { + const before = byId.get(state.id); + if (!before) { + if (state.element !== undefined && sought.get(state.id) === state.element) continue; + return false; + } + for (const device of devicesFor(state, devices)) { + const ref = before.shots[device]; + if (!ref) { + if (state.element !== undefined) continue; // not on this route at that size + return false; + } + if (ref.error || !ref.full) return false; + } + } + return true; +} + +export interface CarryPlan { + /** Routes to shoot: the ones `--only` names, and any the earlier run can't cover. */ + capture: RouteNode[]; + /** Routes whose shots come from the earlier run. */ + carry: RouteNode[]; + /** Routes outside `--only` that are shot anyway, and why. */ + forced: { url: string; reason: "new" | "incomplete" }[]; +} + +/** Split a run's routes into the ones to shoot and the ones to carry from `prev`. */ +export function planCarry( + routes: RouteNode[], + only: string[], + prev: Manifest, + devices: Device[], +): CarryPlan { + const prevByUrl = new Map(prev.routes.map((r) => [r.url, r])); + const plan: CarryPlan = { capture: [], carry: [], forced: [] }; + for (const route of routes) { + if (matchesOnly(route, only)) { + plan.capture.push(route); + continue; + } + const before = prevByUrl.get(route.url); + if (canCarry(route, before, devices, componentsSought(prev, route.url))) { + plan.carry.push(route); + } else { + plan.capture.push(route); + plan.forced.push({ url: route.url, reason: before ? "incomplete" : "new" }); + } + } + return plan; +} + +/** + * Move to `capture` any route whose shots in `source` are missing a file on disk + * (an earlier run cleaned up by hand, or cut short while exporting): carrying it + * would fail after every other page had been shot. + */ +export async function verifyCarry( + plan: CarryPlan, + source: CarrySource, + devices: Device[], +): Promise { + const out: CarryPlan = { capture: [...plan.capture], carry: [], forced: [...plan.forced] }; + for (const route of plan.carry) { + const prev = source.manifest.routes.find((r) => r.url === route.url); + const files = prev ? carriedFiles(route, prev, devices) : []; + const present = await Promise.all(files.map((f) => pathExists(path.join(source.runDir, f)))); + if (present.every(Boolean)) { + out.carry.push(route); + } else { + out.capture.push(route); + out.forced.push({ url: route.url, reason: "incomplete" }); + } + } + return out; +} + +/** The files carrying `route` from `prev` would copy. */ +function carriedFiles(route: RouteNode, prev: RouteNode, devices: Device[]): string[] { + const byId = new Map(prev.states.map((s) => [s.id, s])); + const files: string[] = []; + for (const state of route.states) { + const before = byId.get(state.id); + if (!before) continue; + for (const device of devicesFor(state, devices)) { + const ref = before.shots[device]; + if (ref?.full && !ref.error) files.push(...shotFiles(ref)); + } + } + return files; +} + +/** The files a shot points at, relative to its run directory. */ +function shotFiles(ref: ShotRef): string[] { + return [ref.full, ref.viewport, ref.thumb, ref.mid, ref.meta].filter( + (f): f is string => typeof f === "string", + ); +} + +/** Hard-link `from` to `to`, copying instead where links aren't possible. */ +async function linkOrCopy(from: string, to: string): Promise { + await fs.mkdir(path.dirname(to), { recursive: true }); + await fs.rm(to, { force: true }); + try { + await fs.link(from, to); + } catch { + await fs.copyFile(from, to); + } +} + +/** + * Fill `route`'s shots from the same route in `source`, copying the files into + * `runDir`. Returns how many shots were carried. A shot already carried keeps its + * original provenance: carried twice, it still says where it was really taken. + */ +export async function carryRoute( + route: RouteNode, + source: CarrySource, + runDir: string, + devices: Device[], +): Promise { + const prev = source.manifest.routes.find((r) => r.url === route.url); + if (!prev) return 0; + const byId = new Map(prev.states.map((s) => [s.id, s])); + let carried = 0; + for (const state of route.states) { + const before = byId.get(state.id); + if (!before) continue; + for (const device of devicesFor(state, devices)) { + const ref = before.shots[device]; + if (!ref?.full || ref.error) continue; + try { + for (const rel of shotFiles(ref)) { + await linkOrCopy(path.join(source.runDir, rel), path.join(runDir, rel)); + } + } catch (err) { + // verifyCarry checked the files; one gone since fails this shot, not the run. + state.shots[device] = { + error: `couldn't carry the shot from run ${source.runId}: ${(err as Error).message}`, + }; + continue; + } + state.shots[device] = { + ...ref, + carried: ref.carried ?? { + runId: source.runId, + commit: source.manifest.source?.commit, + capturedAt: source.manifest.source?.capturedAt ?? source.manifest.site.generatedAt, + }, + }; + carried++; + } + } + // As a capture does: a component found on no device was never on this route. + route.states = route.states.filter( + (s) => s.element === undefined || Object.keys(s.shots).length > 0, + ); + return carried; +} diff --git a/src/crawl/changes.ts b/src/crawl/changes.ts new file mode 100644 index 0000000..b640e1b --- /dev/null +++ b/src/crawl/changes.ts @@ -0,0 +1,464 @@ +import path from "node:path"; +import type { SourceInfo } from "../shared/types"; +import { git, isUnder, parsePorcelain, repoRoot, toRepoPath } from "./source"; + +// `visualice changes`: what changed in the project since the commit the catalogue +// was shot from, and what that means for the next run. The mechanical half: git +// knows which files changed, file-based routing says which pages some of them are, +// and a list of known global files says when only a full run will do. Which pages a +// shared component touches is the agent's call; this hands it the facts. + +export type Verdict = "none" | "partial" | "full" | "unknown"; + +export interface ChangedFile { + path: string; + /** git's letter: A added, M modified, D deleted, R renamed, ? untracked. */ + status: string; + /** For a rename, where it came from. */ + from?: string; +} + +/** A changed file that file-based routing ties to pages. */ +export interface PageCandidate { + file: string; + /** The route pattern the file serves: `/blog/*`, or `/docs/**` for a layout. */ + pattern: string; + kind: "page" | "layout"; + /** Pages in the site model the pattern matches. */ + pages: string[]; +} + +export interface ChangesReport { + /** The commit the catalogue was shot from, and where that record was read. */ + baseline?: SourceInfo & { from: "routes-file" | "run" }; + head?: { commit: string; branch?: string; dirty: number }; + verdict: Verdict; + /** Why the verdict is what it is, most important first. */ + reasons: string[]; + /** Commits since the baseline (newest first, capped), and how many in all. */ + commits: { hash: string; subject: string }[]; + commitCount: number; + /** Every changed file that counts (the crawl's own files left out). */ + files: ChangedFile[]; + /** Changed files that change every page: dependencies, global styles, root layouts. */ + global: { path: string; reason: string }[]; + /** Changed files that map to pages by their place in the routes folder. */ + candidates: PageCandidate[]; + /** Changed files that map to no page on their own: the agent traces their importers. */ + unmapped: string[]; + /** Pages found or gone since the catalogue, when a route discovery was run. */ + routes?: { added: string[]; removed: string[] }; + /** What to run next, when the facts alone settle it. */ + suggest?: { only?: string[]; command: string }; +} + +const MAX_COMMITS = 50; + +/* + * Global files: a change to one of these reaches every page, so a partial run would + * leave most of the catalogue stale. + */ + +const DEPENDENCY_FILES = new Set([ + "package.json", + "package-lock.json", + "npm-shrinkwrap.json", + "pnpm-lock.yaml", + "yarn.lock", + "bun.lockb", + "bun.lock", + "composer.json", + "composer.lock", + "Gemfile", + "Gemfile.lock", +]); + +const BUILD_CONFIG = + /^(tailwind|postcss|vite|next|nuxt|astro|svelte|remix|webpack|rollup|uno|panda|stitches|theme|app)\.config\.[cm]?[jt]s$/; + +const GLOBAL_STYLE_NAME = + /^(globals?|app|index|main|site|styles?|style|base|reset|normalize|theme|tokens|variables|vars|typography|fonts?|tailwind|bootstrap)\.(css|scss|sass|less|styl|pcss)$/i; + +const GLOBAL_DIRS = new Set(["styles", "theme", "themes", "tokens", "design-tokens", "fonts"]); + +const ROOT_LAYOUTS = [ + /^(src\/)?app\/(layout|template|global-error|not-found)\.[cm]?[jt]sx?$/, + /^(src\/)?pages\/_(app|document)\.[cm]?[jt]sx?$/, + /^(src\/)?app\.(vue|svelte|html)$/i, + /^(src\/)?App\.(vue|svelte|[cm]?[jt]sx?)$/, + /^(src\/)?(main|index)\.[cm]?[jt]sx?$/, + /^src\/routes\/\+layout(\.server)?\.(svelte|[jt]s)$/, + /^app\/root\.[cm]?[jt]sx?$/, + /^(src\/)?layouts\/default\.vue$/, + /^(public\/)?index\.html$/, +]; + +/** Shared chrome: a header or footer is on every page, whatever folder it's in. */ +const CHROME_NAME = + /^(layout|root-?layout|base-?layout|default-?layout|header|footer|nav|navbar|navigation|sidebar|topbar|top-?nav|site-?header|site-?footer|site-?nav|app-?shell|shell)\.(tsx|jsx|vue|svelte|astro|[cm]?[jt]s)$/i; + +/** Why a change to `file` (repo-relative) reaches every page, or undefined. */ +export function globalReason(file: string): string | undefined { + const base = path.posix.basename(file); + const segments = file.split("/"); + if (DEPENDENCY_FILES.has(base) && segments.length <= 2) return "dependencies"; + if (BUILD_CONFIG.test(base)) return "build or theme config"; + if (/^visualice\.config\.(json|[cm]?[jt]s)$/.test(base)) return "visualice config"; + if (ROOT_LAYOUTS.some((re) => re.test(file))) return "root layout or entry point"; + const inComponents = segments.some((s) => /^components?$/i.test(s)); + if (GLOBAL_STYLE_NAME.test(base) && !inComponents) return "global styles"; + if (segments.slice(0, -1).some((s) => GLOBAL_DIRS.has(s.toLowerCase()))) + return "global styles or theme"; + // A nested route layout (app/blog/layout.tsx) only wraps its own section: routing + // maps it to that section's pages below, rather than to every page. + if (CHROME_NAME.test(base) && guessRoute(file)?.kind !== "layout") + return "shared page chrome (layout, header, footer, nav)"; + if (/^(src\/)?layouts\/[^/]+\.astro$/.test(file)) return "page layout"; + return undefined; +} + +/* + * File-based routing: the frameworks whose folder layout is the URL layout. Each + * rule turns a file into route segments; `*` is one dynamic segment, `**` the rest. + */ + +interface RouteGuess { + segments: string[]; + kind: "page" | "layout"; +} + +const CODE = String.raw`(?:[cm]?[jt]sx?|mdx?|vue|svelte|astro|html)`; + +/** `[slug]` → `*`, `[...rest]` / `[[...rest]]` → `**`, `(group)` / `@slot` → dropped. */ +function bracketSegments(parts: string[]): string[] { + const out: string[] = []; + for (const p of parts) { + if (/^\(.*\)$/.test(p) || p.startsWith("@") || p.startsWith("_")) continue; + if (/^\[\[?\.\.\..+\]\]?$/.test(p)) out.push("**"); + else if (/^\[.+\]$/.test(p)) out.push("*"); + else out.push(p); + } + return out; +} + +/** The route a changed file serves, by the conventions of file-based routers. */ +export function guessRoute(file: string): RouteGuess | undefined { + let m: RegExpExecArray | null; + + // Next.js app router: app/blog/[slug]/page.tsx + m = new RegExp( + String.raw`^(?:src/)?app/(.*?)/?(page|layout|template|loading|error|not-found)\.${CODE}$`, + ).exec(file); + if (m && !/(^|\/)api(\/|$)/.test(m[1])) { + const segs = bracketSegments(m[1] ? m[1].split("/") : []); + return { segments: segs, kind: m[2] === "page" ? "page" : "layout" }; + } + // SvelteKit: src/routes/blog/[slug]/+page.svelte + m = new RegExp(String.raw`^src/routes/(.*?)/?\+(page|layout)(?:\.server)?\.${CODE}$`).exec(file); + if (m) { + return { + segments: bracketSegments(m[1] ? m[1].split("/") : []), + kind: m[2] === "page" ? "page" : "layout", + }; + } + // Remix flat routes: app/routes/blog.$slug.tsx, app/routes/_index.tsx + m = new RegExp(String.raw`^app/routes/([^/]+?)(?:/route)?\.${CODE}$`).exec(file); + if (m) { + const segs: string[] = []; + for (const part of m[1].split(".")) { + if (part === "_index" || part.startsWith("_") || /^\(.*\)$/.test(part)) continue; + if (part === "$") segs.push("**"); + else if (part.startsWith("$")) segs.push("*"); + else segs.push(part.replace(/_$/, "")); + } + return { segments: segs, kind: "page" }; + } + // Pages folders: Next.js pages/, Nuxt pages/, Astro src/pages/, Gatsby src/pages/. + m = new RegExp(String.raw`^(?:src/)?pages/(.+)\.${CODE}$`).exec(file); + if (m) { + const parts = m[1].split("/"); + if (parts[0] === "api" || parts.some((p) => p.startsWith("_"))) return undefined; + if (parts.at(-1) === "index") parts.pop(); + return { segments: bracketSegments(parts), kind: "page" }; + } + // Plain HTML at the root or in public/: about.html, blog/index.html + m = /^(?:public\/|static\/)?(.+)\.html?$/.exec(file); + if (m && !file.includes("node_modules/")) { + const parts = m[1].split("/"); + if (parts.at(-1) === "index") parts.pop(); + return { segments: parts, kind: "page" }; + } + return undefined; +} + +/** A guess as a route pattern string: `/blog/*`; a layout covers everything under it. */ +export function guessPattern(g: RouteGuess): string { + const base = `/${g.segments.join("/")}`; + if (g.kind === "layout") return base === "/" ? "/**" : `${base}/**`; + return base; +} + +/** Does a site-model page match a guessed pattern? */ +export function patternMatches(pattern: string, page: string): boolean { + const pathOnly = + (() => { + try { + return new URL(page, "http://_local_").pathname; + } catch { + return page; + } + })().replace(/\/+$/, "") || "/"; + const variants = new Set([ + pathOnly, + pathOnly.replace(/\.html?$/, ""), + pathOnly.replace(/\/index$/, "") || "/", + ]); + let re = ""; + const p = pattern === "/" ? "/" : pattern.replace(/\/+$/, ""); + for (let i = 0; i < p.length; i++) { + if (p.startsWith("/**", i)) { + re += "(?:/.*)?"; + i += 2; + } else if (p.startsWith("**", i)) { + re += ".*"; + i += 1; + } else if (p[i] === "*") { + re += "[^/]+"; + } else re += p[i].replace(/[.+?^${}()|[\]\\]/g, "\\$&"); + } + const rx = new RegExp(`^${re}$`); + return [...variants].some((v) => rx.test(v)); +} + +/** Parse `git diff --name-status -z`. */ +export function parseNameStatus(out: string): ChangedFile[] { + const parts = out.split("\0").filter((p) => p !== ""); + const files: ChangedFile[] = []; + for (let i = 0; i < parts.length; i++) { + const status = parts[i]; + const letter = status[0]; + if (letter === "R" || letter === "C") { + files.push({ status: letter, from: parts[i + 1], path: parts[i + 2] }); + i += 2; + } else { + files.push({ status: letter, path: parts[i + 1] }); + i += 1; + } + } + return files.filter((f) => f.path); +} + +export interface ChangesInput { + cwd: string; + /** The routes file's record, if it has one. */ + recorded?: SourceInfo; + /** A run manifest's record, used when the routes file has none (and for its dirty files). */ + runSource?: SourceInfo; + /** Paths whose changes don't count: the routes file, the output folder, the config. */ + ignore: string[]; + /** The config file now, and its hash: compared with the record's, not diffed. */ + config?: { file: string; hash?: string }; + /** Pages in the cached site model. */ + pages: string[]; + /** From a route discovery, when one ran. */ + routes?: { added: string[]; removed: string[] }; +} + +/** Work out what changed since the catalogue's commit, and the verdict. */ +export async function computeChanges(input: ChangesInput): Promise { + const report: ChangesReport = { + verdict: "unknown", + reasons: [], + commits: [], + commitCount: 0, + files: [], + global: [], + candidates: [], + unmapped: [], + ...(input.routes ? { routes: input.routes } : {}), + }; + const fullRun = "visualice crawl"; + + const root = await repoRoot(input.cwd); + if (!root) { + report.reasons.push("not a git repository: nothing to diff against"); + report.suggest = { command: fullRun }; + return report; + } + const headCommit = (await git(root, ["rev-parse", "HEAD"]))?.trim(); + const branch = (await git(root, ["symbolic-ref", "--quiet", "--short", "HEAD"]))?.trim(); + + const record = input.recorded ?? input.runSource; + if (!record) { + report.reasons.push( + "no recorded commit: the catalogue predates commit tracking, or was never shot", + ); + report.suggest = { command: fullRun }; + return report; + } + const dirtyFiles = + input.runSource && input.runSource.commit === record.commit + ? input.runSource.dirtyFiles + : undefined; + // A record keeps a capped list; past the cap, the files it left out can't be diffed. + const dirtyListed = record.dirty === 0 || (dirtyFiles?.length ?? 0) >= record.dirty; + report.baseline = { + ...record, + ...(dirtyFiles ? { dirtyFiles } : {}), + from: input.recorded ? "routes-file" : "run", + }; + + if (!(await commitExists(root, record.commit))) { + report.reasons.push( + `commit ${record.commit.slice(0, 12)} isn't in this repository (rebased, squashed or not fetched): nothing to diff against`, + ); + report.suggest = { command: fullRun }; + return report; + } + + const ignore = input.ignore.map((p) => toRepoPath(root, p)).filter(Boolean) as string[]; + const counts = (f: string) => !isUnder(f, ignore); + + const [diffOut, untrackedOut, logOut, countOut, ancestor] = await Promise.all([ + git(root, ["diff", "--name-status", "-z", "-M", record.commit]), + git(root, ["ls-files", "--others", "--exclude-standard", "-z"]), + git(root, ["log", "--format=%H%x09%s", `-n${MAX_COMMITS}`, `${record.commit}..HEAD`]), + git(root, ["rev-list", "--count", `${record.commit}..HEAD`]), + git(root, ["merge-base", "--is-ancestor", record.commit, "HEAD"]), + ]); + const files = parseNameStatus(diffOut ?? "").filter((f) => counts(f.path)); + for (const f of (untrackedOut ?? "").split("\0")) { + if (f && counts(f) && !files.some((x) => x.path === f)) files.push({ path: f, status: "?" }); + } + // Files that were dirty when the catalogue was shot: what was shot isn't the + // commit's version of them, so they count as changed whatever the diff says. + for (const f of dirtyFiles ?? []) { + if (counts(f) && !files.some((x) => x.path === f)) files.push({ path: f, status: "M" }); + } + report.files = files; + report.commits = (logOut ?? "") + .split("\n") + .filter(Boolean) + .map((l) => { + const [hash, ...rest] = l.split("\t"); + return { hash, subject: rest.join("\t") }; + }); + report.commitCount = Number(countOut?.trim() ?? report.commits.length) || 0; + const headStatus = await git(root, ["status", "--porcelain=v1", "-z", "--untracked-files=all"]); + report.head = headCommit + ? { + commit: headCommit, + branch: branch || undefined, + dirty: parsePorcelain(headStatus ?? "").filter(counts).length, + } + : undefined; + + // The config is compared by hash: a first crawl scaffolds it, so it is usually not + // committed yet when the catalogue is shot, and a diff would call it new. + if (input.config && record.config && input.config.hash !== record.config) { + const rel = toRepoPath(root, input.config.file) ?? path.basename(input.config.file); + report.global.push({ path: rel, reason: "visualice config" }); + } + + // Classify every changed file. + for (const f of files) { + const reason = globalReason(f.path) ?? (f.from ? globalReason(f.from) : undefined); + if (reason) { + report.global.push({ path: f.path, reason }); + continue; + } + const guess = guessRoute(f.path); + if (guess) { + const pattern = guessPattern(guess); + report.candidates.push({ + file: f.path, + pattern, + kind: guess.kind, + pages: input.pages.filter((p) => patternMatches(pattern, p)), + }); + continue; + } + report.unmapped.push(f.path); + } + + // The verdict, and the reasons that carry it. + if (ancestor === undefined && headCommit !== record.commit) { + report.reasons.push( + `commit ${record.commit.slice(0, 12)} isn't an ancestor of HEAD (another branch, or history rewritten): the diff is between the two trees`, + ); + } + if (record.target && record.local === false) { + report.reasons.push( + `the catalogue was shot from ${record.target}, not a local server: it shows what was deployed there, which may not be commit ${record.commit.slice(0, 12)}`, + ); + } + if (!dirtyListed) { + report.reasons.push( + `the catalogue was shot with ${record.dirty} uncommitted change(s) that can't all be listed now: a full run is the safe choice`, + ); + } + const removed = input.routes?.removed ?? []; + const added = input.routes?.added ?? []; + if (removed.length) { + // The cached model still has them, and a partial run carries every page it doesn't + // shoot: until the routes file skips them, their old shots carry on. + report.reasons.push( + `${removed.length} page(s) are gone from the site (${removed.slice(0, 5).join(", ")}${removed.length > 5 ? ", …" : ""}): add a "- skip: " line for each to the routes file, or partial runs keep carrying their old shots`, + ); + } + + if (report.global.length) { + report.verdict = "full"; + const kinds = [...new Set(report.global.map((g) => g.reason))]; + report.reasons.unshift( + `${report.global.length} changed file(s) reach every page (${kinds.join("; ")}): ${report.global + .slice(0, 5) + .map((g) => g.path) + .join(", ")}${report.global.length > 5 ? ", …" : ""}`, + ); + } else if (!dirtyListed) { + report.verdict = "full"; + } else if (!files.length && !added.length && !removed.length) { + report.verdict = "none"; + report.reasons.unshift( + headCommit === record.commit + ? "HEAD is the commit the catalogue was shot from, and nothing has changed since" + : `${report.commitCount} commit(s) since, but none of them change a file that counts`, + ); + } else { + report.verdict = "partial"; + report.reasons.unshift( + `${files.length} file(s) changed in ${report.commitCount} commit(s)` + + (report.unmapped.length + ? `; ${report.unmapped.length} of them map to no page on their own — find the pages that import them` + : ""), + ); + } + + if (report.verdict === "full") { + report.suggest = { command: fullRun }; + } else if (report.verdict === "partial") { + const only = [ + ...new Set([ + ...report.candidates.flatMap((c) => (c.pages.length ? c.pages : [c.pattern])), + ...added, + ]), + ]; + if (only.length && !report.unmapped.length) { + report.suggest = { + only, + command: `visualice crawl --replay --only=${shellQuote(only.join(","))}`, + }; + } + } + return report; +} + +/** Quote a value for a POSIX shell when it holds anything a shell would expand. */ +export function shellQuote(value: string): string { + return /^[\w@%+=:,./-]+$/.test(value) ? value : `'${value.replace(/'/g, `'\\''`)}'`; +} + +async function commitExists(root: string, commit: string): Promise { + return (await git(root, ["cat-file", "-t", commit]))?.trim() === "commit"; +} diff --git a/src/crawl/orchestrator.ts b/src/crawl/orchestrator.ts index eeb68f4..30a3ea2 100644 --- a/src/crawl/orchestrator.ts +++ b/src/crawl/orchestrator.ts @@ -1,7 +1,15 @@ import path from "node:path"; import { promises as fs } from "node:fs"; import type { Config } from "../shared/config"; -import type { Manifest, InputManifest, StateNode, ViewerData, ViewerTile } from "../shared/types"; +import type { + Manifest, + InputManifest, + RouteNode, + SourceInfo, + StateNode, + ViewerData, + ViewerTile, +} from "../shared/types"; import { matchesAny } from "../shared/glob"; import { runCapture } from "./capture"; import { renderReport } from "./report"; @@ -41,6 +49,15 @@ import { import { redactConfig } from "../shared/redact"; import { acquireLock } from "../shared/lock"; import { applyHints, describeHints } from "../shared/hints"; +import { + type CarryPlan, + carryRoute, + findCarrySource, + matchesOnly, + planCarry, + verifyCarry, +} from "./carry"; +import { CONFIG_FILES, findConfigFile, readSource } from "./source"; type Log = (msg: string) => void; @@ -65,6 +82,15 @@ export interface CapturePipelineOptions { resume?: string; /** Capture each page's default render only: no UI states, no components. */ pagesOnly?: boolean; + /** + * Shoot only the routes these paths or globs match, and carry every other route's + * shots over from the site's last finished run (`crawl --only`). + */ + only?: string[]; + /** The commit being shot; read from the working directory's git when omitted. */ + source?: SourceInfo | null; + /** The config file the run was loaded from (`--config`), for the source record's hash. */ + configFile?: string; log?: (msg: string) => void; } @@ -114,6 +140,7 @@ export function toViewerData(m: Manifest): ViewerData { meta: ref.meta, a11y: ref.a11y, error: ref.error, + ...(ref.carried ? { carried: ref.carried } : m.partial ? { updated: true } : {}), }); } } @@ -121,6 +148,8 @@ export function toViewerData(m: Manifest): ViewerData { return { site: m.site, + ...(m.source ? { source: m.source } : {}), + ...(m.partial ? { partial: m.partial } : {}), devices: m.devices, tiles, groups: m.routes.map((r) => ({ path: pathSlug(r.url), routeId: r.id, title: r.title })), @@ -272,8 +301,13 @@ async function captureRun(opts: CapturePipelineOptions): Promise n + r.states.length, 0); + result.manifest.stats.carried = carried; + log( + `carried ${carried} shot(s) of ${carry.carry.length} page(s) from run ${carrySource.runId}`, + ); + } + if (result.manifest.stats && wanted.length) { result.manifest.stats.wanted = wanted; log(`${wanted.length} wanted state(s) still need authoring (see routes file / report)`); @@ -317,6 +387,45 @@ async function captureRun(opts: CapturePipelineOptions): Promise { + const routesFile = config.routesFile === false ? undefined : path.resolve(config.routesFile); + // `configFile` is the --config the run was given; without one, the file a first + // crawl may just have scaffolded is looked up now. + const file = await findConfigFile(process.cwd(), configFile); + return readSource(process.cwd(), { + target: baseUrl, + configFile: file, + // The crawl's own files: the output folder, the routes file it rewrites, and the + // config (compared by hash in `changes`, since a first crawl scaffolds it). + ignore: [ + path.resolve(config.output.dir), + ...(routesFile ? [routesFile] : []), + ...CONFIG_FILES.map((name) => path.resolve(name)), + ...(file ? [file] : []), + ], + }); +} + +function logCarryPlan(plan: CarryPlan, only: string[], from: string, log: Log): void { + log( + `--only ${only.join(", ")}: shooting ${plan.capture.length} page(s), ` + + `carrying ${plan.carry.length} from run ${from}`, + ); + if (!plan.capture.some((r) => matchesOnly(r, only))) { + log(`⚠ --only matched no page in the site model (${only.join(", ")})`); + } + for (const f of plan.forced) { + log( + ` + ${f.url}: shot too — ${f.reason === "new" ? "not in the earlier run" : "the earlier run is missing some of its shots"}`, + ); + } +} + export interface CrawlPipelineOptions { config: Config; baseUrl: string; @@ -329,6 +438,14 @@ export interface CrawlPipelineOptions { * cache are kept for the next full crawl rather than dropped. */ pagesOnly?: boolean; + /** + * Shoot only the pages these paths or globs match; every other page's shots are + * carried over from the last finished run. With `replay`, the matched pages' states + * are discovered again, since a changed page may have new ones. + */ + only?: string[]; + /** The config file the run was loaded from (`--config`), for the source record's hash. */ + configFile?: string; log?: (msg: string) => void; } @@ -340,6 +457,16 @@ export interface CrawlPlan { /** Screenshots the capture would take: each state on each device it applies to. */ shots: number; devices: string[]; + /** With `--only`: what would be shot and what carried over, and from which run. */ + partial?: { + only: string[]; + carriedFrom?: string; + capture: string[]; + carry: string[]; + forced: CarryPlan["forced"]; + /** Screenshots carried instead of shot. */ + carriedShots: number; + }; } /** @@ -370,7 +497,8 @@ async function crawlRun( } const { inputManifest, intent, storageState } = await crawlModel(opts, outRoot); - return capturePipeline({ + const source = (await readProjectSource(config, baseUrl, opts.configFile)) ?? null; + const result = await capturePipeline({ config, inputManifest, baseUrl, @@ -379,8 +507,31 @@ async function crawlRun( lock: false, resume, pagesOnly: opts.pagesOnly, + only: opts.only, + configFile: opts.configFile, + source, log, }); + // The routes file now says which commit the catalogue shows, for `changes`. A rerun + // of the same commit keeps the line it has, so the file only changes when the code + // did. (Which files were dirty is in each run's manifest, which `changes` reads.) + if (source) { + const record = sameSource(intent.source, source) ? intent.source : source; + await writeRoutesFile(config, inputManifest, intent, baseUrl, () => {}, record); + } + return result; +} + +/** Do two records describe the same code, shot the same way? Capture time aside. */ +export function sameSource(a: SourceInfo | undefined, b: SourceInfo): boolean { + return ( + !!a && + a.commit === b.commit && + a.branch === b.branch && + a.dirty === b.dirty && + a.target === b.target && + a.config === b.config + ); } /** @@ -396,17 +547,43 @@ export async function planCrawl(opts: CrawlPipelineOptions): Promise const config = applyHints(opts.config, inputManifest.hints); const model = captureModel(inputManifest, intent, config, !!opts.pagesOnly); const names = config.devices.map((d) => d.name); - let shots = 0; - for (const r of model.routes) { - for (const s of r.states) - shots += s.devices ? names.filter((n) => s.devices!.includes(n)).length : names.length; + const shotsOf = (routes: RouteNode[]) => { + let n = 0; + for (const r of routes) { + for (const s of r.states) + n += s.devices ? names.filter((d) => s.devices!.includes(d)).length : names.length; + } + return n; + }; + let partial: CrawlPlan["partial"]; + let shooting = model.routes; + if (opts.only?.length) { + const site = siteDir(outRoot, siteKey(opts.baseUrl, config.output.site)); + const source = await findCarrySource(site); + const plan = source + ? await verifyCarry( + planCarry(model.routes, opts.only, source.manifest, config.devices), + source, + config.devices, + ) + : undefined; + if (plan) shooting = plan.capture; + partial = { + only: opts.only, + carriedFrom: source?.runId, + capture: shooting.map((r) => r.url), + carry: plan?.carry.map((r) => r.url) ?? [], + forced: plan?.forced ?? [], + carriedShots: plan ? shotsOf(plan.carry) : 0, + }; } return { cachePath: modelCachePath(outRoot), routes: model.routes.map((r) => ({ url: r.url, states: r.states.map((s) => s.id) })), states: model.routes.reduce((n, r) => n + r.states.length, 0), - shots, + shots: shotsOf(shooting), devices: names, + ...(partial ? { partial } : {}), }; }); } @@ -420,8 +597,8 @@ async function crawlModel( const cachePath = modelCachePath(outRoot); const keepStates = !!opts.pagesOnly; - const prepared = await prepareModel(config, baseUrl, cachePath, log); - const { intent, storageState, cached } = prepared; + const prepared = await prepareModel(config, baseUrl, cachePath, log, onlyPages(opts.only)); + const { intent, seeded, storageState, cached } = prepared; const effectiveConfig = keepStates ? withoutStates(prepared.effectiveConfig) : prepared.effectiveConfig; @@ -430,18 +607,36 @@ async function crawlModel( log(`replay: reusing cached site model → ${cachePath}`); inputManifest = cached; // Pages added to the routes file since the last discovery: map just those, so - // a replay doesn't silently skip them. - const added = unmappedRoutes(cached, intent); - if (added.length) { - log(`replay: discovering ${added.length} page(s) new in the routes file`); + // a replay doesn't silently skip them. With --only, the pages it names are mapped + // again too: a page that changed may have states the cache doesn't know yet. + const added = unmappedRoutes(cached, seeded); + const changed = opts.only?.length + ? cached.routes.filter((r) => matchesOnly(r, opts.only!)).map((r) => r.url) + : []; + // Known pages first, in the model's order, then the new ones: a site-wide state + // stays with the page that had it, whichever run maps it. + const visit = [...new Set([...changed, ...added])]; + if (visit.length) { + log( + `replay: discovering ${visit.length} page(s)` + + (changed.length + ? ` (${changed.length} named by --only, ${added.length} new)` + : " new in the routes file"), + ); + const remapped = new Set(changed); const found = await discoverPipeline({ config: applyHints(effectiveConfig, cached.hints), baseUrl, storageState, log, - limit: { only: added, takenIds: cached.routes.map((r) => r.id) }, + limit: { + only: visit, + takenIds: cached.routes.filter((r) => !remapped.has(r.url)).map((r) => r.id), + }, existing: cached, }); + keepRouteIds(found.routes, cached.routes); + found.routes = dropMissingSeeds(found.routes, opts.only, intent, cached, log); inputManifest = { ...mergeManifests(cached, found, { keepStates }), site: cached.site ?? found.site, @@ -458,12 +653,39 @@ async function crawlModel( log, keepStates, ); + const kept = dropMissingSeeds(inputManifest.routes, opts.only, intent, cached, log); + if (kept.length !== inputManifest.routes.length) { + inputManifest = { ...inputManifest, routes: kept }; + await saveJson(cachePath, inputManifest); + } } await writeRoutesFile(config, inputManifest, intent, baseUrl, log); return { inputManifest, intent, storageState }; } +/** + * Leave out a page that only `--only` named (not in the routes file, not mapped + * before) when it answered with an error: a mistyped path isn't a page to catalogue. + */ +function dropMissingSeeds( + routes: RouteNode[], + only: string[] | undefined, + intent: RoutesIntent, + cached: InputManifest | undefined, + log: Log, +): RouteNode[] { + const seeds = onlyPages(only).filter( + (p) => !intent.routes.some((r) => r.path === p) && !cached?.routes.some((r) => r.url === p), + ); + if (!seeds.length) return routes; + return routes.filter((r) => { + if (!seeds.includes(r.url) || r.status === undefined || r.status < 400) return true; + log(`⚠ --only ${r.url}: the page answered ${r.status}, so it isn't added to the catalogue`); + return false; + }); +} + /** The config for a pages-only discovery: routes, no state probing. */ function withoutStates(config: Config): Config { return { @@ -475,19 +697,69 @@ function withoutStates(config: Config): Config { }; } +/** + * A page rediscovered by name keeps the id it had: shot folders and tile ids are + * built from it, and a compare between runs matches shots by them. + */ +function keepRouteIds(found: RouteNode[], cached: RouteNode[]): void { + const ids = new Map(cached.map((r) => [r.url, r.id])); + for (const r of found) { + const id = ids.get(r.url); + if (id) r.id = id; + } +} + +/** + * The plain paths among `--only` patterns (no globs), as paths: pages to seed + * discovery with. A full URL is reduced to its path. + */ +export function onlyPages(only: string[] | undefined): string[] { + const out: string[] = []; + for (const p of only ?? []) { + if (p.includes("*")) continue; + if (p.startsWith("/")) out.push(p); + else if (/^https?:/.test(p)) { + try { + const u = new URL(p); + out.push(u.pathname + u.search); + } catch { + // not a URL after all: nothing to seed + } + } + } + return [...new Set(out)]; +} + /** What every model-building run starts from: routes-file intent, session, cache. */ -async function prepareModel(config: Config, baseUrl: string, cachePath: string, log: Log) { - // Routes-file intent: listed routes seed discovery, skip: globs exclude. +async function prepareModel( + config: Config, + baseUrl: string, + cachePath: string, + log: Log, + extraPages: string[] = [], +) { + // Routes-file intent: listed routes seed discovery, skip: globs exclude. Pages + // named on the command line (--only) seed it too, so a new one gets mapped, but only + // in `seeded`: a one-off filter (or a typo in one) isn't written into the routes file. const intent = await loadRoutesIntent(config); + const seeded: RoutesIntent = { + ...intent, + routes: [ + ...intent.routes, + ...extraPages + .filter((page) => !intent.routes.some((r) => r.path === page)) + .map((page) => ({ path: page, states: [] })), + ], + }; const intentOrigin = intent.target ? safeOrigin(intent.target) : undefined; if (intentOrigin && intentOrigin !== safeOrigin(baseUrl)) { log(`⚠ routes file targets ${intent.target} but crawling ${baseUrl} — regenerating for it`); } for (const w of intent.warnings) log(`⚠ routes file: ${w}`); - const effectiveConfig = applyRoutesIntent(config, intent); + const effectiveConfig = applyRoutesIntent(config, seeded); const storageState = await ensureStorageState(config, log); const cached = (await pathExists(cachePath)) ? await loadInputManifest(cachePath) : undefined; - return { intent, effectiveConfig, storageState, cached }; + return { intent, seeded, effectiveConfig, storageState, cached }; } /** Discover the site from scratch and fold it into the cache, which it then saves. */ @@ -570,3 +842,41 @@ function safeOrigin(url: string): string | undefined { return undefined; } } + +/** + * Which pages the site has now against the ones the cached model knows, from a + * route-only discovery (no states, nothing written): `changes --discover`. + */ +export async function previewRoutes(opts: { + config: Config; + baseUrl: string; + log?: Log; +}): Promise<{ added: string[]; removed: string[]; capped: boolean }> { + const { config, baseUrl } = opts; + const log = opts.log ?? (() => {}); + const outRoot = path.resolve(config.output.dir); + const cachePath = modelCachePath(outRoot); + return withLock(outRoot, log, true, async () => { + const { effectiveConfig, storageState, cached, intent } = await prepareModel( + config, + baseUrl, + cachePath, + log, + ); + const found = await discoverPipeline({ + config: applyHints(withoutStates(effectiveConfig), cached?.hints), + baseUrl, + storageState, + log, + }); + const now = new Set(found.routes.map((r) => r.url)); + const known = (cached?.routes ?? []).filter((r) => !matchesAny(r.url, intent.skipRoutes)); + const before = new Set(known.map((r) => r.url)); + return { + added: [...now].filter((u) => !before.has(u)), + removed: [...before].filter((u) => !now.has(u)), + // Discovery stopped at scope.maxPages: a page it didn't reach isn't gone. + capped: found.routes.length >= config.scope.maxPages, + }; + }); +} diff --git a/src/crawl/routes-file.ts b/src/crawl/routes-file.ts index 4506e3f..d8a8710 100644 --- a/src/crawl/routes-file.ts +++ b/src/crawl/routes-file.ts @@ -1,9 +1,17 @@ import path from "node:path"; import { promises as fs } from "node:fs"; import type { Config } from "../shared/config"; -import type { InputManifest, RouteNode, StateNode, Target, WantedState } from "../shared/types"; +import type { + InputManifest, + RouteNode, + SourceInfo, + StateNode, + Target, + WantedState, +} from "../shared/types"; import { matchesAny } from "../shared/glob"; import { isAuthored } from "./manifest"; +import { parseSourceLine, renderSourceLine } from "./source"; // The routes file (`visualice.routes.md`) is the human/LLM-facing site index: a // two-level markdown list where only two tokens are load-bearing — the route path @@ -32,6 +40,8 @@ export interface RoutesIntent { routes: RoutesFileRoute[]; /** Top-level `skip:` globs — excluded from discovery and capture. */ skipRoutes: string[]; + /** The commit the catalogue was last captured from (the `visualice-source` line). */ + source?: SourceInfo; warnings: string[]; } @@ -64,6 +74,11 @@ export function parseRoutesFile(text: string): RoutesIntent { let current: RoutesFileRoute | undefined; for (const raw of text.split("\n")) { + const source = parseSourceLine(raw.trim()); + if (source) { + intent.source ??= source; + continue; + } const header = /^#\s+Routes\s+—\s+(.+?)\s*$/.exec(raw); if (header) { intent.target ??= header[1]; @@ -227,11 +242,16 @@ Generated by visualice — edit freely; regenerated after every crawl. /** * Render the routes file from the cached model + prior intent. Deterministic and * idempotent: an unchanged model re-renders byte-identically, so diffs mean change. + * + * `source` is the commit a capture just shot; without one (a discovery, a dry run, + * `visualice routes`) the line the file already had is kept, since the catalogue is + * still the one shot then. */ export function renderRoutesFile( model: InputManifest, intent: RoutesIntent, target: string, + source: SourceInfo | undefined = intent.source, ): string { const skips = stateSkips(intent); const intentByPath = new Map(intent.routes.map((r) => [r.path, r])); @@ -241,7 +261,9 @@ export function renderRoutesFile( for (const r of model.routes) if (!matchesAny(r.url, intent.skipRoutes)) paths.add(r.url); for (const r of intent.routes) if (!matchesAny(r.path, intent.skipRoutes)) paths.add(r.path); - const lines: string[] = [`# Routes — ${target}`, "", LEGEND, ""]; + const lines: string[] = [`# Routes — ${target}`, ""]; + if (source) lines.push(renderSourceLine(source), ""); + lines.push(LEGEND, ""); for (const p of [...paths].sort(routeOrder)) { const route: RouteNode | undefined = modelByUrl.get(p); @@ -301,9 +323,10 @@ export async function writeRoutesFile( intent: RoutesIntent, target: string, log: (msg: string) => void = () => {}, + source?: SourceInfo, ): Promise { const file = routesFilePath(config); if (!file) return; - await fs.writeFile(file, renderRoutesFile(model, intent, target)); + await fs.writeFile(file, renderRoutesFile(model, intent, target, source ?? intent.source)); log(`routes file → ${file}`); } diff --git a/src/crawl/source.ts b/src/crawl/source.ts new file mode 100644 index 0000000..8058b84 --- /dev/null +++ b/src/crawl/source.ts @@ -0,0 +1,192 @@ +import path from "node:path"; +import { createHash } from "node:crypto"; +import { promises as fs, realpathSync } from "node:fs"; +import { execFile } from "node:child_process"; +import type { SourceInfo } from "../shared/types"; + +// The code a catalogue was shot from: the git commit checked out in the project when +// the crawl ran. Recorded in the routes file (committed) and each run's manifest, so +// a later run can ask git what changed since and recapture only what that touches. + +/** Run git in `cwd`; resolves to stdout, or undefined when git fails or isn't there. */ +export function git(cwd: string, args: string[]): Promise { + return new Promise((resolve) => { + execFile( + "git", + args, + { cwd, timeout: 15_000, maxBuffer: 16 * 1024 * 1024, windowsHide: true }, + (err, stdout) => resolve(err ? undefined : stdout), + ); + }); +} + +/** The repository's top-level directory, if `cwd` is inside one. */ +export async function repoRoot(cwd: string): Promise { + return (await git(cwd, ["rev-parse", "--show-toplevel"]))?.trim() || undefined; +} + +const LOOPBACK = /^(localhost|127(?:\.\d+){3}|\[::1\]|0\.0\.0\.0|[^.]+\.localhost|.+\.local)$/i; + +/** + * Is `target` served from this machine? Only then is HEAD known to be what was shot; + * a staging or production URL runs whatever was last deployed there. + */ +export function isLocalTarget(target: string): boolean { + try { + return LOOPBACK.test(new URL(target).hostname); + } catch { + return false; + } +} + +/** Files listed by `git status --porcelain=v1 -z`, repo-relative. */ +export function parsePorcelain(out: string): string[] { + const files: string[] = []; + const parts = out.split("\0"); + for (let i = 0; i < parts.length; i++) { + const entry = parts[i]; + if (entry.length < 4) continue; + const code = entry.slice(0, 2); + files.push(entry.slice(3)); + // A rename or copy is followed by its original path, as its own NUL-terminated field. + if (code[0] === "R" || code[0] === "C") i++; + } + return files; +} + +/** Does `file` (repo-relative, `/`-separated) sit at or under one of `paths`? */ +export function isUnder(file: string, paths: string[]): boolean { + return paths.some((p) => p && (file === p || file.startsWith(p.endsWith("/") ? p : `${p}/`))); +} + +/** Most dirty files a record keeps; past it, `dirty` still says how many there were. */ +const MAX_DIRTY = 200; + +/** + * Read the commit checked out in `cwd`. `ignore` lists repo-relative paths whose + * changes don't count as a dirty tree (the routes file the crawl itself rewrites, the + * output folder). Undefined outside a git repository, or without git. + */ +export async function readSource( + cwd: string, + opts: { target?: string; ignore?: string[]; configFile?: string; now?: Date } = {}, +): Promise { + const root = await repoRoot(cwd); + if (!root) return undefined; + const head = (await git(root, ["rev-parse", "HEAD"]))?.trim(); + if (!head) return undefined; // a repository with no commits yet + const [branchOut, dateOut, statusOut] = await Promise.all([ + git(root, ["symbolic-ref", "--quiet", "--short", "HEAD"]), + git(root, ["show", "-s", "--format=%cI", "HEAD"]), + git(root, ["status", "--porcelain=v1", "-z", "--untracked-files=all"]), + ]); + const ignore = (opts.ignore ?? []).map((p) => toRepoPath(root, p)).filter(Boolean) as string[]; + const dirtyFiles = parsePorcelain(statusOut ?? "").filter((f) => !isUnder(f, ignore)); + const config = opts.configFile ? await hashFile(opts.configFile) : undefined; + return { + commit: head, + branch: branchOut?.trim() || undefined, + committedAt: dateOut?.trim() || undefined, + dirty: dirtyFiles.length, + ...(dirtyFiles.length ? { dirtyFiles: dirtyFiles.slice(0, MAX_DIRTY) } : {}), + capturedAt: (opts.now ?? new Date()).toISOString(), + ...(opts.target ? { target: opts.target, local: isLocalTarget(opts.target) } : {}), + ...(config ? { config } : {}), + }; +} + +/** The first 12 hex digits of a file's SHA-256, or undefined when it can't be read. */ +export async function hashFile(file: string): Promise { + const data = await fs.readFile(file).catch(() => undefined); + return data ? createHash("sha256").update(data).digest("hex").slice(0, 12) : undefined; +} + +/** Config files picked up from the working directory when `--config` isn't given, in order. */ +export const CONFIG_FILES = [ + "visualice.config.ts", + "visualice.config.mjs", + "visualice.config.js", + "visualice.config.json", +]; + +/** + * The config file a command uses: `explicit` (`--config`) when given, else the first + * of CONFIG_FILES in `cwd`. Absolute; undefined when there is none. + */ +export async function findConfigFile(cwd: string, explicit?: string): Promise { + if (explicit) return path.resolve(cwd, explicit); + for (const name of CONFIG_FILES) { + const file = path.join(cwd, name); + if (await fs.stat(file).catch(() => undefined)) return file; + } + return undefined; +} + +/** An absolute or cwd-relative path as a repo-relative, `/`-separated one (or undefined outside it). */ +export function toRepoPath(root: string, p: string): string | undefined { + // git reports the real path of the repository; `p` may come through a symlink + // (macOS's /var → /private/var), so compare real paths. + const rel = path.relative(realPath(root), realPath(path.resolve(p))); + if (!rel || rel.startsWith("..") || path.isAbsolute(rel)) return undefined; + return rel.split(path.sep).join("/"); +} + +/* + * The routes file carries the record as one comment line under its title: + * + * + * + * One line of key=value pairs: readable in a diff, and parsed back by `changes`. + */ + +const SOURCE_LINE = /^\s*$/; + +export function renderSourceLine(s: SourceInfo): string { + const pairs: [string, string | number | undefined][] = [ + ["commit", s.commit], + ["branch", s.branch], + ["dirty", s.dirty], + ["captured", s.capturedAt], + ["target", s.target], + ["config", s.config], + ]; + const body = pairs + .filter(([, v]) => v !== undefined && v !== "") + .map(([k, v]) => `${k}=${String(v).replace(/\s+/g, "%20")}`) + .join(" "); + return ``; +} + +/** The record in a routes-file line, or undefined when the line isn't one. */ +export function parseSourceLine(line: string): SourceInfo | undefined { + const m = SOURCE_LINE.exec(line); + if (!m) return undefined; + const kv = new Map(); + for (const token of m[1].split(/\s+/)) { + const eq = token.indexOf("="); + if (eq > 0) kv.set(token.slice(0, eq), token.slice(eq + 1).replace(/%20/g, " ")); + } + const commit = kv.get("commit"); + if (!commit || !/^[0-9a-f]{7,64}$/i.test(commit)) return undefined; + const dirty = Number(kv.get("dirty") ?? 0); + const target = kv.get("target"); + const config = kv.get("config"); + return { + commit, + branch: kv.get("branch") || undefined, + dirty: Number.isFinite(dirty) && dirty > 0 ? Math.floor(dirty) : 0, + capturedAt: kv.get("captured") ?? "", + ...(target ? { target, local: isLocalTarget(target) } : {}), + ...(config ? { config } : {}), + }; +} + +/** The real path of `p`, resolving through its nearest existing parent when it doesn't exist. */ +function realPath(p: string): string { + try { + return realpathSync(p); + } catch { + const parent = path.dirname(p); + return parent === p ? p : path.join(realPath(parent), path.basename(p)); + } +} diff --git a/src/shared/manifest-schema.ts b/src/shared/manifest-schema.ts index 3912136..902e9e7 100644 --- a/src/shared/manifest-schema.ts +++ b/src/shared/manifest-schema.ts @@ -93,6 +93,14 @@ export const ShotRefSchema = z.object({ .optional(), a11y: z.number().optional(), error: z.string().optional(), + carried: z + .object({ + runId: z.string(), + commit: z.string().optional(), + capturedAt: z.string().optional(), + }) + .optional() + .describe("Copied from an earlier run by a partial crawl (`--only`), not shot by this one."), }); export const StateNodeSchema = z.object({ diff --git a/src/shared/types.ts b/src/shared/types.ts index 845c6ce..10ac3e6 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -98,6 +98,43 @@ export interface ShotRef { /** Accessibility rules this state broke, when `checks.accessibility` is on. */ a11y?: number; error?: string; // capture was attempted but failed (after retries); no image written + /** + * Not shot by this run: copied from an earlier one by a partial crawl (`--only`), + * because nothing that changed touched this page. Says where it really comes from. + */ + carried?: CarriedFrom; +} + +/** The run (and commit) a carried-over shot was really taken in. */ +export interface CarriedFrom { + runId: string; + /** The commit that run was shot from, when it recorded one. */ + commit?: string; + capturedAt?: string; +} + +/** + * The code a run was shot from: the git commit checked out in the project. Written to + * the routes file and each run's manifest; `visualice changes` diffs against it. + */ +export interface SourceInfo { + /** Full commit hash of HEAD. */ + commit: string; + branch?: string; + /** The commit's committer date (ISO 8601). */ + committedAt?: string; + /** Files with uncommitted changes at capture time (0 = a clean tree). */ + dirty: number; + /** Which files those were (capped). */ + dirtyFiles?: string[]; + /** When the run captured (ISO 8601). */ + capturedAt: string; + /** The URL that was shot. */ + target?: string; + /** A hash of the visualice config file, so a settings change since is noticed. */ + config?: string; + /** The target was served from this machine, so HEAD is known to be what was shot. */ + local?: boolean; } /** One route×state×device capture that failed after all retries. */ @@ -186,6 +223,13 @@ export interface SiteInfo { export interface Manifest { version: number; site: SiteInfo; + /** The commit the run was shot from, when the project is a git repository. */ + source?: SourceInfo; + /** + * A partial run (`crawl --only`): the routes it shot, and the run every other + * route's shots were carried over from. + */ + partial?: { only: string[]; carriedFrom?: string }; configSnapshot?: unknown; devices: Device[]; auth?: { method: string; notes?: string }; @@ -195,6 +239,8 @@ export interface Manifest { states: number; shots: number; failed: number; + /** Shots copied from an earlier run instead of shot (partial runs). */ + carried?: number; durationMs: number; failures: CaptureFailure[]; /** Routes-file requests still unmet after this run (the Explore-mode queue). */ @@ -257,10 +303,18 @@ export interface ViewerTile { a11y?: number; /** Capture failed after retries; the tile is a placeholder carrying the reason. */ error?: string; + /** Copied from an earlier run, not shot by this one; see `ShotRef.carried`. */ + carried?: CarriedFrom; + /** Shot by a partial run, where the other pages' shots were carried over. */ + updated?: boolean; } export interface ViewerData { site: SiteInfo; + /** The commit the run was shot from. */ + source?: SourceInfo; + /** Set on a partial run; see `Manifest.partial`. */ + partial?: Manifest["partial"]; devices: Device[]; tiles: ViewerTile[]; groups: { path: string; routeId: string; title?: string }[]; diff --git a/tests/carry.test.ts b/tests/carry.test.ts new file mode 100644 index 0000000..1663854 --- /dev/null +++ b/tests/carry.test.ts @@ -0,0 +1,243 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + canCarry, + carryRoute, + componentsSought, + findCarrySource, + matchesOnly, + planCarry, + verifyCarry, +} from "../src/crawl/carry"; +import { markRunIncomplete, runDir } from "../src/crawl/runs"; +import type { Device, Manifest, RouteNode, StateNode } from "../src/shared/types"; + +const devices: Device[] = [ + { name: "desktop", width: 1440, height: 900, dpr: 1 }, + { name: "mobile", width: 390, height: 844, dpr: 2, mobile: true }, +]; + +const shot = (route: string, state: string, device: string) => ({ + full: `screenshots/${route}/${state}/${device}.png`, + thumb: `screenshots/${route}/${state}/${device}.thumb.webp`, +}); + +const state = (id: string, extra: Partial = {}): StateNode => ({ + id, + kind: id === "default" ? "default" : "menu", + label: id, + recipe: [], + shots: {}, + ...extra, +}); + +/** A route as an earlier run shot it: every state on every device it applies to. */ +const shotRoute = (id: string, url: string, states: StateNode[]): RouteNode => ({ + id, + url, + depth: 0, + states: states.map((s) => ({ + ...s, + shots: Object.fromEntries( + devices + .filter((d) => !s.devices || s.devices.includes(d.name)) + .map((d) => [d.name, shot(id, s.id, d.name)]), + ), + })), +}); + +const route = (id: string, url: string, states: StateNode[]): RouteNode => ({ + id, + url, + depth: 0, + states: states.map((s) => ({ ...s, shots: {} })), +}); + +const manifest = (routes: RouteNode[]): Manifest => ({ + version: 1, + site: { + baseUrl: "http://x/", + origin: "http://x", + runId: "r1", + generatedAt: "2026-11-09T00:00:00Z", + }, + source: { commit: "a".repeat(40), dirty: 0, capturedAt: "2026-11-09T00:00:00Z" }, + devices, + routes, +}); + +describe("matching --only", () => { + it("takes paths and globs", () => { + expect(matchesOnly({ url: "/about" }, ["/about"])).toBe(true); + expect(matchesOnly({ url: "/blog/hi" }, ["/blog/**"])).toBe(true); + expect(matchesOnly({ url: "/blog" }, ["/blog/**"])).toBe(true); + expect(matchesOnly({ url: "/pricing" }, ["/about"])).toBe(false); + }); +}); + +describe("canCarry", () => { + const prev = shotRoute("home", "/", [state("default"), state("menu", { devices: ["mobile"] })]); + + it("carries a route the earlier run shot completely", () => { + expect( + canCarry( + route("home", "/", [state("default"), state("menu", { devices: ["mobile"] })]), + prev, + devices, + ), + ).toBe(true); + }); + + it("refuses a new route, a new state, a new device and a failed shot", () => { + expect(canCarry(route("x", "/x", [state("default")]), undefined, devices)).toBe(false); + expect(canCarry(route("home", "/", [state("default"), state("modal")]), prev, devices)).toBe( + false, + ); + // A state that used to be mobile-only now applies everywhere. + expect(canCarry(route("home", "/", [state("menu")]), prev, devices)).toBe(false); + const failed = shotRoute("home", "/", [state("default")]); + failed.states[0].shots.mobile = { error: "timeout" }; + expect(canCarry(route("home", "/", [state("default")]), failed, devices)).toBe(false); + }); + + it("doesn't count a component the earlier run sought and didn't find as a gap", () => { + const comp = state("component-card", { element: ".card", kind: "component" }); + const sought = new Map([["component-card", ".card"]]); + expect(canCarry(route("home", "/", [state("default"), comp]), prev, devices, sought)).toBe( + true, + ); + }); + + it("shoots a component the earlier run never looked for, or looked for elsewhere", () => { + const comp = state("component-card", { element: ".card", kind: "component" }); + const r = route("home", "/", [state("default"), comp]); + expect(canCarry(r, prev, devices)).toBe(false); + expect(canCarry(r, prev, devices, new Map([["component-card", ".old-card"]]))).toBe(false); + }); + + it("reads what the earlier run sought from its config snapshot", () => { + const m = { + ...manifest([]), + configSnapshot: { + components: [ + { name: "Card", selector: ".card", routes: ["/**"] }, + { name: "Cart", selector: ".cart", routes: ["/shop/**"] }, + ], + }, + }; + expect([...componentsSought(m, "/about")]).toEqual([["component-card", ".card"]]); + expect(componentsSought(m, "/shop/x").get("component-cart")).toBe(".cart"); + }); +}); + +describe("planCarry", () => { + it("shoots --only pages and anything the earlier run can't cover", () => { + const prev = manifest([ + shotRoute("home", "/", [state("default")]), + shotRoute("about", "/about", [state("default")]), + shotRoute("blog", "/blog", [state("default")]), + ]); + const plan = planCarry( + [ + route("home", "/", [state("default")]), + route("about", "/about", [state("default")]), + route("blog", "/blog", [state("default"), state("menu")]), + route("pricing", "/pricing", [state("default")]), + ], + ["/about"], + prev, + devices, + ); + expect(plan.capture.map((r) => r.url)).toEqual(["/about", "/blog", "/pricing"]); + expect(plan.carry.map((r) => r.url)).toEqual(["/"]); + expect(plan.forced).toEqual([ + { url: "/blog", reason: "incomplete" }, + { url: "/pricing", reason: "new" }, + ]); + }); +}); + +describe("carrying files", () => { + let site: string; + beforeEach(async () => { + site = await fs.mkdtemp(path.join(os.tmpdir(), "vc-carry-")); + }); + afterEach(() => fs.rm(site, { recursive: true, force: true })); + + it("finds the newest finished run and copies its shots with their provenance", async () => { + const earlier = runDir(site, "2026-11-09T00-00-00-000Z"); + const prev = manifest([shotRoute("home", "/", [state("default")])]); + await fs.mkdir(earlier, { recursive: true }); + await fs.writeFile(path.join(earlier, "crawl-map.json"), JSON.stringify(prev)); + for (const s of Object.values(prev.routes[0].states[0].shots)) { + for (const f of [s.full!, s.thumb!]) { + await fs.mkdir(path.dirname(path.join(earlier, f)), { recursive: true }); + await fs.writeFile(path.join(earlier, f), f); + } + } + // An unfinished run after it is skipped. + const broken = runDir(site, "2026-11-09T01-00-00-000Z"); + await fs.mkdir(broken, { recursive: true }); + await markRunIncomplete(broken); + + const source = await findCarrySource(site, "2026-11-09T02-00-00-000Z"); + expect(source?.runId).toBe("2026-11-09T00-00-00-000Z"); + + const now = runDir(site, "2026-11-09T02-00-00-000Z"); + const r = route("home", "/", [state("default")]); + expect(await carryRoute(r, source!, now, devices)).toBe(2); + const desk = r.states[0].shots.desktop; + expect(desk.carried).toEqual({ + runId: "2026-11-09T00-00-00-000Z", + commit: "a".repeat(40), + capturedAt: "2026-11-09T00:00:00Z", + }); + expect(await fs.readFile(path.join(now, desk.full!), "utf8")).toBe(desk.full); + }); + + it("shoots a route instead of carrying it when the earlier run lost a file", async () => { + const earlier = runDir(site, "2026-11-09T00-00-00-000Z"); + const prev = manifest([ + shotRoute("home", "/", [state("default")]), + shotRoute("about", "/about", [state("default")]), + ]); + await fs.mkdir(earlier, { recursive: true }); + await fs.writeFile(path.join(earlier, "crawl-map.json"), JSON.stringify(prev)); + for (const r of prev.routes) { + for (const s of Object.values(r.states[0].shots)) { + for (const f of [s.full!, s.thumb!]) { + if (r.url === "/about" && f.endsWith("mobile.thumb.webp")) continue; // lost + await fs.mkdir(path.dirname(path.join(earlier, f)), { recursive: true }); + await fs.writeFile(path.join(earlier, f), f); + } + } + } + const source = (await findCarrySource(site))!; + const routes = [ + route("home", "/", [state("default")]), + route("about", "/about", [state("default")]), + ]; + const plan = await verifyCarry(planCarry(routes, ["/nothing"], prev, devices), source, devices); + expect(plan.carry.map((r) => r.url)).toEqual(["/"]); + expect(plan.capture.map((r) => r.url)).toEqual(["/about"]); + expect(plan.forced).toEqual([{ url: "/about", reason: "incomplete" }]); + + // And a file gone after the check fails that shot, not the run. + const about = route("about", "/about", [state("default")]); + const now = runDir(site, "2026-11-09T02-00-00-000Z"); + expect(await carryRoute(about, source, now, devices)).toBe(1); + expect(about.states[0].shots.mobile.error).toMatch(/couldn't carry/); + }); +}); + +describe("--only seeds", () => { + it("are the plain paths, a full URL reduced to its path", async () => { + const { onlyPages } = await import("../src/crawl/orchestrator"); + expect(onlyPages(["/about", "/blog/**", "https://site.test/pricing?x=1", "/about"])).toEqual([ + "/about", + "/pricing?x=1", + ]); + }); +}); diff --git a/tests/changes.test.ts b/tests/changes.test.ts new file mode 100644 index 0000000..b7d47cc --- /dev/null +++ b/tests/changes.test.ts @@ -0,0 +1,216 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { execFileSync } from "node:child_process"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + computeChanges, + globalReason, + guessPattern, + guessRoute, + parseNameStatus, + patternMatches, + shellQuote, +} from "../src/crawl/changes"; +import { readSource } from "../src/crawl/source"; + +const pattern = (file: string) => { + const g = guessRoute(file); + return g ? guessPattern(g) : undefined; +}; + +describe("file-based routing", () => { + it("maps Next.js app router files", () => { + expect(pattern("app/page.tsx")).toBe("/"); + expect(pattern("src/app/blog/[slug]/page.tsx")).toBe("/blog/*"); + expect(pattern("app/(marketing)/pricing/page.tsx")).toBe("/pricing"); + expect(pattern("app/docs/layout.tsx")).toBe("/docs/**"); + expect(pattern("app/shop/[...all]/page.tsx")).toBe("/shop/**"); + expect(pattern("app/api/users/route.ts")).toBeUndefined(); + }); + + it("maps pages folders, SvelteKit, Remix and plain HTML", () => { + expect(pattern("pages/about.tsx")).toBe("/about"); + expect(pattern("src/pages/blog/index.astro")).toBe("/blog"); + expect(pattern("pages/_app.tsx")).toBeUndefined(); + expect(pattern("src/routes/blog/[slug]/+page.svelte")).toBe("/blog/*"); + expect(pattern("src/routes/admin/+layout.svelte")).toBe("/admin/**"); + expect(pattern("app/routes/blog.$slug.tsx")).toBe("/blog/*"); + expect(pattern("app/routes/_index.tsx")).toBe("/"); + expect(pattern("about.html")).toBe("/about"); + expect(pattern("public/docs/index.html")).toBe("/docs"); + }); + + it("matches patterns against catalogue pages", () => { + expect(patternMatches("/blog/*", "/blog/hello")).toBe(true); + expect(patternMatches("/blog/*", "/blog")).toBe(false); + expect(patternMatches("/docs/**", "/docs")).toBe(true); + expect(patternMatches("/docs/**", "/docs/a/b")).toBe(true); + expect(patternMatches("/**", "/anything")).toBe(true); + expect(patternMatches("/about", "/about.html")).toBe(true); + expect(patternMatches("/", "/")).toBe(true); + }); +}); + +describe("global files", () => { + it("names the changes that reach every page", () => { + expect(globalReason("package.json")).toBe("dependencies"); + expect(globalReason("tailwind.config.ts")).toBe("build or theme config"); + expect(globalReason("src/app/layout.tsx")).toBe("root layout or entry point"); + expect(globalReason("src/styles/buttons.css")).toBe("global styles or theme"); + expect(globalReason("app/globals.css")).toBe("global styles"); + expect(globalReason("src/components/Header.tsx")).toMatch(/chrome/); + expect(globalReason("visualice.config.json")).toBe("visualice config"); + }); + + it("leaves page files and local components alone", () => { + expect(globalReason("app/pricing/page.tsx")).toBeUndefined(); + // A nested layout wraps its own section only; routing maps it below. + expect(globalReason("app/blog/layout.tsx")).toBeUndefined(); + expect(globalReason("src/app/(shop)/cart/layout.tsx")).toBeUndefined(); + expect(globalReason("src/components/layout.tsx")).toMatch(/chrome/); + expect(globalReason("src/components/PricingCard.tsx")).toBeUndefined(); + expect(globalReason("src/components/card/index.css")).toBeUndefined(); + expect(globalReason("packages/ui/package.json")).toBeUndefined(); + }); +}); + +it("quotes a suggested --only value only when a shell would expand it", () => { + expect(shellQuote("/about,/pricing")).toBe("/about,/pricing"); + expect(shellQuote("/blog/*,/about")).toBe("'/blog/*,/about'"); + expect(shellQuote("/it's")).toBe("'/it'\\''s'"); +}); + +it("parses name-status output with renames", () => { + expect(parseNameStatus("M\0a.ts\0R100\0old.ts\0new.ts\0A\0b.ts\0")).toEqual([ + { status: "M", path: "a.ts" }, + { status: "R", from: "old.ts", path: "new.ts" }, + { status: "A", path: "b.ts" }, + ]); +}); + +describe("computeChanges", () => { + let dir: string; + const git = (...args: string[]) => + execFileSync("git", ["-c", "user.email=t@t", "-c", "user.name=t", ...args], { + cwd: dir, + }).toString(); + const write = async (file: string, text: string) => { + await fs.mkdir(path.dirname(path.join(dir, file)), { recursive: true }); + await fs.writeFile(path.join(dir, file), text); + }; + const commit = (msg: string) => { + git("add", "-A"); + git("commit", "-qm", msg); + }; + const pages = ["/", "/about", "/blog", "/blog/hello", "/pricing"]; + const run = async (recorded: typeof baseline | null = baseline) => + computeChanges({ + cwd: dir, + recorded: recorded ?? undefined, + ignore: [path.join(dir, ".visualice"), path.join(dir, "visualice.routes.md")], + pages, + }); + let baseline: Awaited>; + + beforeEach(async () => { + dir = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "vc-changes-"))); + git("init", "-q", "-b", "main"); + await write("app/page.tsx", "home"); + await write("app/about/page.tsx", "about"); + await write("app/blog/[slug]/page.tsx", "post"); + await write("src/components/Card.tsx", "card"); + commit("init"); + baseline = await readSource(dir, { target: "http://localhost:3000/" }); + }); + afterEach(() => fs.rm(dir, { recursive: true, force: true })); + + it("is none when nothing that counts changed", async () => { + await write("visualice.routes.md", "# Routes"); + commit("catalogue"); + const r = await run(); + expect(r.verdict).toBe("none"); + expect(r.commitCount).toBe(1); + expect(r.files).toEqual([]); + }); + + it("is partial with page candidates, and suggests --only", async () => { + await write("app/about/page.tsx", "about v2"); + await write("app/pricing/page.tsx", "new page"); + commit("about + pricing"); + const r = await run(); + expect(r.verdict).toBe("partial"); + expect(r.candidates.map((c) => [c.file, c.pages])).toEqual([ + ["app/about/page.tsx", ["/about"]], + ["app/pricing/page.tsx", ["/pricing"]], + ]); + expect(r.suggest?.only).toEqual(["/about", "/pricing"]); + expect(r.commits[0].subject).toBe("about + pricing"); + }); + + it("counts uncommitted and untracked files, and leaves the trace to the agent", async () => { + await write("src/components/Card.tsx", "card v2"); + await write("src/components/New.tsx", "new"); + const r = await run(); + expect(r.verdict).toBe("partial"); + expect(r.unmapped.sort()).toEqual(["src/components/Card.tsx", "src/components/New.tsx"]); + expect(r.suggest).toBeUndefined(); + }); + + it("maps a nested layout to its section, and quotes a glob it suggests", async () => { + await write("app/blog/layout.tsx", "blog shell"); + commit("blog layout"); + const r = await run(); + expect(r.verdict).toBe("partial"); + expect(r.candidates[0]).toMatchObject({ pattern: "/blog/**", kind: "layout" }); + expect(r.suggest?.only).toEqual(["/blog", "/blog/hello"]); + + await write("app/shop/[id]/page.tsx", "product"); + commit("shop"); + const next = await run(); + expect(next.suggest?.command).toBe( + "visualice crawl --replay --only='/blog,/blog/hello,/shop/*'", + ); + }); + + it("counts a rename as one uncommitted change", async () => { + git("mv", "src/components/Card.tsx", "src/components/Tile.tsx"); + expect((await run()).head?.dirty).toBe(1); + }); + + it("is full when the record's list of dirty files was cut short", async () => { + const r = await computeChanges({ + cwd: dir, + recorded: { ...baseline!, dirty: 250 }, + runSource: { ...baseline!, dirty: 250, dirtyFiles: ["app/page.tsx"] }, + ignore: [], + pages, + }); + expect(r.verdict).toBe("full"); + expect(r.reasons.some((x) => x.includes("can't all be listed"))).toBe(true); + }); + + it("is full when a global file changed", async () => { + await write("package.json", "{}"); + await write("app/about/page.tsx", "about v2"); + commit("deps"); + const r = await run(); + expect(r.verdict).toBe("full"); + expect(r.global).toEqual([{ path: "package.json", reason: "dependencies" }]); + expect(r.suggest?.command).toBe("visualice crawl"); + }); + + it("is unknown without a record or with a commit that isn't there", async () => { + expect((await run(null)).verdict).toBe("unknown"); + const gone = await run({ ...baseline!, commit: "f".repeat(40) }); + expect(gone.verdict).toBe("unknown"); + expect(gone.reasons[0]).toMatch(/isn't in this repository/); + }); + + it("warns when the catalogue was shot from a deployed site", async () => { + await write("app/about/page.tsx", "about v2"); + commit("about"); + const r = await run({ ...baseline!, target: "https://example.com/", local: false }); + expect(r.reasons.some((x) => x.includes("not a local server"))).toBe(true); + }); +}); diff --git a/tests/integration/partial.test.ts b/tests/integration/partial.test.ts new file mode 100644 index 0000000..41a4140 --- /dev/null +++ b/tests/integration/partial.test.ts @@ -0,0 +1,79 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { serveDirectory } from "../../src/shared/static-server"; +import type { StaticServer } from "../../src/shared/file-serve"; +import { ConfigSchema, type Config } from "../../src/shared/config"; +import { crawlPipeline, planCrawl } from "../../src/crawl/orchestrator"; +import type { ViewerData } from "../../src/shared/types"; +import { FIXTURE_DIR, silent, tempDir } from "./helpers"; + +/** + * `crawl --only` end-to-end against the fixture: a full crawl, then a partial one that + * shoots one page and carries the rest, so the new run is still the whole site. + */ + +let server: StaticServer; +let out: string; +let config: Config; + +beforeAll(async () => { + server = await serveDirectory(FIXTURE_DIR); + out = await tempDir("partial"); + const base = ConfigSchema.parse({ + scope: { maxPages: 3, maxDepth: 1 }, + devices: [{ name: "small", width: 420, height: 640, dpr: 1 }], + discovery: { interactions: { maxPerPage: 2 }, mobileStates: false }, + stabilize: { extraDelayMs: 50 }, + politeness: { minDelayMs: 0, concurrency: 4 }, + }); + config = { ...base, routesFile: false, output: { ...base.output, dir: out } }; +}, 30000); + +afterAll(async () => { + await server.close(); + await fs.rm(out, { recursive: true, force: true }); +}); + +describe("partial runs", () => { + it("shoot the named pages and carry the rest from the last run", async () => { + const full = await crawlPipeline({ config, baseUrl: server.url, log: silent }); + const pages = full.manifest.routes.map((r) => r.url); + expect(pages.length).toBeGreaterThan(1); + const named = pages.find((p) => p !== "/")!; + + const plan = await planCrawl({ config, baseUrl: server.url, replay: true, only: [named] }); + expect(plan.partial?.capture).toEqual([named]); + expect(plan.partial?.carriedFrom).toBe(full.manifest.site.runId); + + const part = await crawlPipeline({ + config, + baseUrl: server.url, + replay: true, + only: [named], + log: silent, + }); + const m = part.manifest; + expect(m.partial).toEqual({ only: [named], carriedFrom: full.manifest.site.runId }); + expect(m.routes.map((r) => r.url)).toEqual(pages); + expect(m.stats?.carried).toBeGreaterThan(0); + expect(m.stats!.shots + m.stats!.carried!).toBe(full.manifest.stats!.shots); + + for (const route of m.routes) { + for (const state of route.states) { + for (const ref of Object.values(state.shots)) { + if (route.url === named) expect(ref.carried).toBeUndefined(); + else expect(ref.carried?.runId).toBe(full.manifest.site.runId); + // Every shot's file is in the new run, carried or not. + await expect(fs.stat(path.join(part.runDir, ref.full!))).resolves.toBeTruthy(); + } + } + } + + const data = JSON.parse( + await fs.readFile(path.join(part.runDir, "viewer-data.json"), "utf8"), + ) as ViewerData; + expect(data.tiles.filter((t) => t.updated).every((t) => t.routeUrl === named)).toBe(true); + expect(data.tiles.some((t) => t.carried)).toBe(true); + }, 120000); +}); diff --git a/tests/source.test.ts b/tests/source.test.ts new file mode 100644 index 0000000..70c1585 --- /dev/null +++ b/tests/source.test.ts @@ -0,0 +1,141 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { execFileSync } from "node:child_process"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + findConfigFile, + isLocalTarget, + isUnder, + parsePorcelain, + parseSourceLine, + readSource, + renderSourceLine, +} from "../src/crawl/source"; +import { parseRoutesFile, renderRoutesFile } from "../src/crawl/routes-file"; +import type { SourceInfo } from "../src/shared/types"; + +const SHA = "3f2a9c1e5b7d4f6a8c0e2b4d6f8a0c2e4b6d8f0a"; + +const record = (extra: Partial = {}): SourceInfo => ({ + commit: SHA, + branch: "main", + dirty: 0, + capturedAt: "2026-11-09T10:00:00.000Z", + target: "http://localhost:3000/", + local: true, + ...extra, +}); + +describe("the routes-file source line", () => { + it("round-trips", () => { + const line = renderSourceLine(record({ config: "bab17dc92481" })); + expect(line).toBe( + ``, + ); + expect(parseSourceLine(line)).toEqual(record({ config: "bab17dc92481" })); + }); + + it("keeps a branch name with spaces in one token", () => { + const back = parseSourceLine(renderSourceLine(record({ branch: "my branch" }))); + expect(back?.branch).toBe("my branch"); + }); + + it("ignores other comments and a line without a real commit", () => { + expect(parseSourceLine("")).toBeUndefined(); + expect(parseSourceLine("")).toBeUndefined(); + }); + + it("is read from the routes file and kept when re-rendered without a new one", () => { + const model = { routes: [{ id: "home", url: "/", depth: 0, states: [] }] }; + const first = renderRoutesFile(model, parseRoutesFile(""), "http://localhost:3000/", record()); + const intent = parseRoutesFile(first); + expect(intent.source?.commit).toBe(SHA); + expect(intent.routes.map((r) => r.path)).toEqual(["/"]); + // A discovery or `visualice routes` rewrites the file: the record stays. + expect(renderRoutesFile(model, intent, "http://localhost:3000/")).toBe(first); + // A capture replaces it. + const next = renderRoutesFile( + model, + intent, + "http://localhost:3000/", + record({ commit: "a".repeat(40) }), + ); + expect(parseRoutesFile(next).source?.commit).toBe("a".repeat(40)); + }); +}); + +describe("helpers", () => { + it("knows a local target from a deployed one", () => { + expect(isLocalTarget("http://localhost:3000/")).toBe(true); + expect(isLocalTarget("http://127.0.0.1:8080")).toBe(true); + expect(isLocalTarget("http://shop.local/")).toBe(true); + expect(isLocalTarget("https://staging.example.com/")).toBe(false); + }); + + it("parses porcelain status, renames included", () => { + expect(parsePorcelain(" M a.ts\0?? b.ts\0R new.ts\0old.ts\0")).toEqual([ + "a.ts", + "b.ts", + "new.ts", + ]); + }); + + it("finds the config file a command uses, --config first", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "vc-config-")); + try { + expect(await findConfigFile(dir)).toBeUndefined(); + await fs.writeFile(path.join(dir, "visualice.config.json"), "{}"); + expect(await findConfigFile(dir)).toBe(path.join(dir, "visualice.config.json")); + expect(await findConfigFile(dir, "ci.json")).toBe(path.join(dir, "ci.json")); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); + + it("matches a path at or under another", () => { + expect(isUnder(".visualice/x.json", [".visualice"])).toBe(true); + expect(isUnder("visualice.routes.md", ["visualice.routes.md"])).toBe(true); + expect(isUnder(".visualice-old/x", [".visualice"])).toBe(false); + }); +}); + +describe("readSource", () => { + let dir: string; + const git = (...args: string[]) => + execFileSync("git", ["-c", "user.email=t@t", "-c", "user.name=t", ...args], { cwd: dir }); + + beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), "vc-source-")); + git("init", "-q", "-b", "main"); + await fs.writeFile(path.join(dir, "index.html"), "

hi

"); + git("add", "-A"); + git("commit", "-qm", "init"); + }); + afterEach(() => fs.rm(dir, { recursive: true, force: true })); + + it("reads HEAD, the branch and a clean tree", async () => { + const s = await readSource(dir, { target: "http://localhost:1/" }); + expect(s?.commit).toMatch(/^[0-9a-f]{40}$/); + expect(s?.branch).toBe("main"); + expect(s?.dirty).toBe(0); + expect(s?.local).toBe(true); + }); + + it("lists dirty files, leaving the crawl's own out", async () => { + await fs.writeFile(path.join(dir, "index.html"), "

changed

"); + await fs.writeFile(path.join(dir, "visualice.routes.md"), "# Routes"); + const s = await readSource(dir, { ignore: [path.join(dir, "visualice.routes.md")] }); + expect(s?.dirty).toBe(1); + expect(s?.dirtyFiles).toEqual(["index.html"]); + }); + + it("is undefined outside a repository", async () => { + const plain = await fs.mkdtemp(path.join(os.tmpdir(), "vc-plain-")); + try { + expect(await readSource(plain)).toBeUndefined(); + } finally { + await fs.rm(plain, { recursive: true, force: true }); + } + }); +}); diff --git a/viewer/src/App.tsx b/viewer/src/App.tsx index d21b520..6af12f8 100644 --- a/viewer/src/App.tsx +++ b/viewer/src/App.tsx @@ -985,6 +985,15 @@ export default function App({ onHome }: { onHome?: () => void } = {}) { closing={!selected} onClosed={() => setDetailed(null)} onFullScreen={openFull} + compare={ + siteEntry && shownSite && shownRun && siteEntry.runs.length > 1 + ? { + runs: siteEntry.runs.filter((r) => r !== shownRun), + baseFor: (r) => runBase(shownSite, r), + current: shownRun, + } + : undefined + } position={ detailedIndex >= 0 ? { index: detailedIndex + 1, total: layout.placed.length } diff --git a/viewer/src/Canvas.tsx b/viewer/src/Canvas.tsx index a4f4812..5c7de09 100644 --- a/viewer/src/Canvas.tsx +++ b/viewer/src/Canvas.tsx @@ -898,6 +898,7 @@ function TileCaption({ p, place }: { p: Placed; place: CaptionPlace }) { {tile.stateLabel} {tile.device} + {tile.updated && updated} {place.screen && ( {`${place.screen.at} / ${place.screen.of}`} @@ -917,6 +918,11 @@ function TileCaption({ p, place }: { p: Placed; place: CaptionPlace }) { {tile.stateLabel} {/* With room for one line only, warnings still show beside the name. */} + {place.compact && tile.updated ? ( + + updated + + ) : null} {place.compact && tile.a11y ? ( {`a11y ${tile.a11y}`} @@ -926,6 +932,7 @@ function TileCaption({ p, place }: { p: Placed; place: CaptionPlace }) { {!place.compact && (
{/* Short of room, the kind gives way first: the warnings matter more. */} + {tile.updated && updated} {tile.stateKind} {tile.device} {p.page ? ( @@ -970,10 +977,13 @@ function DeviceCaption({ p, t, device }: { p: Placed; t: Transform; device?: Dev function TileBadge({ children, warn, + accent, yields, }: { children: string; warn?: boolean; + /** Shot by this (partial) run, where the rest was carried over. */ + accent?: boolean; /** The one that gives up its room when there isn't enough; the others keep theirs. */ yields?: boolean; }) { @@ -983,7 +993,11 @@ function TileBadge({ className={cx( "truncate rounded-[4px] border bg-chip px-1 font-mono text-[10px] leading-[15px] whitespace-nowrap", yields ? "min-w-[2.5ch]" : "shrink-0", - warn ? "border-amber-line text-amber" : "border-line-strong text-dim", + warn + ? "border-amber-line text-amber" + : accent + ? "border-accent-line bg-accent-soft font-semibold text-accent" + : "border-line-strong text-dim", )} > {children} diff --git a/viewer/src/Compare.tsx b/viewer/src/Compare.tsx new file mode 100644 index 0000000..719e867 --- /dev/null +++ b/viewer/src/Compare.tsx @@ -0,0 +1,149 @@ +import { useEffect, useState } from "react"; +import { fetchViewerData } from "./load"; +import { runLabel } from "./format"; +import type { ViewerData, ViewerTile } from "./types"; +import { Caption, cx } from "./ui"; + +/** Other versions of the site this shot can be compared with. */ +export interface CompareRuns { + /** Run ids, newest first, without the one on screen. */ + runs: string[]; + /** Where a run's files are served. */ + baseFor: (run: string) => string; + /** The run on screen. */ + current: string; +} + +/** Each run's data, fetched once: stepping through shots compares against the same run. */ +const cache = new Map>(); +function loadRun(url: string): Promise { + let hit = cache.get(url); + if (!hit) { + hit = fetchViewerData(url); + hit.catch(() => cache.delete(url)); + cache.set(url, hit); + } + return hit; +} + +/** The run a shot was really taken in: its own run, or the one it was carried from. */ +const takenIn = (tile: ViewerTile, run: string): string => tile.carried?.runId ?? run; + +/** + * The same shot in another version of the site, side by side with this one. Shots + * match by id (route, state and device), which stays put across runs. Two shots + * taken in the same run are the same image: a partial run carried it over. + */ +export function Compare({ + tile, + base, + compare, +}: { + tile: ViewerTile; + base: string; + compare: CompareRuns; +}) { + // The version before this one by default: what "what changed?" usually means. + const older = compare.runs.find((r) => r < compare.current); + const [other, setOther] = useState(older ?? compare.runs[0]); + const [open, setOpen] = useState(false); + const [state, setState] = useState< + { run: string; tile?: ViewerTile; error?: string } | undefined + >(); + + // A string, not `compare`: the parent builds that object afresh on every render. + const otherBase = other ? compare.baseFor(other) : ""; + useEffect(() => { + if (!open || !other) return; + let live = true; + setState(undefined); + loadRun(otherBase + "viewer-data.json") + .then( + (data) => live && setState({ run: other, tile: data.tiles.find((t) => t.id === tile.id) }), + ) + .catch((e: Error) => live && setState({ run: other, error: e.message })); + return () => { + live = false; + }; + }, [open, other, otherBase, tile.id]); + + const theirs = state?.tile; + const same = theirs && takenIn(theirs, other) === takenIn(tile, compare.current); + + return ( +
setOpen((e.currentTarget as HTMLDetailsElement).open)} + > + + Compare with another version + + ▾ + + +
+ + {!state ? ( + Loading… + ) : state.error ? ( + + That version couldn’t be read: {state.error} + + ) : ( + <> +
+ + +
+ + {!theirs + ? "This shot isn’t in that version: the page or state is new since." + : same + ? "The same shot: it was carried over unchanged, not taken again." + : "Taken separately in each version."} + + + )} +
+
+ ); +} + +function Side({ label, tile, base }: { label: string; tile?: ViewerTile; base: string }) { + const src = tile?.mid ?? tile?.thumb ?? tile?.full; + return ( +
+
{label}
+ {tile && src && !tile.error ? ( + + {label} + + ) : ( +
+ {tile?.error ? "Capture failed" : "Not captured"} +
+ )} +
+ ); +} diff --git a/viewer/src/Inspector.tsx b/viewer/src/Inspector.tsx index 33b6ed2..64887cb 100644 --- a/viewer/src/Inspector.tsx +++ b/viewer/src/Inspector.tsx @@ -4,6 +4,7 @@ import type { Device, ViewerData, ViewerTile } from "./types"; import { runLabel } from "./format"; import { routePath } from "./layout"; import { ShotActions } from "./ShotActions"; +import { Compare, type CompareRuns } from "./Compare"; import { Caption, cx, deviceIcon, Icon, KindTag } from "./ui"; interface Props { @@ -23,8 +24,12 @@ interface Props { position?: { index: number; total: number }; /** Open this shot full screen. */ onFullScreen?: () => void; + /** Other versions of the site, to compare this shot with. */ + compare?: CompareRuns; } +const short = (commit: string) => commit.slice(0, 7); + /** The label/value grid the details (and the shortcuts popover) use. */ export const DL = "grid gap-x-3 gap-y-2 text-[12.5px] [&>dd]:wrap-anywhere [&>dt]:text-dim"; @@ -52,6 +57,7 @@ export function Inspector({ onClosed, position, onFullScreen, + compare, }: Props) { const device: Device | undefined = data.devices.find((d) => d.name === tile.device); @@ -219,6 +225,34 @@ export function Inspector({ )} + {tile.carried ? ( +
+ Carried over +

+ Not shot in this version: nothing that changed touched this page, so the shot from{" "} + {runLabel(tile.carried.runId)} UTC + {tile.carried.commit && ( + <> + {" "} + (commit {short(tile.carried.commit)}) + + )}{" "} + was kept. +

+
+ ) : tile.updated ? ( +
+ Updated in this version +

+ Shot again because this page changed + {data.partial?.carriedFrom ? ( + <> since {runLabel(data.partial.carriedFrom)} UTC + ) : null} + ; pages nothing touched were carried over. +

+
+ ) : null} + {tile.a11y ? (
@@ -262,6 +296,9 @@ export function Inspector({ )}
) : null} + {compare && compare.runs.length > 0 && ( + + )}
{/* Pinned to the foot, where the hand ends up after reading. */}
@@ -274,7 +311,17 @@ export function Inspector({ />
- Captured {runLabel(data.site.runId)} UTC + + Captured {runLabel(tile.carried?.runId ?? data.site.runId)} UTC + {(tile.carried ? tile.carried.commit : data.source?.commit) && ( + <> + {" · "} + + {short((tile.carried ? tile.carried.commit : data.source?.commit)!)} + + + )} + {position && ( Shot {position.index} of {position.total} diff --git a/viewer/src/load.ts b/viewer/src/load.ts index 4b486b0..cbce670 100644 --- a/viewer/src/load.ts +++ b/viewer/src/load.ts @@ -65,6 +65,39 @@ function parseTile(raw: unknown, i: number): ViewerTile { meta: isString(raw.meta) ? raw.meta : undefined, a11y: typeof raw.a11y === "number" ? raw.a11y : undefined, error: isString(raw.error) ? raw.error : undefined, + carried: parseCarried(raw.carried), + updated: raw.updated === true || undefined, + }; +} + +/** Where a carried-over shot was really taken; dropped unless it names a run. */ +function parseCarried(raw: unknown): ViewerTile["carried"] { + if (!isObject(raw) || !isString(raw.runId)) return undefined; + return { + runId: raw.runId, + commit: isString(raw.commit) ? raw.commit : undefined, + capturedAt: isString(raw.capturedAt) ? raw.capturedAt : undefined, + }; +} + +/** The commit a run was shot from, when it recorded one. */ +function parseSource(raw: unknown): ViewerData["source"] { + if (!isObject(raw) || !isString(raw.commit)) return undefined; + return { + commit: raw.commit, + branch: isString(raw.branch) ? raw.branch : undefined, + dirty: typeof raw.dirty === "number" ? raw.dirty : 0, + capturedAt: isString(raw.capturedAt) ? raw.capturedAt : "", + target: isString(raw.target) ? raw.target : undefined, + local: typeof raw.local === "boolean" ? raw.local : undefined, + }; +} + +function parsePartial(raw: unknown): ViewerData["partial"] { + if (!isObject(raw) || !Array.isArray(raw.only)) return undefined; + return { + only: raw.only.filter(isString), + carriedFrom: isString(raw.carriedFrom) ? raw.carriedFrom : undefined, }; } @@ -110,6 +143,8 @@ export function parseViewerData(raw: unknown): ViewerData { runId: isString(site.runId) ? site.runId : "unknown", generatedAt: isString(site.generatedAt) ? site.generatedAt : "", }, + source: parseSource(raw.source), + partial: parsePartial(raw.partial), devices, tiles, groups, From 8a2607bbbebc585d1afbf349587c9bcf9166c26b Mon Sep 17 00:00:00 2001 From: Alexander Tenepere Date: Tue, 10 Nov 2026 00:12:54 +0800 Subject: [PATCH 2/8] fix: Carry only shot files that stay inside their run folder --- src/crawl/carry.ts | 26 ++++++++++++++++++++++++-- tests/carry.test.ts | 11 +++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/crawl/carry.ts b/src/crawl/carry.ts index d35ef23..a280968 100644 --- a/src/crawl/carry.ts +++ b/src/crawl/carry.ts @@ -163,7 +163,9 @@ export async function verifyCarry( for (const route of plan.carry) { const prev = source.manifest.routes.find((r) => r.url === route.url); const files = prev ? carriedFiles(route, prev, devices) : []; - const present = await Promise.all(files.map((f) => pathExists(path.join(source.runDir, f)))); + const present = await Promise.all( + files.map((f) => (f ? pathExists(path.join(source.runDir, f)) : false)), + ); if (present.every(Boolean)) { out.carry.push(route); } else { @@ -183,7 +185,9 @@ function carriedFiles(route: RouteNode, prev: RouteNode, devices: Device[]): str if (!before) continue; for (const device of devicesFor(state, devices)) { const ref = before.shots[device]; - if (ref?.full && !ref.error) files.push(...shotFiles(ref)); + if (!ref?.full || ref.error) continue; + if (!safeShotFiles(ref)) return [""]; // never carried: verifyCarry shoots the route + files.push(...shotFiles(ref)); } } return files; @@ -196,6 +200,20 @@ function shotFiles(ref: ShotRef): string[] { ); } +/** + * Is every file a shot points at a plain relative path inside its run directory? + * The paths come from an earlier run's crawl-map.json, which is only data: one that + * names `../` or an absolute path must not make carrying read, replace or delete a + * file outside the run. + */ +export function safeShotFiles(ref: ShotRef): boolean { + return shotFiles(ref).every((rel) => { + if (!rel || path.isAbsolute(rel) || path.win32.isAbsolute(rel)) return false; + const norm = path.posix.normalize(rel.replace(/\\/g, "/")); + return norm !== ".." && !norm.startsWith("../") && !norm.startsWith("/"); + }); +} + /** Hard-link `from` to `to`, copying instead where links aren't possible. */ async function linkOrCopy(from: string, to: string): Promise { await fs.mkdir(path.dirname(to), { recursive: true }); @@ -228,6 +246,10 @@ export async function carryRoute( for (const device of devicesFor(state, devices)) { const ref = before.shots[device]; if (!ref?.full || ref.error) continue; + if (!safeShotFiles(ref)) { + state.shots[device] = { error: `run ${source.runId} lists a shot file outside its run` }; + continue; + } try { for (const rel of shotFiles(ref)) { await linkOrCopy(path.join(source.runDir, rel), path.join(runDir, rel)); diff --git a/tests/carry.test.ts b/tests/carry.test.ts index 1663854..9a8b9ed 100644 --- a/tests/carry.test.ts +++ b/tests/carry.test.ts @@ -9,6 +9,7 @@ import { findCarrySource, matchesOnly, planCarry, + safeShotFiles, verifyCarry, } from "../src/crawl/carry"; import { markRunIncomplete, runDir } from "../src/crawl/runs"; @@ -241,3 +242,13 @@ describe("--only seeds", () => { ]); }); }); + +describe("shot paths from an earlier run", () => { + it("are carried only when they stay inside the run", () => { + expect(safeShotFiles({ full: "screenshots/home/default/desktop.png" })).toBe(true); + expect(safeShotFiles({ full: "../../.bashrc" })).toBe(false); + expect(safeShotFiles({ full: "screenshots/../../x.png" })).toBe(false); + expect(safeShotFiles({ full: "/etc/passwd" })).toBe(false); + expect(safeShotFiles({ full: "a.png", thumb: "..\\..\\x.webp" })).toBe(false); + }); +}); From 0028f686dc8434e0c8c7f655f5174b7833df1739 Mon Sep 17 00:00:00 2001 From: Alexander Tenepere Date: Tue, 10 Nov 2026 11:21:49 +0800 Subject: [PATCH 3/8] feat: Tell users when a newer visualice is out, and how to turn on plugin auto-update --- CHANGELOG.md | 9 +++ README.md | 5 ++ docs/crawling.md | 9 +++ skills/visualice/SKILL.md | 4 ++ src/cli.ts | 32 ++++++++++ src/shared/update-check.ts | 122 +++++++++++++++++++++++++++++++++++++ tests/update-check.test.ts | 112 ++++++++++++++++++++++++++++++++++ 7 files changed, 293 insertions(+) create mode 100644 src/shared/update-check.ts create mode 100644 tests/update-check.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 656e587..9e0e878 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,15 @@ - **Compare with another version** in the details panel shows the same shot from another run beside this one. +### Update notice + +- An installed CLI tells you when a newer visualice is out (checked against npm at most + once a day, in the background, never in the way), with the command to update the + plugin or the CLI. The skill passes it on. Off with `--quiet`, in CI, or with + `VISUALICE_NO_UPDATE_CHECK=1`. +- The README says how to turn on auto-update for the plugin: a marketplace plugin + doesn't update itself unless you do. + ## 0.1.1 ### Fixes diff --git a/README.md b/README.md index 6085246..c3d412a 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,11 @@ Needs Node 22.18 or newer. /plugin install visualice@visualice ``` +Plugins from a marketplace like this one don't update on their own unless you ask: turn +it on under `/plugin` → **Marketplaces** → visualice → **Enable auto-update**, or update +by hand with `claude plugin update visualice@visualice`. The CLI also says when a newer +version is out. + Then run `/visualice https://your.site`, or just ask Claude to "run visualice on https://your.site". It maps the site, shows you the plan (the kinds of page, how many screenshots, anything exploring would change on the site) and asks once, explores each diff --git a/docs/crawling.md b/docs/crawling.md index 4aa24cc..ac976eb 100644 --- a/docs/crawling.md +++ b/docs/crawling.md @@ -92,6 +92,15 @@ gitignored folder; once tracked, git keeps it) or restore it from a CI cache, th `visualice crawl --replay --json --quiet --fail-on-error`. Without it, `--replay` falls back to a full discovery and the explored states are missing. +### Update notice + +An installed CLI checks npm for a newer visualice at most once a day, in the background, +and says so on stderr at the start of a later command (`ℹ visualice 0.3.0 is available +…`). It never delays a command for more than a moment, and a failed lookup is ignored. +It is off with `--quiet`, in CI (`CI` set), in a source checkout, and with +`VISUALICE_NO_UPDATE_CHECK=1` (or `NO_UPDATE_NOTIFIER=1`). The last answer is cached in +`~/.cache/visualice/update-check.json` (`$XDG_CACHE_HOME` when set). + ## Output Output lands under the configured output directory (default `.visualice/`), one folder diff --git a/skills/visualice/SKILL.md b/skills/visualice/SKILL.md index 7bb8618..0a441dc 100644 --- a/skills/visualice/SKILL.md +++ b/skills/visualice/SKILL.md @@ -19,6 +19,10 @@ from any directory, no repo checkout needed. In a source checkout of this repo i use the npm scripts instead: `npm run vc -- ` for anything below, `npm run crawl -- ` for crawls. The commands below say `visualice`; substitute whichever applies. +If a command prints `ℹ visualice X is available (you have Y)…` on stderr, tell the user +once, with the update command it gives (`claude plugin update visualice@visualice`), and +carry on with the task: don't update anything yourself. + Before the first crawl in a project, run `visualice doctor` — an idempotent preflight that checks Node/Playwright chromium/sharp/the viewer build and self-heals what it safely can (installs chromium, builds the viewer; `--no-fix` to only report). This diff --git a/src/cli.ts b/src/cli.ts index 5efc4f8..a3e753e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -34,6 +34,7 @@ import { } from "./crawl/routes-file"; import { ensureStorageState } from "./crawl/auth"; import { pathExists } from "./shared/fsx"; +import { startUpdateCheck, updateCheckDisabled, type UpdateCheck } from "./shared/update-check"; import { nodeBinCommand } from "./shared/node-bin"; import { serveDirectory } from "./shared/static-server"; import { serveViewer } from "./serve"; @@ -95,6 +96,9 @@ Usage: visualice serve [opts] Launch the zoomable results viewer visualice --version Print the version +An installed CLI says on stderr when a newer version is out (checked at most daily); +VISUALICE_NO_UPDATE_CHECK=1, --quiet or CI turns that off. + Project files: visualice.config.json controls behavior; visualice.routes.md lists pages + UI states and is regenerated after every crawl (edit it: 'skip:' excludes, '[ ]' requests a state). A visualice.config.* in the cwd is picked up automatically. @@ -1014,6 +1018,34 @@ async function main(): Promise { } const flags = parseFlags(cmd, rest); setHeaded(flags.headed === true); + const update = await updateCheck(flags); + if (update?.notice) console.error(`ℹ ${update.notice}`); + try { + await runCommand(cmd, flags); + } finally { + // Let a background registry lookup save its answer, but never hold a finished + // command up for long: an unanswered lookup is simply tried again next time. + if (update) await Promise.race([update.refreshed, delay(1500)]); + } +} + +/** + * The update check, for an installed CLI. A source checkout (this repo, or one + * `npm link`ed) is never "behind" in a way worth telling, and `--quiet` asks for silence. + */ +async function updateCheck(flags: Flags): Promise { + if (flags.quiet === true || updateCheckDisabled()) return undefined; + if (await pathExists(path.join(PROJECT_ROOT, ".git"))) return undefined; + try { + return await startUpdateCheck(await readVersion()); + } catch { + return undefined; + } +} + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms).unref()); + +async function runCommand(cmd: string, flags: Flags): Promise { switch (cmd) { case "init": return cmdInit(flags); diff --git a/src/shared/update-check.ts b/src/shared/update-check.ts new file mode 100644 index 0000000..909699b --- /dev/null +++ b/src/shared/update-check.ts @@ -0,0 +1,122 @@ +import os from "node:os"; +import path from "node:path"; +import { promises as fs } from "node:fs"; + +// "A newer visualice is out" notice. A plugin install doesn't auto-update by default, +// and the plugin's skill pins the CLI version it runs, so without this a user only +// learns about a release by going to look. The check is cheap and never in the way: +// the notice comes from a cached answer, the registry is asked at most once a day, +// and a slow or failed lookup is dropped without a word. + +const REGISTRY_URL = "https://registry.npmjs.org/visualice/latest"; +const DAY_MS = 24 * 60 * 60 * 1000; +const FETCH_TIMEOUT_MS = 2000; + +interface Cache { + checkedAt: number; + latest?: string; +} + +/** Where the last answer is kept: the user's cache folder, outside any project. */ +export function cacheFile(env: NodeJS.ProcessEnv = process.env): string { + const base = + env.XDG_CACHE_HOME || + (process.platform === "win32" && env.LOCALAPPDATA) || + path.join(os.homedir(), ".cache"); + return path.join(base, "visualice", "update-check.json"); +} + +/** Is the check switched off here: opted out, CI, or a test run? */ +export function updateCheckDisabled(env: NodeJS.ProcessEnv = process.env): boolean { + return !!( + env.VISUALICE_NO_UPDATE_CHECK || + env.NO_UPDATE_NOTIFIER || + env.CI || + env.VITEST || + env.NODE_ENV === "test" + ); +} + +/** `a` > `b` for plain x.y.z versions; a prerelease is never "newer". */ +export function isNewer(a: string, b: string): boolean { + const parse = (v: string) => { + const m = /^v?(\d+)\.(\d+)\.(\d+)$/.exec(v.trim()); + return m ? m.slice(1).map(Number) : undefined; + }; + const x = parse(a); + const y = parse(b); + if (!x || !y) return false; + for (let i = 0; i < 3; i++) if (x[i] !== y[i]) return x[i] > y[i]; + return false; +} + +/** The notice for a user on `current` when `latest` is out. */ +export function updateNotice(current: string, latest: string): string { + return ( + `visualice ${latest} is available (you have ${current}). Update the plugin: ` + + `claude plugin update visualice@visualice — or the CLI: npm i -g visualice@latest` + ); +} + +async function readCache(file: string): Promise { + try { + const c = JSON.parse(await fs.readFile(file, "utf8")) as Cache; + return typeof c.checkedAt === "number" ? c : undefined; + } catch { + return undefined; + } +} + +export interface UpdateCheck { + /** The notice to show now, from the cached answer; undefined when up to date. */ + notice?: string; + /** Settles once a stale cache has been refreshed (never rejects). */ + refreshed: Promise; +} + +/** + * Start an update check for `current`. The notice comes from the last answer at + * once; when that answer is a day old, the registry is asked again in the background + * and the next run shows what it said. + */ +export async function startUpdateCheck( + current: string, + opts: { file?: string; now?: number; fetchFn?: typeof fetch } = {}, +): Promise { + const file = opts.file ?? cacheFile(); + const now = opts.now ?? Date.now(); + const cache = await readCache(file); + const notice = + cache?.latest && isNewer(cache.latest, current) + ? updateNotice(current, cache.latest) + : undefined; + if (cache && now - cache.checkedAt < DAY_MS) return { notice, refreshed: Promise.resolve() }; + return { notice, refreshed: refresh(file, now, opts.fetchFn ?? fetch, cache) }; +} + +async function refresh( + file: string, + now: number, + fetchFn: typeof fetch, + previous: Cache | undefined, +): Promise { + let latest = previous?.latest; + try { + const res = await fetchFn(REGISTRY_URL, { + headers: { accept: "application/json" }, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); + if (res.ok) { + const body = (await res.json()) as { version?: unknown }; + if (typeof body.version === "string") latest = body.version; + } + } catch { + // offline, blocked or slow: try again tomorrow + } + try { + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, JSON.stringify({ checkedAt: now, latest } satisfies Cache)); + } catch { + // a read-only home: the check just runs again next time + } +} diff --git a/tests/update-check.test.ts b/tests/update-check.test.ts new file mode 100644 index 0000000..ce2707a --- /dev/null +++ b/tests/update-check.test.ts @@ -0,0 +1,112 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + cacheFile, + isNewer, + startUpdateCheck, + updateCheckDisabled, + updateNotice, +} from "../src/shared/update-check"; + +const DAY = 24 * 60 * 60 * 1000; + +/** A fetch that answers with `version`, and counts its calls. */ +function registry(version: string | Error) { + const calls: string[] = []; + const fn = (async (url: string) => { + calls.push(url); + if (version instanceof Error) throw version; + return new Response(JSON.stringify({ version }), { status: 200 }); + }) as unknown as typeof fetch; + return { fn, calls }; +} + +describe("versions", () => { + it("compares plain x.y.z versions", () => { + expect(isNewer("0.2.0", "0.1.1")).toBe(true); + expect(isNewer("0.10.0", "0.9.9")).toBe(true); + expect(isNewer("1.0.0", "1.0.0")).toBe(false); + expect(isNewer("0.1.0", "0.1.1")).toBe(false); + expect(isNewer("0.3.0-beta.1", "0.2.0")).toBe(false); + expect(isNewer("nonsense", "0.2.0")).toBe(false); + }); + + it("names both update commands", () => { + const n = updateNotice("0.2.0", "0.3.0"); + expect(n).toContain("visualice 0.3.0 is available (you have 0.2.0)"); + expect(n).toContain("claude plugin update visualice@visualice"); + expect(n).toContain("npm i -g visualice@latest"); + }); +}); + +describe("when it runs", () => { + it("is off when opted out, in CI and in tests", () => { + expect(updateCheckDisabled({})).toBe(false); + expect(updateCheckDisabled({ VISUALICE_NO_UPDATE_CHECK: "1" })).toBe(true); + expect(updateCheckDisabled({ NO_UPDATE_NOTIFIER: "1" })).toBe(true); + expect(updateCheckDisabled({ CI: "true" })).toBe(true); + }); + + it("caches under XDG_CACHE_HOME when set", () => { + expect(cacheFile({ XDG_CACHE_HOME: "/c" })).toBe( + path.join("/c", "visualice", "update-check.json"), + ); + }); +}); + +describe("startUpdateCheck", () => { + let dir: string; + let file: string; + beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), "vc-update-")); + file = path.join(dir, "nested", "update-check.json"); + }); + afterEach(() => fs.rm(dir, { recursive: true, force: true })); + + it("says nothing on the first run, asks the registry, and tells the next run", async () => { + const reg = registry("0.3.0"); + const first = await startUpdateCheck("0.2.0", { file, now: 1000, fetchFn: reg.fn }); + expect(first.notice).toBeUndefined(); + await first.refreshed; + expect(reg.calls).toEqual(["https://registry.npmjs.org/visualice/latest"]); + + const next = await startUpdateCheck("0.2.0", { file, now: 2000, fetchFn: reg.fn }); + expect(next.notice).toContain("0.3.0 is available"); + await next.refreshed; + expect(reg.calls).toHaveLength(1); // a fresh answer isn't asked for again + }); + + it("asks again once the answer is a day old, and says nothing when up to date", async () => { + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, JSON.stringify({ checkedAt: 0, latest: "0.2.0" })); + const reg = registry("0.2.0"); + const check = await startUpdateCheck("0.2.0", { file, now: DAY + 1, fetchFn: reg.fn }); + expect(check.notice).toBeUndefined(); + await check.refreshed; + expect(reg.calls).toHaveLength(1); + }); + + it("keeps the last answer when the registry can't be reached", async () => { + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, JSON.stringify({ checkedAt: 0, latest: "0.3.0" })); + const check = await startUpdateCheck("0.2.0", { + file, + now: DAY + 1, + fetchFn: registry(new Error("offline")).fn, + }); + expect(check.notice).toContain("0.3.0"); + await expect(check.refreshed).resolves.toBeUndefined(); + const saved = JSON.parse(await fs.readFile(file, "utf8")); + expect(saved).toEqual({ checkedAt: DAY + 1, latest: "0.3.0" }); + }); + + it("ignores a cache it can't read", async () => { + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, "{not json"); + const check = await startUpdateCheck("0.2.0", { file, now: 1, fetchFn: registry("0.2.0").fn }); + expect(check.notice).toBeUndefined(); + await check.refreshed; + }); +}); From 32b0b1f509d0f6ce29c10b29f90f204e6f29f60f Mon Sep 17 00:00:00 2001 From: Alexander Tenepere Date: Tue, 10 Nov 2026 14:23:24 +0800 Subject: [PATCH 4/8] chore: End-to-end test of the update loop through the CLI in a git repository --- tests/integration/update-flow.test.ts | 157 ++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 tests/integration/update-flow.test.ts diff --git a/tests/integration/update-flow.test.ts b/tests/integration/update-flow.test.ts new file mode 100644 index 0000000..e88f412 --- /dev/null +++ b/tests/integration/update-flow.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it, beforeAll, afterAll } from "vitest"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { serveDirectory } from "../../src/shared/static-server"; +import type { StaticServer } from "../../src/shared/file-serve"; +import type { ChangesReport } from "../../src/crawl/changes"; +import type { Manifest } from "../../src/shared/types"; +import { FIXTURE_DIR, tempDir } from "./helpers"; + +const exec = promisify(execFile); + +const REPO = path.resolve(import.meta.dirname, "..", ".."); +const BIN = path.join(REPO, "cli", "visualice.mjs"); + +/** + * The update loop end to end, through the real CLI in a real git repository, the + * way a project uses it: a full crawl records the commit it shot; `changes` reads + * that record back out of the routes file and diffs against it; the partial crawl + * it suggests shoots the changed pages and carries the rest; and a change that + * reaches every page asks for a full run. The unit tests cover each piece; this + * covers the hand-offs between them (routes-file line, run manifests, the CLI). + */ + +let site: string; +let server: StaticServer; + +const env = { ...process.env, NO_COLOR: "1", VISUALICE_NO_UPDATE_CHECK: "1" }; + +const git = async (...args: string[]) => + (await exec("git", args, { cwd: site, env })).stdout.trim(); + +const visualice = async (...args: string[]) => + (await exec(process.execPath, [BIN, ...args], { cwd: site, env, maxBuffer: 1 << 26 })).stdout; + +const changes = async (): Promise => + JSON.parse(await visualice("changes", "--json", "--quiet")) as ChangesReport; + +const commit = async (message: string) => { + await git("add", "-A"); + await git("commit", "-q", "-m", message); +}; + +const sourceLine = async () => + /visualice-source commit=([0-9a-f]+)/.exec( + await fs.readFile(path.join(site, "visualice.routes.md"), "utf8"), + )?.[1]; + +beforeAll(async () => { + site = await tempDir("update-flow"); + await fs.cp(FIXTURE_DIR, site, { recursive: true }); + await fs.rm(path.join(site, "crawl-map.input.json"), { force: true }); + await fs.writeFile(path.join(site, ".gitignore"), ".visualice/\n"); + // Small and quick: one device, few interactions. The site is the repository. + await fs.writeFile( + path.join(site, "visualice.config.json"), + JSON.stringify( + { + scope: { maxPages: 6, maxDepth: 1 }, + devices: [{ name: "small", width: 420, height: 640, dpr: 1 }], + discovery: { interactions: { maxPerPage: 1 }, mobileStates: false }, + stabilize: { extraDelayMs: 50 }, + politeness: { minDelayMs: 0, concurrency: 4 }, + }, + null, + 2, + ), + ); + await git("init", "-q"); + // A Windows runner converts line endings by default, which would make every + // checked-out file look modified. + await git("config", "core.autocrlf", "false"); + await git("config", "user.email", "test@example.com"); + await git("config", "user.name", "test"); + await commit("site"); + server = await serveDirectory(site); +}, 60000); + +afterAll(async () => { + await server?.close(); + if (site) await fs.rm(site, { recursive: true, force: true }); +}); + +describe("updating a catalogue after code changes", () => { + it("records, diffs, reruns only what changed, and asks for everything when it should", async () => { + // 1. A full crawl records the commit it shot in the routes file. + const first = JSON.parse(await visualice("crawl", server.url, "--json", "--quiet")); + expect(first.shots).toBeGreaterThan(0); + expect(first.failed).toBe(0); + expect(await sourceLine()).toBe(await git("rev-parse", "HEAD")); + const firstManifest = JSON.parse(await fs.readFile(first.manifest, "utf8")) as Manifest; + const pages = firstManifest.routes.map((r) => r.url); + expect(pages).toContain("/about"); + + // 2. Committing the catalogue changes nothing that counts. + await commit("catalogue"); + expect((await changes()).verdict).toBe("none"); + + // 3. One page edited, one added: partial, with exactly those pages. + const about = path.join(site, "about.html"); + const before = await fs.readFile(about, "utf8"); + const edited = before.replace("", "

We now roast on Sundays too.

"); + expect(edited).not.toBe(before); + await fs.writeFile(about, edited); + await fs.writeFile( + path.join(site, "pricing.html"), + "Pricing

Pricing

", + ); + await commit("about + pricing"); + const partial = await changes(); + expect(partial.verdict).toBe("partial"); + expect(partial.global).toEqual([]); + expect(partial.candidates.map((c) => c.file).sort()).toEqual(["about.html", "pricing.html"]); + expect(partial.suggest?.only?.sort()).toEqual(["/about", "/pricing"]); + expect(partial.suggest?.command).toBe("visualice crawl --replay --only=/about,/pricing"); + + // 4. The suggested run shoots those two pages and carries every other one. + const part = JSON.parse( + await visualice( + "crawl", + server.url, + "--replay", + `--only=${partial.suggest!.only!.join(",")}`, + "--json", + "--quiet", + ), + ); + expect(part.failed).toBe(0); + expect(part.carried).toBeGreaterThan(0); + const m = JSON.parse(await fs.readFile(part.manifest, "utf8")) as Manifest; + expect(m.partial?.carriedFrom).toBe(first.runId); + const urls = m.routes.map((r) => r.url); + expect(urls).toEqual(expect.arrayContaining([...pages, "/pricing"])); + for (const route of m.routes) { + const shots = route.states.flatMap((s) => Object.values(s.shots)); + const fresh = route.url === "/about" || route.url === "/pricing"; + for (const ref of shots) { + if (fresh) expect(ref.carried).toBeUndefined(); + else expect(ref.carried?.runId).toBe(first.runId); + } + } + expect(await sourceLine()).toBe(await git("rev-parse", "HEAD")); + + // 5. With the new catalogue committed, it's up to date again. + await commit("catalogue after partial"); + const after = await changes(); + expect(after.verdict).toBe("none"); + + // 6. A change every page sees asks for a full run. + await fs.appendFile(path.join(site, "assets", "tailwind.css"), "\nbody{color:red}\n"); + const full = await changes(); + expect(full.verdict).toBe("full"); + expect(full.global.map((g) => g.path)).toContain("assets/tailwind.css"); + expect(full.suggest?.command).toBe("visualice crawl"); + }, 240000); +}); From 83298f629e26a85ca616f946dc1cefb93565bebf Mon Sep 17 00:00:00 2001 From: Alexander Tenepere Date: Tue, 10 Nov 2026 16:24:02 +0800 Subject: [PATCH 5/8] fix: Match repository paths through Windows short names, so the routes file isn't counted as a change --- src/crawl/source.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/crawl/source.ts b/src/crawl/source.ts index 8058b84..499aeda 100644 --- a/src/crawl/source.ts +++ b/src/crawl/source.ts @@ -184,7 +184,9 @@ export function parseSourceLine(line: string): SourceInfo | undefined { /** The real path of `p`, resolving through its nearest existing parent when it doesn't exist. */ function realPath(p: string): string { try { - return realpathSync(p); + // `.native` asks the OS: on Windows it also expands 8.3 short names + // (C:\Users\RUNNER~1), which git never reports and the JS version leaves alone. + return realpathSync.native(p); } catch { const parent = path.dirname(p); return parent === p ? p : path.join(realPath(parent), path.basename(p)); From 218d99315a7b2aa3b26fded37971b174df907cea Mon Sep 17 00:00:00 2001 From: Alexander Tenepere Date: Tue, 10 Nov 2026 16:24:02 +0800 Subject: [PATCH 6/8] feat: Update banner in the viewer, with the steps to update --- CHANGELOG.md | 3 + docs/crawling.md | 3 + src/cli.ts | 25 +++++- src/export.ts | 4 + src/serve.ts | 14 ++++ src/shared/update-check.ts | 19 +++++ tests/serve.test.ts | 27 +++++++ tests/update-check.test.ts | 16 ++++ viewer/src/Root.tsx | 22 +++-- viewer/src/UpdateBanner.tsx | 156 ++++++++++++++++++++++++++++++++++++ 10 files changed, 281 insertions(+), 8 deletions(-) create mode 100644 viewer/src/UpdateBanner.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e0e878..009499e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,9 @@ once a day, in the background, never in the way), with the command to update the plugin or the CLI. The skill passes it on. Off with `--quiet`, in CI, or with `VISUALICE_NO_UPDATE_CHECK=1`. +- `visualice serve` shows the same news as a banner across the top of the viewer, with + the update steps and a copy button for each command. Dismissed, it stays away until + the next version; a shared export never shows it. - The README says how to turn on auto-update for the plugin: a marketplace plugin doesn't update itself unless you do. diff --git a/docs/crawling.md b/docs/crawling.md index ac976eb..9503975 100644 --- a/docs/crawling.md +++ b/docs/crawling.md @@ -101,6 +101,9 @@ It is off with `--quiet`, in CI (`CI` set), in a source checkout, and with `VISUALICE_NO_UPDATE_CHECK=1` (or `NO_UPDATE_NOTIFIER=1`). The last answer is cached in `~/.cache/visualice/update-check.json` (`$XDG_CACHE_HOME` when set). +`visualice serve` passes the same answer to the viewer, which shows it as a banner with +the update steps (the viewer never contacts npm itself; an export never shows it). + ## Output Output lands under the configured output directory (default `.visualice/`), one folder diff --git a/src/cli.ts b/src/cli.ts index a3e753e..c0243bc 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -34,7 +34,12 @@ import { } from "./crawl/routes-file"; import { ensureStorageState } from "./crawl/auth"; import { pathExists } from "./shared/fsx"; -import { startUpdateCheck, updateCheckDisabled, type UpdateCheck } from "./shared/update-check"; +import { + knownUpdate, + startUpdateCheck, + updateCheckDisabled, + type UpdateCheck, +} from "./shared/update-check"; import { nodeBinCommand } from "./shared/node-bin"; import { serveDirectory } from "./shared/static-server"; import { serveViewer } from "./serve"; @@ -429,6 +434,14 @@ async function cmdServe(flags: Flags): Promise { const viewerDist = path.join(PROJECT_ROOT, "viewer", "dist"); await ensureViewerBuilt(viewerDist, log); + // The viewer's update banner, from the same check as the terminal notice. Not in + // a --screenshot, which is a picture of the catalogue, not of this install. + const version = await readVersion(); + const update = + str(flags, "screenshot") === undefined && (await updatesOn(flags)) + ? () => knownUpdate(version) + : undefined; + const server = await serveViewer({ runDir: dir, viewerDist, @@ -437,6 +450,7 @@ async function cmdServe(flags: Flags): Promise { site, runId, port: parsePort(flags), + update, }); // Naming a site or run opens it; otherwise the page decides (the dashboard, when @@ -1034,8 +1048,7 @@ async function main(): Promise { * `npm link`ed) is never "behind" in a way worth telling, and `--quiet` asks for silence. */ async function updateCheck(flags: Flags): Promise { - if (flags.quiet === true || updateCheckDisabled()) return undefined; - if (await pathExists(path.join(PROJECT_ROOT, ".git"))) return undefined; + if (!(await updatesOn(flags))) return undefined; try { return await startUpdateCheck(await readVersion()); } catch { @@ -1043,6 +1056,12 @@ async function updateCheck(flags: Flags): Promise { } } +/** Is the update check (terminal notice and viewer banner) on for this run? */ +async function updatesOn(flags: Flags): Promise { + if (flags.quiet === true || updateCheckDisabled()) return false; + return !(await pathExists(path.join(PROJECT_ROOT, ".git"))); +} + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms).unref()); async function runCommand(cmd: string, flags: Flags): Promise { diff --git a/src/export.ts b/src/export.ts index f60283e..46f6389 100644 --- a/src/export.ts +++ b/src/export.ts @@ -13,6 +13,7 @@ import type { SiteList } from "./serve"; * index.html, assets/ the built viewer * data/ the run * sites.json the (one-entry) site and run list + * update.json `{}`: a shared copy has no update banner to show */ export interface ExportOptions { @@ -72,6 +73,9 @@ export async function exportRun(opts: ExportOptions): Promise { sites: [{ key: site, latest: opts.runId, runs: [opts.runId] }], }; await fs.writeFile(path.join(dest, "sites.json"), JSON.stringify(list)); + // The viewer asks `serve` whether a newer visualice is out. Whoever opens an export + // isn't running this install, so the answer is always "nothing to say". + await fs.writeFile(path.join(dest, "update.json"), "{}"); return { dest, runId: opts.runId, files: await countFiles(dest) }; } diff --git a/src/serve.ts b/src/serve.ts index b80c76e..649cc18 100644 --- a/src/serve.ts +++ b/src/serve.ts @@ -154,6 +154,12 @@ export async function serveViewer(opts: { /** Run id shown as current in `/sites.json`. */ runId?: string; port?: number; + /** + * A newer visualice, if one is out, for the viewer's update banner at + * `/update.json`. Asked on every request, so a check that lands while the server + * runs shows on the next page load. Omitted: the banner never shows. + */ + update?: () => Promise<{ current: string; latest: string } | undefined>; }): Promise { const runDir = path.resolve(opts.runDir); const viewerDist = path.resolve(opts.viewerDist); @@ -211,6 +217,14 @@ export async function serveViewer(opts: { return; } + if (pathname === "/update.json") { + const info = opts.update ? await opts.update().catch(() => undefined) : undefined; + res.setHeader("content-type", "application/json; charset=utf-8"); + res.setHeader("cache-control", "no-store"); + res.end(JSON.stringify(info ?? {})); + return; + } + if (pathname.startsWith("/data/")) { // Never immutable: `/data/` is whichever run this server was pointed at, which // by default is `latest`, and a re-crawl changes it without changing a URL. diff --git a/src/shared/update-check.ts b/src/shared/update-check.ts index 909699b..22c682a 100644 --- a/src/shared/update-check.ts +++ b/src/shared/update-check.ts @@ -67,6 +67,25 @@ async function readCache(file: string): Promise { } } +/** What the viewer is told: the version running, and the newer one out. */ +export interface UpdateInfo { + current: string; + latest: string; +} + +/** + * The newer version the last check found, if any: what `visualice serve` hands the + * viewer for its banner. Only the cache is read, so it costs a file read and never + * reaches the network; the CLI's own check keeps the cache fresh. + */ +export async function knownUpdate( + current: string, + file: string = cacheFile(), +): Promise { + const latest = (await readCache(file))?.latest; + return latest && isNewer(latest, current) ? { current, latest } : undefined; +} + export interface UpdateCheck { /** The notice to show now, from the cached answer; undefined when up to date. */ notice?: string; diff --git a/tests/serve.test.ts b/tests/serve.test.ts index 813f21f..6d789da 100644 --- a/tests/serve.test.ts +++ b/tests/serve.test.ts @@ -190,6 +190,33 @@ describe("serveViewer sites and runs", () => { }); }); +describe("serveViewer update banner", () => { + it("answers /update.json from the update getter, uncached, and {} without one", async () => { + expect(await (await fetch(`${server.url}/update.json`)).json()).toEqual({}); + + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "vc-serve-update-")); + let latest: string | undefined; + const withUpdate = await serveViewer({ + runDir: dir, + viewerDist: dir, + update: async () => (latest ? { current: "0.2.0", latest } : undefined), + }); + try { + const first = await fetch(`${withUpdate.url}/update.json`); + expect(first.headers.get("cache-control")).toBe("no-store"); + expect(await first.json()).toEqual({}); + latest = "0.3.0"; // a check that lands while the server runs + expect(await (await fetch(`${withUpdate.url}/update.json`)).json()).toEqual({ + current: "0.2.0", + latest: "0.3.0", + }); + } finally { + await withUpdate.close(); + await fs.rm(dir, { recursive: true, force: true }); + } + }); +}); + describe("isChild", () => { it("accepts a plain name and rejects ones that leave the directory", () => { const dir = path.resolve("/out/sites"); diff --git a/tests/update-check.test.ts b/tests/update-check.test.ts index ce2707a..0383c9d 100644 --- a/tests/update-check.test.ts +++ b/tests/update-check.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { cacheFile, isNewer, + knownUpdate, startUpdateCheck, updateCheckDisabled, updateNotice, @@ -110,3 +111,18 @@ describe("startUpdateCheck", () => { await check.refreshed; }); }); + +describe("knownUpdate", () => { + it("reads the cached answer for the viewer, without asking the registry", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "vc-known-")); + const file = path.join(dir, "update-check.json"); + try { + expect(await knownUpdate("0.2.0", file)).toBeUndefined(); + await fs.writeFile(file, JSON.stringify({ checkedAt: 0, latest: "0.3.0" })); + expect(await knownUpdate("0.2.0", file)).toEqual({ current: "0.2.0", latest: "0.3.0" }); + expect(await knownUpdate("0.3.0", file)).toBeUndefined(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/viewer/src/Root.tsx b/viewer/src/Root.tsx index 6d51f5e..6373ed3 100644 --- a/viewer/src/Root.tsx +++ b/viewer/src/Root.tsx @@ -2,6 +2,7 @@ import { useCallback, useEffect, useState } from "react"; import App from "./App"; import { Dashboard } from "./Dashboard"; import { fetchSites, runHref, type SiteList } from "./sites"; +import { UpdateBanner } from "./UpdateBanner"; /** Any of these in the URL asks for the viewer: a run, a shot, a filter, a data file. */ const VIEWER_PARAMS = ["site", "run", "tile", "route", "q", "devices", "kinds", "mode", "data"]; @@ -41,15 +42,26 @@ export function Root() { }, []); const several = (list?.sites.length ?? 0) > 1; - if (viewer) return ; - // A blank page for the moment `sites.json` takes, not a loader that restarts. - if (list === undefined) return
; - if (!list || !several) return ; - return ( + const page = viewer ? ( + + ) : list === undefined ? ( + // A blank page for the moment `sites.json` takes, not a loader that restarts. +
+ ) : !list || !several ? ( + + ) : ( void fetchSites().then((next) => setList(next))} /> ); + // The update banner sits above whichever page is showing, and takes its height + // from it rather than covering it. + return ( +
+ +
{page}
+
+ ); } diff --git a/viewer/src/UpdateBanner.tsx b/viewer/src/UpdateBanner.tsx new file mode 100644 index 0000000..9bf3cba --- /dev/null +++ b/viewer/src/UpdateBanner.tsx @@ -0,0 +1,156 @@ +import { useEffect, useState } from "react"; +import { Icon, LINK, cx, EASE } from "./ui"; + +/** + * "A newer visualice is out" across the top of the page, with how to update. + * + * `visualice serve` answers `/update.json` from the CLI's once-a-day check, so the + * page itself never contacts npm. A shared export has no such endpoint and never + * shows the banner. Dismissing it hides it for that version: the next release + * brings it back. + */ + +interface UpdateInfo { + current: string; + latest: string; +} + +const DISMISS_KEY = "visualice:update-dismissed"; +const RELEASES = "https://github.com/alexdevlabs/visualice/releases/tag/v"; + +function dismissedVersion(): string | null { + try { + return localStorage.getItem(DISMISS_KEY); + } catch { + return null; + } +} + +async function fetchUpdate(): Promise { + try { + const r = await fetch("./update.json", { cache: "no-store" }); + if (!r.ok) return undefined; + const body = (await r.json()) as Partial; + return typeof body.latest === "string" && typeof body.current === "string" + ? { current: body.current, latest: body.latest } + : undefined; + } catch { + return undefined; // an export, or an older server: no banner + } +} + +/** A command with a copy button. */ +function Command({ text }: { text: string }) { + const [copied, setCopied] = useState(false); + const copy = () => + void navigator.clipboard?.writeText(text).then( + () => { + setCopied(true); + setTimeout(() => setCopied(false), 1400); + }, + () => {}, + ); + return ( + + {text} + + + ); +} + +export function UpdateBanner() { + const [info, setInfo] = useState(); + const [open, setOpen] = useState(false); + const [hidden, setHidden] = useState(false); + + useEffect(() => { + void fetchUpdate().then((u) => { + if (u && dismissedVersion() !== u.latest) setInfo(u); + }); + }, []); + + if (!info || hidden) return null; + + const dismiss = () => { + try { + localStorage.setItem(DISMISS_KEY, info.latest); + } catch { + // private window: hidden for this page load only + } + setHidden(true); + }; + + return ( +
+
+ + visualice {info.latest} is available + · you have {info.current} + + + + What's new + + +
+ {open && ( +
+
+ With Claude Code (the plugin) + + + Then restart Claude Code, or run /reload-plugins. + To get updates on their own: /plugin → Marketplaces + → visualice → Enable auto-update. + +
+
+ With the CLI on its own + + + With npx visualice, the next run already uses{" "} + {info.latest}. Restart visualice serve afterwards. + +
+
+ )} +
+ ); +} From a0ca18f38b9d120c940016575d4e14888b35ef1c Mon Sep 17 00:00:00 2001 From: Alexander Tenepere Date: Tue, 10 Nov 2026 18:08:16 +0800 Subject: [PATCH 7/8] feat: Update banner actions as badges on the left, with the steps in a modal --- viewer/src/UpdateBanner.tsx | 148 ++++++++++++++++++++++++------------ 1 file changed, 100 insertions(+), 48 deletions(-) diff --git a/viewer/src/UpdateBanner.tsx b/viewer/src/UpdateBanner.tsx index 9bf3cba..7638f8e 100644 --- a/viewer/src/UpdateBanner.tsx +++ b/viewer/src/UpdateBanner.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { Icon, LINK, cx, EASE } from "./ui"; /** @@ -69,6 +69,78 @@ function Command({ text }: { text: string }) { ); } +/** The banner's actions: small pill badges, right after the message. */ +const BADGE = cx( + "inline-flex h-6 shrink-0 items-center gap-1 rounded-full border border-accent-line bg-panel px-2.5 text-[12px] font-medium text-accent no-underline hover:bg-accent-soft", + EASE, +); + +/** The update steps, in a modal: one way per kind of install, each command copyable. */ +function UpdateSteps({ info, onClose }: { info: UpdateInfo; onClose: () => void }) { + const ref = useRef(null); + useEffect(() => { + const dialog = ref.current; + if (dialog && !dialog.open) dialog.showModal(); + }, []); + return ( + e.target === ref.current && ref.current?.close()} + className="m-auto w-[min(520px,calc(100vw-32px))] rounded-xl border border-line-strong bg-panel p-0 text-[12.5px] text-fg shadow-pop backdrop:bg-scrim" + > +
+
+

+ Update to visualice {info.latest} +

+

You have {info.current}.

+
+ +
+
+
+

With Claude Code (the plugin)

+ +

+ Then restart Claude Code, or run /reload-plugins. To + get every update on its own: /plugin → Marketplaces → + visualice → Enable auto-update. +

+
+
+

With the CLI on its own

+ +

+ With npx visualice, the next run already uses{" "} + {info.latest}. Restart visualice serve afterwards. +

+
+ + What's new in {info.latest} + +
+
+ ); +} + export function UpdateBanner() { const [info, setInfo] = useState(); const [open, setOpen] = useState(false); @@ -92,31 +164,31 @@ export function UpdateBanner() { }; return ( -
-
- - visualice {info.latest} is available - · you have {info.current} - - - - What's new - + <> +
+ {/* Message and actions together on the left, where the eye starts. */} +
+ + visualice {info.latest} is available + · you have {info.current} + + + + + What's new + + +
- {open && ( -
-
- With Claude Code (the plugin) - - - Then restart Claude Code, or run /reload-plugins. - To get updates on their own: /plugin → Marketplaces - → visualice → Enable auto-update. - -
-
- With the CLI on its own - - - With npx visualice, the next run already uses{" "} - {info.latest}. Restart visualice serve afterwards. - -
-
- )} -
+ {open && setOpen(false)} />} + ); } From dfa346665a5e7ee9b6964ee8a69957a572540c06 Mon Sep 17 00:00:00 2001 From: Alexander Tenepere Date: Wed, 11 Nov 2026 00:35:35 +0800 Subject: [PATCH 8/8] fix: Long pages on high-density phones keep their shots: WebP previews scale down to fit the format's limit --- CHANGELOG.md | 7 +++++ cli/visualice.mjs | 0 src/crawl/thumbnails.ts | 24 +++++++++++++- tests/integration/capture-safety.test.ts | 40 ++++++++++++++++++++++++ tests/thumbnails.test.ts | 32 +++++++++++++++++++ 5 files changed, 102 insertions(+), 1 deletion(-) mode change 100644 => 100755 cli/visualice.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 009499e..d47bc61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,13 @@ - The README says how to turn on auto-update for the plugin: a marketplace plugin doesn't update itself unless you do. +### Fixes + +- A very long page, above all on a high-density phone, no longer fails its shot with + "Processed image is too large for the WebP format": the viewer's preview images are + scaled down to fit WebP's 16383px limit, and the full PNG is untouched. With + `screenshot.format: "webp"`, a page too tall for WebP now says so and how to fix it. + ## 0.1.1 ### Fixes diff --git a/cli/visualice.mjs b/cli/visualice.mjs old mode 100644 new mode 100755 diff --git a/src/crawl/thumbnails.ts b/src/crawl/thumbnails.ts index 72a05c5..ad075e6 100644 --- a/src/crawl/thumbnails.ts +++ b/src/crawl/thumbnails.ts @@ -10,6 +10,13 @@ sharp.cache({ files: 0 }); type Thumb = Config["screenshot"]["thumbnail"]; type Format = "png" | "jpeg" | "webp"; +/** + * The tallest image each format can hold, in pixels. A long page captured on a + * high-density phone runs to tens of thousands of pixels: PNG holds it, but a WebP + * stops at 16383 and a JPEG at 65535, and sharp then throws instead of writing. + */ +export const MAX_SIDE: Record = { webp: 16383, jpeg: 65535, png: Infinity }; + function encode(img: sharp.Sharp, format: Format, quality: number): sharp.Sharp { if (format === "webp") return img.webp({ quality }); if (format === "jpeg") return img.jpeg({ quality }); @@ -24,6 +31,13 @@ export async function writeFull( quality: number, ): Promise { if (format === "webp") { + const { width = 0, height = 0 } = await sharp(buf).metadata(); + if (Math.max(width, height) > MAX_SIDE.webp) { + throw new Error( + `the page is ${height}px tall, and a WebP image holds at most ${MAX_SIDE.webp}px: ` + + `set screenshot.format to "png", or lower screenshot.maxHeight`, + ); + } await sharp(buf).webp({ quality }).toFile(destPath); } else { await fs.writeFile(destPath, buf); // already encoded as png/jpeg by Playwright @@ -36,7 +50,15 @@ export async function makeThumb( thumb: Thumb, width = thumb.width, ): Promise { - const img = sharp(buf).resize({ width, withoutEnlargement: true }); + // A very long page would come out taller than the format holds at this width: it + // is scaled down further to fit, keeping its shape, so the whole page still shows + // (just softer) rather than the shot failing over a preview image. + const max = MAX_SIDE[thumb.format]; + const img = sharp(buf, { limitInputPixels: false }).resize({ + width, + ...(Number.isFinite(max) ? { height: max, fit: "inside" as const } : {}), + withoutEnlargement: true, + }); await encode(img, thumb.format, thumb.quality).toFile(destPath); } diff --git a/tests/integration/capture-safety.test.ts b/tests/integration/capture-safety.test.ts index 739caa7..8452ddd 100644 --- a/tests/integration/capture-safety.test.ts +++ b/tests/integration/capture-safety.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { promises as fs } from "node:fs"; import path from "node:path"; +import sharp from "sharp"; import { ConfigSchema, type Config } from "../../src/shared/config"; import { capturePipeline } from "../../src/crawl/orchestrator"; import type { InputManifest } from "../../src/shared/types"; @@ -65,6 +66,45 @@ describe("screenshot safety", () => { } }, 120_000); + // The bug: a long page on a high-density phone failed its whole shot with "Processed + // image is too large for the WebP format". The full PNG was fine; the viewer's WebP + // previews, scaled to a fixed width, came out taller than WebP's 16383px limit. + it("shoots a long page on a 3x phone, previews and all, though WebP can't hold it", async () => { + const { server, dir: siteDir } = await serveFiles({ "index.html": TALL }); + const out = await tempDir("safety"); + try { + const result = await capturePipeline({ + config: config(out, { + // 430 CSS px at 3x is 1290px wide (wide enough to get a 1024px preview); + // the 9000px page is 27000px tall, so that preview would be 21433px tall. + devices: [{ name: "small", width: 430, height: 300, dpr: 3 }], + screenshot: { fullPage: "single", maxHeight: 0 }, + }), + inputManifest: single(), + baseUrl: server.url, + log: silent, + }); + const shot = result.manifest.routes[0].states[0].shots.small; + expect(shot.error).toBeUndefined(); + expect(result.manifest.stats?.failed).toBe(0); + + const size = async (rel: string) => sharp(path.join(result.runDir, rel)).metadata(); + const full = await size(shot.full!); + expect(full.height).toBeGreaterThan(16383); // the full PNG keeps every pixel + for (const rel of [shot.thumb!, shot.mid!]) { + const m = await size(rel); + expect(m.format).toBe("webp"); + expect(m.height).toBeLessThanOrEqual(16383); + // Scaled to fit, not cropped: the whole page, in the same shape. + expect(m.width! / m.height!).toBeCloseTo(full.width! / full.height!, 2); + } + } finally { + await server.close(); + await fs.rm(siteDir, { recursive: true, force: true }); + await fs.rm(out, { recursive: true, force: true }); + } + }, 120_000); + it("stops an unbounded page at maxHeight, screen by screen, and flags the shot", async () => { const { server, dir: siteDir } = await serveFiles({ "index.html": TALL }); const out = await tempDir("safety"); diff --git a/tests/thumbnails.test.ts b/tests/thumbnails.test.ts index a06fa51..0cfc808 100644 --- a/tests/thumbnails.test.ts +++ b/tests/thumbnails.test.ts @@ -101,6 +101,38 @@ describe("makeThumb", () => { }); }); +describe("a page taller than WebP holds", () => { + // A long page on a 3x phone: 1170px wide and far past WebP's 16383px. One flat + // colour, so it encodes fast and small. + const tall = () => + sharp({ + create: { width: 1170, height: 40000, channels: 3, background: "#ffffff" }, + limitInputPixels: false, + }) + .png() + .toBuffer(); + + it("still gets its preview images, scaled down to fit, whole and in shape", async () => { + const buf = await tall(); + const mid = path.join(dir, "tall-mid.webp"); + await makeThumb(buf, mid, THUMB, 1024); + const m = await sharp(mid).metadata(); + expect(m.format).toBe("webp"); + expect(m.height).toBe(16383); + expect(m.width).toBe(Math.round((1170 * 16383) / 40000)); + + const thumb = path.join(dir, "tall-thumb.webp"); + await makeThumb(buf, thumb, THUMB); // 480 wide → 16410 tall: just over, so it fits too + expect((await sharp(thumb).metadata()).height).toBeLessThanOrEqual(16383); + }, 60000); + + it("explains itself when the full image is set to WebP", async () => { + await expect(writeFull(await tall(), path.join(dir, "tall.webp"), "webp", 80)).rejects.toThrow( + /40000px tall.*screenshot\.format/, + ); + }, 60000); +}); + describe("imageSize", () => { it("reports the intrinsic pixel dimensions of the buffer", async () => { expect(await imageSize(await page(640, 480, 100))).toEqual({ width: 640, height: 480 });