From a67db2ebc1604bd1484605d2a42be36be11ca677 Mon Sep 17 00:00:00 2001
From: Alexander Tenepere
Date: Tue, 10 Nov 2026 00:10:52 +0800
Subject: [PATCH 1/8] feat: Track the commit a catalogue shows, visualice
changes, and partial runs with crawl --only
---
CHANGELOG.md | 27 ++
commands/visualice.md | 4 +
docs/crawling.md | 4 +
docs/viewer.md | 11 +
manifest.schema.json | 18 +
site/.vitepress/config.mts | 2 +
skills/visualice/SKILL.md | 15 +-
skills/visualice/references/cli.md | 41 +-
skills/visualice/references/update-mode.md | 114 +++++
src/cli-args.ts | 4 +
src/cli.ts | 187 ++++++++-
src/crawl/carry.ts | 258 ++++++++++++
src/crawl/changes.ts | 464 +++++++++++++++++++++
src/crawl/orchestrator.ts | 346 ++++++++++++++-
src/crawl/routes-file.ts | 29 +-
src/crawl/source.ts | 192 +++++++++
src/shared/manifest-schema.ts | 8 +
src/shared/types.ts | 54 +++
tests/carry.test.ts | 243 +++++++++++
tests/changes.test.ts | 216 ++++++++++
tests/integration/partial.test.ts | 79 ++++
tests/source.test.ts | 141 +++++++
viewer/src/App.tsx | 9 +
viewer/src/Canvas.tsx | 16 +-
viewer/src/Compare.tsx | 149 +++++++
viewer/src/Inspector.tsx | 49 ++-
viewer/src/load.ts | 35 ++
27 files changed, 2653 insertions(+), 62 deletions(-)
create mode 100644 skills/visualice/references/update-mode.md
create mode 100644 src/crawl/carry.ts
create mode 100644 src/crawl/changes.ts
create mode 100644 src/crawl/source.ts
create mode 100644 tests/carry.test.ts
create mode 100644 tests/changes.test.ts
create mode 100644 tests/integration/partial.test.ts
create mode 100644 tests/source.test.ts
create mode 100644 viewer/src/Compare.tsx
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6710d9c..656e587 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,32 @@
# Changelog
+## Unreleased
+
+### Updating after code changes
+
+- Each crawl records the git commit it shot, on a `visualice-source` line in
+ `visualice.routes.md` and as `source` in the run's `crawl-map.json` (branch, files with
+ uncommitted changes, whether the target was local, a hash of the config).
+- `visualice changes` diffs the project against that commit: the commits and files since,
+ the changes that reach every page (dependencies, global styles, root layouts, header
+ and footer, the config), the pages file-based routing ties changed files to, and the
+ files left for an agent to trace. It gives a verdict (`none`, `partial`, `full`,
+ `unknown`) and the command to run; `--discover` also lists new and removed pages.
+- `crawl --only=` shoots only those pages (paths or globs) and carries every
+ other page's shots over from the last finished run, as hard links where the disk
+ allows, so the new run is still the whole site. A page the last run can't cover (new,
+ or missing shots) is shot anyway. With `--replay` the named pages' states are
+ discovered again. `--dry-run` shows what would be shot and carried.
+- The skill and `/visualice` follow it: run `changes`, trace the affected pages, ask
+ whether to rerun those or everything, then run it.
+
+### Viewer
+
+- Shots a partial run took carry an **updated** tag; a carried shot's details say which
+ run and commit it was really taken in.
+- **Compare with another version** in the details panel shows the same shot from another
+ run beside this one.
+
## 0.1.1
### Fixes
diff --git a/commands/visualice.md b/commands/visualice.md
index 2890179..f304cf0 100644
--- a/commands/visualice.md
+++ b/commands/visualice.md
@@ -17,6 +17,10 @@ configured in `visualice.config.json` in the current directory.
map with a dry run, show the user the page types, shot count and any writes and
confirm once, dispatch one `visualice:visualice-explorer` agent per page type,
then `visualice author` and `visualice crawl --replay`.
+ - **Already mapped, and the routes file records a commit** (a `visualice-source`
+ line) → run `visualice changes --json` first and follow the skill's
+ `references/update-mode.md`: trace the affected pages, ask the user whether to
+ rerun just those (`crawl --replay --only=…`, carrying the rest over) or everything.
- **Already mapped** → `visualice crawl --replay --json --quiet`, then explore only
what's new (pages added to the routes file, `wanted` lines), per "Later runs" in
the same reference. `/visualice-explore` is for widening it on purpose.
diff --git a/docs/crawling.md b/docs/crawling.md
index a3580d4..4aa24cc 100644
--- a/docs/crawling.md
+++ b/docs/crawling.md
@@ -17,6 +17,10 @@ visualice crawl --resume # continue the newest interrupted run, keepin
visualice crawl --dry-run # discover, then list what would be captured; capture nothing
visualice crawl --no-states # pages only, no UI states (fast; cached states are kept)
+# after code changes (the routes file records the commit each crawl shot):
+visualice changes # what changed since, and whether some pages or all need reshooting
+visualice crawl --replay --only=/pricing,/blog/** # reshoot those; carry every other page over
+
# other commands:
visualice discover https://your.site # discovery only, no screenshots
visualice init https://your.site # scaffold the config without crawling
diff --git a/docs/viewer.md b/docs/viewer.md
index 27a0035..fa39139 100644
--- a/docs/viewer.md
+++ b/docs/viewer.md
@@ -71,6 +71,17 @@ back to it. A card's trash button deletes the site and all its runs from disk af
confirmation; the site `serve` was started on can't be deleted (409) while that server
runs.
+### Partial runs and comparing versions
+
+A partial run (`crawl --only`, see [updating after code changes](../skills/visualice/references/update-mode.md))
+shoots the pages that changed and carries the rest over. Its shots carry an **updated**
+tag on the canvas; a carried shot's details panel says so, with the run and commit it
+was really taken in. The details panel's footer shows the commit a shot came from.
+
+**Compare with another version**, in the details panel, puts the same shot from another
+run of the site beside this one (the previous version by default). A shot that was
+carried over unchanged says so instead of looking like a coincidence.
+
## Share a run
```bash
diff --git a/manifest.schema.json b/manifest.schema.json
index fc961c0..247e987 100644
--- a/manifest.schema.json
+++ b/manifest.schema.json
@@ -740,6 +740,24 @@
},
"error": {
"type": "string"
+ },
+ "carried": {
+ "description": "Copied from an earlier run by a partial crawl (`--only`), not shot by this one.",
+ "type": "object",
+ "properties": {
+ "runId": {
+ "type": "string"
+ },
+ "commit": {
+ "type": "string"
+ },
+ "capturedAt": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "runId"
+ ]
}
}
}
diff --git a/site/.vitepress/config.mts b/site/.vitepress/config.mts
index 8264647..70867dc 100644
--- a/site/.vitepress/config.mts
+++ b/site/.vitepress/config.mts
@@ -17,6 +17,7 @@ const PAGES: Record = {
"site/guide/getting-started.md": "guide/getting-started.md",
"skills/visualice/references/explore-mode.md": "guide/explore.md",
"docs/crawling.md": "guide/crawling.md",
+ "skills/visualice/references/update-mode.md": "guide/updating.md",
"docs/configuration.md": "guide/configuration.md",
"skills/visualice/references/auth.md": "guide/auth.md",
"docs/viewer.md": "guide/viewer.md",
@@ -127,6 +128,7 @@ export default defineConfig({
text: "Guide",
items: [
{ text: "Crawling", link: "/guide/crawling" },
+ { text: "Updating after code changes", link: "/guide/updating" },
{ text: "Configuration", link: "/guide/configuration" },
{ text: "Logging in", link: "/guide/auth" },
{ text: "The viewer", link: "/guide/viewer" },
diff --git a/skills/visualice/SKILL.md b/skills/visualice/SKILL.md
index d4ad53c..7bb8618 100644
--- a/skills/visualice/SKILL.md
+++ b/skills/visualice/SKILL.md
@@ -88,10 +88,18 @@ rediscovery and loses any Explore-mode states authored into it.
hasn't been mapped: map it with a dry run, plan the page types with the user,
explore each type, then capture with `--replay`. Read
[`references/explore-mode.md`](references/explore-mode.md) and follow it.
-2. **Replay or extend.** The site is mapped. To recapture, run
+2. **Update after code changes.** The site is mapped and the project is a git repo
+ whose code changed since the last crawl. Run `visualice changes --json`: it diffs
+ against the commit the catalogue was shot from (recorded in the routes file) and
+ says whether nothing, some pages, or everything needs reshooting. Work out the
+ affected pages, ask the user whether to rerun just those or everything, then run
+ `visualice crawl --replay --only=`, which carries every other page over from
+ the last run. Read [`references/update-mode.md`](references/update-mode.md) and
+ follow it.
+3. **Replay or extend.** The site is mapped. To recapture, run
`visualice crawl --replay --json --quiet`. For new pages, `wanted` lines, or a focus the user names, explore
just those page types first (explore-mode.md, "Later runs").
-3. **Quick.** Only when the user asks for it ("quick", "just screenshot it", pages only)
+4. **Quick.** Only when the user asks for it ("quick", "just screenshot it", pages only)
or you're scripting CI: `visualice crawl --json --quiet` runs the heuristics
and captures in one go, with no exploring. Mention that exploring would find more.
@@ -161,3 +169,6 @@ Full flag reference: [`references/cli.md`](references/cli.md).
by default never publish, delete, pay, sign out or change account settings.
- The cached model makes re-runs cheap — once a site is mapped, default to `--replay`
and explore only what's new.
+- After code changes, never reshoot part of a catalogue without asking: lay out what
+ `visualice changes` found and the pages you traced, and let the user choose between
+ those pages and a full run. When in doubt, recommend the full run.
diff --git a/skills/visualice/references/cli.md b/skills/visualice/references/cli.md
index 5b94813..9b514e0 100644
--- a/skills/visualice/references/cli.md
+++ b/skills/visualice/references/cli.md
@@ -7,25 +7,26 @@ the dedicated script where one exists, e.g. `npm run crawl -- `).
## Commands
-| Command | Purpose |
-| --------------------------- | ------------------------------------------------------------------------- |
-| `init [url]` | Scaffold `visualice.config.json` (+ `.gitignore` entry) without crawling |
-| `crawl [url]` | Discover routes + UI states, then capture across devices |
-| `discover [url]` | Auto-discover routes + UI states only — no screenshots |
-| `routes` | Regenerate `visualice.routes.md` from the cached site model |
-| `validate` | Validate a crawl manifest / cached site model |
-| `auth [--force]` | Run the configured login recipe, save storage state |
-| `capture --manifest=` | Replay a given manifest's recipes and capture screenshots |
-| `demo` | Capture the bundled fixture from a hand-written manifest |
-| `demo-crawl` | Full discover → capture run against the bundled fixture |
-| `fixture` | Serve the bundled fixture blog so you can browse it |
-| `doctor [--no-fix]` | Idempotent preflight: check Node/Playwright/sharp/viewer, self-heal |
-| `serve` | Launch the zoomable results viewer |
-| `export ` | Write the viewer + a run as a static folder |
-| `probe ` | JSON: the page's inventory, or (`--recipe`/`--state`) what a recipe does |
-| `author [dir\|file]` | Merge explorer state files (default `.visualice/explore/`) into the cache |
-| `--version` | Print the installed version |
-| `help` | Print usage |
+| Command | Purpose |
+| --------------------------- | -------------------------------------------------------------------------- |
+| `init [url]` | Scaffold `visualice.config.json` (+ `.gitignore` entry) without crawling |
+| `crawl [url]` | Discover routes + UI states, then capture across devices |
+| `discover [url]` | Auto-discover routes + UI states only — no screenshots |
+| `routes` | Regenerate `visualice.routes.md` from the cached site model |
+| `validate` | Validate a crawl manifest / cached site model |
+| `auth [--force]` | Run the configured login recipe, save storage state |
+| `capture --manifest=` | Replay a given manifest's recipes and capture screenshots |
+| `demo` | Capture the bundled fixture from a hand-written manifest |
+| `demo-crawl` | Full discover → capture run against the bundled fixture |
+| `fixture` | Serve the bundled fixture blog so you can browse it |
+| `doctor [--no-fix]` | Idempotent preflight: check Node/Playwright/sharp/viewer, self-heal |
+| `serve` | Launch the zoomable results viewer |
+| `export ` | Write the viewer + a run as a static folder |
+| `probe ` | JSON: the page's inventory, or (`--recipe`/`--state`) what a recipe does |
+| `author [dir\|file]` | Merge explorer state files (default `.visualice/explore/`) into the cache |
+| `changes [--discover]` | JSON/text: what changed in git since the catalogue's commit, and a verdict |
+| `--version` | Print the installed version |
+| `help` | Print usage |
## Flags
@@ -43,6 +44,8 @@ command are accepted by it.
| `--manifest=` | `capture`/`validate` | Input manifest path |
| `--replay` | `crawl`/`demo-crawl` | Reuse the cached site model, skip discovery |
| `--resume` | `crawl` | Continue the newest interrupted run, keeping its shots |
+| `--only=` | `crawl` | Shoot only these pages (paths or globs, comma-separated); every other page's shots are carried over from the last run. With `--replay`, rediscovers their states |
+| `--discover` | `changes` | Also map the site's pages (no states, nothing written) to list new and removed ones |
| `--json` | most | Print a machine-readable JSON summary to stdout |
| `--quiet` | most | Suppress progress output on stderr |
| `--fail-on-error` | capture cmds | Exit 3 if any capture failed |
diff --git a/skills/visualice/references/update-mode.md b/skills/visualice/references/update-mode.md
new file mode 100644
index 0000000..e32e6e6
--- /dev/null
+++ b/skills/visualice/references/update-mode.md
@@ -0,0 +1,114 @@
+# Updating a catalogue after code changes
+
+A crawl records the commit it shot in `visualice.routes.md`, on a line under the
+title:
+
+```md
+
+```
+
+Each run's `crawl-map.json` has the same record as `source`, with the list of files
+that had uncommitted changes. When the project moves on (new pages, edited ones), you
+don't have to reshoot everything: find out what changed, work out which pages it
+touches, and reshoot only those. A partial run carries every other page's shots over
+from the last run, so the new run is still the whole site.
+
+## 1. Ask what changed
+
+```bash
+visualice changes --json # add --discover to also map the site for new/removed pages
+```
+
+The answer, in short:
+
+| Field | What it is |
+| ------------ | ---------------------------------------------------------------------------------------------------------------- |
+| `verdict` | `none` (nothing to do), `partial` (some pages), `full` (everything), `unknown` (no baseline: run a full crawl) |
+| `reasons` | Why, most important first. Read these out to the user |
+| `baseline` | The commit the catalogue shows, its branch, when it was shot, and whether the target was local |
+| `commits` | Commits since (hash, subject), newest first; `commitCount` is the total |
+| `files` | Every changed file that counts (committed since, uncommitted now, untracked, and files dirty at capture time) |
+| `global` | Changed files that reach every page: dependencies, global styles, theme, root layouts, header/footer/nav, config |
+| `candidates` | Changed files that file-based routing ties to pages: `{file, pattern, kind, pages}` |
+| `unmapped` | Changed files that map to no page on their own: components, hooks, utilities, data |
+| `routes` | With `--discover`: `added` and `removed` pages |
+| `suggest` | The command to run, when the facts settle it (no unmapped files) |
+
+The visualice files themselves (`visualice.routes.md`, `.visualice/`) never count.
+The config is compared by its hash: changing devices or capture settings means a full
+run.
+
+## 2. Work out the affected pages
+
+`verdict: "full"` or `"unknown"` → skip to step 3 and recommend a full run.
+
+For `partial`:
+
+- **`candidates`** already list their pages. A `layout` candidate covers every page
+ under its pattern (`/docs/**`). A candidate with no `pages` is a page the catalogue
+ doesn't have yet: include its `pattern` as a new page.
+- **`unmapped`** files are yours to trace. For each one, find who imports it (search
+ for its module name or path) and follow the chain up to page or layout files, then
+ map those the same way the candidates are mapped. Read the diff
+ (`git diff -- `) when the reach is unclear: a renamed CSS
+ class in a shared component reaches every page that renders it, a fixed typo in one
+ prop doesn't.
+ - A component used by most pages (a button, a card used everywhere, a design-system
+ primitive) is effectively global: say so and recommend a full run.
+ - Data files, API handlers and server code can change what a page shows without a
+ UI file changing. If a page reads that data, include it.
+ - Tests, docs, CI config, and server code no page renders don't affect screenshots:
+ leave them out, and say you did.
+- **`routes.removed`** pages are gone: suggest a `- skip: /path` line in the routes
+ file so they stop being carried over.
+
+Map files to the page paths the routes file uses (`/pricing`, not `pricing.tsx`). Globs
+work: `/blog/**` covers every post.
+
+## 3. Ask the user
+
+Lay it out in one short message, then ask: rerun the affected pages, or everything?
+Recommend one.
+
+> Since 3f2a9c1 (6 commits): `/pricing` and `/compare` changed (PricingCard.tsx is used
+> by both), and `/blog` is new. Nothing global changed.
+> → **Rerun 3 pages (≈24 shots)**, carrying the other 38 pages over · or rerun
+> everything (≈310 shots)
+
+Recommend everything when:
+
+- `verdict` is `full` or `unknown`;
+- you couldn't trace an unmapped file with confidence;
+- the baseline wasn't shot from a local server (`baseline.local: false`): it shows what
+ was deployed, which may not be that commit;
+- the catalogue has gone through several partial runs since the last full one (check
+ the earlier runs' `partial` field): a full run now and then catches what tracing
+ missed.
+
+Add `--dry-run --json` to the command to show exact numbers before running it. With
+`--only`, its `partial` field lists the pages to shoot and the pages to carry.
+
+## 4. Run it
+
+```bash
+# Affected pages only: rediscovers their states (a changed page may have new ones),
+# maps new pages, shoots them, carries every other page from the last run.
+visualice crawl --replay --only=/pricing,/compare,/blog --json --quiet
+
+# Everything
+visualice crawl --replay --json --quiet # or without --replay to rediscover the whole site
+```
+
+`--only` takes comma-separated paths or globs. A page outside it is still shot when the
+last run can't cover it: a page new since, or one whose shots are missing or failed
+there (a device added, a state added). The summary's `carried` count says how many
+shots came from the earlier run.
+
+When the run finishes, the routes file records the new commit. Tell the user to commit
+`visualice.routes.md` with their change, so the next `changes` starts from here.
+
+## In the viewer
+
+A partial run marks the shots it took with an **updated** tag. A carried shot says so
+in its details panel, with the run and commit it was really taken in. **Compare with
+another version** in the details panel puts the same shot from another run beside it.
diff --git a/src/cli-args.ts b/src/cli-args.ts
index c3b0190..e57e2fd 100644
--- a/src/cli-args.ts
+++ b/src/cli-args.ts
@@ -48,6 +48,8 @@ const FLAG_TYPES: Record = {
inventory: "boolean",
from: "string",
"all-devices": "boolean",
+ only: "string",
+ discover: "boolean",
};
/**
@@ -93,6 +95,7 @@ export const COMMAND_FLAGS: Record = {
"dry-run",
"replay",
"resume",
+ "only",
"json",
"quiet",
"fail-on-error",
@@ -112,6 +115,7 @@ export const COMMAND_FLAGS: Record = {
doctor: ["json", "quiet", "no-fix"],
export: ["config", "site", "run", "dest", "json", "quiet", "dir"],
author: ["config", "target", "from", "json", "quiet", "dir"],
+ changes: ["config", "target", "discover", "site", "headed", "json", "quiet", "dir"],
probe: [
"config",
"target",
diff --git a/src/cli.ts b/src/cli.ts
index e3576ee..5efc4f8 100644
--- a/src/cli.ts
+++ b/src/cli.ts
@@ -4,7 +4,15 @@ import { promises as fs } from "node:fs";
import { spawn } from "node:child_process";
import { chromium } from "playwright";
import { ConfigLoadError, loadConfig, type Config } from "./shared/config";
-import { isCommand, UsageError, applyOverrides, parseFlags, str, type Flags } from "./cli-args";
+import {
+ isCommand,
+ UsageError,
+ applyOverrides,
+ csv,
+ parseFlags,
+ str,
+ type Flags,
+} from "./cli-args";
import { countStates, loadInputManifest, modelCachePath } from "./crawl/manifest";
import { InputManifestSchema, formatManifestIssues } from "./shared/manifest-schema";
import {
@@ -12,8 +20,11 @@ import {
crawlPipeline,
discoverOnlyPipeline,
planCrawl,
+ previewRoutes,
type CrawlPlan,
} from "./crawl/orchestrator";
+import { computeChanges, type ChangesReport } from "./crawl/changes";
+import { CONFIG_FILES, findConfigFile, hashFile } from "./crawl/source";
import { setHeaded } from "./shared/launch";
import {
computeWanted,
@@ -29,6 +40,7 @@ import { serveViewer } from "./serve";
import { formatDoctor, runDoctor } from "./doctor";
import {
latestRunId,
+ listRuns,
listSites,
migrateFlatRuns,
newestSite,
@@ -36,7 +48,7 @@ import {
siteDir,
siteKey,
} from "./crawl/runs";
-import type { Manifest, RecipeStep } from "./shared/types";
+import type { Manifest, RecipeStep, SourceInfo } from "./shared/types";
import { exportRun } from "./export";
import { probePage } from "./crawl/probe";
import { listAuthoredFiles, mergeAuthored, readAuthoredFile } from "./crawl/author";
@@ -49,22 +61,9 @@ const HERE = path.dirname(fileURLToPath(import.meta.url));
const PROJECT_ROOT = path.resolve(HERE, "..");
const FIXTURE_DIR = path.join(PROJECT_ROOT, "tests", "fixture");
-/** Config files picked up automatically from the working directory. */
-const CONFIG_CANDIDATES = [
- "visualice.config.ts",
- "visualice.config.mjs",
- "visualice.config.js",
- "visualice.config.json",
-];
-
-async function detectConfigFile(): Promise {
- for (const c of CONFIG_CANDIDATES) if (await pathExists(path.resolve(c))) return c;
- return undefined;
-}
-
/** Explicit --config wins; otherwise a visualice.config.* in the cwd is picked up. */
async function resolveConfigPath(flags: Flags): Promise {
- return str(flags, "config") ?? (await detectConfigFile());
+ return findConfigFile(process.cwd(), str(flags, "config"));
}
async function loadConfigWithOverrides(flags: Flags): Promise {
@@ -92,6 +91,7 @@ Usage:
visualice export Write the viewer + a run as a static folder to share
visualice probe Report a page's inventory, or what a recipe does on it (JSON)
visualice author [dir|file] Merge explorer state files (.visualice/explore/) into the cache
+ visualice changes [opts] What changed in git since the catalogue was shot, and what to rerun
visualice serve [opts] Launch the zoomable results viewer
visualice --version Print the version
@@ -109,6 +109,9 @@ Options (flags accept --name=value or --name value):
--manifest= input manifest for 'capture' / 'validate'
--replay for 'crawl'/'demo-crawl': reuse cached site model, skip discovery
--resume for 'crawl': continue the newest interrupted run
+ --only= for 'crawl': shoot only these pages (paths or globs, comma-separated);
+ every other page's shots are carried over from the last run
+ --discover for 'changes': also map the site's pages, to list new and removed ones
--json print a machine-readable JSON summary to stdout
--quiet suppress progress output on stderr
--fail-on-error exit 3 if any capture failed
@@ -156,6 +159,7 @@ function reportDone(runDir: string, manifest: Manifest, flags: Flags, log: Log):
routes: stats?.routes ?? 0,
states: stats?.states ?? 0,
shots: stats?.shots ?? 0,
+ ...(stats?.carried !== undefined ? { carried: stats.carried } : {}),
failed: stats?.failed ?? 0,
failures: stats?.failures ?? [],
wanted: stats?.wanted ?? [],
@@ -165,7 +169,9 @@ function reportDone(runDir: string, manifest: Manifest, flags: Flags, log: Log):
} else {
log("");
log(
- `done — ${stats?.shots ?? 0} screenshots${stats?.failed ? `, ${stats.failed} FAILED` : ""}`,
+ `done — ${stats?.shots ?? 0} screenshots` +
+ (stats?.carried ? ` (+${stats.carried} carried over)` : "") +
+ (stats?.failed ? `, ${stats.failed} FAILED` : ""),
);
console.log(runDir);
log(` manifest : ${path.join(runDir, "crawl-map.json")}`);
@@ -186,7 +192,14 @@ async function cmdCapture(flags: Flags): Promise {
if (!baseUrl) throw new UsageError("no target URL: pass --target= or set target in config");
const inputManifest = await loadInputManifest(manifestPath);
- const { runDir, manifest } = await capturePipeline({ config, inputManifest, baseUrl, log });
+ const configFile = await resolveConfigPath(flags);
+ const { runDir, manifest } = await capturePipeline({
+ config,
+ inputManifest,
+ baseUrl,
+ configFile,
+ log,
+ });
reportDone(runDir, manifest, flags, log);
}
@@ -502,6 +515,9 @@ async function cmdCrawl(flags: Flags): Promise {
await scaffoldProject(baseUrl, log);
}
const pagesOnly = flags["no-states"] === true;
+ const onlyFlag = str(flags, "only");
+ const only = onlyFlag === undefined ? undefined : csv(onlyFlag);
+ if (only && !only.length) throw new UsageError("--only needs at least one path or glob");
if (flags["dry-run"] === true) {
if (flags.resume === true) throw new UsageError("pass --dry-run or --resume, not both");
const plan = await planCrawl({
@@ -509,6 +525,7 @@ async function cmdCrawl(flags: Flags): Promise {
baseUrl,
replay: flags.replay === true,
pagesOnly,
+ only,
log,
});
reportPlan(plan, flags, log);
@@ -520,6 +537,8 @@ async function cmdCrawl(flags: Flags): Promise {
replay: flags.replay === true,
resume: flags.resume === true,
pagesOnly,
+ only,
+ configFile: configPath,
log,
});
reportDone(runDir, manifest, flags, log);
@@ -540,9 +559,139 @@ function reportPlan(plan: CrawlPlan, flags: Flags, log: Log): void {
`dry run — ${plan.routes.length} page(s), ${plan.states} state(s), ` +
`${plan.shots} screenshot(s) across ${plan.devices.join(", ")}; nothing captured`,
);
+ const p = plan.partial;
+ if (p) {
+ if (p.carriedFrom) {
+ log(
+ ` --only: shoot ${p.capture.length} page(s), carry ${p.carry.length} page(s) ` +
+ `(${p.carriedShots} screenshot(s)) from run ${p.carriedFrom}`,
+ );
+ for (const f of p.forced)
+ log(` + ${f.url} shot too (${f.reason === "new" ? "new" : "incomplete in that run"})`);
+ } else {
+ log(" --only: no finished run to carry from, so every page would be shot");
+ }
+ }
log(` model : ${plan.cachePath}`);
}
+/** The commit records in a site's runs, newest first. */
+async function runSources(config: Config, baseUrl: string | undefined): Promise {
+ const outRoot = path.resolve(config.output.dir);
+ const site = baseUrl ? siteKey(baseUrl, config.output.site) : config.output.site;
+ if (!site) return [];
+ const root = siteDir(outRoot, site);
+ const out: SourceInfo[] = [];
+ for (const id of [...(await listRuns(root))].reverse()) {
+ try {
+ const m = JSON.parse(
+ await fs.readFile(path.join(runDir(root, id), "crawl-map.json"), "utf8"),
+ ) as Manifest;
+ if (m.source?.commit) out.push(m.source);
+ } catch {
+ // An unfinished or unreadable run has no record to offer.
+ }
+ }
+ return out;
+}
+
+async function cmdChanges(flags: Flags): Promise {
+ const log = makeLog(flags);
+ const config = await loadConfigWithOverrides(flags);
+ const intent = await loadRoutesIntent(config);
+ const baseUrl = str(flags, "target") ?? config.target ?? intent.target;
+ const recorded = intent.source;
+ // The run that matches the routes file's record holds its list of dirty files; with
+ // no record in the file (not committed, or an older catalogue), the newest run's.
+ const sources = await runSources(config, baseUrl);
+ const runSource =
+ (recorded &&
+ (sources.find((s) => s.commit === recorded.commit && s.capturedAt === recorded.capturedAt) ??
+ sources.find((s) => s.commit === recorded.commit))) ||
+ (recorded ? undefined : sources[0]);
+
+ const cachePath = modelCachePath(path.resolve(config.output.dir));
+ const model = (await pathExists(cachePath)) ? await loadInputManifest(cachePath) : undefined;
+ const pages = model?.routes.map((r) => r.url) ?? intent.routes.map((r) => r.path);
+
+ let routes: ChangesReport["routes"];
+ if (flags.discover === true) {
+ if (!baseUrl)
+ throw new UsageError("--discover needs a target: pass --target= or set target");
+ const preview = await previewRoutes({ config, baseUrl, log });
+ routes = { added: preview.added, removed: preview.capped ? [] : preview.removed };
+ if (preview.capped)
+ log(
+ `⚠ discovery stopped at scope.maxPages (${config.scope.maxPages}): removed pages aren't listed`,
+ );
+ }
+
+ const routesFile = routesFilePath(config);
+ const configFile = (await resolveConfigPath(flags)) ?? path.resolve("visualice.config.json");
+ const report = await computeChanges({
+ cwd: process.cwd(),
+ recorded,
+ runSource,
+ ignore: [
+ path.resolve(config.output.dir),
+ ...(routesFile ? [routesFile] : []),
+ ...CONFIG_FILES.map((name) => path.resolve(name)),
+ configFile,
+ ],
+ config: { file: configFile, hash: await hashFile(configFile) },
+ pages,
+ routes,
+ });
+ if (flags.json === true) {
+ console.log(JSON.stringify(report, null, 2));
+ return;
+ }
+ printChanges(report);
+}
+
+function printChanges(r: ChangesReport): void {
+ const short = (c: string) => c.slice(0, 12);
+ if (r.baseline) {
+ const b = r.baseline;
+ console.log(
+ `catalogue: ${short(b.commit)}${b.branch ? ` (${b.branch})` : ""}` +
+ `${b.dirty ? `, ${b.dirty} uncommitted change(s)` : ""}` +
+ `${b.capturedAt ? `, shot ${b.capturedAt}` : ""}`,
+ );
+ }
+ if (r.head) {
+ console.log(
+ `now: ${short(r.head.commit)}${r.head.branch ? ` (${r.head.branch})` : ""}` +
+ `${r.head.dirty ? `, ${r.head.dirty} uncommitted change(s)` : ""}` +
+ (r.baseline ? ` — ${r.commitCount} commit(s) later` : ""),
+ );
+ }
+ console.log(`verdict: ${r.verdict}`);
+ for (const reason of r.reasons) console.log(` · ${reason}`);
+ if (r.global.length) {
+ console.log("\nchanges every page:");
+ for (const g of r.global) console.log(` ${g.path} (${g.reason})`);
+ }
+ if (r.candidates.length) {
+ console.log("\npages by file:");
+ for (const c of r.candidates) {
+ const pages = c.pages.length ? c.pages.join(", ") : "no page in the catalogue yet";
+ console.log(
+ ` ${c.file} → ${c.pattern}${c.kind === "layout" ? " (layout)" : ""}: ${pages}`,
+ );
+ }
+ }
+ if (r.unmapped.length) {
+ console.log("\nno page on their own (trace who imports them):");
+ for (const f of r.unmapped) console.log(` ${f}`);
+ }
+ if (r.routes) {
+ if (r.routes.added.length) console.log(`\nnew pages: ${r.routes.added.join(", ")}`);
+ if (r.routes.removed.length) console.log(`gone pages: ${r.routes.removed.join(", ")}`);
+ }
+ if (r.suggest) console.log(`\nnext: ${r.suggest.command}`);
+}
+
async function cmdRoutes(flags: Flags): Promise {
const log = makeLog(flags);
const config = await loadConfigWithOverrides(flags);
@@ -896,6 +1045,8 @@ async function main(): Promise {
return cmdProbe(flags);
case "author":
return cmdAuthor(flags);
+ case "changes":
+ return cmdChanges(flags);
}
}
diff --git a/src/crawl/carry.ts b/src/crawl/carry.ts
new file mode 100644
index 0000000..d35ef23
--- /dev/null
+++ b/src/crawl/carry.ts
@@ -0,0 +1,258 @@
+import path from "node:path";
+import { promises as fs } from "node:fs";
+import type { Device, Manifest, RouteNode, ShotRef, StateNode } from "../shared/types";
+import { matchesAny } from "../shared/glob";
+import { pathExists } from "../shared/fsx";
+import { stateSlug } from "../shared/slug";
+import { isRunComplete, listRuns, runDir as runDirFor } from "./runs";
+import { loadManifest } from "./manifest";
+
+// Partial runs (`crawl --only`): shoot the pages a change touched, and carry every
+// other page's shots over from the last finished run, so the new run is still the
+// whole site. A carried shot is a hard link to the earlier file where the disk
+// allows it (no extra space, and it outlives the earlier run being pruned), a copy
+// where it doesn't, and records the run and commit it was really shot in.
+
+/** The finished run a partial run carries from. */
+export interface CarrySource {
+ runId: string;
+ runDir: string;
+ manifest: Manifest;
+}
+
+/**
+ * The newest finished run of a site with a manifest to carry from, other than
+ * `exclude` (the run being written, which a `--resume` may have reopened).
+ */
+export async function findCarrySource(
+ site: string,
+ exclude?: string,
+): Promise {
+ for (const id of [...(await listRuns(site))].reverse()) {
+ if (id === exclude) continue;
+ const dir = runDirFor(site, id);
+ if (!(await isRunComplete(dir))) continue;
+ const file = path.join(dir, "crawl-map.json");
+ if (!(await pathExists(file))) continue;
+ try {
+ return { runId: id, runDir: dir, manifest: await loadManifest(file) };
+ } catch {
+ // A map that doesn't parse can't be carried from; try the run before it.
+ }
+ }
+ return undefined;
+}
+
+/** A route's path, for matching `--only` globs against: `/pricing`, never a full URL. */
+function routePath(url: string): string {
+ try {
+ return new URL(url, "http://_local_").pathname;
+ } catch {
+ return url;
+ }
+}
+
+/** Does a route match one of the `--only` paths or globs? */
+export function matchesOnly(route: Pick, only: string[]): boolean {
+ return matchesAny(route.url, only) || matchesAny(routePath(route.url), only);
+}
+
+/** The devices a state is captured on. */
+function devicesFor(state: StateNode, devices: Device[]): string[] {
+ const names = devices.map((d) => d.name);
+ return state.devices ? names.filter((n) => state.devices!.includes(n)) : names;
+}
+
+/**
+ * The component states the earlier run looked for on a route, by state id → selector,
+ * from its config snapshot: a component it sought there and didn't find isn't on the
+ * route, but one added to the config (or pointed at the route) since was never sought.
+ */
+export function componentsSought(prev: Manifest, url: string): Map {
+ const sought = new Map();
+ const snapshot = prev.configSnapshot as
+ | { components?: { name?: unknown; selector?: unknown; routes?: unknown }[] }
+ | undefined;
+ for (const c of snapshot?.components ?? []) {
+ if (typeof c.name !== "string" || typeof c.selector !== "string") continue;
+ const globs = Array.isArray(c.routes) ? c.routes.filter((g) => typeof g === "string") : [];
+ if (matchesAny(routePath(url), globs)) sought.set(`component-${stateSlug(c.name)}`, c.selector);
+ }
+ return sought;
+}
+
+/**
+ * Can every shot this route's capture would take be carried from `prev`? Only if
+ * the earlier run shot the same state on each device it applies to, and the shot
+ * succeeded: a state new since then, a device added since, or a shot that failed
+ * then all mean the route is shot again instead. A component the earlier run sought
+ * on the route (`sought`) and didn't find was dropped from it, so that is no gap; a
+ * component it never looked for is.
+ */
+export function canCarry(
+ route: RouteNode,
+ prev: RouteNode | undefined,
+ devices: Device[],
+ sought: Map = new Map(),
+): boolean {
+ if (!prev) return false;
+ const byId = new Map(prev.states.map((s) => [s.id, s]));
+ for (const state of route.states) {
+ const before = byId.get(state.id);
+ if (!before) {
+ if (state.element !== undefined && sought.get(state.id) === state.element) continue;
+ return false;
+ }
+ for (const device of devicesFor(state, devices)) {
+ const ref = before.shots[device];
+ if (!ref) {
+ if (state.element !== undefined) continue; // not on this route at that size
+ return false;
+ }
+ if (ref.error || !ref.full) return false;
+ }
+ }
+ return true;
+}
+
+export interface CarryPlan {
+ /** Routes to shoot: the ones `--only` names, and any the earlier run can't cover. */
+ capture: RouteNode[];
+ /** Routes whose shots come from the earlier run. */
+ carry: RouteNode[];
+ /** Routes outside `--only` that are shot anyway, and why. */
+ forced: { url: string; reason: "new" | "incomplete" }[];
+}
+
+/** Split a run's routes into the ones to shoot and the ones to carry from `prev`. */
+export function planCarry(
+ routes: RouteNode[],
+ only: string[],
+ prev: Manifest,
+ devices: Device[],
+): CarryPlan {
+ const prevByUrl = new Map(prev.routes.map((r) => [r.url, r]));
+ const plan: CarryPlan = { capture: [], carry: [], forced: [] };
+ for (const route of routes) {
+ if (matchesOnly(route, only)) {
+ plan.capture.push(route);
+ continue;
+ }
+ const before = prevByUrl.get(route.url);
+ if (canCarry(route, before, devices, componentsSought(prev, route.url))) {
+ plan.carry.push(route);
+ } else {
+ plan.capture.push(route);
+ plan.forced.push({ url: route.url, reason: before ? "incomplete" : "new" });
+ }
+ }
+ return plan;
+}
+
+/**
+ * Move to `capture` any route whose shots in `source` are missing a file on disk
+ * (an earlier run cleaned up by hand, or cut short while exporting): carrying it
+ * would fail after every other page had been shot.
+ */
+export async function verifyCarry(
+ plan: CarryPlan,
+ source: CarrySource,
+ devices: Device[],
+): Promise {
+ const out: CarryPlan = { capture: [...plan.capture], carry: [], forced: [...plan.forced] };
+ for (const route of plan.carry) {
+ const prev = source.manifest.routes.find((r) => r.url === route.url);
+ const files = prev ? carriedFiles(route, prev, devices) : [];
+ const present = await Promise.all(files.map((f) => pathExists(path.join(source.runDir, f))));
+ if (present.every(Boolean)) {
+ out.carry.push(route);
+ } else {
+ out.capture.push(route);
+ out.forced.push({ url: route.url, reason: "incomplete" });
+ }
+ }
+ return out;
+}
+
+/** The files carrying `route` from `prev` would copy. */
+function carriedFiles(route: RouteNode, prev: RouteNode, devices: Device[]): string[] {
+ const byId = new Map(prev.states.map((s) => [s.id, s]));
+ const files: string[] = [];
+ for (const state of route.states) {
+ const before = byId.get(state.id);
+ if (!before) continue;
+ for (const device of devicesFor(state, devices)) {
+ const ref = before.shots[device];
+ if (ref?.full && !ref.error) files.push(...shotFiles(ref));
+ }
+ }
+ return files;
+}
+
+/** The files a shot points at, relative to its run directory. */
+function shotFiles(ref: ShotRef): string[] {
+ return [ref.full, ref.viewport, ref.thumb, ref.mid, ref.meta].filter(
+ (f): f is string => typeof f === "string",
+ );
+}
+
+/** Hard-link `from` to `to`, copying instead where links aren't possible. */
+async function linkOrCopy(from: string, to: string): Promise {
+ await fs.mkdir(path.dirname(to), { recursive: true });
+ await fs.rm(to, { force: true });
+ try {
+ await fs.link(from, to);
+ } catch {
+ await fs.copyFile(from, to);
+ }
+}
+
+/**
+ * Fill `route`'s shots from the same route in `source`, copying the files into
+ * `runDir`. Returns how many shots were carried. A shot already carried keeps its
+ * original provenance: carried twice, it still says where it was really taken.
+ */
+export async function carryRoute(
+ route: RouteNode,
+ source: CarrySource,
+ runDir: string,
+ devices: Device[],
+): Promise {
+ const prev = source.manifest.routes.find((r) => r.url === route.url);
+ if (!prev) return 0;
+ const byId = new Map(prev.states.map((s) => [s.id, s]));
+ let carried = 0;
+ for (const state of route.states) {
+ const before = byId.get(state.id);
+ if (!before) continue;
+ for (const device of devicesFor(state, devices)) {
+ const ref = before.shots[device];
+ if (!ref?.full || ref.error) continue;
+ try {
+ for (const rel of shotFiles(ref)) {
+ await linkOrCopy(path.join(source.runDir, rel), path.join(runDir, rel));
+ }
+ } catch (err) {
+ // verifyCarry checked the files; one gone since fails this shot, not the run.
+ state.shots[device] = {
+ error: `couldn't carry the shot from run ${source.runId}: ${(err as Error).message}`,
+ };
+ continue;
+ }
+ state.shots[device] = {
+ ...ref,
+ carried: ref.carried ?? {
+ runId: source.runId,
+ commit: source.manifest.source?.commit,
+ capturedAt: source.manifest.source?.capturedAt ?? source.manifest.site.generatedAt,
+ },
+ };
+ carried++;
+ }
+ }
+ // As a capture does: a component found on no device was never on this route.
+ route.states = route.states.filter(
+ (s) => s.element === undefined || Object.keys(s.shots).length > 0,
+ );
+ return carried;
+}
diff --git a/src/crawl/changes.ts b/src/crawl/changes.ts
new file mode 100644
index 0000000..b640e1b
--- /dev/null
+++ b/src/crawl/changes.ts
@@ -0,0 +1,464 @@
+import path from "node:path";
+import type { SourceInfo } from "../shared/types";
+import { git, isUnder, parsePorcelain, repoRoot, toRepoPath } from "./source";
+
+// `visualice changes`: what changed in the project since the commit the catalogue
+// was shot from, and what that means for the next run. The mechanical half: git
+// knows which files changed, file-based routing says which pages some of them are,
+// and a list of known global files says when only a full run will do. Which pages a
+// shared component touches is the agent's call; this hands it the facts.
+
+export type Verdict = "none" | "partial" | "full" | "unknown";
+
+export interface ChangedFile {
+ path: string;
+ /** git's letter: A added, M modified, D deleted, R renamed, ? untracked. */
+ status: string;
+ /** For a rename, where it came from. */
+ from?: string;
+}
+
+/** A changed file that file-based routing ties to pages. */
+export interface PageCandidate {
+ file: string;
+ /** The route pattern the file serves: `/blog/*`, or `/docs/**` for a layout. */
+ pattern: string;
+ kind: "page" | "layout";
+ /** Pages in the site model the pattern matches. */
+ pages: string[];
+}
+
+export interface ChangesReport {
+ /** The commit the catalogue was shot from, and where that record was read. */
+ baseline?: SourceInfo & { from: "routes-file" | "run" };
+ head?: { commit: string; branch?: string; dirty: number };
+ verdict: Verdict;
+ /** Why the verdict is what it is, most important first. */
+ reasons: string[];
+ /** Commits since the baseline (newest first, capped), and how many in all. */
+ commits: { hash: string; subject: string }[];
+ commitCount: number;
+ /** Every changed file that counts (the crawl's own files left out). */
+ files: ChangedFile[];
+ /** Changed files that change every page: dependencies, global styles, root layouts. */
+ global: { path: string; reason: string }[];
+ /** Changed files that map to pages by their place in the routes folder. */
+ candidates: PageCandidate[];
+ /** Changed files that map to no page on their own: the agent traces their importers. */
+ unmapped: string[];
+ /** Pages found or gone since the catalogue, when a route discovery was run. */
+ routes?: { added: string[]; removed: string[] };
+ /** What to run next, when the facts alone settle it. */
+ suggest?: { only?: string[]; command: string };
+}
+
+const MAX_COMMITS = 50;
+
+/*
+ * Global files: a change to one of these reaches every page, so a partial run would
+ * leave most of the catalogue stale.
+ */
+
+const DEPENDENCY_FILES = new Set([
+ "package.json",
+ "package-lock.json",
+ "npm-shrinkwrap.json",
+ "pnpm-lock.yaml",
+ "yarn.lock",
+ "bun.lockb",
+ "bun.lock",
+ "composer.json",
+ "composer.lock",
+ "Gemfile",
+ "Gemfile.lock",
+]);
+
+const BUILD_CONFIG =
+ /^(tailwind|postcss|vite|next|nuxt|astro|svelte|remix|webpack|rollup|uno|panda|stitches|theme|app)\.config\.[cm]?[jt]s$/;
+
+const GLOBAL_STYLE_NAME =
+ /^(globals?|app|index|main|site|styles?|style|base|reset|normalize|theme|tokens|variables|vars|typography|fonts?|tailwind|bootstrap)\.(css|scss|sass|less|styl|pcss)$/i;
+
+const GLOBAL_DIRS = new Set(["styles", "theme", "themes", "tokens", "design-tokens", "fonts"]);
+
+const ROOT_LAYOUTS = [
+ /^(src\/)?app\/(layout|template|global-error|not-found)\.[cm]?[jt]sx?$/,
+ /^(src\/)?pages\/_(app|document)\.[cm]?[jt]sx?$/,
+ /^(src\/)?app\.(vue|svelte|html)$/i,
+ /^(src\/)?App\.(vue|svelte|[cm]?[jt]sx?)$/,
+ /^(src\/)?(main|index)\.[cm]?[jt]sx?$/,
+ /^src\/routes\/\+layout(\.server)?\.(svelte|[jt]s)$/,
+ /^app\/root\.[cm]?[jt]sx?$/,
+ /^(src\/)?layouts\/default\.vue$/,
+ /^(public\/)?index\.html$/,
+];
+
+/** Shared chrome: a header or footer is on every page, whatever folder it's in. */
+const CHROME_NAME =
+ /^(layout|root-?layout|base-?layout|default-?layout|header|footer|nav|navbar|navigation|sidebar|topbar|top-?nav|site-?header|site-?footer|site-?nav|app-?shell|shell)\.(tsx|jsx|vue|svelte|astro|[cm]?[jt]s)$/i;
+
+/** Why a change to `file` (repo-relative) reaches every page, or undefined. */
+export function globalReason(file: string): string | undefined {
+ const base = path.posix.basename(file);
+ const segments = file.split("/");
+ if (DEPENDENCY_FILES.has(base) && segments.length <= 2) return "dependencies";
+ if (BUILD_CONFIG.test(base)) return "build or theme config";
+ if (/^visualice\.config\.(json|[cm]?[jt]s)$/.test(base)) return "visualice config";
+ if (ROOT_LAYOUTS.some((re) => re.test(file))) return "root layout or entry point";
+ const inComponents = segments.some((s) => /^components?$/i.test(s));
+ if (GLOBAL_STYLE_NAME.test(base) && !inComponents) return "global styles";
+ if (segments.slice(0, -1).some((s) => GLOBAL_DIRS.has(s.toLowerCase())))
+ return "global styles or theme";
+ // A nested route layout (app/blog/layout.tsx) only wraps its own section: routing
+ // maps it to that section's pages below, rather than to every page.
+ if (CHROME_NAME.test(base) && guessRoute(file)?.kind !== "layout")
+ return "shared page chrome (layout, header, footer, nav)";
+ if (/^(src\/)?layouts\/[^/]+\.astro$/.test(file)) return "page layout";
+ return undefined;
+}
+
+/*
+ * File-based routing: the frameworks whose folder layout is the URL layout. Each
+ * rule turns a file into route segments; `*` is one dynamic segment, `**` the rest.
+ */
+
+interface RouteGuess {
+ segments: string[];
+ kind: "page" | "layout";
+}
+
+const CODE = String.raw`(?:[cm]?[jt]sx?|mdx?|vue|svelte|astro|html)`;
+
+/** `[slug]` → `*`, `[...rest]` / `[[...rest]]` → `**`, `(group)` / `@slot` → dropped. */
+function bracketSegments(parts: string[]): string[] {
+ const out: string[] = [];
+ for (const p of parts) {
+ if (/^\(.*\)$/.test(p) || p.startsWith("@") || p.startsWith("_")) continue;
+ if (/^\[\[?\.\.\..+\]\]?$/.test(p)) out.push("**");
+ else if (/^\[.+\]$/.test(p)) out.push("*");
+ else out.push(p);
+ }
+ return out;
+}
+
+/** The route a changed file serves, by the conventions of file-based routers. */
+export function guessRoute(file: string): RouteGuess | undefined {
+ let m: RegExpExecArray | null;
+
+ // Next.js app router: app/blog/[slug]/page.tsx
+ m = new RegExp(
+ String.raw`^(?:src/)?app/(.*?)/?(page|layout|template|loading|error|not-found)\.${CODE}$`,
+ ).exec(file);
+ if (m && !/(^|\/)api(\/|$)/.test(m[1])) {
+ const segs = bracketSegments(m[1] ? m[1].split("/") : []);
+ return { segments: segs, kind: m[2] === "page" ? "page" : "layout" };
+ }
+ // SvelteKit: src/routes/blog/[slug]/+page.svelte
+ m = new RegExp(String.raw`^src/routes/(.*?)/?\+(page|layout)(?:\.server)?\.${CODE}$`).exec(file);
+ if (m) {
+ return {
+ segments: bracketSegments(m[1] ? m[1].split("/") : []),
+ kind: m[2] === "page" ? "page" : "layout",
+ };
+ }
+ // Remix flat routes: app/routes/blog.$slug.tsx, app/routes/_index.tsx
+ m = new RegExp(String.raw`^app/routes/([^/]+?)(?:/route)?\.${CODE}$`).exec(file);
+ if (m) {
+ const segs: string[] = [];
+ for (const part of m[1].split(".")) {
+ if (part === "_index" || part.startsWith("_") || /^\(.*\)$/.test(part)) continue;
+ if (part === "$") segs.push("**");
+ else if (part.startsWith("$")) segs.push("*");
+ else segs.push(part.replace(/_$/, ""));
+ }
+ return { segments: segs, kind: "page" };
+ }
+ // Pages folders: Next.js pages/, Nuxt pages/, Astro src/pages/, Gatsby src/pages/.
+ m = new RegExp(String.raw`^(?:src/)?pages/(.+)\.${CODE}$`).exec(file);
+ if (m) {
+ const parts = m[1].split("/");
+ if (parts[0] === "api" || parts.some((p) => p.startsWith("_"))) return undefined;
+ if (parts.at(-1) === "index") parts.pop();
+ return { segments: bracketSegments(parts), kind: "page" };
+ }
+ // Plain HTML at the root or in public/: about.html, blog/index.html
+ m = /^(?:public\/|static\/)?(.+)\.html?$/.exec(file);
+ if (m && !file.includes("node_modules/")) {
+ const parts = m[1].split("/");
+ if (parts.at(-1) === "index") parts.pop();
+ return { segments: parts, kind: "page" };
+ }
+ return undefined;
+}
+
+/** A guess as a route pattern string: `/blog/*`; a layout covers everything under it. */
+export function guessPattern(g: RouteGuess): string {
+ const base = `/${g.segments.join("/")}`;
+ if (g.kind === "layout") return base === "/" ? "/**" : `${base}/**`;
+ return base;
+}
+
+/** Does a site-model page match a guessed pattern? */
+export function patternMatches(pattern: string, page: string): boolean {
+ const pathOnly =
+ (() => {
+ try {
+ return new URL(page, "http://_local_").pathname;
+ } catch {
+ return page;
+ }
+ })().replace(/\/+$/, "") || "/";
+ const variants = new Set([
+ pathOnly,
+ pathOnly.replace(/\.html?$/, ""),
+ pathOnly.replace(/\/index$/, "") || "/",
+ ]);
+ let re = "";
+ const p = pattern === "/" ? "/" : pattern.replace(/\/+$/, "");
+ for (let i = 0; i < p.length; i++) {
+ if (p.startsWith("/**", i)) {
+ re += "(?:/.*)?";
+ i += 2;
+ } else if (p.startsWith("**", i)) {
+ re += ".*";
+ i += 1;
+ } else if (p[i] === "*") {
+ re += "[^/]+";
+ } else re += p[i].replace(/[.+?^${}()|[\]\\]/g, "\\$&");
+ }
+ const rx = new RegExp(`^${re}$`);
+ return [...variants].some((v) => rx.test(v));
+}
+
+/** Parse `git diff --name-status -z`. */
+export function parseNameStatus(out: string): ChangedFile[] {
+ const parts = out.split("\0").filter((p) => p !== "");
+ const files: ChangedFile[] = [];
+ for (let i = 0; i < parts.length; i++) {
+ const status = parts[i];
+ const letter = status[0];
+ if (letter === "R" || letter === "C") {
+ files.push({ status: letter, from: parts[i + 1], path: parts[i + 2] });
+ i += 2;
+ } else {
+ files.push({ status: letter, path: parts[i + 1] });
+ i += 1;
+ }
+ }
+ return files.filter((f) => f.path);
+}
+
+export interface ChangesInput {
+ cwd: string;
+ /** The routes file's record, if it has one. */
+ recorded?: SourceInfo;
+ /** A run manifest's record, used when the routes file has none (and for its dirty files). */
+ runSource?: SourceInfo;
+ /** Paths whose changes don't count: the routes file, the output folder, the config. */
+ ignore: string[];
+ /** The config file now, and its hash: compared with the record's, not diffed. */
+ config?: { file: string; hash?: string };
+ /** Pages in the cached site model. */
+ pages: string[];
+ /** From a route discovery, when one ran. */
+ routes?: { added: string[]; removed: string[] };
+}
+
+/** Work out what changed since the catalogue's commit, and the verdict. */
+export async function computeChanges(input: ChangesInput): Promise {
+ const report: ChangesReport = {
+ verdict: "unknown",
+ reasons: [],
+ commits: [],
+ commitCount: 0,
+ files: [],
+ global: [],
+ candidates: [],
+ unmapped: [],
+ ...(input.routes ? { routes: input.routes } : {}),
+ };
+ const fullRun = "visualice crawl";
+
+ const root = await repoRoot(input.cwd);
+ if (!root) {
+ report.reasons.push("not a git repository: nothing to diff against");
+ report.suggest = { command: fullRun };
+ return report;
+ }
+ const headCommit = (await git(root, ["rev-parse", "HEAD"]))?.trim();
+ const branch = (await git(root, ["symbolic-ref", "--quiet", "--short", "HEAD"]))?.trim();
+
+ const record = input.recorded ?? input.runSource;
+ if (!record) {
+ report.reasons.push(
+ "no recorded commit: the catalogue predates commit tracking, or was never shot",
+ );
+ report.suggest = { command: fullRun };
+ return report;
+ }
+ const dirtyFiles =
+ input.runSource && input.runSource.commit === record.commit
+ ? input.runSource.dirtyFiles
+ : undefined;
+ // A record keeps a capped list; past the cap, the files it left out can't be diffed.
+ const dirtyListed = record.dirty === 0 || (dirtyFiles?.length ?? 0) >= record.dirty;
+ report.baseline = {
+ ...record,
+ ...(dirtyFiles ? { dirtyFiles } : {}),
+ from: input.recorded ? "routes-file" : "run",
+ };
+
+ if (!(await commitExists(root, record.commit))) {
+ report.reasons.push(
+ `commit ${record.commit.slice(0, 12)} isn't in this repository (rebased, squashed or not fetched): nothing to diff against`,
+ );
+ report.suggest = { command: fullRun };
+ return report;
+ }
+
+ const ignore = input.ignore.map((p) => toRepoPath(root, p)).filter(Boolean) as string[];
+ const counts = (f: string) => !isUnder(f, ignore);
+
+ const [diffOut, untrackedOut, logOut, countOut, ancestor] = await Promise.all([
+ git(root, ["diff", "--name-status", "-z", "-M", record.commit]),
+ git(root, ["ls-files", "--others", "--exclude-standard", "-z"]),
+ git(root, ["log", "--format=%H%x09%s", `-n${MAX_COMMITS}`, `${record.commit}..HEAD`]),
+ git(root, ["rev-list", "--count", `${record.commit}..HEAD`]),
+ git(root, ["merge-base", "--is-ancestor", record.commit, "HEAD"]),
+ ]);
+ const files = parseNameStatus(diffOut ?? "").filter((f) => counts(f.path));
+ for (const f of (untrackedOut ?? "").split("\0")) {
+ if (f && counts(f) && !files.some((x) => x.path === f)) files.push({ path: f, status: "?" });
+ }
+ // Files that were dirty when the catalogue was shot: what was shot isn't the
+ // commit's version of them, so they count as changed whatever the diff says.
+ for (const f of dirtyFiles ?? []) {
+ if (counts(f) && !files.some((x) => x.path === f)) files.push({ path: f, status: "M" });
+ }
+ report.files = files;
+ report.commits = (logOut ?? "")
+ .split("\n")
+ .filter(Boolean)
+ .map((l) => {
+ const [hash, ...rest] = l.split("\t");
+ return { hash, subject: rest.join("\t") };
+ });
+ report.commitCount = Number(countOut?.trim() ?? report.commits.length) || 0;
+ const headStatus = await git(root, ["status", "--porcelain=v1", "-z", "--untracked-files=all"]);
+ report.head = headCommit
+ ? {
+ commit: headCommit,
+ branch: branch || undefined,
+ dirty: parsePorcelain(headStatus ?? "").filter(counts).length,
+ }
+ : undefined;
+
+ // The config is compared by hash: a first crawl scaffolds it, so it is usually not
+ // committed yet when the catalogue is shot, and a diff would call it new.
+ if (input.config && record.config && input.config.hash !== record.config) {
+ const rel = toRepoPath(root, input.config.file) ?? path.basename(input.config.file);
+ report.global.push({ path: rel, reason: "visualice config" });
+ }
+
+ // Classify every changed file.
+ for (const f of files) {
+ const reason = globalReason(f.path) ?? (f.from ? globalReason(f.from) : undefined);
+ if (reason) {
+ report.global.push({ path: f.path, reason });
+ continue;
+ }
+ const guess = guessRoute(f.path);
+ if (guess) {
+ const pattern = guessPattern(guess);
+ report.candidates.push({
+ file: f.path,
+ pattern,
+ kind: guess.kind,
+ pages: input.pages.filter((p) => patternMatches(pattern, p)),
+ });
+ continue;
+ }
+ report.unmapped.push(f.path);
+ }
+
+ // The verdict, and the reasons that carry it.
+ if (ancestor === undefined && headCommit !== record.commit) {
+ report.reasons.push(
+ `commit ${record.commit.slice(0, 12)} isn't an ancestor of HEAD (another branch, or history rewritten): the diff is between the two trees`,
+ );
+ }
+ if (record.target && record.local === false) {
+ report.reasons.push(
+ `the catalogue was shot from ${record.target}, not a local server: it shows what was deployed there, which may not be commit ${record.commit.slice(0, 12)}`,
+ );
+ }
+ if (!dirtyListed) {
+ report.reasons.push(
+ `the catalogue was shot with ${record.dirty} uncommitted change(s) that can't all be listed now: a full run is the safe choice`,
+ );
+ }
+ const removed = input.routes?.removed ?? [];
+ const added = input.routes?.added ?? [];
+ if (removed.length) {
+ // The cached model still has them, and a partial run carries every page it doesn't
+ // shoot: until the routes file skips them, their old shots carry on.
+ report.reasons.push(
+ `${removed.length} page(s) are gone from the site (${removed.slice(0, 5).join(", ")}${removed.length > 5 ? ", …" : ""}): add a "- skip: " line for each to the routes file, or partial runs keep carrying their old shots`,
+ );
+ }
+
+ if (report.global.length) {
+ report.verdict = "full";
+ const kinds = [...new Set(report.global.map((g) => g.reason))];
+ report.reasons.unshift(
+ `${report.global.length} changed file(s) reach every page (${kinds.join("; ")}): ${report.global
+ .slice(0, 5)
+ .map((g) => g.path)
+ .join(", ")}${report.global.length > 5 ? ", …" : ""}`,
+ );
+ } else if (!dirtyListed) {
+ report.verdict = "full";
+ } else if (!files.length && !added.length && !removed.length) {
+ report.verdict = "none";
+ report.reasons.unshift(
+ headCommit === record.commit
+ ? "HEAD is the commit the catalogue was shot from, and nothing has changed since"
+ : `${report.commitCount} commit(s) since, but none of them change a file that counts`,
+ );
+ } else {
+ report.verdict = "partial";
+ report.reasons.unshift(
+ `${files.length} file(s) changed in ${report.commitCount} commit(s)` +
+ (report.unmapped.length
+ ? `; ${report.unmapped.length} of them map to no page on their own — find the pages that import them`
+ : ""),
+ );
+ }
+
+ if (report.verdict === "full") {
+ report.suggest = { command: fullRun };
+ } else if (report.verdict === "partial") {
+ const only = [
+ ...new Set([
+ ...report.candidates.flatMap((c) => (c.pages.length ? c.pages : [c.pattern])),
+ ...added,
+ ]),
+ ];
+ if (only.length && !report.unmapped.length) {
+ report.suggest = {
+ only,
+ command: `visualice crawl --replay --only=${shellQuote(only.join(","))}`,
+ };
+ }
+ }
+ return report;
+}
+
+/** Quote a value for a POSIX shell when it holds anything a shell would expand. */
+export function shellQuote(value: string): string {
+ return /^[\w@%+=:,./-]+$/.test(value) ? value : `'${value.replace(/'/g, `'\\''`)}'`;
+}
+
+async function commitExists(root: string, commit: string): Promise {
+ return (await git(root, ["cat-file", "-t", commit]))?.trim() === "commit";
+}
diff --git a/src/crawl/orchestrator.ts b/src/crawl/orchestrator.ts
index eeb68f4..30a3ea2 100644
--- a/src/crawl/orchestrator.ts
+++ b/src/crawl/orchestrator.ts
@@ -1,7 +1,15 @@
import path from "node:path";
import { promises as fs } from "node:fs";
import type { Config } from "../shared/config";
-import type { Manifest, InputManifest, StateNode, ViewerData, ViewerTile } from "../shared/types";
+import type {
+ Manifest,
+ InputManifest,
+ RouteNode,
+ SourceInfo,
+ StateNode,
+ ViewerData,
+ ViewerTile,
+} from "../shared/types";
import { matchesAny } from "../shared/glob";
import { runCapture } from "./capture";
import { renderReport } from "./report";
@@ -41,6 +49,15 @@ import {
import { redactConfig } from "../shared/redact";
import { acquireLock } from "../shared/lock";
import { applyHints, describeHints } from "../shared/hints";
+import {
+ type CarryPlan,
+ carryRoute,
+ findCarrySource,
+ matchesOnly,
+ planCarry,
+ verifyCarry,
+} from "./carry";
+import { CONFIG_FILES, findConfigFile, readSource } from "./source";
type Log = (msg: string) => void;
@@ -65,6 +82,15 @@ export interface CapturePipelineOptions {
resume?: string;
/** Capture each page's default render only: no UI states, no components. */
pagesOnly?: boolean;
+ /**
+ * Shoot only the routes these paths or globs match, and carry every other route's
+ * shots over from the site's last finished run (`crawl --only`).
+ */
+ only?: string[];
+ /** The commit being shot; read from the working directory's git when omitted. */
+ source?: SourceInfo | null;
+ /** The config file the run was loaded from (`--config`), for the source record's hash. */
+ configFile?: string;
log?: (msg: string) => void;
}
@@ -114,6 +140,7 @@ export function toViewerData(m: Manifest): ViewerData {
meta: ref.meta,
a11y: ref.a11y,
error: ref.error,
+ ...(ref.carried ? { carried: ref.carried } : m.partial ? { updated: true } : {}),
});
}
}
@@ -121,6 +148,8 @@ export function toViewerData(m: Manifest): ViewerData {
return {
site: m.site,
+ ...(m.source ? { source: m.source } : {}),
+ ...(m.partial ? { partial: m.partial } : {}),
devices: m.devices,
tiles,
groups: m.routes.map((r) => ({ path: pathSlug(r.url), routeId: r.id, title: r.title })),
@@ -272,8 +301,13 @@ async function captureRun(opts: CapturePipelineOptions): Promise n + r.states.length, 0);
+ result.manifest.stats.carried = carried;
+ log(
+ `carried ${carried} shot(s) of ${carry.carry.length} page(s) from run ${carrySource.runId}`,
+ );
+ }
+
if (result.manifest.stats && wanted.length) {
result.manifest.stats.wanted = wanted;
log(`${wanted.length} wanted state(s) still need authoring (see routes file / report)`);
@@ -317,6 +387,45 @@ async function captureRun(opts: CapturePipelineOptions): Promise {
+ const routesFile = config.routesFile === false ? undefined : path.resolve(config.routesFile);
+ // `configFile` is the --config the run was given; without one, the file a first
+ // crawl may just have scaffolded is looked up now.
+ const file = await findConfigFile(process.cwd(), configFile);
+ return readSource(process.cwd(), {
+ target: baseUrl,
+ configFile: file,
+ // The crawl's own files: the output folder, the routes file it rewrites, and the
+ // config (compared by hash in `changes`, since a first crawl scaffolds it).
+ ignore: [
+ path.resolve(config.output.dir),
+ ...(routesFile ? [routesFile] : []),
+ ...CONFIG_FILES.map((name) => path.resolve(name)),
+ ...(file ? [file] : []),
+ ],
+ });
+}
+
+function logCarryPlan(plan: CarryPlan, only: string[], from: string, log: Log): void {
+ log(
+ `--only ${only.join(", ")}: shooting ${plan.capture.length} page(s), ` +
+ `carrying ${plan.carry.length} from run ${from}`,
+ );
+ if (!plan.capture.some((r) => matchesOnly(r, only))) {
+ log(`⚠ --only matched no page in the site model (${only.join(", ")})`);
+ }
+ for (const f of plan.forced) {
+ log(
+ ` + ${f.url}: shot too — ${f.reason === "new" ? "not in the earlier run" : "the earlier run is missing some of its shots"}`,
+ );
+ }
+}
+
export interface CrawlPipelineOptions {
config: Config;
baseUrl: string;
@@ -329,6 +438,14 @@ export interface CrawlPipelineOptions {
* cache are kept for the next full crawl rather than dropped.
*/
pagesOnly?: boolean;
+ /**
+ * Shoot only the pages these paths or globs match; every other page's shots are
+ * carried over from the last finished run. With `replay`, the matched pages' states
+ * are discovered again, since a changed page may have new ones.
+ */
+ only?: string[];
+ /** The config file the run was loaded from (`--config`), for the source record's hash. */
+ configFile?: string;
log?: (msg: string) => void;
}
@@ -340,6 +457,16 @@ export interface CrawlPlan {
/** Screenshots the capture would take: each state on each device it applies to. */
shots: number;
devices: string[];
+ /** With `--only`: what would be shot and what carried over, and from which run. */
+ partial?: {
+ only: string[];
+ carriedFrom?: string;
+ capture: string[];
+ carry: string[];
+ forced: CarryPlan["forced"];
+ /** Screenshots carried instead of shot. */
+ carriedShots: number;
+ };
}
/**
@@ -370,7 +497,8 @@ async function crawlRun(
}
const { inputManifest, intent, storageState } = await crawlModel(opts, outRoot);
- return capturePipeline({
+ const source = (await readProjectSource(config, baseUrl, opts.configFile)) ?? null;
+ const result = await capturePipeline({
config,
inputManifest,
baseUrl,
@@ -379,8 +507,31 @@ async function crawlRun(
lock: false,
resume,
pagesOnly: opts.pagesOnly,
+ only: opts.only,
+ configFile: opts.configFile,
+ source,
log,
});
+ // The routes file now says which commit the catalogue shows, for `changes`. A rerun
+ // of the same commit keeps the line it has, so the file only changes when the code
+ // did. (Which files were dirty is in each run's manifest, which `changes` reads.)
+ if (source) {
+ const record = sameSource(intent.source, source) ? intent.source : source;
+ await writeRoutesFile(config, inputManifest, intent, baseUrl, () => {}, record);
+ }
+ return result;
+}
+
+/** Do two records describe the same code, shot the same way? Capture time aside. */
+export function sameSource(a: SourceInfo | undefined, b: SourceInfo): boolean {
+ return (
+ !!a &&
+ a.commit === b.commit &&
+ a.branch === b.branch &&
+ a.dirty === b.dirty &&
+ a.target === b.target &&
+ a.config === b.config
+ );
}
/**
@@ -396,17 +547,43 @@ export async function planCrawl(opts: CrawlPipelineOptions): Promise
const config = applyHints(opts.config, inputManifest.hints);
const model = captureModel(inputManifest, intent, config, !!opts.pagesOnly);
const names = config.devices.map((d) => d.name);
- let shots = 0;
- for (const r of model.routes) {
- for (const s of r.states)
- shots += s.devices ? names.filter((n) => s.devices!.includes(n)).length : names.length;
+ const shotsOf = (routes: RouteNode[]) => {
+ let n = 0;
+ for (const r of routes) {
+ for (const s of r.states)
+ n += s.devices ? names.filter((d) => s.devices!.includes(d)).length : names.length;
+ }
+ return n;
+ };
+ let partial: CrawlPlan["partial"];
+ let shooting = model.routes;
+ if (opts.only?.length) {
+ const site = siteDir(outRoot, siteKey(opts.baseUrl, config.output.site));
+ const source = await findCarrySource(site);
+ const plan = source
+ ? await verifyCarry(
+ planCarry(model.routes, opts.only, source.manifest, config.devices),
+ source,
+ config.devices,
+ )
+ : undefined;
+ if (plan) shooting = plan.capture;
+ partial = {
+ only: opts.only,
+ carriedFrom: source?.runId,
+ capture: shooting.map((r) => r.url),
+ carry: plan?.carry.map((r) => r.url) ?? [],
+ forced: plan?.forced ?? [],
+ carriedShots: plan ? shotsOf(plan.carry) : 0,
+ };
}
return {
cachePath: modelCachePath(outRoot),
routes: model.routes.map((r) => ({ url: r.url, states: r.states.map((s) => s.id) })),
states: model.routes.reduce((n, r) => n + r.states.length, 0),
- shots,
+ shots: shotsOf(shooting),
devices: names,
+ ...(partial ? { partial } : {}),
};
});
}
@@ -420,8 +597,8 @@ async function crawlModel(
const cachePath = modelCachePath(outRoot);
const keepStates = !!opts.pagesOnly;
- const prepared = await prepareModel(config, baseUrl, cachePath, log);
- const { intent, storageState, cached } = prepared;
+ const prepared = await prepareModel(config, baseUrl, cachePath, log, onlyPages(opts.only));
+ const { intent, seeded, storageState, cached } = prepared;
const effectiveConfig = keepStates
? withoutStates(prepared.effectiveConfig)
: prepared.effectiveConfig;
@@ -430,18 +607,36 @@ async function crawlModel(
log(`replay: reusing cached site model → ${cachePath}`);
inputManifest = cached;
// Pages added to the routes file since the last discovery: map just those, so
- // a replay doesn't silently skip them.
- const added = unmappedRoutes(cached, intent);
- if (added.length) {
- log(`replay: discovering ${added.length} page(s) new in the routes file`);
+ // a replay doesn't silently skip them. With --only, the pages it names are mapped
+ // again too: a page that changed may have states the cache doesn't know yet.
+ const added = unmappedRoutes(cached, seeded);
+ const changed = opts.only?.length
+ ? cached.routes.filter((r) => matchesOnly(r, opts.only!)).map((r) => r.url)
+ : [];
+ // Known pages first, in the model's order, then the new ones: a site-wide state
+ // stays with the page that had it, whichever run maps it.
+ const visit = [...new Set([...changed, ...added])];
+ if (visit.length) {
+ log(
+ `replay: discovering ${visit.length} page(s)` +
+ (changed.length
+ ? ` (${changed.length} named by --only, ${added.length} new)`
+ : " new in the routes file"),
+ );
+ const remapped = new Set(changed);
const found = await discoverPipeline({
config: applyHints(effectiveConfig, cached.hints),
baseUrl,
storageState,
log,
- limit: { only: added, takenIds: cached.routes.map((r) => r.id) },
+ limit: {
+ only: visit,
+ takenIds: cached.routes.filter((r) => !remapped.has(r.url)).map((r) => r.id),
+ },
existing: cached,
});
+ keepRouteIds(found.routes, cached.routes);
+ found.routes = dropMissingSeeds(found.routes, opts.only, intent, cached, log);
inputManifest = {
...mergeManifests(cached, found, { keepStates }),
site: cached.site ?? found.site,
@@ -458,12 +653,39 @@ async function crawlModel(
log,
keepStates,
);
+ const kept = dropMissingSeeds(inputManifest.routes, opts.only, intent, cached, log);
+ if (kept.length !== inputManifest.routes.length) {
+ inputManifest = { ...inputManifest, routes: kept };
+ await saveJson(cachePath, inputManifest);
+ }
}
await writeRoutesFile(config, inputManifest, intent, baseUrl, log);
return { inputManifest, intent, storageState };
}
+/**
+ * Leave out a page that only `--only` named (not in the routes file, not mapped
+ * before) when it answered with an error: a mistyped path isn't a page to catalogue.
+ */
+function dropMissingSeeds(
+ routes: RouteNode[],
+ only: string[] | undefined,
+ intent: RoutesIntent,
+ cached: InputManifest | undefined,
+ log: Log,
+): RouteNode[] {
+ const seeds = onlyPages(only).filter(
+ (p) => !intent.routes.some((r) => r.path === p) && !cached?.routes.some((r) => r.url === p),
+ );
+ if (!seeds.length) return routes;
+ return routes.filter((r) => {
+ if (!seeds.includes(r.url) || r.status === undefined || r.status < 400) return true;
+ log(`⚠ --only ${r.url}: the page answered ${r.status}, so it isn't added to the catalogue`);
+ return false;
+ });
+}
+
/** The config for a pages-only discovery: routes, no state probing. */
function withoutStates(config: Config): Config {
return {
@@ -475,19 +697,69 @@ function withoutStates(config: Config): Config {
};
}
+/**
+ * A page rediscovered by name keeps the id it had: shot folders and tile ids are
+ * built from it, and a compare between runs matches shots by them.
+ */
+function keepRouteIds(found: RouteNode[], cached: RouteNode[]): void {
+ const ids = new Map(cached.map((r) => [r.url, r.id]));
+ for (const r of found) {
+ const id = ids.get(r.url);
+ if (id) r.id = id;
+ }
+}
+
+/**
+ * The plain paths among `--only` patterns (no globs), as paths: pages to seed
+ * discovery with. A full URL is reduced to its path.
+ */
+export function onlyPages(only: string[] | undefined): string[] {
+ const out: string[] = [];
+ for (const p of only ?? []) {
+ if (p.includes("*")) continue;
+ if (p.startsWith("/")) out.push(p);
+ else if (/^https?:/.test(p)) {
+ try {
+ const u = new URL(p);
+ out.push(u.pathname + u.search);
+ } catch {
+ // not a URL after all: nothing to seed
+ }
+ }
+ }
+ return [...new Set(out)];
+}
+
/** What every model-building run starts from: routes-file intent, session, cache. */
-async function prepareModel(config: Config, baseUrl: string, cachePath: string, log: Log) {
- // Routes-file intent: listed routes seed discovery, skip: globs exclude.
+async function prepareModel(
+ config: Config,
+ baseUrl: string,
+ cachePath: string,
+ log: Log,
+ extraPages: string[] = [],
+) {
+ // Routes-file intent: listed routes seed discovery, skip: globs exclude. Pages
+ // named on the command line (--only) seed it too, so a new one gets mapped, but only
+ // in `seeded`: a one-off filter (or a typo in one) isn't written into the routes file.
const intent = await loadRoutesIntent(config);
+ const seeded: RoutesIntent = {
+ ...intent,
+ routes: [
+ ...intent.routes,
+ ...extraPages
+ .filter((page) => !intent.routes.some((r) => r.path === page))
+ .map((page) => ({ path: page, states: [] })),
+ ],
+ };
const intentOrigin = intent.target ? safeOrigin(intent.target) : undefined;
if (intentOrigin && intentOrigin !== safeOrigin(baseUrl)) {
log(`⚠ routes file targets ${intent.target} but crawling ${baseUrl} — regenerating for it`);
}
for (const w of intent.warnings) log(`⚠ routes file: ${w}`);
- const effectiveConfig = applyRoutesIntent(config, intent);
+ const effectiveConfig = applyRoutesIntent(config, seeded);
const storageState = await ensureStorageState(config, log);
const cached = (await pathExists(cachePath)) ? await loadInputManifest(cachePath) : undefined;
- return { intent, effectiveConfig, storageState, cached };
+ return { intent, seeded, effectiveConfig, storageState, cached };
}
/** Discover the site from scratch and fold it into the cache, which it then saves. */
@@ -570,3 +842,41 @@ function safeOrigin(url: string): string | undefined {
return undefined;
}
}
+
+/**
+ * Which pages the site has now against the ones the cached model knows, from a
+ * route-only discovery (no states, nothing written): `changes --discover`.
+ */
+export async function previewRoutes(opts: {
+ config: Config;
+ baseUrl: string;
+ log?: Log;
+}): Promise<{ added: string[]; removed: string[]; capped: boolean }> {
+ const { config, baseUrl } = opts;
+ const log = opts.log ?? (() => {});
+ const outRoot = path.resolve(config.output.dir);
+ const cachePath = modelCachePath(outRoot);
+ return withLock(outRoot, log, true, async () => {
+ const { effectiveConfig, storageState, cached, intent } = await prepareModel(
+ config,
+ baseUrl,
+ cachePath,
+ log,
+ );
+ const found = await discoverPipeline({
+ config: applyHints(withoutStates(effectiveConfig), cached?.hints),
+ baseUrl,
+ storageState,
+ log,
+ });
+ const now = new Set(found.routes.map((r) => r.url));
+ const known = (cached?.routes ?? []).filter((r) => !matchesAny(r.url, intent.skipRoutes));
+ const before = new Set(known.map((r) => r.url));
+ return {
+ added: [...now].filter((u) => !before.has(u)),
+ removed: [...before].filter((u) => !now.has(u)),
+ // Discovery stopped at scope.maxPages: a page it didn't reach isn't gone.
+ capped: found.routes.length >= config.scope.maxPages,
+ };
+ });
+}
diff --git a/src/crawl/routes-file.ts b/src/crawl/routes-file.ts
index 4506e3f..d8a8710 100644
--- a/src/crawl/routes-file.ts
+++ b/src/crawl/routes-file.ts
@@ -1,9 +1,17 @@
import path from "node:path";
import { promises as fs } from "node:fs";
import type { Config } from "../shared/config";
-import type { InputManifest, RouteNode, StateNode, Target, WantedState } from "../shared/types";
+import type {
+ InputManifest,
+ RouteNode,
+ SourceInfo,
+ StateNode,
+ Target,
+ WantedState,
+} from "../shared/types";
import { matchesAny } from "../shared/glob";
import { isAuthored } from "./manifest";
+import { parseSourceLine, renderSourceLine } from "./source";
// The routes file (`visualice.routes.md`) is the human/LLM-facing site index: a
// two-level markdown list where only two tokens are load-bearing — the route path
@@ -32,6 +40,8 @@ export interface RoutesIntent {
routes: RoutesFileRoute[];
/** Top-level `skip:` globs — excluded from discovery and capture. */
skipRoutes: string[];
+ /** The commit the catalogue was last captured from (the `visualice-source` line). */
+ source?: SourceInfo;
warnings: string[];
}
@@ -64,6 +74,11 @@ export function parseRoutesFile(text: string): RoutesIntent {
let current: RoutesFileRoute | undefined;
for (const raw of text.split("\n")) {
+ const source = parseSourceLine(raw.trim());
+ if (source) {
+ intent.source ??= source;
+ continue;
+ }
const header = /^#\s+Routes\s+—\s+(.+?)\s*$/.exec(raw);
if (header) {
intent.target ??= header[1];
@@ -227,11 +242,16 @@ Generated by visualice — edit freely; regenerated after every crawl.
/**
* Render the routes file from the cached model + prior intent. Deterministic and
* idempotent: an unchanged model re-renders byte-identically, so diffs mean change.
+ *
+ * `source` is the commit a capture just shot; without one (a discovery, a dry run,
+ * `visualice routes`) the line the file already had is kept, since the catalogue is
+ * still the one shot then.
*/
export function renderRoutesFile(
model: InputManifest,
intent: RoutesIntent,
target: string,
+ source: SourceInfo | undefined = intent.source,
): string {
const skips = stateSkips(intent);
const intentByPath = new Map(intent.routes.map((r) => [r.path, r]));
@@ -241,7 +261,9 @@ export function renderRoutesFile(
for (const r of model.routes) if (!matchesAny(r.url, intent.skipRoutes)) paths.add(r.url);
for (const r of intent.routes) if (!matchesAny(r.path, intent.skipRoutes)) paths.add(r.path);
- const lines: string[] = [`# Routes — ${target}`, "", LEGEND, ""];
+ const lines: string[] = [`# Routes — ${target}`, ""];
+ if (source) lines.push(renderSourceLine(source), "");
+ lines.push(LEGEND, "");
for (const p of [...paths].sort(routeOrder)) {
const route: RouteNode | undefined = modelByUrl.get(p);
@@ -301,9 +323,10 @@ export async function writeRoutesFile(
intent: RoutesIntent,
target: string,
log: (msg: string) => void = () => {},
+ source?: SourceInfo,
): Promise {
const file = routesFilePath(config);
if (!file) return;
- await fs.writeFile(file, renderRoutesFile(model, intent, target));
+ await fs.writeFile(file, renderRoutesFile(model, intent, target, source ?? intent.source));
log(`routes file → ${file}`);
}
diff --git a/src/crawl/source.ts b/src/crawl/source.ts
new file mode 100644
index 0000000..8058b84
--- /dev/null
+++ b/src/crawl/source.ts
@@ -0,0 +1,192 @@
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { promises as fs, realpathSync } from "node:fs";
+import { execFile } from "node:child_process";
+import type { SourceInfo } from "../shared/types";
+
+// The code a catalogue was shot from: the git commit checked out in the project when
+// the crawl ran. Recorded in the routes file (committed) and each run's manifest, so
+// a later run can ask git what changed since and recapture only what that touches.
+
+/** Run git in `cwd`; resolves to stdout, or undefined when git fails or isn't there. */
+export function git(cwd: string, args: string[]): Promise {
+ return new Promise((resolve) => {
+ execFile(
+ "git",
+ args,
+ { cwd, timeout: 15_000, maxBuffer: 16 * 1024 * 1024, windowsHide: true },
+ (err, stdout) => resolve(err ? undefined : stdout),
+ );
+ });
+}
+
+/** The repository's top-level directory, if `cwd` is inside one. */
+export async function repoRoot(cwd: string): Promise {
+ return (await git(cwd, ["rev-parse", "--show-toplevel"]))?.trim() || undefined;
+}
+
+const LOOPBACK = /^(localhost|127(?:\.\d+){3}|\[::1\]|0\.0\.0\.0|[^.]+\.localhost|.+\.local)$/i;
+
+/**
+ * Is `target` served from this machine? Only then is HEAD known to be what was shot;
+ * a staging or production URL runs whatever was last deployed there.
+ */
+export function isLocalTarget(target: string): boolean {
+ try {
+ return LOOPBACK.test(new URL(target).hostname);
+ } catch {
+ return false;
+ }
+}
+
+/** Files listed by `git status --porcelain=v1 -z`, repo-relative. */
+export function parsePorcelain(out: string): string[] {
+ const files: string[] = [];
+ const parts = out.split("\0");
+ for (let i = 0; i < parts.length; i++) {
+ const entry = parts[i];
+ if (entry.length < 4) continue;
+ const code = entry.slice(0, 2);
+ files.push(entry.slice(3));
+ // A rename or copy is followed by its original path, as its own NUL-terminated field.
+ if (code[0] === "R" || code[0] === "C") i++;
+ }
+ return files;
+}
+
+/** Does `file` (repo-relative, `/`-separated) sit at or under one of `paths`? */
+export function isUnder(file: string, paths: string[]): boolean {
+ return paths.some((p) => p && (file === p || file.startsWith(p.endsWith("/") ? p : `${p}/`)));
+}
+
+/** Most dirty files a record keeps; past it, `dirty` still says how many there were. */
+const MAX_DIRTY = 200;
+
+/**
+ * Read the commit checked out in `cwd`. `ignore` lists repo-relative paths whose
+ * changes don't count as a dirty tree (the routes file the crawl itself rewrites, the
+ * output folder). Undefined outside a git repository, or without git.
+ */
+export async function readSource(
+ cwd: string,
+ opts: { target?: string; ignore?: string[]; configFile?: string; now?: Date } = {},
+): Promise {
+ const root = await repoRoot(cwd);
+ if (!root) return undefined;
+ const head = (await git(root, ["rev-parse", "HEAD"]))?.trim();
+ if (!head) return undefined; // a repository with no commits yet
+ const [branchOut, dateOut, statusOut] = await Promise.all([
+ git(root, ["symbolic-ref", "--quiet", "--short", "HEAD"]),
+ git(root, ["show", "-s", "--format=%cI", "HEAD"]),
+ git(root, ["status", "--porcelain=v1", "-z", "--untracked-files=all"]),
+ ]);
+ const ignore = (opts.ignore ?? []).map((p) => toRepoPath(root, p)).filter(Boolean) as string[];
+ const dirtyFiles = parsePorcelain(statusOut ?? "").filter((f) => !isUnder(f, ignore));
+ const config = opts.configFile ? await hashFile(opts.configFile) : undefined;
+ return {
+ commit: head,
+ branch: branchOut?.trim() || undefined,
+ committedAt: dateOut?.trim() || undefined,
+ dirty: dirtyFiles.length,
+ ...(dirtyFiles.length ? { dirtyFiles: dirtyFiles.slice(0, MAX_DIRTY) } : {}),
+ capturedAt: (opts.now ?? new Date()).toISOString(),
+ ...(opts.target ? { target: opts.target, local: isLocalTarget(opts.target) } : {}),
+ ...(config ? { config } : {}),
+ };
+}
+
+/** The first 12 hex digits of a file's SHA-256, or undefined when it can't be read. */
+export async function hashFile(file: string): Promise {
+ const data = await fs.readFile(file).catch(() => undefined);
+ return data ? createHash("sha256").update(data).digest("hex").slice(0, 12) : undefined;
+}
+
+/** Config files picked up from the working directory when `--config` isn't given, in order. */
+export const CONFIG_FILES = [
+ "visualice.config.ts",
+ "visualice.config.mjs",
+ "visualice.config.js",
+ "visualice.config.json",
+];
+
+/**
+ * The config file a command uses: `explicit` (`--config`) when given, else the first
+ * of CONFIG_FILES in `cwd`. Absolute; undefined when there is none.
+ */
+export async function findConfigFile(cwd: string, explicit?: string): Promise {
+ if (explicit) return path.resolve(cwd, explicit);
+ for (const name of CONFIG_FILES) {
+ const file = path.join(cwd, name);
+ if (await fs.stat(file).catch(() => undefined)) return file;
+ }
+ return undefined;
+}
+
+/** An absolute or cwd-relative path as a repo-relative, `/`-separated one (or undefined outside it). */
+export function toRepoPath(root: string, p: string): string | undefined {
+ // git reports the real path of the repository; `p` may come through a symlink
+ // (macOS's /var → /private/var), so compare real paths.
+ const rel = path.relative(realPath(root), realPath(path.resolve(p)));
+ if (!rel || rel.startsWith("..") || path.isAbsolute(rel)) return undefined;
+ return rel.split(path.sep).join("/");
+}
+
+/*
+ * The routes file carries the record as one comment line under its title:
+ *
+ *
+ *
+ * One line of key=value pairs: readable in a diff, and parsed back by `changes`.
+ */
+
+const SOURCE_LINE = /^\s*$/;
+
+export function renderSourceLine(s: SourceInfo): string {
+ const pairs: [string, string | number | undefined][] = [
+ ["commit", s.commit],
+ ["branch", s.branch],
+ ["dirty", s.dirty],
+ ["captured", s.capturedAt],
+ ["target", s.target],
+ ["config", s.config],
+ ];
+ const body = pairs
+ .filter(([, v]) => v !== undefined && v !== "")
+ .map(([k, v]) => `${k}=${String(v).replace(/\s+/g, "%20")}`)
+ .join(" ");
+ return ``;
+}
+
+/** The record in a routes-file line, or undefined when the line isn't one. */
+export function parseSourceLine(line: string): SourceInfo | undefined {
+ const m = SOURCE_LINE.exec(line);
+ if (!m) return undefined;
+ const kv = new Map();
+ for (const token of m[1].split(/\s+/)) {
+ const eq = token.indexOf("=");
+ if (eq > 0) kv.set(token.slice(0, eq), token.slice(eq + 1).replace(/%20/g, " "));
+ }
+ const commit = kv.get("commit");
+ if (!commit || !/^[0-9a-f]{7,64}$/i.test(commit)) return undefined;
+ const dirty = Number(kv.get("dirty") ?? 0);
+ const target = kv.get("target");
+ const config = kv.get("config");
+ return {
+ commit,
+ branch: kv.get("branch") || undefined,
+ dirty: Number.isFinite(dirty) && dirty > 0 ? Math.floor(dirty) : 0,
+ capturedAt: kv.get("captured") ?? "",
+ ...(target ? { target, local: isLocalTarget(target) } : {}),
+ ...(config ? { config } : {}),
+ };
+}
+
+/** The real path of `p`, resolving through its nearest existing parent when it doesn't exist. */
+function realPath(p: string): string {
+ try {
+ return realpathSync(p);
+ } catch {
+ const parent = path.dirname(p);
+ return parent === p ? p : path.join(realPath(parent), path.basename(p));
+ }
+}
diff --git a/src/shared/manifest-schema.ts b/src/shared/manifest-schema.ts
index 3912136..902e9e7 100644
--- a/src/shared/manifest-schema.ts
+++ b/src/shared/manifest-schema.ts
@@ -93,6 +93,14 @@ export const ShotRefSchema = z.object({
.optional(),
a11y: z.number().optional(),
error: z.string().optional(),
+ carried: z
+ .object({
+ runId: z.string(),
+ commit: z.string().optional(),
+ capturedAt: z.string().optional(),
+ })
+ .optional()
+ .describe("Copied from an earlier run by a partial crawl (`--only`), not shot by this one."),
});
export const StateNodeSchema = z.object({
diff --git a/src/shared/types.ts b/src/shared/types.ts
index 845c6ce..10ac3e6 100644
--- a/src/shared/types.ts
+++ b/src/shared/types.ts
@@ -98,6 +98,43 @@ export interface ShotRef {
/** Accessibility rules this state broke, when `checks.accessibility` is on. */
a11y?: number;
error?: string; // capture was attempted but failed (after retries); no image written
+ /**
+ * Not shot by this run: copied from an earlier one by a partial crawl (`--only`),
+ * because nothing that changed touched this page. Says where it really comes from.
+ */
+ carried?: CarriedFrom;
+}
+
+/** The run (and commit) a carried-over shot was really taken in. */
+export interface CarriedFrom {
+ runId: string;
+ /** The commit that run was shot from, when it recorded one. */
+ commit?: string;
+ capturedAt?: string;
+}
+
+/**
+ * The code a run was shot from: the git commit checked out in the project. Written to
+ * the routes file and each run's manifest; `visualice changes` diffs against it.
+ */
+export interface SourceInfo {
+ /** Full commit hash of HEAD. */
+ commit: string;
+ branch?: string;
+ /** The commit's committer date (ISO 8601). */
+ committedAt?: string;
+ /** Files with uncommitted changes at capture time (0 = a clean tree). */
+ dirty: number;
+ /** Which files those were (capped). */
+ dirtyFiles?: string[];
+ /** When the run captured (ISO 8601). */
+ capturedAt: string;
+ /** The URL that was shot. */
+ target?: string;
+ /** A hash of the visualice config file, so a settings change since is noticed. */
+ config?: string;
+ /** The target was served from this machine, so HEAD is known to be what was shot. */
+ local?: boolean;
}
/** One route×state×device capture that failed after all retries. */
@@ -186,6 +223,13 @@ export interface SiteInfo {
export interface Manifest {
version: number;
site: SiteInfo;
+ /** The commit the run was shot from, when the project is a git repository. */
+ source?: SourceInfo;
+ /**
+ * A partial run (`crawl --only`): the routes it shot, and the run every other
+ * route's shots were carried over from.
+ */
+ partial?: { only: string[]; carriedFrom?: string };
configSnapshot?: unknown;
devices: Device[];
auth?: { method: string; notes?: string };
@@ -195,6 +239,8 @@ export interface Manifest {
states: number;
shots: number;
failed: number;
+ /** Shots copied from an earlier run instead of shot (partial runs). */
+ carried?: number;
durationMs: number;
failures: CaptureFailure[];
/** Routes-file requests still unmet after this run (the Explore-mode queue). */
@@ -257,10 +303,18 @@ export interface ViewerTile {
a11y?: number;
/** Capture failed after retries; the tile is a placeholder carrying the reason. */
error?: string;
+ /** Copied from an earlier run, not shot by this one; see `ShotRef.carried`. */
+ carried?: CarriedFrom;
+ /** Shot by a partial run, where the other pages' shots were carried over. */
+ updated?: boolean;
}
export interface ViewerData {
site: SiteInfo;
+ /** The commit the run was shot from. */
+ source?: SourceInfo;
+ /** Set on a partial run; see `Manifest.partial`. */
+ partial?: Manifest["partial"];
devices: Device[];
tiles: ViewerTile[];
groups: { path: string; routeId: string; title?: string }[];
diff --git a/tests/carry.test.ts b/tests/carry.test.ts
new file mode 100644
index 0000000..1663854
--- /dev/null
+++ b/tests/carry.test.ts
@@ -0,0 +1,243 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import {
+ canCarry,
+ carryRoute,
+ componentsSought,
+ findCarrySource,
+ matchesOnly,
+ planCarry,
+ verifyCarry,
+} from "../src/crawl/carry";
+import { markRunIncomplete, runDir } from "../src/crawl/runs";
+import type { Device, Manifest, RouteNode, StateNode } from "../src/shared/types";
+
+const devices: Device[] = [
+ { name: "desktop", width: 1440, height: 900, dpr: 1 },
+ { name: "mobile", width: 390, height: 844, dpr: 2, mobile: true },
+];
+
+const shot = (route: string, state: string, device: string) => ({
+ full: `screenshots/${route}/${state}/${device}.png`,
+ thumb: `screenshots/${route}/${state}/${device}.thumb.webp`,
+});
+
+const state = (id: string, extra: Partial = {}): StateNode => ({
+ id,
+ kind: id === "default" ? "default" : "menu",
+ label: id,
+ recipe: [],
+ shots: {},
+ ...extra,
+});
+
+/** A route as an earlier run shot it: every state on every device it applies to. */
+const shotRoute = (id: string, url: string, states: StateNode[]): RouteNode => ({
+ id,
+ url,
+ depth: 0,
+ states: states.map((s) => ({
+ ...s,
+ shots: Object.fromEntries(
+ devices
+ .filter((d) => !s.devices || s.devices.includes(d.name))
+ .map((d) => [d.name, shot(id, s.id, d.name)]),
+ ),
+ })),
+});
+
+const route = (id: string, url: string, states: StateNode[]): RouteNode => ({
+ id,
+ url,
+ depth: 0,
+ states: states.map((s) => ({ ...s, shots: {} })),
+});
+
+const manifest = (routes: RouteNode[]): Manifest => ({
+ version: 1,
+ site: {
+ baseUrl: "http://x/",
+ origin: "http://x",
+ runId: "r1",
+ generatedAt: "2026-11-09T00:00:00Z",
+ },
+ source: { commit: "a".repeat(40), dirty: 0, capturedAt: "2026-11-09T00:00:00Z" },
+ devices,
+ routes,
+});
+
+describe("matching --only", () => {
+ it("takes paths and globs", () => {
+ expect(matchesOnly({ url: "/about" }, ["/about"])).toBe(true);
+ expect(matchesOnly({ url: "/blog/hi" }, ["/blog/**"])).toBe(true);
+ expect(matchesOnly({ url: "/blog" }, ["/blog/**"])).toBe(true);
+ expect(matchesOnly({ url: "/pricing" }, ["/about"])).toBe(false);
+ });
+});
+
+describe("canCarry", () => {
+ const prev = shotRoute("home", "/", [state("default"), state("menu", { devices: ["mobile"] })]);
+
+ it("carries a route the earlier run shot completely", () => {
+ expect(
+ canCarry(
+ route("home", "/", [state("default"), state("menu", { devices: ["mobile"] })]),
+ prev,
+ devices,
+ ),
+ ).toBe(true);
+ });
+
+ it("refuses a new route, a new state, a new device and a failed shot", () => {
+ expect(canCarry(route("x", "/x", [state("default")]), undefined, devices)).toBe(false);
+ expect(canCarry(route("home", "/", [state("default"), state("modal")]), prev, devices)).toBe(
+ false,
+ );
+ // A state that used to be mobile-only now applies everywhere.
+ expect(canCarry(route("home", "/", [state("menu")]), prev, devices)).toBe(false);
+ const failed = shotRoute("home", "/", [state("default")]);
+ failed.states[0].shots.mobile = { error: "timeout" };
+ expect(canCarry(route("home", "/", [state("default")]), failed, devices)).toBe(false);
+ });
+
+ it("doesn't count a component the earlier run sought and didn't find as a gap", () => {
+ const comp = state("component-card", { element: ".card", kind: "component" });
+ const sought = new Map([["component-card", ".card"]]);
+ expect(canCarry(route("home", "/", [state("default"), comp]), prev, devices, sought)).toBe(
+ true,
+ );
+ });
+
+ it("shoots a component the earlier run never looked for, or looked for elsewhere", () => {
+ const comp = state("component-card", { element: ".card", kind: "component" });
+ const r = route("home", "/", [state("default"), comp]);
+ expect(canCarry(r, prev, devices)).toBe(false);
+ expect(canCarry(r, prev, devices, new Map([["component-card", ".old-card"]]))).toBe(false);
+ });
+
+ it("reads what the earlier run sought from its config snapshot", () => {
+ const m = {
+ ...manifest([]),
+ configSnapshot: {
+ components: [
+ { name: "Card", selector: ".card", routes: ["/**"] },
+ { name: "Cart", selector: ".cart", routes: ["/shop/**"] },
+ ],
+ },
+ };
+ expect([...componentsSought(m, "/about")]).toEqual([["component-card", ".card"]]);
+ expect(componentsSought(m, "/shop/x").get("component-cart")).toBe(".cart");
+ });
+});
+
+describe("planCarry", () => {
+ it("shoots --only pages and anything the earlier run can't cover", () => {
+ const prev = manifest([
+ shotRoute("home", "/", [state("default")]),
+ shotRoute("about", "/about", [state("default")]),
+ shotRoute("blog", "/blog", [state("default")]),
+ ]);
+ const plan = planCarry(
+ [
+ route("home", "/", [state("default")]),
+ route("about", "/about", [state("default")]),
+ route("blog", "/blog", [state("default"), state("menu")]),
+ route("pricing", "/pricing", [state("default")]),
+ ],
+ ["/about"],
+ prev,
+ devices,
+ );
+ expect(plan.capture.map((r) => r.url)).toEqual(["/about", "/blog", "/pricing"]);
+ expect(plan.carry.map((r) => r.url)).toEqual(["/"]);
+ expect(plan.forced).toEqual([
+ { url: "/blog", reason: "incomplete" },
+ { url: "/pricing", reason: "new" },
+ ]);
+ });
+});
+
+describe("carrying files", () => {
+ let site: string;
+ beforeEach(async () => {
+ site = await fs.mkdtemp(path.join(os.tmpdir(), "vc-carry-"));
+ });
+ afterEach(() => fs.rm(site, { recursive: true, force: true }));
+
+ it("finds the newest finished run and copies its shots with their provenance", async () => {
+ const earlier = runDir(site, "2026-11-09T00-00-00-000Z");
+ const prev = manifest([shotRoute("home", "/", [state("default")])]);
+ await fs.mkdir(earlier, { recursive: true });
+ await fs.writeFile(path.join(earlier, "crawl-map.json"), JSON.stringify(prev));
+ for (const s of Object.values(prev.routes[0].states[0].shots)) {
+ for (const f of [s.full!, s.thumb!]) {
+ await fs.mkdir(path.dirname(path.join(earlier, f)), { recursive: true });
+ await fs.writeFile(path.join(earlier, f), f);
+ }
+ }
+ // An unfinished run after it is skipped.
+ const broken = runDir(site, "2026-11-09T01-00-00-000Z");
+ await fs.mkdir(broken, { recursive: true });
+ await markRunIncomplete(broken);
+
+ const source = await findCarrySource(site, "2026-11-09T02-00-00-000Z");
+ expect(source?.runId).toBe("2026-11-09T00-00-00-000Z");
+
+ const now = runDir(site, "2026-11-09T02-00-00-000Z");
+ const r = route("home", "/", [state("default")]);
+ expect(await carryRoute(r, source!, now, devices)).toBe(2);
+ const desk = r.states[0].shots.desktop;
+ expect(desk.carried).toEqual({
+ runId: "2026-11-09T00-00-00-000Z",
+ commit: "a".repeat(40),
+ capturedAt: "2026-11-09T00:00:00Z",
+ });
+ expect(await fs.readFile(path.join(now, desk.full!), "utf8")).toBe(desk.full);
+ });
+
+ it("shoots a route instead of carrying it when the earlier run lost a file", async () => {
+ const earlier = runDir(site, "2026-11-09T00-00-00-000Z");
+ const prev = manifest([
+ shotRoute("home", "/", [state("default")]),
+ shotRoute("about", "/about", [state("default")]),
+ ]);
+ await fs.mkdir(earlier, { recursive: true });
+ await fs.writeFile(path.join(earlier, "crawl-map.json"), JSON.stringify(prev));
+ for (const r of prev.routes) {
+ for (const s of Object.values(r.states[0].shots)) {
+ for (const f of [s.full!, s.thumb!]) {
+ if (r.url === "/about" && f.endsWith("mobile.thumb.webp")) continue; // lost
+ await fs.mkdir(path.dirname(path.join(earlier, f)), { recursive: true });
+ await fs.writeFile(path.join(earlier, f), f);
+ }
+ }
+ }
+ const source = (await findCarrySource(site))!;
+ const routes = [
+ route("home", "/", [state("default")]),
+ route("about", "/about", [state("default")]),
+ ];
+ const plan = await verifyCarry(planCarry(routes, ["/nothing"], prev, devices), source, devices);
+ expect(plan.carry.map((r) => r.url)).toEqual(["/"]);
+ expect(plan.capture.map((r) => r.url)).toEqual(["/about"]);
+ expect(plan.forced).toEqual([{ url: "/about", reason: "incomplete" }]);
+
+ // And a file gone after the check fails that shot, not the run.
+ const about = route("about", "/about", [state("default")]);
+ const now = runDir(site, "2026-11-09T02-00-00-000Z");
+ expect(await carryRoute(about, source, now, devices)).toBe(1);
+ expect(about.states[0].shots.mobile.error).toMatch(/couldn't carry/);
+ });
+});
+
+describe("--only seeds", () => {
+ it("are the plain paths, a full URL reduced to its path", async () => {
+ const { onlyPages } = await import("../src/crawl/orchestrator");
+ expect(onlyPages(["/about", "/blog/**", "https://site.test/pricing?x=1", "/about"])).toEqual([
+ "/about",
+ "/pricing?x=1",
+ ]);
+ });
+});
diff --git a/tests/changes.test.ts b/tests/changes.test.ts
new file mode 100644
index 0000000..b7d47cc
--- /dev/null
+++ b/tests/changes.test.ts
@@ -0,0 +1,216 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { execFileSync } from "node:child_process";
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import {
+ computeChanges,
+ globalReason,
+ guessPattern,
+ guessRoute,
+ parseNameStatus,
+ patternMatches,
+ shellQuote,
+} from "../src/crawl/changes";
+import { readSource } from "../src/crawl/source";
+
+const pattern = (file: string) => {
+ const g = guessRoute(file);
+ return g ? guessPattern(g) : undefined;
+};
+
+describe("file-based routing", () => {
+ it("maps Next.js app router files", () => {
+ expect(pattern("app/page.tsx")).toBe("/");
+ expect(pattern("src/app/blog/[slug]/page.tsx")).toBe("/blog/*");
+ expect(pattern("app/(marketing)/pricing/page.tsx")).toBe("/pricing");
+ expect(pattern("app/docs/layout.tsx")).toBe("/docs/**");
+ expect(pattern("app/shop/[...all]/page.tsx")).toBe("/shop/**");
+ expect(pattern("app/api/users/route.ts")).toBeUndefined();
+ });
+
+ it("maps pages folders, SvelteKit, Remix and plain HTML", () => {
+ expect(pattern("pages/about.tsx")).toBe("/about");
+ expect(pattern("src/pages/blog/index.astro")).toBe("/blog");
+ expect(pattern("pages/_app.tsx")).toBeUndefined();
+ expect(pattern("src/routes/blog/[slug]/+page.svelte")).toBe("/blog/*");
+ expect(pattern("src/routes/admin/+layout.svelte")).toBe("/admin/**");
+ expect(pattern("app/routes/blog.$slug.tsx")).toBe("/blog/*");
+ expect(pattern("app/routes/_index.tsx")).toBe("/");
+ expect(pattern("about.html")).toBe("/about");
+ expect(pattern("public/docs/index.html")).toBe("/docs");
+ });
+
+ it("matches patterns against catalogue pages", () => {
+ expect(patternMatches("/blog/*", "/blog/hello")).toBe(true);
+ expect(patternMatches("/blog/*", "/blog")).toBe(false);
+ expect(patternMatches("/docs/**", "/docs")).toBe(true);
+ expect(patternMatches("/docs/**", "/docs/a/b")).toBe(true);
+ expect(patternMatches("/**", "/anything")).toBe(true);
+ expect(patternMatches("/about", "/about.html")).toBe(true);
+ expect(patternMatches("/", "/")).toBe(true);
+ });
+});
+
+describe("global files", () => {
+ it("names the changes that reach every page", () => {
+ expect(globalReason("package.json")).toBe("dependencies");
+ expect(globalReason("tailwind.config.ts")).toBe("build or theme config");
+ expect(globalReason("src/app/layout.tsx")).toBe("root layout or entry point");
+ expect(globalReason("src/styles/buttons.css")).toBe("global styles or theme");
+ expect(globalReason("app/globals.css")).toBe("global styles");
+ expect(globalReason("src/components/Header.tsx")).toMatch(/chrome/);
+ expect(globalReason("visualice.config.json")).toBe("visualice config");
+ });
+
+ it("leaves page files and local components alone", () => {
+ expect(globalReason("app/pricing/page.tsx")).toBeUndefined();
+ // A nested layout wraps its own section only; routing maps it below.
+ expect(globalReason("app/blog/layout.tsx")).toBeUndefined();
+ expect(globalReason("src/app/(shop)/cart/layout.tsx")).toBeUndefined();
+ expect(globalReason("src/components/layout.tsx")).toMatch(/chrome/);
+ expect(globalReason("src/components/PricingCard.tsx")).toBeUndefined();
+ expect(globalReason("src/components/card/index.css")).toBeUndefined();
+ expect(globalReason("packages/ui/package.json")).toBeUndefined();
+ });
+});
+
+it("quotes a suggested --only value only when a shell would expand it", () => {
+ expect(shellQuote("/about,/pricing")).toBe("/about,/pricing");
+ expect(shellQuote("/blog/*,/about")).toBe("'/blog/*,/about'");
+ expect(shellQuote("/it's")).toBe("'/it'\\''s'");
+});
+
+it("parses name-status output with renames", () => {
+ expect(parseNameStatus("M\0a.ts\0R100\0old.ts\0new.ts\0A\0b.ts\0")).toEqual([
+ { status: "M", path: "a.ts" },
+ { status: "R", from: "old.ts", path: "new.ts" },
+ { status: "A", path: "b.ts" },
+ ]);
+});
+
+describe("computeChanges", () => {
+ let dir: string;
+ const git = (...args: string[]) =>
+ execFileSync("git", ["-c", "user.email=t@t", "-c", "user.name=t", ...args], {
+ cwd: dir,
+ }).toString();
+ const write = async (file: string, text: string) => {
+ await fs.mkdir(path.dirname(path.join(dir, file)), { recursive: true });
+ await fs.writeFile(path.join(dir, file), text);
+ };
+ const commit = (msg: string) => {
+ git("add", "-A");
+ git("commit", "-qm", msg);
+ };
+ const pages = ["/", "/about", "/blog", "/blog/hello", "/pricing"];
+ const run = async (recorded: typeof baseline | null = baseline) =>
+ computeChanges({
+ cwd: dir,
+ recorded: recorded ?? undefined,
+ ignore: [path.join(dir, ".visualice"), path.join(dir, "visualice.routes.md")],
+ pages,
+ });
+ let baseline: Awaited>;
+
+ beforeEach(async () => {
+ dir = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "vc-changes-")));
+ git("init", "-q", "-b", "main");
+ await write("app/page.tsx", "home");
+ await write("app/about/page.tsx", "about");
+ await write("app/blog/[slug]/page.tsx", "post");
+ await write("src/components/Card.tsx", "card");
+ commit("init");
+ baseline = await readSource(dir, { target: "http://localhost:3000/" });
+ });
+ afterEach(() => fs.rm(dir, { recursive: true, force: true }));
+
+ it("is none when nothing that counts changed", async () => {
+ await write("visualice.routes.md", "# Routes");
+ commit("catalogue");
+ const r = await run();
+ expect(r.verdict).toBe("none");
+ expect(r.commitCount).toBe(1);
+ expect(r.files).toEqual([]);
+ });
+
+ it("is partial with page candidates, and suggests --only", async () => {
+ await write("app/about/page.tsx", "about v2");
+ await write("app/pricing/page.tsx", "new page");
+ commit("about + pricing");
+ const r = await run();
+ expect(r.verdict).toBe("partial");
+ expect(r.candidates.map((c) => [c.file, c.pages])).toEqual([
+ ["app/about/page.tsx", ["/about"]],
+ ["app/pricing/page.tsx", ["/pricing"]],
+ ]);
+ expect(r.suggest?.only).toEqual(["/about", "/pricing"]);
+ expect(r.commits[0].subject).toBe("about + pricing");
+ });
+
+ it("counts uncommitted and untracked files, and leaves the trace to the agent", async () => {
+ await write("src/components/Card.tsx", "card v2");
+ await write("src/components/New.tsx", "new");
+ const r = await run();
+ expect(r.verdict).toBe("partial");
+ expect(r.unmapped.sort()).toEqual(["src/components/Card.tsx", "src/components/New.tsx"]);
+ expect(r.suggest).toBeUndefined();
+ });
+
+ it("maps a nested layout to its section, and quotes a glob it suggests", async () => {
+ await write("app/blog/layout.tsx", "blog shell");
+ commit("blog layout");
+ const r = await run();
+ expect(r.verdict).toBe("partial");
+ expect(r.candidates[0]).toMatchObject({ pattern: "/blog/**", kind: "layout" });
+ expect(r.suggest?.only).toEqual(["/blog", "/blog/hello"]);
+
+ await write("app/shop/[id]/page.tsx", "product");
+ commit("shop");
+ const next = await run();
+ expect(next.suggest?.command).toBe(
+ "visualice crawl --replay --only='/blog,/blog/hello,/shop/*'",
+ );
+ });
+
+ it("counts a rename as one uncommitted change", async () => {
+ git("mv", "src/components/Card.tsx", "src/components/Tile.tsx");
+ expect((await run()).head?.dirty).toBe(1);
+ });
+
+ it("is full when the record's list of dirty files was cut short", async () => {
+ const r = await computeChanges({
+ cwd: dir,
+ recorded: { ...baseline!, dirty: 250 },
+ runSource: { ...baseline!, dirty: 250, dirtyFiles: ["app/page.tsx"] },
+ ignore: [],
+ pages,
+ });
+ expect(r.verdict).toBe("full");
+ expect(r.reasons.some((x) => x.includes("can't all be listed"))).toBe(true);
+ });
+
+ it("is full when a global file changed", async () => {
+ await write("package.json", "{}");
+ await write("app/about/page.tsx", "about v2");
+ commit("deps");
+ const r = await run();
+ expect(r.verdict).toBe("full");
+ expect(r.global).toEqual([{ path: "package.json", reason: "dependencies" }]);
+ expect(r.suggest?.command).toBe("visualice crawl");
+ });
+
+ it("is unknown without a record or with a commit that isn't there", async () => {
+ expect((await run(null)).verdict).toBe("unknown");
+ const gone = await run({ ...baseline!, commit: "f".repeat(40) });
+ expect(gone.verdict).toBe("unknown");
+ expect(gone.reasons[0]).toMatch(/isn't in this repository/);
+ });
+
+ it("warns when the catalogue was shot from a deployed site", async () => {
+ await write("app/about/page.tsx", "about v2");
+ commit("about");
+ const r = await run({ ...baseline!, target: "https://example.com/", local: false });
+ expect(r.reasons.some((x) => x.includes("not a local server"))).toBe(true);
+ });
+});
diff --git a/tests/integration/partial.test.ts b/tests/integration/partial.test.ts
new file mode 100644
index 0000000..41a4140
--- /dev/null
+++ b/tests/integration/partial.test.ts
@@ -0,0 +1,79 @@
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { serveDirectory } from "../../src/shared/static-server";
+import type { StaticServer } from "../../src/shared/file-serve";
+import { ConfigSchema, type Config } from "../../src/shared/config";
+import { crawlPipeline, planCrawl } from "../../src/crawl/orchestrator";
+import type { ViewerData } from "../../src/shared/types";
+import { FIXTURE_DIR, silent, tempDir } from "./helpers";
+
+/**
+ * `crawl --only` end-to-end against the fixture: a full crawl, then a partial one that
+ * shoots one page and carries the rest, so the new run is still the whole site.
+ */
+
+let server: StaticServer;
+let out: string;
+let config: Config;
+
+beforeAll(async () => {
+ server = await serveDirectory(FIXTURE_DIR);
+ out = await tempDir("partial");
+ const base = ConfigSchema.parse({
+ scope: { maxPages: 3, maxDepth: 1 },
+ devices: [{ name: "small", width: 420, height: 640, dpr: 1 }],
+ discovery: { interactions: { maxPerPage: 2 }, mobileStates: false },
+ stabilize: { extraDelayMs: 50 },
+ politeness: { minDelayMs: 0, concurrency: 4 },
+ });
+ config = { ...base, routesFile: false, output: { ...base.output, dir: out } };
+}, 30000);
+
+afterAll(async () => {
+ await server.close();
+ await fs.rm(out, { recursive: true, force: true });
+});
+
+describe("partial runs", () => {
+ it("shoot the named pages and carry the rest from the last run", async () => {
+ const full = await crawlPipeline({ config, baseUrl: server.url, log: silent });
+ const pages = full.manifest.routes.map((r) => r.url);
+ expect(pages.length).toBeGreaterThan(1);
+ const named = pages.find((p) => p !== "/")!;
+
+ const plan = await planCrawl({ config, baseUrl: server.url, replay: true, only: [named] });
+ expect(plan.partial?.capture).toEqual([named]);
+ expect(plan.partial?.carriedFrom).toBe(full.manifest.site.runId);
+
+ const part = await crawlPipeline({
+ config,
+ baseUrl: server.url,
+ replay: true,
+ only: [named],
+ log: silent,
+ });
+ const m = part.manifest;
+ expect(m.partial).toEqual({ only: [named], carriedFrom: full.manifest.site.runId });
+ expect(m.routes.map((r) => r.url)).toEqual(pages);
+ expect(m.stats?.carried).toBeGreaterThan(0);
+ expect(m.stats!.shots + m.stats!.carried!).toBe(full.manifest.stats!.shots);
+
+ for (const route of m.routes) {
+ for (const state of route.states) {
+ for (const ref of Object.values(state.shots)) {
+ if (route.url === named) expect(ref.carried).toBeUndefined();
+ else expect(ref.carried?.runId).toBe(full.manifest.site.runId);
+ // Every shot's file is in the new run, carried or not.
+ await expect(fs.stat(path.join(part.runDir, ref.full!))).resolves.toBeTruthy();
+ }
+ }
+ }
+
+ const data = JSON.parse(
+ await fs.readFile(path.join(part.runDir, "viewer-data.json"), "utf8"),
+ ) as ViewerData;
+ expect(data.tiles.filter((t) => t.updated).every((t) => t.routeUrl === named)).toBe(true);
+ expect(data.tiles.some((t) => t.carried)).toBe(true);
+ }, 120000);
+});
diff --git a/tests/source.test.ts b/tests/source.test.ts
new file mode 100644
index 0000000..70c1585
--- /dev/null
+++ b/tests/source.test.ts
@@ -0,0 +1,141 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { execFileSync } from "node:child_process";
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import {
+ findConfigFile,
+ isLocalTarget,
+ isUnder,
+ parsePorcelain,
+ parseSourceLine,
+ readSource,
+ renderSourceLine,
+} from "../src/crawl/source";
+import { parseRoutesFile, renderRoutesFile } from "../src/crawl/routes-file";
+import type { SourceInfo } from "../src/shared/types";
+
+const SHA = "3f2a9c1e5b7d4f6a8c0e2b4d6f8a0c2e4b6d8f0a";
+
+const record = (extra: Partial = {}): SourceInfo => ({
+ commit: SHA,
+ branch: "main",
+ dirty: 0,
+ capturedAt: "2026-11-09T10:00:00.000Z",
+ target: "http://localhost:3000/",
+ local: true,
+ ...extra,
+});
+
+describe("the routes-file source line", () => {
+ it("round-trips", () => {
+ const line = renderSourceLine(record({ config: "bab17dc92481" }));
+ expect(line).toBe(
+ ``,
+ );
+ expect(parseSourceLine(line)).toEqual(record({ config: "bab17dc92481" }));
+ });
+
+ it("keeps a branch name with spaces in one token", () => {
+ const back = parseSourceLine(renderSourceLine(record({ branch: "my branch" })));
+ expect(back?.branch).toBe("my branch");
+ });
+
+ it("ignores other comments and a line without a real commit", () => {
+ expect(parseSourceLine("")).toBeUndefined();
+ expect(parseSourceLine("")).toBeUndefined();
+ });
+
+ it("is read from the routes file and kept when re-rendered without a new one", () => {
+ const model = { routes: [{ id: "home", url: "/", depth: 0, states: [] }] };
+ const first = renderRoutesFile(model, parseRoutesFile(""), "http://localhost:3000/", record());
+ const intent = parseRoutesFile(first);
+ expect(intent.source?.commit).toBe(SHA);
+ expect(intent.routes.map((r) => r.path)).toEqual(["/"]);
+ // A discovery or `visualice routes` rewrites the file: the record stays.
+ expect(renderRoutesFile(model, intent, "http://localhost:3000/")).toBe(first);
+ // A capture replaces it.
+ const next = renderRoutesFile(
+ model,
+ intent,
+ "http://localhost:3000/",
+ record({ commit: "a".repeat(40) }),
+ );
+ expect(parseRoutesFile(next).source?.commit).toBe("a".repeat(40));
+ });
+});
+
+describe("helpers", () => {
+ it("knows a local target from a deployed one", () => {
+ expect(isLocalTarget("http://localhost:3000/")).toBe(true);
+ expect(isLocalTarget("http://127.0.0.1:8080")).toBe(true);
+ expect(isLocalTarget("http://shop.local/")).toBe(true);
+ expect(isLocalTarget("https://staging.example.com/")).toBe(false);
+ });
+
+ it("parses porcelain status, renames included", () => {
+ expect(parsePorcelain(" M a.ts\0?? b.ts\0R new.ts\0old.ts\0")).toEqual([
+ "a.ts",
+ "b.ts",
+ "new.ts",
+ ]);
+ });
+
+ it("finds the config file a command uses, --config first", async () => {
+ const dir = await fs.mkdtemp(path.join(os.tmpdir(), "vc-config-"));
+ try {
+ expect(await findConfigFile(dir)).toBeUndefined();
+ await fs.writeFile(path.join(dir, "visualice.config.json"), "{}");
+ expect(await findConfigFile(dir)).toBe(path.join(dir, "visualice.config.json"));
+ expect(await findConfigFile(dir, "ci.json")).toBe(path.join(dir, "ci.json"));
+ } finally {
+ await fs.rm(dir, { recursive: true, force: true });
+ }
+ });
+
+ it("matches a path at or under another", () => {
+ expect(isUnder(".visualice/x.json", [".visualice"])).toBe(true);
+ expect(isUnder("visualice.routes.md", ["visualice.routes.md"])).toBe(true);
+ expect(isUnder(".visualice-old/x", [".visualice"])).toBe(false);
+ });
+});
+
+describe("readSource", () => {
+ let dir: string;
+ const git = (...args: string[]) =>
+ execFileSync("git", ["-c", "user.email=t@t", "-c", "user.name=t", ...args], { cwd: dir });
+
+ beforeEach(async () => {
+ dir = await fs.mkdtemp(path.join(os.tmpdir(), "vc-source-"));
+ git("init", "-q", "-b", "main");
+ await fs.writeFile(path.join(dir, "index.html"), "hi ");
+ git("add", "-A");
+ git("commit", "-qm", "init");
+ });
+ afterEach(() => fs.rm(dir, { recursive: true, force: true }));
+
+ it("reads HEAD, the branch and a clean tree", async () => {
+ const s = await readSource(dir, { target: "http://localhost:1/" });
+ expect(s?.commit).toMatch(/^[0-9a-f]{40}$/);
+ expect(s?.branch).toBe("main");
+ expect(s?.dirty).toBe(0);
+ expect(s?.local).toBe(true);
+ });
+
+ it("lists dirty files, leaving the crawl's own out", async () => {
+ await fs.writeFile(path.join(dir, "index.html"), "changed ");
+ await fs.writeFile(path.join(dir, "visualice.routes.md"), "# Routes");
+ const s = await readSource(dir, { ignore: [path.join(dir, "visualice.routes.md")] });
+ expect(s?.dirty).toBe(1);
+ expect(s?.dirtyFiles).toEqual(["index.html"]);
+ });
+
+ it("is undefined outside a repository", async () => {
+ const plain = await fs.mkdtemp(path.join(os.tmpdir(), "vc-plain-"));
+ try {
+ expect(await readSource(plain)).toBeUndefined();
+ } finally {
+ await fs.rm(plain, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/viewer/src/App.tsx b/viewer/src/App.tsx
index d21b520..6af12f8 100644
--- a/viewer/src/App.tsx
+++ b/viewer/src/App.tsx
@@ -985,6 +985,15 @@ export default function App({ onHome }: { onHome?: () => void } = {}) {
closing={!selected}
onClosed={() => setDetailed(null)}
onFullScreen={openFull}
+ compare={
+ siteEntry && shownSite && shownRun && siteEntry.runs.length > 1
+ ? {
+ runs: siteEntry.runs.filter((r) => r !== shownRun),
+ baseFor: (r) => runBase(shownSite, r),
+ current: shownRun,
+ }
+ : undefined
+ }
position={
detailedIndex >= 0
? { index: detailedIndex + 1, total: layout.placed.length }
diff --git a/viewer/src/Canvas.tsx b/viewer/src/Canvas.tsx
index a4f4812..5c7de09 100644
--- a/viewer/src/Canvas.tsx
+++ b/viewer/src/Canvas.tsx
@@ -898,6 +898,7 @@ function TileCaption({ p, place }: { p: Placed; place: CaptionPlace }) {
{tile.stateLabel}
{tile.device}
+ {tile.updated && updated }
{place.screen && (
{`${place.screen.at} / ${place.screen.of}`}
@@ -917,6 +918,11 @@ function TileCaption({ p, place }: { p: Placed; place: CaptionPlace }) {
{tile.stateLabel}
{/* With room for one line only, warnings still show beside the name. */}
+ {place.compact && tile.updated ? (
+
+ updated
+
+ ) : null}
{place.compact && tile.a11y ? (
{`a11y ${tile.a11y}`}
@@ -926,6 +932,7 @@ function TileCaption({ p, place }: { p: Placed; place: CaptionPlace }) {
{!place.compact && (
{/* Short of room, the kind gives way first: the warnings matter more. */}
+ {tile.updated &&
updated }
{tile.stateKind}
{tile.device}
{p.page ? (
@@ -970,10 +977,13 @@ function DeviceCaption({ p, t, device }: { p: Placed; t: Transform; device?: Dev
function TileBadge({
children,
warn,
+ accent,
yields,
}: {
children: string;
warn?: boolean;
+ /** Shot by this (partial) run, where the rest was carried over. */
+ accent?: boolean;
/** The one that gives up its room when there isn't enough; the others keep theirs. */
yields?: boolean;
}) {
@@ -983,7 +993,11 @@ function TileBadge({
className={cx(
"truncate rounded-[4px] border bg-chip px-1 font-mono text-[10px] leading-[15px] whitespace-nowrap",
yields ? "min-w-[2.5ch]" : "shrink-0",
- warn ? "border-amber-line text-amber" : "border-line-strong text-dim",
+ warn
+ ? "border-amber-line text-amber"
+ : accent
+ ? "border-accent-line bg-accent-soft font-semibold text-accent"
+ : "border-line-strong text-dim",
)}
>
{children}
diff --git a/viewer/src/Compare.tsx b/viewer/src/Compare.tsx
new file mode 100644
index 0000000..719e867
--- /dev/null
+++ b/viewer/src/Compare.tsx
@@ -0,0 +1,149 @@
+import { useEffect, useState } from "react";
+import { fetchViewerData } from "./load";
+import { runLabel } from "./format";
+import type { ViewerData, ViewerTile } from "./types";
+import { Caption, cx } from "./ui";
+
+/** Other versions of the site this shot can be compared with. */
+export interface CompareRuns {
+ /** Run ids, newest first, without the one on screen. */
+ runs: string[];
+ /** Where a run's files are served. */
+ baseFor: (run: string) => string;
+ /** The run on screen. */
+ current: string;
+}
+
+/** Each run's data, fetched once: stepping through shots compares against the same run. */
+const cache = new Map
>();
+function loadRun(url: string): Promise {
+ let hit = cache.get(url);
+ if (!hit) {
+ hit = fetchViewerData(url);
+ hit.catch(() => cache.delete(url));
+ cache.set(url, hit);
+ }
+ return hit;
+}
+
+/** The run a shot was really taken in: its own run, or the one it was carried from. */
+const takenIn = (tile: ViewerTile, run: string): string => tile.carried?.runId ?? run;
+
+/**
+ * The same shot in another version of the site, side by side with this one. Shots
+ * match by id (route, state and device), which stays put across runs. Two shots
+ * taken in the same run are the same image: a partial run carried it over.
+ */
+export function Compare({
+ tile,
+ base,
+ compare,
+}: {
+ tile: ViewerTile;
+ base: string;
+ compare: CompareRuns;
+}) {
+ // The version before this one by default: what "what changed?" usually means.
+ const older = compare.runs.find((r) => r < compare.current);
+ const [other, setOther] = useState(older ?? compare.runs[0]);
+ const [open, setOpen] = useState(false);
+ const [state, setState] = useState<
+ { run: string; tile?: ViewerTile; error?: string } | undefined
+ >();
+
+ // A string, not `compare`: the parent builds that object afresh on every render.
+ const otherBase = other ? compare.baseFor(other) : "";
+ useEffect(() => {
+ if (!open || !other) return;
+ let live = true;
+ setState(undefined);
+ loadRun(otherBase + "viewer-data.json")
+ .then(
+ (data) => live && setState({ run: other, tile: data.tiles.find((t) => t.id === tile.id) }),
+ )
+ .catch((e: Error) => live && setState({ run: other, error: e.message }));
+ return () => {
+ live = false;
+ };
+ }, [open, other, otherBase, tile.id]);
+
+ const theirs = state?.tile;
+ const same = theirs && takenIn(theirs, other) === takenIn(tile, compare.current);
+
+ return (
+ setOpen((e.currentTarget as HTMLDetailsElement).open)}
+ >
+
+ Compare with another version
+
+ ▾
+
+
+
+
+ Version
+ setOther(e.target.value)}
+ >
+ {compare.runs.map((r) => (
+
+ {runLabel(r)}
+ {r < compare.current ? "" : " (newer)"}
+
+ ))}
+
+
+ {!state ? (
+
Loading…
+ ) : state.error ? (
+
+ That version couldn’t be read: {state.error}
+
+ ) : (
+ <>
+
+
+
+
+
+ {!theirs
+ ? "This shot isn’t in that version: the page or state is new since."
+ : same
+ ? "The same shot: it was carried over unchanged, not taken again."
+ : "Taken separately in each version."}
+
+ >
+ )}
+
+
+ );
+}
+
+function Side({ label, tile, base }: { label: string; tile?: ViewerTile; base: string }) {
+ const src = tile?.mid ?? tile?.thumb ?? tile?.full;
+ return (
+
+ {label}
+ {tile && src && !tile.error ? (
+
+
+
+ ) : (
+
+ {tile?.error ? "Capture failed" : "Not captured"}
+
+ )}
+
+ );
+}
diff --git a/viewer/src/Inspector.tsx b/viewer/src/Inspector.tsx
index 33b6ed2..64887cb 100644
--- a/viewer/src/Inspector.tsx
+++ b/viewer/src/Inspector.tsx
@@ -4,6 +4,7 @@ import type { Device, ViewerData, ViewerTile } from "./types";
import { runLabel } from "./format";
import { routePath } from "./layout";
import { ShotActions } from "./ShotActions";
+import { Compare, type CompareRuns } from "./Compare";
import { Caption, cx, deviceIcon, Icon, KindTag } from "./ui";
interface Props {
@@ -23,8 +24,12 @@ interface Props {
position?: { index: number; total: number };
/** Open this shot full screen. */
onFullScreen?: () => void;
+ /** Other versions of the site, to compare this shot with. */
+ compare?: CompareRuns;
}
+const short = (commit: string) => commit.slice(0, 7);
+
/** The label/value grid the details (and the shortcuts popover) use. */
export const DL = "grid gap-x-3 gap-y-2 text-[12.5px] [&>dd]:wrap-anywhere [&>dt]:text-dim";
@@ -52,6 +57,7 @@ export function Inspector({
onClosed,
position,
onFullScreen,
+ compare,
}: Props) {
const device: Device | undefined = data.devices.find((d) => d.name === tile.device);
@@ -219,6 +225,34 @@ export function Inspector({
)}
+ {tile.carried ? (
+
+ Carried over
+
+ Not shot in this version: nothing that changed touched this page, so the shot from{" "}
+ {runLabel(tile.carried.runId)} UTC
+ {tile.carried.commit && (
+ <>
+ {" "}
+ (commit {short(tile.carried.commit)})
+ >
+ )}{" "}
+ was kept.
+
+
+ ) : tile.updated ? (
+
+ Updated in this version
+
+ Shot again because this page changed
+ {data.partial?.carriedFrom ? (
+ <> since {runLabel(data.partial.carriedFrom)} UTC>
+ ) : null}
+ ; pages nothing touched were carried over.
+
+
+ ) : null}
+
{tile.a11y ? (
@@ -262,6 +296,9 @@ export function Inspector({
)}
) : null}
+ {compare && compare.runs.length > 0 && (
+
+ )}
{/* Pinned to the foot, where the hand ends up after reading. */}
@@ -274,7 +311,17 @@ export function Inspector({
/>
-
Captured {runLabel(data.site.runId)} UTC
+
+ Captured {runLabel(tile.carried?.runId ?? data.site.runId)} UTC
+ {(tile.carried ? tile.carried.commit : data.source?.commit) && (
+ <>
+ {" · "}
+
+ {short((tile.carried ? tile.carried.commit : data.source?.commit)!)}
+
+ >
+ )}
+
{position && (
Shot {position.index} of {position.total}
diff --git a/viewer/src/load.ts b/viewer/src/load.ts
index 4b486b0..cbce670 100644
--- a/viewer/src/load.ts
+++ b/viewer/src/load.ts
@@ -65,6 +65,39 @@ function parseTile(raw: unknown, i: number): ViewerTile {
meta: isString(raw.meta) ? raw.meta : undefined,
a11y: typeof raw.a11y === "number" ? raw.a11y : undefined,
error: isString(raw.error) ? raw.error : undefined,
+ carried: parseCarried(raw.carried),
+ updated: raw.updated === true || undefined,
+ };
+}
+
+/** Where a carried-over shot was really taken; dropped unless it names a run. */
+function parseCarried(raw: unknown): ViewerTile["carried"] {
+ if (!isObject(raw) || !isString(raw.runId)) return undefined;
+ return {
+ runId: raw.runId,
+ commit: isString(raw.commit) ? raw.commit : undefined,
+ capturedAt: isString(raw.capturedAt) ? raw.capturedAt : undefined,
+ };
+}
+
+/** The commit a run was shot from, when it recorded one. */
+function parseSource(raw: unknown): ViewerData["source"] {
+ if (!isObject(raw) || !isString(raw.commit)) return undefined;
+ return {
+ commit: raw.commit,
+ branch: isString(raw.branch) ? raw.branch : undefined,
+ dirty: typeof raw.dirty === "number" ? raw.dirty : 0,
+ capturedAt: isString(raw.capturedAt) ? raw.capturedAt : "",
+ target: isString(raw.target) ? raw.target : undefined,
+ local: typeof raw.local === "boolean" ? raw.local : undefined,
+ };
+}
+
+function parsePartial(raw: unknown): ViewerData["partial"] {
+ if (!isObject(raw) || !Array.isArray(raw.only)) return undefined;
+ return {
+ only: raw.only.filter(isString),
+ carriedFrom: isString(raw.carriedFrom) ? raw.carriedFrom : undefined,
};
}
@@ -110,6 +143,8 @@ export function parseViewerData(raw: unknown): ViewerData {
runId: isString(site.runId) ? site.runId : "unknown",
generatedAt: isString(site.generatedAt) ? site.generatedAt : "",
},
+ source: parseSource(raw.source),
+ partial: parsePartial(raw.partial),
devices,
tiles,
groups,
From 8a2607bbbebc585d1afbf349587c9bcf9166c26b Mon Sep 17 00:00:00 2001
From: Alexander Tenepere
Date: Tue, 10 Nov 2026 00:12:54 +0800
Subject: [PATCH 2/8] fix: Carry only shot files that stay inside their run
folder
---
src/crawl/carry.ts | 26 ++++++++++++++++++++++++--
tests/carry.test.ts | 11 +++++++++++
2 files changed, 35 insertions(+), 2 deletions(-)
diff --git a/src/crawl/carry.ts b/src/crawl/carry.ts
index d35ef23..a280968 100644
--- a/src/crawl/carry.ts
+++ b/src/crawl/carry.ts
@@ -163,7 +163,9 @@ export async function verifyCarry(
for (const route of plan.carry) {
const prev = source.manifest.routes.find((r) => r.url === route.url);
const files = prev ? carriedFiles(route, prev, devices) : [];
- const present = await Promise.all(files.map((f) => pathExists(path.join(source.runDir, f))));
+ const present = await Promise.all(
+ files.map((f) => (f ? pathExists(path.join(source.runDir, f)) : false)),
+ );
if (present.every(Boolean)) {
out.carry.push(route);
} else {
@@ -183,7 +185,9 @@ function carriedFiles(route: RouteNode, prev: RouteNode, devices: Device[]): str
if (!before) continue;
for (const device of devicesFor(state, devices)) {
const ref = before.shots[device];
- if (ref?.full && !ref.error) files.push(...shotFiles(ref));
+ if (!ref?.full || ref.error) continue;
+ if (!safeShotFiles(ref)) return [""]; // never carried: verifyCarry shoots the route
+ files.push(...shotFiles(ref));
}
}
return files;
@@ -196,6 +200,20 @@ function shotFiles(ref: ShotRef): string[] {
);
}
+/**
+ * Is every file a shot points at a plain relative path inside its run directory?
+ * The paths come from an earlier run's crawl-map.json, which is only data: one that
+ * names `../` or an absolute path must not make carrying read, replace or delete a
+ * file outside the run.
+ */
+export function safeShotFiles(ref: ShotRef): boolean {
+ return shotFiles(ref).every((rel) => {
+ if (!rel || path.isAbsolute(rel) || path.win32.isAbsolute(rel)) return false;
+ const norm = path.posix.normalize(rel.replace(/\\/g, "/"));
+ return norm !== ".." && !norm.startsWith("../") && !norm.startsWith("/");
+ });
+}
+
/** Hard-link `from` to `to`, copying instead where links aren't possible. */
async function linkOrCopy(from: string, to: string): Promise {
await fs.mkdir(path.dirname(to), { recursive: true });
@@ -228,6 +246,10 @@ export async function carryRoute(
for (const device of devicesFor(state, devices)) {
const ref = before.shots[device];
if (!ref?.full || ref.error) continue;
+ if (!safeShotFiles(ref)) {
+ state.shots[device] = { error: `run ${source.runId} lists a shot file outside its run` };
+ continue;
+ }
try {
for (const rel of shotFiles(ref)) {
await linkOrCopy(path.join(source.runDir, rel), path.join(runDir, rel));
diff --git a/tests/carry.test.ts b/tests/carry.test.ts
index 1663854..9a8b9ed 100644
--- a/tests/carry.test.ts
+++ b/tests/carry.test.ts
@@ -9,6 +9,7 @@ import {
findCarrySource,
matchesOnly,
planCarry,
+ safeShotFiles,
verifyCarry,
} from "../src/crawl/carry";
import { markRunIncomplete, runDir } from "../src/crawl/runs";
@@ -241,3 +242,13 @@ describe("--only seeds", () => {
]);
});
});
+
+describe("shot paths from an earlier run", () => {
+ it("are carried only when they stay inside the run", () => {
+ expect(safeShotFiles({ full: "screenshots/home/default/desktop.png" })).toBe(true);
+ expect(safeShotFiles({ full: "../../.bashrc" })).toBe(false);
+ expect(safeShotFiles({ full: "screenshots/../../x.png" })).toBe(false);
+ expect(safeShotFiles({ full: "/etc/passwd" })).toBe(false);
+ expect(safeShotFiles({ full: "a.png", thumb: "..\\..\\x.webp" })).toBe(false);
+ });
+});
From 0028f686dc8434e0c8c7f655f5174b7833df1739 Mon Sep 17 00:00:00 2001
From: Alexander Tenepere
Date: Tue, 10 Nov 2026 11:21:49 +0800
Subject: [PATCH 3/8] feat: Tell users when a newer visualice is out, and how
to turn on plugin auto-update
---
CHANGELOG.md | 9 +++
README.md | 5 ++
docs/crawling.md | 9 +++
skills/visualice/SKILL.md | 4 ++
src/cli.ts | 32 ++++++++++
src/shared/update-check.ts | 122 +++++++++++++++++++++++++++++++++++++
tests/update-check.test.ts | 112 ++++++++++++++++++++++++++++++++++
7 files changed, 293 insertions(+)
create mode 100644 src/shared/update-check.ts
create mode 100644 tests/update-check.test.ts
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 656e587..9e0e878 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -27,6 +27,15 @@
- **Compare with another version** in the details panel shows the same shot from another
run beside this one.
+### Update notice
+
+- An installed CLI tells you when a newer visualice is out (checked against npm at most
+ once a day, in the background, never in the way), with the command to update the
+ plugin or the CLI. The skill passes it on. Off with `--quiet`, in CI, or with
+ `VISUALICE_NO_UPDATE_CHECK=1`.
+- The README says how to turn on auto-update for the plugin: a marketplace plugin
+ doesn't update itself unless you do.
+
## 0.1.1
### Fixes
diff --git a/README.md b/README.md
index 6085246..c3d412a 100644
--- a/README.md
+++ b/README.md
@@ -30,6 +30,11 @@ Needs Node 22.18 or newer.
/plugin install visualice@visualice
```
+Plugins from a marketplace like this one don't update on their own unless you ask: turn
+it on under `/plugin` → **Marketplaces** → visualice → **Enable auto-update**, or update
+by hand with `claude plugin update visualice@visualice`. The CLI also says when a newer
+version is out.
+
Then run `/visualice https://your.site`, or just ask Claude to "run visualice on
https://your.site". It maps the site, shows you the plan (the kinds of page, how many
screenshots, anything exploring would change on the site) and asks once, explores each
diff --git a/docs/crawling.md b/docs/crawling.md
index 4aa24cc..ac976eb 100644
--- a/docs/crawling.md
+++ b/docs/crawling.md
@@ -92,6 +92,15 @@ gitignored folder; once tracked, git keeps it) or restore it from a CI cache, th
`visualice crawl --replay --json --quiet --fail-on-error`. Without it, `--replay` falls
back to a full discovery and the explored states are missing.
+### Update notice
+
+An installed CLI checks npm for a newer visualice at most once a day, in the background,
+and says so on stderr at the start of a later command (`ℹ visualice 0.3.0 is available
+…`). It never delays a command for more than a moment, and a failed lookup is ignored.
+It is off with `--quiet`, in CI (`CI` set), in a source checkout, and with
+`VISUALICE_NO_UPDATE_CHECK=1` (or `NO_UPDATE_NOTIFIER=1`). The last answer is cached in
+`~/.cache/visualice/update-check.json` (`$XDG_CACHE_HOME` when set).
+
## Output
Output lands under the configured output directory (default `.visualice/`), one folder
diff --git a/skills/visualice/SKILL.md b/skills/visualice/SKILL.md
index 7bb8618..0a441dc 100644
--- a/skills/visualice/SKILL.md
+++ b/skills/visualice/SKILL.md
@@ -19,6 +19,10 @@ from any directory, no repo checkout needed. In a source checkout of this repo i
use the npm scripts instead: `npm run vc -- ` for anything below, `npm run crawl --
` for crawls. The commands below say `visualice`; substitute whichever applies.
+If a command prints `ℹ visualice X is available (you have Y)…` on stderr, tell the user
+once, with the update command it gives (`claude plugin update visualice@visualice`), and
+carry on with the task: don't update anything yourself.
+
Before the first crawl in a project, run `visualice doctor` — an idempotent preflight
that checks Node/Playwright chromium/sharp/the viewer build and self-heals what it
safely can (installs chromium, builds the viewer; `--no-fix` to only report). This
diff --git a/src/cli.ts b/src/cli.ts
index 5efc4f8..a3e753e 100644
--- a/src/cli.ts
+++ b/src/cli.ts
@@ -34,6 +34,7 @@ import {
} from "./crawl/routes-file";
import { ensureStorageState } from "./crawl/auth";
import { pathExists } from "./shared/fsx";
+import { startUpdateCheck, updateCheckDisabled, type UpdateCheck } from "./shared/update-check";
import { nodeBinCommand } from "./shared/node-bin";
import { serveDirectory } from "./shared/static-server";
import { serveViewer } from "./serve";
@@ -95,6 +96,9 @@ Usage:
visualice serve [opts] Launch the zoomable results viewer
visualice --version Print the version
+An installed CLI says on stderr when a newer version is out (checked at most daily);
+VISUALICE_NO_UPDATE_CHECK=1, --quiet or CI turns that off.
+
Project files: visualice.config.json controls behavior; visualice.routes.md lists
pages + UI states and is regenerated after every crawl (edit it: 'skip:' excludes,
'[ ]' requests a state). A visualice.config.* in the cwd is picked up automatically.
@@ -1014,6 +1018,34 @@ async function main(): Promise {
}
const flags = parseFlags(cmd, rest);
setHeaded(flags.headed === true);
+ const update = await updateCheck(flags);
+ if (update?.notice) console.error(`ℹ ${update.notice}`);
+ try {
+ await runCommand(cmd, flags);
+ } finally {
+ // Let a background registry lookup save its answer, but never hold a finished
+ // command up for long: an unanswered lookup is simply tried again next time.
+ if (update) await Promise.race([update.refreshed, delay(1500)]);
+ }
+}
+
+/**
+ * The update check, for an installed CLI. A source checkout (this repo, or one
+ * `npm link`ed) is never "behind" in a way worth telling, and `--quiet` asks for silence.
+ */
+async function updateCheck(flags: Flags): Promise {
+ if (flags.quiet === true || updateCheckDisabled()) return undefined;
+ if (await pathExists(path.join(PROJECT_ROOT, ".git"))) return undefined;
+ try {
+ return await startUpdateCheck(await readVersion());
+ } catch {
+ return undefined;
+ }
+}
+
+const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms).unref());
+
+async function runCommand(cmd: string, flags: Flags): Promise {
switch (cmd) {
case "init":
return cmdInit(flags);
diff --git a/src/shared/update-check.ts b/src/shared/update-check.ts
new file mode 100644
index 0000000..909699b
--- /dev/null
+++ b/src/shared/update-check.ts
@@ -0,0 +1,122 @@
+import os from "node:os";
+import path from "node:path";
+import { promises as fs } from "node:fs";
+
+// "A newer visualice is out" notice. A plugin install doesn't auto-update by default,
+// and the plugin's skill pins the CLI version it runs, so without this a user only
+// learns about a release by going to look. The check is cheap and never in the way:
+// the notice comes from a cached answer, the registry is asked at most once a day,
+// and a slow or failed lookup is dropped without a word.
+
+const REGISTRY_URL = "https://registry.npmjs.org/visualice/latest";
+const DAY_MS = 24 * 60 * 60 * 1000;
+const FETCH_TIMEOUT_MS = 2000;
+
+interface Cache {
+ checkedAt: number;
+ latest?: string;
+}
+
+/** Where the last answer is kept: the user's cache folder, outside any project. */
+export function cacheFile(env: NodeJS.ProcessEnv = process.env): string {
+ const base =
+ env.XDG_CACHE_HOME ||
+ (process.platform === "win32" && env.LOCALAPPDATA) ||
+ path.join(os.homedir(), ".cache");
+ return path.join(base, "visualice", "update-check.json");
+}
+
+/** Is the check switched off here: opted out, CI, or a test run? */
+export function updateCheckDisabled(env: NodeJS.ProcessEnv = process.env): boolean {
+ return !!(
+ env.VISUALICE_NO_UPDATE_CHECK ||
+ env.NO_UPDATE_NOTIFIER ||
+ env.CI ||
+ env.VITEST ||
+ env.NODE_ENV === "test"
+ );
+}
+
+/** `a` > `b` for plain x.y.z versions; a prerelease is never "newer". */
+export function isNewer(a: string, b: string): boolean {
+ const parse = (v: string) => {
+ const m = /^v?(\d+)\.(\d+)\.(\d+)$/.exec(v.trim());
+ return m ? m.slice(1).map(Number) : undefined;
+ };
+ const x = parse(a);
+ const y = parse(b);
+ if (!x || !y) return false;
+ for (let i = 0; i < 3; i++) if (x[i] !== y[i]) return x[i] > y[i];
+ return false;
+}
+
+/** The notice for a user on `current` when `latest` is out. */
+export function updateNotice(current: string, latest: string): string {
+ return (
+ `visualice ${latest} is available (you have ${current}). Update the plugin: ` +
+ `claude plugin update visualice@visualice — or the CLI: npm i -g visualice@latest`
+ );
+}
+
+async function readCache(file: string): Promise {
+ try {
+ const c = JSON.parse(await fs.readFile(file, "utf8")) as Cache;
+ return typeof c.checkedAt === "number" ? c : undefined;
+ } catch {
+ return undefined;
+ }
+}
+
+export interface UpdateCheck {
+ /** The notice to show now, from the cached answer; undefined when up to date. */
+ notice?: string;
+ /** Settles once a stale cache has been refreshed (never rejects). */
+ refreshed: Promise;
+}
+
+/**
+ * Start an update check for `current`. The notice comes from the last answer at
+ * once; when that answer is a day old, the registry is asked again in the background
+ * and the next run shows what it said.
+ */
+export async function startUpdateCheck(
+ current: string,
+ opts: { file?: string; now?: number; fetchFn?: typeof fetch } = {},
+): Promise {
+ const file = opts.file ?? cacheFile();
+ const now = opts.now ?? Date.now();
+ const cache = await readCache(file);
+ const notice =
+ cache?.latest && isNewer(cache.latest, current)
+ ? updateNotice(current, cache.latest)
+ : undefined;
+ if (cache && now - cache.checkedAt < DAY_MS) return { notice, refreshed: Promise.resolve() };
+ return { notice, refreshed: refresh(file, now, opts.fetchFn ?? fetch, cache) };
+}
+
+async function refresh(
+ file: string,
+ now: number,
+ fetchFn: typeof fetch,
+ previous: Cache | undefined,
+): Promise {
+ let latest = previous?.latest;
+ try {
+ const res = await fetchFn(REGISTRY_URL, {
+ headers: { accept: "application/json" },
+ signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+ });
+ if (res.ok) {
+ const body = (await res.json()) as { version?: unknown };
+ if (typeof body.version === "string") latest = body.version;
+ }
+ } catch {
+ // offline, blocked or slow: try again tomorrow
+ }
+ try {
+ await fs.mkdir(path.dirname(file), { recursive: true });
+ await fs.writeFile(file, JSON.stringify({ checkedAt: now, latest } satisfies Cache));
+ } catch {
+ // a read-only home: the check just runs again next time
+ }
+}
diff --git a/tests/update-check.test.ts b/tests/update-check.test.ts
new file mode 100644
index 0000000..ce2707a
--- /dev/null
+++ b/tests/update-check.test.ts
@@ -0,0 +1,112 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import {
+ cacheFile,
+ isNewer,
+ startUpdateCheck,
+ updateCheckDisabled,
+ updateNotice,
+} from "../src/shared/update-check";
+
+const DAY = 24 * 60 * 60 * 1000;
+
+/** A fetch that answers with `version`, and counts its calls. */
+function registry(version: string | Error) {
+ const calls: string[] = [];
+ const fn = (async (url: string) => {
+ calls.push(url);
+ if (version instanceof Error) throw version;
+ return new Response(JSON.stringify({ version }), { status: 200 });
+ }) as unknown as typeof fetch;
+ return { fn, calls };
+}
+
+describe("versions", () => {
+ it("compares plain x.y.z versions", () => {
+ expect(isNewer("0.2.0", "0.1.1")).toBe(true);
+ expect(isNewer("0.10.0", "0.9.9")).toBe(true);
+ expect(isNewer("1.0.0", "1.0.0")).toBe(false);
+ expect(isNewer("0.1.0", "0.1.1")).toBe(false);
+ expect(isNewer("0.3.0-beta.1", "0.2.0")).toBe(false);
+ expect(isNewer("nonsense", "0.2.0")).toBe(false);
+ });
+
+ it("names both update commands", () => {
+ const n = updateNotice("0.2.0", "0.3.0");
+ expect(n).toContain("visualice 0.3.0 is available (you have 0.2.0)");
+ expect(n).toContain("claude plugin update visualice@visualice");
+ expect(n).toContain("npm i -g visualice@latest");
+ });
+});
+
+describe("when it runs", () => {
+ it("is off when opted out, in CI and in tests", () => {
+ expect(updateCheckDisabled({})).toBe(false);
+ expect(updateCheckDisabled({ VISUALICE_NO_UPDATE_CHECK: "1" })).toBe(true);
+ expect(updateCheckDisabled({ NO_UPDATE_NOTIFIER: "1" })).toBe(true);
+ expect(updateCheckDisabled({ CI: "true" })).toBe(true);
+ });
+
+ it("caches under XDG_CACHE_HOME when set", () => {
+ expect(cacheFile({ XDG_CACHE_HOME: "/c" })).toBe(
+ path.join("/c", "visualice", "update-check.json"),
+ );
+ });
+});
+
+describe("startUpdateCheck", () => {
+ let dir: string;
+ let file: string;
+ beforeEach(async () => {
+ dir = await fs.mkdtemp(path.join(os.tmpdir(), "vc-update-"));
+ file = path.join(dir, "nested", "update-check.json");
+ });
+ afterEach(() => fs.rm(dir, { recursive: true, force: true }));
+
+ it("says nothing on the first run, asks the registry, and tells the next run", async () => {
+ const reg = registry("0.3.0");
+ const first = await startUpdateCheck("0.2.0", { file, now: 1000, fetchFn: reg.fn });
+ expect(first.notice).toBeUndefined();
+ await first.refreshed;
+ expect(reg.calls).toEqual(["https://registry.npmjs.org/visualice/latest"]);
+
+ const next = await startUpdateCheck("0.2.0", { file, now: 2000, fetchFn: reg.fn });
+ expect(next.notice).toContain("0.3.0 is available");
+ await next.refreshed;
+ expect(reg.calls).toHaveLength(1); // a fresh answer isn't asked for again
+ });
+
+ it("asks again once the answer is a day old, and says nothing when up to date", async () => {
+ await fs.mkdir(path.dirname(file), { recursive: true });
+ await fs.writeFile(file, JSON.stringify({ checkedAt: 0, latest: "0.2.0" }));
+ const reg = registry("0.2.0");
+ const check = await startUpdateCheck("0.2.0", { file, now: DAY + 1, fetchFn: reg.fn });
+ expect(check.notice).toBeUndefined();
+ await check.refreshed;
+ expect(reg.calls).toHaveLength(1);
+ });
+
+ it("keeps the last answer when the registry can't be reached", async () => {
+ await fs.mkdir(path.dirname(file), { recursive: true });
+ await fs.writeFile(file, JSON.stringify({ checkedAt: 0, latest: "0.3.0" }));
+ const check = await startUpdateCheck("0.2.0", {
+ file,
+ now: DAY + 1,
+ fetchFn: registry(new Error("offline")).fn,
+ });
+ expect(check.notice).toContain("0.3.0");
+ await expect(check.refreshed).resolves.toBeUndefined();
+ const saved = JSON.parse(await fs.readFile(file, "utf8"));
+ expect(saved).toEqual({ checkedAt: DAY + 1, latest: "0.3.0" });
+ });
+
+ it("ignores a cache it can't read", async () => {
+ await fs.mkdir(path.dirname(file), { recursive: true });
+ await fs.writeFile(file, "{not json");
+ const check = await startUpdateCheck("0.2.0", { file, now: 1, fetchFn: registry("0.2.0").fn });
+ expect(check.notice).toBeUndefined();
+ await check.refreshed;
+ });
+});
From 32b0b1f509d0f6ce29c10b29f90f204e6f29f60f Mon Sep 17 00:00:00 2001
From: Alexander Tenepere
Date: Tue, 10 Nov 2026 14:23:24 +0800
Subject: [PATCH 4/8] chore: End-to-end test of the update loop through the CLI
in a git repository
---
tests/integration/update-flow.test.ts | 157 ++++++++++++++++++++++++++
1 file changed, 157 insertions(+)
create mode 100644 tests/integration/update-flow.test.ts
diff --git a/tests/integration/update-flow.test.ts b/tests/integration/update-flow.test.ts
new file mode 100644
index 0000000..e88f412
--- /dev/null
+++ b/tests/integration/update-flow.test.ts
@@ -0,0 +1,157 @@
+import { describe, expect, it, beforeAll, afterAll } from "vitest";
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { serveDirectory } from "../../src/shared/static-server";
+import type { StaticServer } from "../../src/shared/file-serve";
+import type { ChangesReport } from "../../src/crawl/changes";
+import type { Manifest } from "../../src/shared/types";
+import { FIXTURE_DIR, tempDir } from "./helpers";
+
+const exec = promisify(execFile);
+
+const REPO = path.resolve(import.meta.dirname, "..", "..");
+const BIN = path.join(REPO, "cli", "visualice.mjs");
+
+/**
+ * The update loop end to end, through the real CLI in a real git repository, the
+ * way a project uses it: a full crawl records the commit it shot; `changes` reads
+ * that record back out of the routes file and diffs against it; the partial crawl
+ * it suggests shoots the changed pages and carries the rest; and a change that
+ * reaches every page asks for a full run. The unit tests cover each piece; this
+ * covers the hand-offs between them (routes-file line, run manifests, the CLI).
+ */
+
+let site: string;
+let server: StaticServer;
+
+const env = { ...process.env, NO_COLOR: "1", VISUALICE_NO_UPDATE_CHECK: "1" };
+
+const git = async (...args: string[]) =>
+ (await exec("git", args, { cwd: site, env })).stdout.trim();
+
+const visualice = async (...args: string[]) =>
+ (await exec(process.execPath, [BIN, ...args], { cwd: site, env, maxBuffer: 1 << 26 })).stdout;
+
+const changes = async (): Promise =>
+ JSON.parse(await visualice("changes", "--json", "--quiet")) as ChangesReport;
+
+const commit = async (message: string) => {
+ await git("add", "-A");
+ await git("commit", "-q", "-m", message);
+};
+
+const sourceLine = async () =>
+ /visualice-source commit=([0-9a-f]+)/.exec(
+ await fs.readFile(path.join(site, "visualice.routes.md"), "utf8"),
+ )?.[1];
+
+beforeAll(async () => {
+ site = await tempDir("update-flow");
+ await fs.cp(FIXTURE_DIR, site, { recursive: true });
+ await fs.rm(path.join(site, "crawl-map.input.json"), { force: true });
+ await fs.writeFile(path.join(site, ".gitignore"), ".visualice/\n");
+ // Small and quick: one device, few interactions. The site is the repository.
+ await fs.writeFile(
+ path.join(site, "visualice.config.json"),
+ JSON.stringify(
+ {
+ scope: { maxPages: 6, maxDepth: 1 },
+ devices: [{ name: "small", width: 420, height: 640, dpr: 1 }],
+ discovery: { interactions: { maxPerPage: 1 }, mobileStates: false },
+ stabilize: { extraDelayMs: 50 },
+ politeness: { minDelayMs: 0, concurrency: 4 },
+ },
+ null,
+ 2,
+ ),
+ );
+ await git("init", "-q");
+ // A Windows runner converts line endings by default, which would make every
+ // checked-out file look modified.
+ await git("config", "core.autocrlf", "false");
+ await git("config", "user.email", "test@example.com");
+ await git("config", "user.name", "test");
+ await commit("site");
+ server = await serveDirectory(site);
+}, 60000);
+
+afterAll(async () => {
+ await server?.close();
+ if (site) await fs.rm(site, { recursive: true, force: true });
+});
+
+describe("updating a catalogue after code changes", () => {
+ it("records, diffs, reruns only what changed, and asks for everything when it should", async () => {
+ // 1. A full crawl records the commit it shot in the routes file.
+ const first = JSON.parse(await visualice("crawl", server.url, "--json", "--quiet"));
+ expect(first.shots).toBeGreaterThan(0);
+ expect(first.failed).toBe(0);
+ expect(await sourceLine()).toBe(await git("rev-parse", "HEAD"));
+ const firstManifest = JSON.parse(await fs.readFile(first.manifest, "utf8")) as Manifest;
+ const pages = firstManifest.routes.map((r) => r.url);
+ expect(pages).toContain("/about");
+
+ // 2. Committing the catalogue changes nothing that counts.
+ await commit("catalogue");
+ expect((await changes()).verdict).toBe("none");
+
+ // 3. One page edited, one added: partial, with exactly those pages.
+ const about = path.join(site, "about.html");
+ const before = await fs.readFile(about, "utf8");
+ const edited = before.replace("
Pricing We now roast on Sundays too.
");
+ expect(edited).not.toBe(before);
+ await fs.writeFile(about, edited);
+ await fs.writeFile(
+ path.join(site, "pricing.html"),
+ "