diff --git a/site/astro.config.ts b/site/astro.config.ts index abf8388a..bd69dce8 100644 --- a/site/astro.config.ts +++ b/site/astro.config.ts @@ -19,6 +19,7 @@ import { baseRawLinks } from "./src/lib/base-raw-links.js" import { citationLinks } from "./src/lib/citation-links.js" import { focusableScrollers } from "./src/lib/focusable-scrollers.js" import mermaid from "./src/lib/mermaid.js" +import { relativeMdLinks } from "./src/lib/relative-md-links.js" import { robotsPolicyFile } from "./src/lib/robots.js" import { rootTwin } from "./src/lib/root-twin.js" @@ -60,6 +61,7 @@ const here = fileURLToPath(new URL(".", import.meta.url)) const REPO_ROOT = resolve(here, "..") const TREE_ROOT = resolve(REPO_ROOT, "docs") const MANIFEST = "src/content/docs/.sync-manifest.json" +const CONTENT_ROOT = resolve(here, "src/content/docs") /** Where `scripts/gen-reference.mjs` writes the two SDK references, as absolute directories. */ const SDK_REFERENCE_DIRS = ["python", "typescript"].map( @@ -204,6 +206,12 @@ export default defineConfig({ processor: satteri({ mdastPlugins: [ agentNotePlugin(), + /* + * Relative `.md` links become page routes here, before `starlightBasePath()` (pushed after + * these) prefixes the base onto them. Emitted as written, `rpc-tools.md` on + * `/reference/public-api/` resolves one directory too deep and 404s. + */ + relativeMdLinks(CONTENT_ROOT), citationLinks({ commit: manifest.commit, repoUrl: REPO_URL, @@ -550,20 +558,23 @@ export default defineConfig({ /* * The raw `.md` twins come from an injected dynamic route, so they are absent from the page * list this validator checks against and every link to one reports `InvalidLink` while - * resolving perfectly in the browser. The exclusion is only allowed because two stricter gates - * replace it: `tests/raw-route-links.test.ts` and `tests/agent-surface.test.ts` resolve every - * link against the files actually present in `dist/`, which needs no side table at all. + * resolving perfectly in the browser. Only root-relative twin links are excluded: a relative + * `.md` link is one `relativeMdLinks` missed, and it reaches the check below. The exclusion is + * only allowed because stricter gates + * replace it: `tests/raw-route-links.test.ts` resolves every link in the twins and + * `tests/built-links.test.ts` every link on every built surface, each + * against the files actually present in `dist/`, which needs no side table at all. */ - exclude: ({ link }) => link.endsWith(".md"), + exclude: ({ link }) => link.startsWith("/") && link.endsWith(".md") /* - * The generated tree cross-links with relative targets — `../insights/contract-map.md` and - * siblings, hundreds of them — and that is the correct form here rather than something to - * migrate. A relative target is the ONE link shape that resolves on both surfaces under a base - * segment: Astro rewrites it to a route in the rendered page, and on the raw twin it resolves - * against the twin's own directory, which already carries the base. + * `errorOnRelativeLinks` keeps its default, `true`. The tree cross-links with relative `.md` + * targets, which resolve on GitHub and on the raw twins, and `relativeMdLinks` above rewrites + * each one to its page route before this validator reads the tree, because in the rendered + * page a relative target resolves one directory too deep. A relative link reaching this + * validator is therefore one that rewrite missed, and it fails the build. + * `tests/built-links.test.ts` then resolves every link on every built surface against `dist/`. */ - errorOnRelativeLinks: false }), // A return-to-top control on long pages; Platform alone runs past a thousand lines of source. diff --git a/site/src/components/Head.astro b/site/src/components/Head.astro index 3d40510c..fea06613 100644 --- a/site/src/components/Head.astro +++ b/site/src/components/Head.astro @@ -24,10 +24,11 @@ * the `/microvms-agentd` segment and yields a 404 that only appears in production. * `import.meta.env.BASE_URL` is the accessor that includes it. */ +import { getCollection } from "astro:content" import Default from "@astrojs/starlight/components/Head.astro" import { APPLE_TOUCH_SIZE, ICO_SIZES, ICON_PNG_SIZE } from "../branding/mark.js" -import { OG_HEIGHT, OG_WIDTH, ogAlt, ogSlug } from "../branding/og-card.js" +import { OG_HEIGHT, OG_WIDTH, ogAlt, ogId, ogSlug } from "../branding/og-card.js" import { discoveryLinks, siteUrl } from "../lib/agent-surface.js" import { jsonLdGraph, jsonLdText, type SiteDescription } from "../lib/jsonld.js" @@ -72,6 +73,13 @@ const asset = (file: string): string => siteUrl(file, context).pathname /* `og:image` is the one URL that must be absolute: a crawler resolves it against nothing. */ const card = siteUrl(`og/${ogSlug(entry.id)}`, context) +/* + * The card route renders one card per `docs` collection entry, keyed by `ogId`, which maps the root + * page's two id spellings to one card; this reads the collection the same way. + * Starlight's built-in 404 page is a fallback entry outside that collection, so it has no card and + * names none. + */ +const hasCard = (await getCollection("docs", ({ id }) => ogId(id) === ogId(entry.id))).length > 0 const cardAlt = ogAlt(entry.data.title) const icoSizes = ICO_SIZES.map((size) => `${size}x${size}`).join(" ") @@ -119,13 +127,19 @@ const graph = jsonLdGraph( Starlight emits `twitter:card: summary_large_image` and no image (read from its `utils/head.ts`), so these are additions and not a second `og:image` beside a default one. */} - - - - - - - +{ + hasCard && ( + <> + + + + + + + + + ) +} { links.map((link) => ( diff --git a/site/src/lib/jsonld.ts b/site/src/lib/jsonld.ts index de31fae1..60ebf870 100644 --- a/site/src/lib/jsonld.ts +++ b/site/src/lib/jsonld.ts @@ -4,8 +4,8 @@ * for the site, joined in a single `@graph`. * * This module is pure and takes the site's origin and base segment as arguments rather than reading - * `import.meta.env`, so every URL and every node it emits is asserted in a unit test rather than only - * in a validator run against a deployed page. + * `import.meta.env`, so every URL it emits can be checked without a deployed page: + * `tests/built-links.test.ts` resolves every `url` and `@id` in the built graph against `dist/`. * * Every URL is built through `siteUrl`, which resolves with `new URL()`. A base segment joined to a * path by string concatenation produces `//microvms-agentd/index.md` — a protocol-relative URL naming @@ -26,7 +26,7 @@ * present one as asserted. */ -import { type SiteContext, siteUrl } from "./agent-surface.js" +import { rawMarkdownUrl, type SiteContext, siteUrl } from "./agent-surface.js" /** * The site node, described once. @@ -89,7 +89,8 @@ const pageUrl = (entryId: string, context: SiteContext): URL => * neither makes the other redundant, because they are read by different consumers. */ const markdownEncoding = (entryId: string, context: SiteContext) => { - const href = siteUrl(`${entryId === "index" ? "" : entryId}.md`, context).href + // The root twin is `index.md`: `rawMarkdownUrl` owns that mapping, as it does for the head link. + const href = rawMarkdownUrl(entryId, context).href return { "@type": "MediaObject" as const, "@id": href, diff --git a/site/src/lib/relative-md-links.ts b/site/src/lib/relative-md-links.ts new file mode 100644 index 00000000..d8a4d8f7 --- /dev/null +++ b/site/src/lib/relative-md-links.ts @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: Apache-2.0 +import { existsSync } from "node:fs" +import { posix, relative, sep } from "node:path" +import { fileURLToPath } from "node:url" + +import { defineMdastPlugin, type MdastPluginDefinition } from "satteri" + +/** + * Rewrites a relative link to another page's Markdown source into the route that page is served at. + * + * The tree cross-links with relative `.md` targets such as `rpc-tools.md` and + * `../internals/insights/contract-map.md`. That form resolves on GitHub and on the raw `.md` twins, + * because there each target sits beside its source. The rendered page sits one directory deeper than + * its source: `reference/public-api.md` is served at `/reference/public-api/`. Sätteri emits a link + * target as written, so the browser resolved `rpc-tools.md` on that page to + * `/reference/public-api/rpc-tools.md`, a 404. Every relative `.md` link in the rendered tree broke + * this way, 148 of them on 29 pages, and the llms bundles inherited each one, because + * `starlight-llms-txt` flattens the rendered page back to Markdown. + * + * This resolves the target against the source document's path under the content root and emits the + * target's route root-relative, `/reference/rpc-tools/`, with any fragment kept. `starlightBasePath()` + * then prefixes the base onto it, exactly as it does for an authored `/agents/`; its Sätteri pass is + * pushed after the plugins configured in `astro.config.ts`, and Sätteri applies each plugin's edits + * before the next one runs, so it reads the rewritten URL. + * + * The raw twins never pass through this processor and keep the relative form, which resolves there + * for every page except an `index.md`: its twin is served one level up (`reference/index.md` at + * `reference.md`), so a relative link written in an index page resolves one level too high on the + * twin. The index pages link root-relative, and `tests/built-links.test.ts` fails on the first + * relative link that breaks a twin. + * + * A relative `.md` target that names no file under the content root fails the build with the document + * and line. `scripts/sync-docs.mjs` already turns a link that leaves the published corpus into a + * permalink, so a target missing here is a defect, and emitting it would ship a 404. + * + * The route is the content path lowercased, which is the route Astro gives a file whose path + * segments use only `[a-z0-9_-]`: the glob loader runs each segment through github-slugger, which + * leaves those characters alone and drops or replaces most others (`v1.2-notes` becomes `v12-notes`). + * Every page in the tree meets that rule, so a target outside it fails the build here rather than + * through a guessed route. A frontmatter `slug:` override is not read; the link validator fails the + * build on the route it would leave unserved. + * + * Only `link` nodes are rewritten. A reference definition (`[x]: page.md`) would need the base added + * here as well, because `starlightBasePath()` rewrites `link` nodes only; the tree uses none, and + * `tests/built-links.test.ts` fails on the first one that ships unresolved. + */ + +type LinkVisitor = NonNullable +type LinkNode = Parameters[0] + +/** A target this plugin owns: no scheme, not root-relative, not an in-page anchor, ending `.md`. */ +const RELATIVE_MD = /^(?![a-z][a-z0-9+.-]*:)(?![/#])(?[^?#]*\.md)(?[?#].*)?$/i + +/** A path segment whose github-slugger slug is itself once lowercased. */ +const SLUG_SAFE = /^[a-z0-9][a-z0-9_-]*$/ + +/** A content path as the route it is served at: `reference/cli.md` is `/reference/cli/`. */ +const routeOfContentPath = (contentPath: string): string => + `/${contentPath.slice(0, -".md".length).toLowerCase()}/`.replace(/(^|\/)index\/$/, "$1") + +/** + * @param contentRoot the absolute path of Starlight's content directory, `src/content/docs`. A + * document outside it is left alone: its route is not derived from its path. + */ +export const relativeMdLinks = (contentRoot: string): MdastPluginDefinition => { + const root = contentRoot.endsWith(sep) ? contentRoot : `${contentRoot}${sep}` + + return defineMdastPlugin({ + name: "docs-relative-md-links", + // Positions are read for the error message, which names the line the link sits on. + options: { position: true }, + + link(node: LinkNode, ctx) { + const groups = RELATIVE_MD.exec(node.url)?.groups + if (groups?.path === undefined || ctx.fileURL === undefined) return + const document = fileURLToPath(ctx.fileURL) + if (!document.startsWith(root)) return + + const line = node.position?.start.line + const refuse = (why: string): never => { + throw new Error( + `docs-relative-md-links: ${line === undefined ? document : `${document}:${line}`}: ` + + `\`${node.url}\` ${why}` + ) + } + let path = groups.path + try { + path = decodeURI(groups.path) + } catch { + refuse("is not valid percent-encoding.") + } + const from = posix.dirname(relative(root, document).split(sep).join("/")) + const target = posix.normalize(posix.join(from, path)) + if (target.startsWith("../") || !existsSync(`${root}${target}`)) { + refuse( + `resolves to \`${target}\`, which is not a page under ${root}. Link to a published ` + + "page, or let `scripts/sync-docs.mjs` turn the link into a permalink." + ) + } + const segments = target.slice(0, -".md".length).toLowerCase().split("/") + if (!segments.every((segment) => SLUG_SAFE.test(segment))) { + refuse( + `resolves to \`${target}\`, whose route this rewrite cannot derive: Astro slugs a path ` + + "segment with github-slugger, which changes characters outside `[a-z0-9_-]`. Rename " + + "the page, or link to its route." + ) + } + ctx.setProperty(node, "url", `${routeOfContentPath(target)}${groups.suffix ?? ""}`) + } + }) +} diff --git a/site/tests/built-links.test.ts b/site/tests/built-links.test.ts new file mode 100644 index 00000000..ea25a78f --- /dev/null +++ b/site/tests/built-links.test.ts @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: Apache-2.0 +import { readdirSync, readFileSync } from "node:fs" +import { dirname, join } from "node:path" +import { fileURLToPath } from "node:url" +import { describe, expect, it } from "vitest" + +import { BASE, DIST_DIR } from "../src/gates.js" + +/** + * Every link on every built surface resolves to a file the build emitted, and every fragment into a + * rendered page names an id on that page. + * + * Each link is resolved the way a browser resolves it: against the URL its file is served at, so + * `reference/public-api/index.html` resolves its links against `/microvms-agentd/reference/public-api/`. + * No other gate does that for the rendered pages or the llms bundles. `starlight-links-validator` reads + * the links the Markdown pipeline produced and skips `.md` targets, and before + * `src/lib/relative-md-links.ts` existed a relative `.md` link passed every gate on 29 pages while the + * browser resolved it one directory too deep: `rpc-tools.md` on `/reference/public-api/` fetched + * `/reference/public-api/rpc-tools.md`, a 404. The llms bundles carried the same links from the site + * root, where `../` climbed above the base. + * + * `tests/raw-route-links.test.ts` asserts the twins' own link forms (the base exactly once, twins + * reachable from twins). This file holds every surface to one rule with one resolver, twins included. + * External links are lychee's: `mise run docs:links` and the weekly `links.yml` check them. + */ + +/* ================================================================================================= + * CONFIG: the whole site-specific surface of this file. + * ================================================================================================= */ + +const CONFIG = { + /** The origin the site is published at, read from the same variable `astro.config.ts` reads. */ + origin: new URL(process.env.DOCS_SITE ?? "https://laithalsaadoon.github.io").origin, + /** Output directories that hold assets, not documents. Their files are link targets, not sources. */ + ignore: ["_astro/", "pagefind/"], + /** + * The page GitHub Pages serves, with status 404, for every path that has no file. A relative link on + * it resolves against whichever missing URL was requested, so it may carry none. + */ + notFoundPage: "404.html", + /** + * Links that name a missing file on purpose, each with the reason. Starlight gives the 404 page a + * canonical URL and an `og:url` of its own route, and that route has no file because the page is + * served at every missing path instead. + */ + allowed: [{ from: "404.html", target: `${BASE}404/` }] +} as const + +/* ================================================================================================= + * The built files and the URLs they are served at. + * ================================================================================================= */ + +const distDir = join(dirname(dirname(fileURLToPath(import.meta.url))), DIST_DIR) + +const files = (directory: string): ReadonlyArray => + readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const path = join(directory, entry.name) + return entry.isDirectory() ? files(path) : [path.slice(distDir.length + 1)] + }) + +const built = files(distDir) +const documents = built.filter((path) => !CONFIG.ignore.some((prefix) => path.startsWith(prefix))) +const pages = documents.filter((path) => path.endsWith(".html")) +const twins = documents.filter((path) => path.endsWith(".md")) +const bundles = documents.filter((path) => /^llms[^/]*\.txt$/.test(path)) +const sitemaps = documents.filter((path) => /^sitemap[^/]*\.xml$/.test(path)) + +/** `reference/cli/index.html` is served at `/microvms-agentd/reference/cli/`; anything else as named. */ +const servedAt = (path: string): URL => + new URL(`${BASE}${path.replace(/(^|\/)index\.html$/, "$1")}`, CONFIG.origin) + +const builtFiles: ReadonlySet = new Set(built) +const isFile = (path: string): boolean => builtFiles.has(path) + +/** + * The built file a decoded same-origin path is served from, or `undefined` when GitHub Pages would + * answer 404. Pages serves `x/index.html` at `x/`, redirects `x` to `x/` when that directory has an + * index (the bare base included), and serves `x.html` at `x`. A doubled slash inside the path is + * reported even though Pages collapses it, because it is how a base joined on twice shows up. + */ +const fileAt = (pathname: string): string | undefined => { + if (`${pathname}/` === BASE) return "index.html" + if (!pathname.startsWith(BASE)) return undefined + const path = pathname.slice(BASE.length) + const candidates = + path === "" || path.endsWith("/") + ? [`${path}index.html`] + : [path, `${path}/index.html`, `${path}.html`] + return candidates.find(isFile) +} + +/* ================================================================================================= + * Reading links out of each kind of file. + * ================================================================================================= */ + +const decodeEntities = (value: string): string => + value.replace(/&(#x[0-9a-f]+|#\d+|amp|quot|apos|lt|gt);/gi, (_, entity: string) => { + const named: Record = { amp: "&", quot: '"', apos: "'", lt: "<", gt: ">" } + if (entity[0] !== "#") return named[entity.toLowerCase()] ?? "" + return String.fromCodePoint( + entity[1] === "x" || entity[1] === "X" + ? Number.parseInt(entity.slice(2), 16) + : Number.parseInt(entity.slice(1), 10) + ) + }) + +/** + * Every start tag in a document, with its attributes. Script and style bodies and comments are removed + * first, and the opening tags of scripts and styles are kept, so a `