From a2cc15d6f07dac702bf35a252a94a20763063d73 Mon Sep 17 00:00:00 2001 From: Martin Beckert Date: Wed, 9 Sep 2026 09:36:20 +0200 Subject: [PATCH] Add versioned HTML index sources and systemd documentation --- .changeset/tidy-manual-examples.md | 5 + .gitignore | 1 + packages/context/src/html.test.ts | 37 +++ packages/context/src/html.ts | 16 + packages/registry/package.json | 1 + packages/registry/src/build.ts | 26 +- packages/registry/src/cli.ts | 4 +- packages/registry/src/definition.ts | 46 ++- .../registry/src/html-index-build.test.ts | 154 +++++++++ packages/registry/src/html-index.test.ts | 284 ++++++++++++++++ packages/registry/src/html-index.ts | 309 ++++++++++++++++++ packages/registry/src/index.ts | 4 + packages/registry/src/version-check.ts | 8 +- pnpm-lock.yaml | 3 + registry/README.md | 45 ++- registry/systemd/systemd-guides.yaml | 9 + registry/systemd/systemd.yaml | 12 + 17 files changed, 939 insertions(+), 25 deletions(-) create mode 100644 .changeset/tidy-manual-examples.md create mode 100644 packages/context/src/html.test.ts create mode 100644 packages/registry/src/html-index-build.test.ts create mode 100644 packages/registry/src/html-index.test.ts create mode 100644 packages/registry/src/html-index.ts create mode 100644 registry/systemd/systemd-guides.yaml create mode 100644 registry/systemd/systemd.yaml diff --git a/.changeset/tidy-manual-examples.md b/.changeset/tidy-manual-examples.md new file mode 100644 index 0000000..b50ec7d --- /dev/null +++ b/.changeset/tidy-manual-examples.md @@ -0,0 +1,5 @@ +--- +"@neuledge/context": patch +--- + +Preserve code formatting in HTML documentation that uses bare preformatted blocks, including systemd's rendered DocBook manuals. diff --git a/.gitignore b/.gitignore index 7647369..b0ea31b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ dist-packages dist-test node_modules .turbo +**/.cache/context/html-index # system specific .DS_Store diff --git a/packages/context/src/html.test.ts b/packages/context/src/html.test.ts new file mode 100644 index 0000000..961ce73 --- /dev/null +++ b/packages/context/src/html.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { parseHtml } from "./html.js"; + +describe("DocBook HTML examples", () => { + it("preserves bare preformatted blocks, indentation and inline markup as code", () => { + const parsed = parseHtml( + `

systemd.service

Example

+ 
[Service]\nExecStart=/usr/bin/example \\\n        --flag=<value>
`, + "systemd.service.html", + ); + expect(parsed.sections[0]?.hasCode).toBe(true); + expect(parsed.sections[0]?.content).toContain( + "[Service]\nExecStart=/usr/bin/example", + ); + expect(parsed.sections[0]?.content).toContain(" --flag="); + }); + + it("keeps backtick runs inside a single fenced code block", () => { + const parsed = parseHtml( + "

Guide

Example

first\n```\nlast
", + "guide.html", + ); + expect(parsed.sections).toHaveLength(1); + expect(parsed.sections[0]?.hasCode).toBe(true); + expect(parsed.sections[0]?.content).toContain( + "````\nfirst\n```\nlast\n````", + ); + }); + + it("preserves language detection for existing pre/code blocks", () => { + const parsed = parseHtml( + '

Guide

Example

echo hello
', + "guide.html", + ); + expect(parsed.sections[0]?.content).toContain("```sh\necho hello\n```"); + }); +}); diff --git a/packages/context/src/html.ts b/packages/context/src/html.ts index 75ac857..cabfcf5 100644 --- a/packages/context/src/html.ts +++ b/packages/context/src/html.ts @@ -38,6 +38,22 @@ for (const tag of REMOVED_TAGS) { turndown.remove(tag); } +// DocBook emits bare 
 elements; Turndown's code rule requires 
.
+// Preserve their whitespace and prevent Markdown escaping of unit-file examples.
+turndown.addRule("barePre", {
+  filter: (node) =>
+    node.nodeName === "PRE" && node.firstChild?.nodeName !== "CODE",
+  replacement: (_content, node) => {
+    const code: string = node.textContent ?? "";
+    const longestRun = (code.match(/`+/g) ?? []).reduce(
+      (longest, run) => Math.max(longest, run.length),
+      0,
+    );
+    const fence = "`".repeat(Math.max(3, longestRun + 1));
+    return `\n\n${fence}\n${code.replace(/\n$/, "")}\n${fence}\n\n`;
+  },
+});
+
 /**
  * Parse an HTML file by converting to Markdown, then using the existing
  * Markdown parser for section extraction and chunking.
diff --git a/packages/registry/package.json b/packages/registry/package.json
index fd611ba..a428c95 100644
--- a/packages/registry/package.json
+++ b/packages/registry/package.json
@@ -27,6 +27,7 @@
   "dependencies": {
     "@neuledge/context": "workspace:*",
     "commander": "^14.0.0",
+    "linkedom": "^0.18.12",
     "p-retry": "^8.0.0",
     "yaml": "^2.8.2",
     "zod": "^4.3.6"
diff --git a/packages/registry/src/build.ts b/packages/registry/src/build.ts
index 092c3f1..adb70ba 100644
--- a/packages/registry/src/build.ts
+++ b/packages/registry/src/build.ts
@@ -5,7 +5,7 @@
  * to clone repos, read docs, and build SQLite packages.
  *
  * Supports both versioned (clone at specific tag) and unversioned
- * (clone default branch) definitions. Supports git and zip sources.
+ * (clone default branch) definitions. Supports git, zip and HTML index sources.
  */
 
 import { execSync } from "node:child_process";
@@ -25,6 +25,7 @@ import {
   type VersionedDefinition,
 } from "./definition.js";
 import { excludeFiles } from "./glob.js";
+import { downloadHtmlIndex } from "./html-index.js";
 import { downloadAndExtractZip } from "./zip.js";
 
 export interface RegistryBuildResult extends BuildResult {
@@ -91,19 +92,22 @@ export async function buildFromDefinition(
     );
   }
 
-  // Zip source: resolve URL template and download
+  // Explicit releases download an archive or a pinned HTML index.
   const url = resolveUrl(entry.source.url, version);
-  const docsPath = entry.source.docs_path
-    ? resolveUrl(entry.source.docs_path, version)
-    : undefined;
-
-  const files = await downloadAndExtractZip(url, {
-    docsPath,
-    excludePaths: entry.source.exclude_paths,
-  });
+  const files =
+    entry.source.type === "html-index"
+      ? await downloadHtmlIndex(entry.source, version)
+      : await downloadAndExtractZip(url, {
+          docsPath: entry.source.docs_path
+            ? resolveUrl(entry.source.docs_path, version)
+            : undefined,
+          excludePaths: entry.source.exclude_paths,
+        });
 
   if (files.length === 0) {
-    throw new Error(`No documentation files found in ZIP from ${url}`);
+    throw new Error(
+      `No documentation files found in ${entry.source.type} source from ${url}`,
+    );
   }
 
   const result = buildPackage(outputPath, files, {
diff --git a/packages/registry/src/cli.ts b/packages/registry/src/cli.ts
index a956772..b92e6dd 100644
--- a/packages/registry/src/cli.ts
+++ b/packages/registry/src/cli.ts
@@ -15,8 +15,8 @@ import {
   getHeadCommit,
 } from "./build.js";
 import {
+  isExplicitVersionEntry,
   isVersioned,
-  isZipVersionEntry,
   listDefinitions,
 } from "./definition.js";
 import { checkPackageExists, publishPackage } from "./publish.js";
@@ -60,7 +60,7 @@ program
       if (isVersioned(def)) {
         const ranges = def.versions
           .map((v) => {
-            if (isZipVersionEntry(v)) {
+            if (isExplicitVersionEntry(v)) {
               return v.versions.join(", ");
             }
             return `${v.min_version}${v.max_version ? `-${v.max_version}` : "+"}`;
diff --git a/packages/registry/src/definition.ts b/packages/registry/src/definition.ts
index ffd6e8b..e18498c 100644
--- a/packages/registry/src/definition.ts
+++ b/packages/registry/src/definition.ts
@@ -17,12 +17,14 @@
  * Source types:
  * - **git**: Clone a git repository at a specific tag.
  * - **zip**: Download a ZIP archive from a URL. Supports {version} placeholder.
+ * - **html-index**: Download a pinned HTML index and its reference pages.
  */
 
 import { readdirSync, readFileSync } from "node:fs";
 import { basename, dirname, join, relative } from "node:path";
 import { parse as parseYaml } from "yaml";
 import { z } from "zod/v4";
+import { resolveIndexUrl } from "./html-index.js";
 
 const GitSourceSchema = z.object({
   type: z.literal("git"),
@@ -47,11 +49,26 @@ const ZipSourceSchema = z.object({
   lang: z.string().default("en"),
 });
 
-const SourceSchema = z.discriminatedUnion("type", [
+const UnversionedSourceSchema = z.discriminatedUnion("type", [
   GitSourceSchema,
   ZipSourceSchema,
 ]);
 
+const HtmlIndexSourceSchema = z.object({
+  type: z.literal("html-index"),
+  url: z.string().refine((url) => {
+    try {
+      resolveIndexUrl(url, "1");
+      return true;
+    } catch {
+      return false;
+    }
+  }, "HTML index URL must be HTTPS with a {version} directory and no credentials, query, or fragment"),
+  exclude_paths: z.array(z.string()).optional(),
+  concurrency: z.int().min(1).max(10).default(4),
+  max_pages: z.int().min(1).max(5000).default(2000),
+});
+
 // Git version entry: semver range matching
 const GitVersionEntrySchema = z.object({
   min_version: z.string(),
@@ -66,9 +83,15 @@ const ZipVersionEntrySchema = z.object({
   source: ZipSourceSchema,
 });
 
+const HtmlIndexVersionEntrySchema = z.object({
+  versions: z.array(z.string().regex(/^\d+(?:[._-][A-Za-z0-9]+)*$/)).min(1),
+  source: HtmlIndexSourceSchema,
+});
+
 const VersionEntrySchema = z.union([
   GitVersionEntrySchema,
   ZipVersionEntrySchema,
+  HtmlIndexVersionEntrySchema,
 ]);
 
 // A definition has either `versions` (versioned) or `source` (unversioned), not both.
@@ -78,7 +101,7 @@ const DefinitionFileSchema = z
     description: z.string().optional(),
     repository: z.url().optional(),
     versions: z.array(VersionEntrySchema).min(1).optional(),
-    source: SourceSchema.optional(),
+    source: UnversionedSourceSchema.optional(),
   })
   .check((ctx) => {
     const hasVersions = ctx.value.versions != null;
@@ -96,9 +119,11 @@ const DefinitionFileSchema = z
 
 export type GitSource = z.infer;
 export type ZipSource = z.infer;
-export type Source = z.infer;
+export type HtmlIndexSource = z.infer;
+export type Source = GitSource | ZipSource | HtmlIndexSource;
 export type GitVersionEntry = z.infer;
 export type ZipVersionEntry = z.infer;
+export type HtmlIndexVersionEntry = z.infer;
 export type VersionEntry = z.infer;
 export type DefinitionFile = z.infer;
 
@@ -109,10 +134,17 @@ export function isGitVersionEntry(
   return "min_version" in entry;
 }
 
-/** Type guard for zip version entries (have versions array). */
+/** Type guard for ZIP archive releases. */
 export function isZipVersionEntry(
   entry: VersionEntry,
 ): entry is ZipVersionEntry {
+  return entry.source.type === "zip";
+}
+
+/** Entries with an explicit release list, independent of package-manager APIs. */
+export function isExplicitVersionEntry(
+  entry: VersionEntry,
+): entry is ZipVersionEntry | HtmlIndexVersionEntry {
   return "versions" in entry;
 }
 
@@ -130,7 +162,7 @@ export interface VersionedDefinition extends BaseDefinition {
 }
 
 export interface UnversionedDefinition extends BaseDefinition {
-  source: Source;
+  source: GitSource | ZipSource;
   versions?: undefined;
 }
 
@@ -231,7 +263,7 @@ export function listDefinitions(registryDir: string): PackageDefinition[] {
  * Find the first version entry that matches a given version.
  * For git entries: ranges are evaluated top-to-bottom; first match wins.
  *   A version matches if: min_version <= version (< max_version if set).
- * For zip entries: exact match against the versions array.
+ * For ZIP and HTML index entries: exact match against the versions array.
  * Only applicable to versioned definitions.
  */
 export function resolveVersionEntry(
@@ -239,7 +271,7 @@ export function resolveVersionEntry(
   version: string,
 ): VersionEntry | undefined {
   return definition.versions.find((entry) => {
-    if (isZipVersionEntry(entry)) {
+    if (isExplicitVersionEntry(entry)) {
       return entry.versions.includes(version);
     }
     if (compareSemver(version, entry.min_version) < 0) return false;
diff --git a/packages/registry/src/html-index-build.test.ts b/packages/registry/src/html-index-build.test.ts
new file mode 100644
index 0000000..7a6afa1
--- /dev/null
+++ b/packages/registry/src/html-index-build.test.ts
@@ -0,0 +1,154 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { initDatabase } from "@neuledge/context";
+import {
+  afterEach,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  it,
+  vi,
+} from "vitest";
+import { buildFromDefinition } from "./build.js";
+import {
+  isExplicitVersionEntry,
+  isVersioned,
+  isZipVersionEntry,
+  loadDefinition,
+  resolveVersionEntry,
+  type VersionedDefinition,
+} from "./definition.js";
+import { downloadHtmlIndex } from "./html-index.js";
+import { discoverVersions } from "./version-check.js";
+
+vi.mock("./html-index.js", async (importOriginal) => ({
+  ...(await importOriginal()),
+  downloadHtmlIndex: vi.fn(),
+}));
+
+describe("HTML index registry integration", () => {
+  let dir: string;
+
+  beforeAll(async () => {
+    await initDatabase();
+  });
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), "html-registry-"));
+    mkdirSync(join(dir, "systemd"));
+  });
+  afterEach(() => {
+    vi.clearAllMocks();
+    vi.unstubAllGlobals();
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  function definition(
+    source = 'type: html-index\n      url: "https://docs.example/man/{version}/"',
+    versions = '["258"]',
+  ) {
+    const path = join(dir, "systemd", "systemd.yaml");
+    writeFileSync(
+      path,
+      `name: systemd\nversions:\n  - versions: ${versions}\n    source:\n      ${source}\n`,
+    );
+    return loadDefinition(path);
+  }
+
+  it("discovers explicit releases without registry API requests, including in nightly builds", async () => {
+    const fetchMock = vi.fn();
+    vi.stubGlobal("fetch", fetchMock);
+    const def = definition();
+    expect(isVersioned(def)).toBe(true);
+    const entry = resolveVersionEntry(def as VersionedDefinition, "258");
+    expect(entry && isExplicitVersionEntry(entry)).toBe(true);
+    expect(entry && isZipVersionEntry(entry)).toBe(false);
+    expect(
+      resolveVersionEntry(def as VersionedDefinition, "257"),
+    ).toBeUndefined();
+    expect(await discoverVersions(def, { since: 2 })).toEqual([
+      { name: "systemd", registry: "systemd", version: "258" },
+    ]);
+    expect(fetchMock).not.toHaveBeenCalled();
+    expect(entry?.source).toMatchObject({ concurrency: 4, max_pages: 2000 });
+  });
+
+  it.each([
+    ['type: html-index\n      url: "https://docs.example/latest/"', '["258"]'],
+    [
+      'type: html-index\n      url: "https://docs.example/{version}/"',
+      '["latest"]',
+    ],
+    [
+      'type: html-index\n      url: "https://docs.example/{version}/"\n      concurrency: 0',
+      '["258"]',
+    ],
+    [
+      'type: html-index\n      url: "https://docs.example/{version}/"\n      max_pages: 5001',
+      '["258"]',
+    ],
+  ])("rejects invalid HTML source definitions", (source, versions) => {
+    expect(() => definition(source, versions)).toThrow();
+  });
+
+  it("uses explicit HTML releases even when a package-manager API exists", async () => {
+    const fetchMock = vi.fn();
+    vi.stubGlobal("fetch", fetchMock);
+    const def = { ...definition(), registry: "npm" };
+    expect(await discoverVersions(def)).toEqual([
+      { name: "systemd", registry: "npm", version: "258" },
+    ]);
+    expect(fetchMock).not.toHaveBeenCalled();
+  });
+
+  it("rejects unversioned HTML sources", () => {
+    const path = join(dir, "systemd", "systemd.yaml");
+    writeFileSync(
+      path,
+      'name: systemd\nsource:\n  type: html-index\n  url: "https://docs.example/man/258/"\n',
+    );
+    expect(() => loadDefinition(path)).toThrow();
+  });
+
+  it("builds an HTML release into a searchable package through the registry pipeline", async () => {
+    vi.mocked(downloadHtmlIndex).mockResolvedValue([
+      {
+        path: "systemd.service.html",
+        content:
+          "
systemd.service
Options
ExecStart= specifies the command to run.
",
+      },
+    ]);
+    const result = await buildFromDefinition(
+      definition() as VersionedDefinition,
+      "258",
+      dir,
+    );
+    expect(result).toMatchObject({
+      name: "systemd",
+      registry: "systemd",
+      version: "258",
+    });
+    expect(result.sectionCount).toBeGreaterThan(0);
+    expect(result.totalTokens).toBeGreaterThan(0);
+    expect(downloadHtmlIndex).toHaveBeenCalledWith(
+      expect.objectContaining({ type: "html-index" }),
+      "258",
+    );
+  });
+
+  it("loads both shipped systemd definitions", async () => {
+    const root = resolve(import.meta.dirname, "../../..", "registry/systemd");
+    const manuals = loadDefinition(join(root, "systemd.yaml"));
+    expect(await discoverVersions(manuals)).toEqual([
+      { name: "systemd", registry: "systemd", version: "258" },
+    ]);
+    const guides = loadDefinition(join(root, "systemd-guides.yaml"));
+    expect(guides.source).toMatchObject({
+      type: "git",
+      ref: "v258",
+      docs_path: "docs",
+    });
+  });
+});
diff --git a/packages/registry/src/html-index.test.ts b/packages/registry/src/html-index.test.ts
new file mode 100644
index 0000000..c33d3ea
--- /dev/null
+++ b/packages/registry/src/html-index.test.ts
@@ -0,0 +1,284 @@
+import { mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { parseDocument } from "@neuledge/context";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { HtmlIndexSource } from "./definition.js";
+import { downloadHtmlIndex, resolveIndexUrl } from "./html-index.js";
+
+const base = "https://docs.example/man/258/";
+const source: HtmlIndexSource = {
+  type: "html-index",
+  url: "https://docs.example/man/{version}/",
+  concurrency: 2,
+  max_pages: 20,
+};
+const html = (body: string) =>
+  new Response(body, {
+    headers: { "content-type": "text/html; charset=utf-8" },
+  });
+const page = (name: string) =>
+  `
${name}
Reference for ${name}.
`;
+
+describe("HTML index downloads", () => {
+  let cacheDir: string;
+  let fetchMock: ReturnType>;
+
+  beforeEach(() => {
+    cacheDir = mkdtempSync(join(tmpdir(), "html-index-"));
+    fetchMock = vi.fn();
+    vi.stubGlobal("fetch", fetchMock);
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+    vi.restoreAllMocks();
+    vi.useRealTimers();
+    rmSync(cacheDir, { recursive: true, force: true });
+  });
+
+  const download = (overrides: Partial = {}) =>
+    downloadHtmlIndex({ ...source, ...overrides }, "258", { cacheDir });
+
+  it("follows only scoped HTML links once, excluding fragments, navigation and queries", async () => {
+    fetchMock.mockImplementation(async (input) =>
+      input.toString() === base
+        ? html(`one
+          twothree
+          indexAold
+          latestexternal
+          portscript
+          queryescape
+          separatordouble
+          credentials
+          httpinvalid`)
+        : html(page("systemctl")),
+    );
+    const files = await download({ exclude_paths: ["index.html"] });
+    expect(files.map((file) => file.path)).toEqual(["systemctl.html"]);
+    expect(fetchMock.mock.calls.map(([url]) => url.toString())).toEqual([
+      base,
+      `${base}systemctl.html`,
+    ]);
+  });
+
+  it("does not crawl links found inside manual pages", async () => {
+    fetchMock
+      .mockResolvedValueOnce(html('one'))
+      .mockResolvedValueOnce(html(`${page("one")}two`));
+    expect(await download()).toHaveLength(1);
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+  });
+
+  it("reuses pinned downloads and refetches a corrupted cache entry", async () => {
+    fetchMock.mockImplementation(async (input) =>
+      html(
+        input.toString() === base ? 'one' : page("one"),
+      ),
+    );
+    const first = await download();
+    expect(await download()).toEqual(first);
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    const cached = readdirSync(cacheDir)[0];
+    expect(cached).toBeDefined();
+    writeFileSync(join(cacheDir, cached as string), "interrupted JSON");
+    expect(await download()).toEqual(first);
+    expect(fetchMock).toHaveBeenCalledTimes(3);
+  });
+
+  it("deduplicates identical aliases deterministically and limits concurrency", async () => {
+    let active = 0;
+    let maximum = 0;
+    fetchMock.mockImplementation(async (input) => {
+      if (input.toString() === base)
+        return html(
+          'bac',
+        );
+      active++;
+      maximum = Math.max(maximum, active);
+      await new Promise((resolve) => setTimeout(resolve, 5));
+      active--;
+      return html(page("shared"));
+    });
+    expect((await download()).map((file) => file.path)).toEqual(["a.html"]);
+    expect(maximum).toBe(2);
+  });
+
+  it.each([
+    "https://other.example/page.html",
+    "../257/page.html",
+    "../latest/page.html",
+    "/man/258evil/page.html",
+    "page.html?view=print",
+  ])("rejects redirects outside the pinned scope: %s", async (location) => {
+    fetchMock
+      .mockResolvedValueOnce(html('one'))
+      .mockResolvedValueOnce(
+        new Response(null, { status: 302, headers: { location } }),
+      );
+    await expect(download()).rejects.toThrow(
+      "redirect leaves the pinned directory",
+    );
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+  });
+
+  it("allows scoped redirects and resolves links relative to the final index", async () => {
+    fetchMock
+      .mockResolvedValueOnce(
+        new Response(null, {
+          status: 302,
+          headers: { location: "sub/index.html" },
+        }),
+      )
+      .mockResolvedValueOnce(html('one'))
+      .mockResolvedValueOnce(html(page("one")));
+    expect((await download())[0]?.path).toBe("sub/one.html");
+    expect(fetchMock.mock.calls[2]?.[0].toString()).toBe(`${base}sub/one.html`);
+  });
+
+  it("bounds redirect loops", async () => {
+    fetchMock.mockResolvedValue(
+      new Response(null, { status: 302, headers: { location: base } }),
+    );
+    await expect(download()).rejects.toThrow("Too many HTML redirects");
+    expect(fetchMock).toHaveBeenCalledTimes(6);
+  });
+
+  it("fails on empty or over-limit indexes without fetching partial documentation", async () => {
+    fetchMock.mockResolvedValueOnce(
+      html('ab'),
+    );
+    await expect(download({ max_pages: 1 })).rejects.toThrow("max_pages");
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+    await expect(download({ exclude_paths: ["*.html"] })).rejects.toThrow(
+      "No documentation links",
+    );
+  });
+
+  it("fails the whole download on a missing page and retains completed cache entries", async () => {
+    fetchMock
+      .mockResolvedValueOnce(
+        html('ab'),
+      )
+      .mockResolvedValueOnce(html(page("a")))
+      .mockResolvedValueOnce(new Response("missing", { status: 404 }));
+    await expect(download({ concurrency: 1 })).rejects.toThrow("HTTP 404");
+    fetchMock.mockResolvedValueOnce(html(page("b")));
+    expect(await download()).toHaveLength(2);
+    expect(fetchMock).toHaveBeenCalledTimes(4);
+  });
+
+  it("retries transient responses without caching errors", async () => {
+    fetchMock
+      .mockResolvedValueOnce(new Response("busy", { status: 503 }))
+      .mockResolvedValueOnce(html('a'))
+      .mockResolvedValueOnce(html(page("a")));
+    expect(await download()).toHaveLength(1);
+    expect(fetchMock).toHaveBeenCalledTimes(3);
+    expect(readdirSync(cacheDir)).toHaveLength(2);
+  });
+
+  it.each([
+    [() => new Response("plain text"), "Expected HTML"],
+    [() => html(" "), "Empty HTML"],
+    [
+      () =>
+        new Response("partial", {
+          status: 206,
+          headers: { "content-type": "text/html" },
+        }),
+      "HTTP 206",
+    ],
+    [
+      () =>
+        new Response("large", {
+          headers: {
+            "content-type": "text/html",
+            "content-length": String(11 * 1024 * 1024),
+          },
+        }),
+      "exceeds 10 MiB",
+    ],
+    [() => html("a".repeat(10 * 1024 * 1024 + 1)), "exceeds 10 MiB"],
+  ] as const)("rejects invalid or oversized responses", async (response, message) => {
+    fetchMock.mockResolvedValueOnce(response());
+    await expect(download()).rejects.toThrow(message);
+    expect(readdirSync(cacheDir)).toEqual([]);
+  });
+
+  it("times out stalled response bodies and stops after two retries", async () => {
+    const realTimeout = AbortSignal.timeout.bind(AbortSignal);
+    vi.spyOn(AbortSignal, "timeout").mockImplementation((milliseconds) => {
+      expect(milliseconds).toBe(30_000);
+      return realTimeout(30);
+    });
+    fetchMock.mockImplementation(async (_input, init) => {
+      const body = new ReadableStream({
+        start(controller) {
+          controller.enqueue(new TextEncoder().encode(""));
+          init?.signal?.addEventListener(
+            "abort",
+            () => {
+              controller.error(init.signal?.reason);
+            },
+            { once: true },
+          );
+        },
+      });
+      return new Response(body, { headers: { "content-type": "text/html" } });
+    });
+    await expect(download()).rejects.toThrow(/timeout/i);
+    expect(fetchMock).toHaveBeenCalledTimes(3);
+    expect(readdirSync(cacheDir)).toEqual([]);
+  });
+
+  it("keeps options, tables and code from rendered DocBook HTML", async () => {
+    fetchMock
+      .mockResolvedValueOnce(html('service'))
+      .mockResolvedValueOnce(
+        html(`
systemd.service

+        
Options
Type=
Controls startup.

+        
SpecifierMeaning
%nFull unit name

+        
[Service]\nExecStart=/usr/bin/example
`),
+      );
+    const [file] = await download();
+    expect(file).toBeDefined();
+    const parsed = parseDocument(file?.content ?? "", file?.path ?? "");
+    expect(parsed.sections.some((section) => section.hasCode)).toBe(true);
+    const result = JSON.stringify(parsed);
+    for (const text of [
+      "Type=",
+      "Controls startup",
+      "%n",
+      "Full unit name",
+      "ExecStart=/usr/bin/example",
+    ]) {
+      expect(result).toContain(text);
+    }
+  });
+});
+
+describe("pinned HTML index URLs", () => {
+  it.each([
+    "latest",
+    "stable",
+    "main",
+    "../258",
+    "258/../../latest",
+    "258?x=1",
+  ])("rejects release %s", (version) => {
+    expect(() => resolveIndexUrl(source.url, version)).toThrow();
+  });
+  it.each([
+    "http://docs.example/{version}/",
+    "https://docs.example/latest/",
+    "https://docs.example/?version={version}",
+    "https://user:secret@docs.example/{version}/",
+    "https://docs.example/{version}/?x=1",
+    "https://docs.example/{version}/#a",
+    "https://docs.example/{version}/../latest/",
+    "https://{version}.example/latest/",
+  ])("rejects unpinned template %s", (url) => {
+    expect(() => resolveIndexUrl(url, "258")).toThrow();
+  });
+});
diff --git a/packages/registry/src/html-index.ts b/packages/registry/src/html-index.ts
new file mode 100644
index 0000000..feb609c
--- /dev/null
+++ b/packages/registry/src/html-index.ts
@@ -0,0 +1,309 @@
+/** Download a pinned table of contents and its same-directory HTML pages. */
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { parseHTML } from "linkedom";
+import type { HtmlIndexSource } from "./definition.js";
+import { compileGlob } from "./glob.js";
+
+const MAX_BYTES = 10 * 1024 * 1024;
+const MAX_TOTAL_BYTES = 128 * 1024 * 1024;
+const CACHE_DIR = resolve(".cache/context/html-index");
+const HTML_PATH = /\.html?$/i;
+
+export function resolveIndexUrl(template: string, version: string): URL {
+  if (!/^\d+(?:[._-][A-Za-z0-9]+)*$/.test(version)) {
+    throw new Error(`HTML index requires a pinned numeric release: ${version}`);
+  }
+  if (template.split("{version}").length !== 2) {
+    throw new Error("HTML index URL requires one {version} directory");
+  }
+  const url = new URL(template.replace("{version}", version));
+  if (
+    url.protocol !== "https:" ||
+    url.username ||
+    url.password ||
+    url.search ||
+    url.hash ||
+    !template.includes("/{version}/") ||
+    !url.pathname.includes(`/${version}/`) ||
+    (!url.pathname.endsWith("/") && !HTML_PATH.test(url.pathname)) ||
+    /%(?:2f|5c|25|00)/i.test(url.pathname)
+  ) {
+    throw new Error("HTML index URL must pin an HTTPS version directory");
+  }
+  return url;
+}
+
+/** URL parsing normalizes dot segments; encoded separators remain forbidden. */
+function scopedUrl(href: string, base: URL, root: URL): URL | undefined {
+  try {
+    const url = new URL(href, base);
+    if (
+      url.origin !== root.origin ||
+      url.username ||
+      url.password ||
+      url.search ||
+      !url.pathname.startsWith(root.pathname) ||
+      /%(?:2f|5c|25|00)/i.test(url.pathname)
+    )
+      return;
+    decodeURIComponent(url.pathname); // Reject malformed escapes as well.
+    url.hash = "";
+    return url;
+  } catch {
+    return;
+  }
+}
+
+function indexLinks(
+  html: string,
+  index: URL,
+  root: URL,
+  source: HtmlIndexSource,
+): URL[] {
+  const { document } = parseHTML(html);
+  const excluded = source.exclude_paths?.map(compileGlob) ?? [];
+  const urls = new Set();
+  for (const anchor of document.querySelectorAll("a[href]")) {
+    const url = scopedUrl(anchor.getAttribute("href") ?? "", index, root);
+    if (!url || !HTML_PATH.test(url.pathname) || url.href === index.href)
+      continue;
+    const path = decodeURIComponent(url.pathname.slice(root.pathname.length));
+    if (excluded.some((pattern) => pattern.test(path))) continue;
+    urls.add(url.href);
+    if (urls.size > source.max_pages) {
+      throw new Error(
+        `HTML index exceeds max_pages (${source.max_pages}): ${index}`,
+      );
+    }
+  }
+  if (!urls.size)
+    throw new Error(`No documentation links found in HTML index: ${index}`);
+  return [...urls].sort().map((url) => new URL(url));
+}
+
+interface CachedPage {
+  url: string;
+  finalUrl: string;
+  content: string;
+}
+
+function digest(value: string): string {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+async function readCache(
+  path: string,
+  url: URL,
+  root: URL,
+): Promise {
+  try {
+    const page: CachedPage = JSON.parse(await readFile(path, "utf8"));
+    if (
+      page.url === url.href &&
+      typeof page.finalUrl === "string" &&
+      scopedUrl(page.finalUrl, url, root) &&
+      typeof page.content === "string" &&
+      page.content.trim() &&
+      Buffer.byteLength(page.content) <= MAX_BYTES
+    )
+      return page;
+  } catch {
+    // Missing or interrupted cache entries are refetched.
+  }
+  return;
+}
+
+async function writeCache(path: string, page: CachedPage): Promise {
+  const temporary = `${path}.${randomUUID()}.tmp`;
+  try {
+    await writeFile(temporary, JSON.stringify(page));
+    await rename(temporary, path);
+  } finally {
+    await rm(temporary, { force: true });
+  }
+}
+
+class RetryableError extends Error {}
+
+async function readHtml(response: Response, url: URL): Promise {
+  const type = response.headers
+    .get("content-type")
+    ?.split(";")[0]
+    ?.trim()
+    .toLowerCase();
+  if (type !== "text/html" && type !== "application/xhtml+xml") {
+    await response.body?.cancel();
+    throw new Error(
+      `Expected HTML from ${url}, received ${type ?? "no content type"}`,
+    );
+  }
+  if (Number(response.headers.get("content-length")) > MAX_BYTES) {
+    await response.body?.cancel();
+    throw new Error(`HTML response exceeds 10 MiB: ${url}`);
+  }
+  const reader = response.body?.getReader();
+  if (!reader) throw new Error(`Empty HTML response: ${url}`);
+  const chunks: Uint8Array[] = [];
+  let bytes = 0;
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      bytes += value.byteLength;
+      if (bytes > MAX_BYTES)
+        throw new Error(`HTML response exceeds 10 MiB: ${url}`);
+      chunks.push(value);
+    }
+  } finally {
+    await reader.cancel().catch(() => {});
+    reader.releaseLock();
+  }
+  const content = Buffer.concat(chunks).toString("utf8");
+  if (!content.trim()) throw new Error(`Empty HTML response: ${url}`);
+  return content;
+}
+
+async function requestPage(
+  url: URL,
+  root: URL,
+  signal: AbortSignal,
+): Promise {
+  let current = url;
+  // This deadline covers redirects AND the streamed body, not just headers.
+  const timeout = AbortSignal.any([signal, AbortSignal.timeout(30_000)]);
+  for (let redirects = 0; redirects <= 5; redirects++) {
+    let response: Response;
+    try {
+      response = await fetch(current, {
+        redirect: "manual",
+        signal: timeout,
+        headers: {
+          Accept: "text/html,application/xhtml+xml",
+          "User-Agent":
+            "context-registry/1.0 (+https://github.com/neuledge/context)",
+        },
+      });
+    } catch (error) {
+      throw new RetryableError(`Could not fetch ${current}: ${String(error)}`);
+    }
+    if ([301, 302, 303, 307, 308].includes(response.status)) {
+      await response.body?.cancel();
+      const location = response.headers.get("location");
+      const next = location ? scopedUrl(location, current, root) : undefined;
+      if (
+        !next ||
+        (next.pathname !== root.pathname && !HTML_PATH.test(next.pathname))
+      ) {
+        throw new Error(
+          `HTML redirect leaves the pinned directory: ${current} -> ${location}`,
+        );
+      }
+      current = next;
+      continue;
+    }
+    if (response.status !== 200) {
+      await response.body?.cancel();
+      const message = `Failed to fetch ${current}: HTTP ${response.status}`;
+      if (response.status === 429 || response.status >= 500)
+        throw new RetryableError(message);
+      throw new Error(message);
+    }
+    try {
+      return {
+        url: url.href,
+        finalUrl: current.href,
+        content: await readHtml(response, current),
+      };
+    } catch (error) {
+      if (timeout.aborted || error instanceof TypeError) {
+        throw new RetryableError(`Could not read ${current}: ${String(error)}`);
+      }
+      throw error;
+    }
+  }
+  throw new Error(`Too many HTML redirects: ${url}`);
+}
+
+async function fetchPage(
+  url: URL,
+  root: URL,
+  cacheDir: string,
+  signal: AbortSignal,
+): Promise {
+  const path = join(cacheDir, `${digest(url.href)}.json`);
+  const cached = await readCache(path, url, root);
+  if (cached) return cached;
+  for (let attempt = 0; ; attempt++) {
+    signal.throwIfAborted();
+    try {
+      const page = await requestPage(url, root, signal);
+      await writeCache(path, page);
+      return page;
+    } catch (error) {
+      if (!(error instanceof RetryableError) || attempt === 2 || signal.aborted)
+        throw error;
+      await delay(1000 * 2 ** attempt, undefined, { signal });
+    }
+  }
+}
+
+export async function downloadHtmlIndex(
+  source: HtmlIndexSource,
+  version: string,
+  options: { cacheDir?: string } = {},
+): Promise> {
+  const index = resolveIndexUrl(source.url, version);
+  const root = new URL(".", index);
+  const cacheDir = options.cacheDir ?? CACHE_DIR;
+  await mkdir(cacheDir, { recursive: true });
+  const controller = new AbortController();
+  const indexPage = await fetchPage(index, root, cacheDir, controller.signal);
+  const urls = indexLinks(
+    indexPage.content,
+    new URL(indexPage.finalUrl),
+    root,
+    source,
+  );
+  const pages = new Map();
+  let totalBytes = Buffer.byteLength(indexPage.content);
+  let cursor = 0;
+  let failure: unknown;
+  const worker = async () => {
+    try {
+      while (!controller.signal.aborted) {
+        const url = urls[cursor++];
+        if (!url) return;
+        const page = await fetchPage(url, root, cacheDir, controller.signal);
+        totalBytes += Buffer.byteLength(page.content);
+        if (totalBytes > MAX_TOTAL_BYTES)
+          throw new Error("HTML index exceeds 128 MiB total");
+        pages.set(url.href, page);
+      }
+    } catch (error) {
+      if (!controller.signal.aborted) failure = error;
+      controller.abort();
+    }
+  };
+  await Promise.all(
+    Array.from({ length: Math.min(source.concurrency, urls.length) }, worker),
+  );
+  if (failure) throw failure;
+  const seen = new Set();
+  const files: Array<{ path: string; content: string }> = [];
+  // Stable order chooses the same alias regardless of download completion order.
+  for (const url of urls) {
+    const page = pages.get(url.href);
+    if (!page) throw new Error(`HTML page was not downloaded: ${url}`);
+    const hash = digest(page.content);
+    if (seen.has(hash)) continue;
+    seen.add(hash);
+    files.push({
+      path: decodeURIComponent(url.pathname.slice(root.pathname.length)),
+      content: page.content,
+    });
+  }
+  return files;
+}
diff --git a/packages/registry/src/index.ts b/packages/registry/src/index.ts
index 14bff5e..72c54db 100644
--- a/packages/registry/src/index.ts
+++ b/packages/registry/src/index.ts
@@ -7,6 +7,9 @@ export {
   constructTag,
   type GitSource,
   type GitVersionEntry,
+  type HtmlIndexSource,
+  type HtmlIndexVersionEntry,
+  isExplicitVersionEntry,
   isGitVersionEntry,
   isVersioned,
   isZipVersionEntry,
@@ -21,6 +24,7 @@ export {
   type ZipSource,
   type ZipVersionEntry,
 } from "./definition.js";
+export { downloadHtmlIndex } from "./html-index.js";
 export {
   checkPackageExists,
   type PackageMetadata,
diff --git a/packages/registry/src/version-check.ts b/packages/registry/src/version-check.ts
index ace0b17..1b70658 100644
--- a/packages/registry/src/version-check.ts
+++ b/packages/registry/src/version-check.ts
@@ -8,8 +8,8 @@
 import pRetry, { AbortError } from "p-retry";
 import {
   compareSemver,
+  isExplicitVersionEntry,
   isVersioned,
-  isZipVersionEntry,
   type PackageDefinition,
   resolveVersionEntry,
 } from "./definition.js";
@@ -61,12 +61,12 @@ export async function discoverVersions(
   }
 
   const fetcher = registryFetchers[definition.registry];
-  if (!fetcher) {
+  if (!fetcher || definition.versions.every(isExplicitVersionEntry)) {
     // For registries without API fetchers (e.g., python, java),
-    // extract versions from zip version entries directly
+    // explicit ZIP/HTML releases do not require a package-manager API.
     const explicitVersions: AvailableVersion[] = [];
     for (const entry of definition.versions) {
-      if (isZipVersionEntry(entry)) {
+      if (isExplicitVersionEntry(entry)) {
         for (const v of entry.versions) {
           explicitVersions.push({
             name: definition.name,
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index b60a0d3..41c3bca 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -109,6 +109,9 @@ importers:
       commander:
         specifier: ^14.0.0
         version: 14.0.2
+      linkedom:
+        specifier: ^0.18.12
+        version: 0.18.12
       p-retry:
         specifier: ^8.0.0
         version: 8.0.0
diff --git a/registry/README.md b/registry/README.md
index 1beac55..a91bdfe 100644
--- a/registry/README.md
+++ b/registry/README.md
@@ -90,7 +90,50 @@ versions:
 > to ask, so the build fails with `Unsupported registry: ` — and because one bad
 > definition fails the whole nightly publish, it takes every other package down with it.
 >
-> Outside those four directories, use **unversioned** or **versioned-by-zip**.
+> Outside those four directories, use **unversioned**, **versioned-by-zip**, or **versioned HTML index**.
+
+
+### Versioned HTML index — for published reference manuals
+
+Use `html-index` when a single HTML table of contents links to all reference
+pages. It downloads those links and uses the existing HTML parser:
+
+```yaml
+versions:
+  - versions: ["258"]
+    source:
+      type: html-index
+      url: "https://www.freedesktop.org/software/systemd/man/{version}/"
+      exclude_paths:
+        - "index.html"
+        - "systemd.directives.html"
+```
+
+This source requires explicit numeric release versions (for example `258` or
+`3.14.0`) and an HTTPS URL containing a `{version}` directory segment. Moving
+aliases such as `latest` are rejected. It works in any registry directory.
+The index may be a directory URL or an `.html`/`.htm` file. Only HTML links
+within that index's directory and origin are downloaded; fragments are removed,
+query links are ignored, and linked pages are not crawled recursively. Redirects
+must stay inside the same directory. Exclusions are relative to that directory.
+
+Downloads use four workers by default (`concurrency: 1..10`) and allow up to
+2,000 pages (`max_pages: 1..5000`). Exceeding that limit, a failed page, or an
+empty index fails the build instead of publishing partial documentation.
+Requests have a 30-second timeout, transient failures are retried twice, and
+responses are limited to 10 MiB each and 128 MiB per build. Identical pages are
+indexed once, which avoids duplicate man-page aliases.
+
+Pinned downloads are reused from `.cache/context/html-index` across builds.
+Delete that directory to refetch a corrected upstream release. The nightly
+publisher skips releases already present in the registry. Check the publisher's crawling policy
+before adding an index source.
+
+For systemd, `systemd/systemd` contains the versioned reference manuals and
+`systemd/systemd-guides` contains the Markdown architecture and integration
+guides from Git. These are separate sources and packages; `docs_path` selects
+a directory within one Git or ZIP source.
+
 
 ## Excluding parts of a source
 
diff --git a/registry/systemd/systemd-guides.yaml b/registry/systemd/systemd-guides.yaml
new file mode 100644
index 0000000..5a46d09
--- /dev/null
+++ b/registry/systemd/systemd-guides.yaml
@@ -0,0 +1,9 @@
+name: systemd-guides
+description: "systemd architecture, integration, and developer guides; reference manuals are in systemd/systemd"
+repository: https://github.com/systemd/systemd
+
+source:
+  type: git
+  url: https://github.com/systemd/systemd
+  ref: v258
+  docs_path: docs
diff --git a/registry/systemd/systemd.yaml b/registry/systemd/systemd.yaml
new file mode 100644
index 0000000..520f182
--- /dev/null
+++ b/registry/systemd/systemd.yaml
@@ -0,0 +1,12 @@
+name: systemd
+description: "systemd reference manuals: services, units, systemctl, journalctl, networking, and APIs"
+repository: https://github.com/systemd/systemd
+
+versions:
+  - versions: ["258"]
+    source:
+      type: html-index
+      url: "https://www.freedesktop.org/software/systemd/man/{version}/"
+      exclude_paths:
+        - "index.html"
+        - "systemd.directives.html"