Add versioned HTML index sources and systemd documentation

.changeset/tidy-manual-examples.md

@@ -0,0 +1,5 @@
+---
+"@neuledge/context": patch
+---
+
+Preserve code formatting in HTML documentation that uses bare preformatted blocks, including systemd's rendered DocBook manuals.

.gitignore

@@ -4,6 +4,7 @@ dist-packages
 dist-test
 node_modules
 .turbo
+**/.cache/context/html-index
 
 # system specific
 .DS_Store

packages/context/src/html.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from "vitest";
+import { parseHtml } from "./html.js";
+
+describe("DocBook HTML examples", () => {
+  it("preserves bare preformatted blocks, indentation and inline markup as code", () => {
+    const parsed = parseHtml(
+      `<h1>systemd.service</h1><h2>Example</h2>
+      <pre class="programlisting">[Service]\nExecStart=/usr/bin/<em>example</em> \\\n        --flag=&lt;value&gt;</pre>`,
+      "systemd.service.html",
+    );
+    expect(parsed.sections[0]?.hasCode).toBe(true);
+    expect(parsed.sections[0]?.content).toContain(
+      "[Service]\nExecStart=/usr/bin/example",
+    );
+    expect(parsed.sections[0]?.content).toContain("        --flag=<value>");
+  });
+
+  it("keeps backtick runs inside a single fenced code block", () => {
+    const parsed = parseHtml(
+      "<h1>Guide</h1><h2>Example</h2><pre>first\n```\nlast</pre>",
+      "guide.html",
+    );
+    expect(parsed.sections).toHaveLength(1);
+    expect(parsed.sections[0]?.hasCode).toBe(true);
+    expect(parsed.sections[0]?.content).toContain(
+      "````\nfirst\n```\nlast\n````",
+    );
+  });
+
+  it("preserves language detection for existing pre/code blocks", () => {
+    const parsed = parseHtml(
+      '<h1>Guide</h1><h2>Example</h2><pre><code class="language-sh">echo hello</code></pre>',
+      "guide.html",
+    );
+    expect(parsed.sections[0]?.content).toContain("```sh\necho hello\n```");
+  });
+});

packages/context/src/html.ts

@@ -38,6 +38,22 @@ for (const tag of REMOVED_TAGS) {
   turndown.remove(tag);
 }
 
+// DocBook emits bare <pre> elements; Turndown's code rule requires <pre><code>.
+// Preserve their whitespace and prevent Markdown escaping of unit-file examples.
+turndown.addRule("barePre", {
+  filter: (node) =>
+    node.nodeName === "PRE" && node.firstChild?.nodeName !== "CODE",
+  replacement: (_content, node) => {
+    const code: string = node.textContent ?? "";
+    const longestRun = (code.match(/`+/g) ?? []).reduce(
+      (longest, run) => Math.max(longest, run.length),
+      0,
+    );
+    const fence = "`".repeat(Math.max(3, longestRun + 1));
+    return `\n\n${fence}\n${code.replace(/\n$/, "")}\n${fence}\n\n`;
+  },
+});
+
 /**
  * Parse an HTML file by converting to Markdown, then using the existing
  * Markdown parser for section extraction and chunking.

packages/registry/package.json

@@ -27,6 +27,7 @@
   "dependencies": {
     "@neuledge/context": "workspace:*",
     "commander": "^14.0.0",
+    "linkedom": "^0.18.12",
     "p-retry": "^8.0.0",
     "yaml": "^2.8.2",
     "zod": "^4.3.6"

packages/registry/src/build.ts

@@ -5,7 +5,7 @@
  * to clone repos, read docs, and build SQLite packages.
  *
  * Supports both versioned (clone at specific tag) and unversioned
- * (clone default branch) definitions. Supports git and zip sources.
+ * (clone default branch) definitions. Supports git, zip and HTML index sources.
  */
 
 import { execSync } from "node:child_process";
@@ -25,6 +25,7 @@ import {
   type VersionedDefinition,
 } from "./definition.js";
 import { excludeFiles } from "./glob.js";
+import { downloadHtmlIndex } from "./html-index.js";
 import { downloadAndExtractZip } from "./zip.js";
 
 export interface RegistryBuildResult extends BuildResult {
@@ -91,19 +92,22 @@ export async function buildFromDefinition(
     );
   }
 
-  // Zip source: resolve URL template and download
+  // Explicit releases download an archive or a pinned HTML index.
   const url = resolveUrl(entry.source.url, version);
-  const docsPath = entry.source.docs_path
-    ? resolveUrl(entry.source.docs_path, version)
-    : undefined;
-
-  const files = await downloadAndExtractZip(url, {
-    docsPath,
-    excludePaths: entry.source.exclude_paths,
-  });
+  const files =
+    entry.source.type === "html-index"
+      ? await downloadHtmlIndex(entry.source, version)
+      : await downloadAndExtractZip(url, {
+          docsPath: entry.source.docs_path
+            ? resolveUrl(entry.source.docs_path, version)
+            : undefined,
+          excludePaths: entry.source.exclude_paths,
+        });
 
   if (files.length === 0) {
-    throw new Error(`No documentation files found in ZIP from ${url}`);
+    throw new Error(
+      `No documentation files found in ${entry.source.type} source from ${url}`,
+    );
   }
 
   const result = buildPackage(outputPath, files, {

packages/registry/src/cli.ts

@@ -15,8 +15,8 @@ import {
   getHeadCommit,
 } from "./build.js";
 import {
+  isExplicitVersionEntry,
   isVersioned,
-  isZipVersionEntry,
   listDefinitions,
 } from "./definition.js";
 import { checkPackageExists, publishPackage } from "./publish.js";
@@ -60,7 +60,7 @@ program
       if (isVersioned(def)) {
         const ranges = def.versions
           .map((v) => {
-            if (isZipVersionEntry(v)) {
+            if (isExplicitVersionEntry(v)) {
               return v.versions.join(", ");
             }
             return `${v.min_version}${v.max_version ? `-${v.max_version}` : "+"}`;

packages/registry/src/definition.ts

@@ -17,12 +17,14 @@
  * Source types:
  * - **git**: Clone a git repository at a specific tag.
  * - **zip**: Download a ZIP archive from a URL. Supports {version} placeholder.
+ * - **html-index**: Download a pinned HTML index and its reference pages.
  */
 
 import { readdirSync, readFileSync } from "node:fs";
 import { basename, dirname, join, relative } from "node:path";
 import { parse as parseYaml } from "yaml";
 import { z } from "zod/v4";
+import { resolveIndexUrl } from "./html-index.js";
 
 const GitSourceSchema = z.object({
   type: z.literal("git"),
@@ -47,11 +49,26 @@ const ZipSourceSchema = z.object({
   lang: z.string().default("en"),
 });
 
-const SourceSchema = z.discriminatedUnion("type", [
+const UnversionedSourceSchema = z.discriminatedUnion("type", [
   GitSourceSchema,
   ZipSourceSchema,
 ]);
 
+const HtmlIndexSourceSchema = z.object({
+  type: z.literal("html-index"),
+  url: z.string().refine((url) => {
+    try {
+      resolveIndexUrl(url, "1");
+      return true;
+    } catch {
+      return false;
+    }
+  }, "HTML index URL must be HTTPS with a {version} directory and no credentials, query, or fragment"),
+  exclude_paths: z.array(z.string()).optional(),
+  concurrency: z.int().min(1).max(10).default(4),
+  max_pages: z.int().min(1).max(5000).default(2000),
+});
+
 // Git version entry: semver range matching
 const GitVersionEntrySchema = z.object({
   min_version: z.string(),
@@ -66,9 +83,15 @@ const ZipVersionEntrySchema = z.object({
   source: ZipSourceSchema,
 });
 
+const HtmlIndexVersionEntrySchema = z.object({
+  versions: z.array(z.string().regex(/^\d+(?:[._-][A-Za-z0-9]+)*$/)).min(1),
+  source: HtmlIndexSourceSchema,
+});
+
 const VersionEntrySchema = z.union([
   GitVersionEntrySchema,
   ZipVersionEntrySchema,
+  HtmlIndexVersionEntrySchema,
 ]);
 
 // A definition has either `versions` (versioned) or `source` (unversioned), not both.
@@ -78,7 +101,7 @@ const DefinitionFileSchema = z
     description: z.string().optional(),
     repository: z.url().optional(),
     versions: z.array(VersionEntrySchema).min(1).optional(),
-    source: SourceSchema.optional(),
+    source: UnversionedSourceSchema.optional(),
   })
   .check((ctx) => {
     const hasVersions = ctx.value.versions != null;
@@ -96,9 +119,11 @@ const DefinitionFileSchema = z
 
 export type GitSource = z.infer<typeof GitSourceSchema>;
 export type ZipSource = z.infer<typeof ZipSourceSchema>;
-export type Source = z.infer<typeof SourceSchema>;
+export type HtmlIndexSource = z.infer<typeof HtmlIndexSourceSchema>;
+export type Source = GitSource | ZipSource | HtmlIndexSource;
 export type GitVersionEntry = z.infer<typeof GitVersionEntrySchema>;
 export type ZipVersionEntry = z.infer<typeof ZipVersionEntrySchema>;
+export type HtmlIndexVersionEntry = z.infer<typeof HtmlIndexVersionEntrySchema>;
 export type VersionEntry = z.infer<typeof VersionEntrySchema>;
 export type DefinitionFile = z.infer<typeof DefinitionFileSchema>;
 
@@ -109,10 +134,17 @@ export function isGitVersionEntry(
   return "min_version" in entry;
 }
 
-/** Type guard for zip version entries (have versions array). */
+/** Type guard for ZIP archive releases. */
 export function isZipVersionEntry(
   entry: VersionEntry,
 ): entry is ZipVersionEntry {
+  return entry.source.type === "zip";
+}
+
+/** Entries with an explicit release list, independent of package-manager APIs. */
+export function isExplicitVersionEntry(
+  entry: VersionEntry,
+): entry is ZipVersionEntry | HtmlIndexVersionEntry {
   return "versions" in entry;
 }
 
@@ -130,7 +162,7 @@ export interface VersionedDefinition extends BaseDefinition {
 }
 
 export interface UnversionedDefinition extends BaseDefinition {
-  source: Source;
+  source: GitSource | ZipSource;
   versions?: undefined;
 }
 
@@ -231,15 +263,15 @@ export function listDefinitions(registryDir: string): PackageDefinition[] {
  * Find the first version entry that matches a given version.
  * For git entries: ranges are evaluated top-to-bottom; first match wins.
  *   A version matches if: min_version <= version (< max_version if set).
- * For zip entries: exact match against the versions array.
+ * For ZIP and HTML index entries: exact match against the versions array.
  * Only applicable to versioned definitions.
  */
 export function resolveVersionEntry(
   definition: VersionedDefinition,
   version: string,
 ): VersionEntry | undefined {
   return definition.versions.find((entry) => {
-    if (isZipVersionEntry(entry)) {
+    if (isExplicitVersionEntry(entry)) {
       return entry.versions.includes(version);
     }
     if (compareSemver(version, entry.min_version) < 0) return false;