diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index 6700bc46..c39c7890 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -83,14 +83,30 @@ jobs: - name: Comment PR with results if: failure() && github.event_name == 'pull_request' uses: actions/github-script@v7 + env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} with: script: | const fs = require('fs'); + // GitHub rejects PR comments longer than 65536 chars. The lychee + // report often exceeds this when many formula URLs are broken, so + // we cap the body and point readers to the uploaded artifact. + const MAX_BODY = 60000; + let comment = '## 🔗 Link Check Failed\n\n'; + comment += `Full report: [workflow run](${process.env.RUN_URL}) → download the \`link-check-results\` artifact.\n\n`; if (fs.existsSync('docs/link-check-results.md')) { const results = fs.readFileSync('docs/link-check-results.md', 'utf8'); - comment += '### Results\n\n' + results + '\n\n'; + if (comment.length + results.length <= MAX_BODY) { + comment += '### Results\n\n' + results; + } else { + const budget = MAX_BODY - comment.length - 200; + const truncated = results.substring(0, budget); + const lastNewline = truncated.lastIndexOf('\n'); + const safe = lastNewline > 0 ? truncated.substring(0, lastNewline) : truncated; + comment += '### Results (truncated — see artifact for full report)\n\n' + safe; + } } github.rest.issues.createComment({ diff --git a/docs/build-batch.sh b/docs/build-batch.sh deleted file mode 100755 index e2032e2f..00000000 --- a/docs/build-batch.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Build a range of formula pages in sequential batches. -# Usage: START=0 END=1070 BATCH_SIZE=500 bash build-batch.sh -# Intended to be called from CI matrix jobs. - -cd "$(dirname "$0")" - -BROWSE_DIR="browse" -STAGING_DIR=".browse-staging" -DIST_DIR=".vitepress/dist" -MAX_OLD_SPACE="${MAX_OLD_SPACE:-6144}" -BATCH_SIZE="${BATCH_SIZE:-500}" -START="${START:-0}" -END="${END:-0}" - -# Stage all browse pages (move to staging) -echo "=== Staging browse pages ===" -rm -rf "$STAGING_DIR" -mkdir -p "$STAGING_DIR" -find "$BROWSE_DIR" -name "*.md" ! -name "index.md" | while read -r f; do - relpath="${f#$BROWSE_DIR/}" - mkdir -p "$STAGING_DIR/$(dirname "$relpath")" - mv "$f" "$STAGING_DIR/$relpath" -done - -# Enumerate all staged files -mapfile -t all_files < <(find "$STAGING_DIR" -name "*.md" | sort) -total=${#all_files[@]} -actual_end=$((END > total ? total : END)) -echo "=== Building pages $((START+1))-$actual_end of $total (batch size: $BATCH_SIZE) ===" - -# Create the first batch's directory structure -for ((i=START; i/dev/null || true - -batch_num=0 -for ((batch_start=START; batch_start/dev/null || true - added=$(find "$DIST_DIR/browse" -name "*.html" ! -name "index.html" | wc -l | tr -d ' ') - echo " Added $added browse pages" - fi - - # Remove this batch - find "$BROWSE_DIR" -name "*.md" ! -name "index.md" -delete - find "$BROWSE_DIR" -type d -empty -delete 2>/dev/null || true - rm -rf "$DIST_DIR" -done - -# Replace dist with merged output -rm -rf "$DIST_DIR" -mv "$DIST_DIR-saved" "$DIST_DIR" - -# Clean up -rm -rf "$STAGING_DIR" - -echo "=== Batch build complete ===" diff --git a/docs/build.js b/docs/build.js new file mode 100644 index 00000000..52e5c166 --- /dev/null +++ b/docs/build.js @@ -0,0 +1,166 @@ +#!/usr/bin/env node +// Multi-pass VitePress build for the formula site. +// +// VitePress runs out of memory when rendering all ~4,300 generated formula +// pages in a single pass, so we render them in batches. The pipeline: +// 1. node generate.js -> writes browse/.md, licenses/*.md, public/*.json +// 2. stage browse pages -> move them out of the source tree +// 3. vitepress build (pass 0) -> renders the main site (guide, licenses, browse/index) +// 4. for each batch of N pages: +// - move the batch back into browse/ +// - vitepress build -> renders just this batch +// - merge browse/ output into the final dist +// - move the batch back to staging +// 5. promote the final dist +import { mkdir, rm, readdir, rename, cp, access } from "node:fs/promises"; +import { join, dirname, relative } from "node:path"; +import { spawnSync } from "node:child_process"; + +const BROWSE_DIR = "browse"; +const STAGING_DIR = ".browse-staging"; +const DIST_DIR = ".vitepress/dist"; +const FINAL_DIR = ".vitepress/dist-final"; + +const MAX_OLD_SPACE = process.env.MAX_OLD_SPACE ?? "6144"; +const BATCH_SIZE = Number.parseInt(process.env.BATCH_SIZE ?? "500", 10); + +async function pathExists(p) { + try { await access(p); return true; } catch { return false; } +} + +async function findMarkdownFiles(dir) { + const out = []; + async function walk(d) { + const entries = await readdir(d, { withFileTypes: true }); + for (const e of entries) { + const full = join(d, e.name); + if (e.isDirectory()) await walk(full); + else if (e.isFile() && e.name.endsWith(".md")) out.push(full); + } + } + await walk(dir); + return out; +} + +async function countHtmlFiles(dir) { + let n = 0; + async function walk(d) { + const entries = await readdir(d, { withFileTypes: true }); + for (const e of entries) { + const full = join(d, e.name); + if (e.isDirectory()) await walk(full); + else if (e.isFile() && e.name.endsWith(".html")) n++; + } + } + await walk(dir); + return n; +} + +function run(cmd, args, env = process.env) { + const result = spawnSync(cmd, args, { + stdio: "inherit", + env, + shell: process.platform === "win32", + }); + if (result.status !== 0) process.exit(result.status ?? 1); +} + +function runGenerate() { + console.log("=== Generating formula pages ==="); + run("node", ["generate.js"]); +} + +function runVitepressBuild() { + run("npx", ["vitepress", "build"], { + ...process.env, + NODE_OPTIONS: `--max-old-space-size=${MAX_OLD_SPACE}`, + }); +} + +async function moveFile(src, dest) { + await mkdir(dirname(dest), { recursive: true }); + await rename(src, dest); +} + +async function mergeBrowseOutput(distRoot, finalRoot) { + const distBrowse = join(distRoot, "browse"); + if (!(await pathExists(distBrowse))) return 0; + + const finalBrowse = join(finalRoot, "browse"); + await mkdir(finalBrowse, { recursive: true }); + + let added = 0; + const entries = await readdir(distBrowse, { withFileTypes: true }); + for (const e of entries) { + const src = join(distBrowse, e.name); + const dest = join(finalBrowse, e.name); + if (e.isDirectory()) { + await mkdir(dest, { recursive: true }); + await cp(src, dest, { recursive: true }).catch(() => {}); + } else if (e.isFile() && e.name.endsWith(".html") && e.name !== "index.html") { + await cp(src, dest).catch(() => {}); + added++; + } + } + return added; +} + +async function main() { + runGenerate(); + + console.log("=== Staging browse pages ==="); + await rm(STAGING_DIR, { recursive: true, force: true }); + await rm(FINAL_DIR, { recursive: true, force: true }); + await mkdir(STAGING_DIR, { recursive: true }); + + const browseFiles = (await findMarkdownFiles(BROWSE_DIR)) + .filter((f) => !f.endsWith("index.md")); + for (const f of browseFiles) { + await moveFile(f, join(STAGING_DIR, relative(BROWSE_DIR, f))); + } + + console.log("=== Pass 0: Main site (guide, licenses, browse/index) ==="); + runVitepressBuild(); + await rename(DIST_DIR, FINAL_DIR); + console.log(`Main site built: ${await countHtmlFiles(FINAL_DIR)} pages`); + + const staged = (await findMarkdownFiles(STAGING_DIR)).sort(); + const total = staged.length; + const numBatches = Math.ceil(total / BATCH_SIZE); + console.log( + `=== ${total} formula pages in ${numBatches} batches (batch size: ${BATCH_SIZE}) ===` + ); + + for (let batch = 0; batch < numBatches; batch++) { + const start = batch * BATCH_SIZE; + const end = Math.min(start + BATCH_SIZE, total); + console.log(`\n=== Pass ${batch + 1}: ${end - start} pages (${start + 1}-${end} of ${total}) ===`); + + for (let i = start; i < end; i++) { + await moveFile(staged[i], join(BROWSE_DIR, relative(STAGING_DIR, staged[i]))); + } + + runVitepressBuild(); + + const added = await mergeBrowseOutput(DIST_DIR, FINAL_DIR); + console.log(` Added ${added} browse pages`); + + for (let i = start; i < end; i++) { + const f = staged[i]; + await moveFile(join(BROWSE_DIR, relative(STAGING_DIR, f)), f); + } + + await rm(DIST_DIR, { recursive: true, force: true }); + } + + await rm(STAGING_DIR, { recursive: true, force: true }); + await rm(DIST_DIR, { recursive: true, force: true }); + await rename(FINAL_DIR, DIST_DIR); + + console.log(`\n=== Build complete: ${await countHtmlFiles(DIST_DIR)} total HTML pages ===`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/docs/build.sh b/docs/build.sh deleted file mode 100755 index 27495026..00000000 --- a/docs/build.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Multi-pass VitePress build for large formula sites. -# Moves individual .md files in batches to avoid OOM. - -cd "$(dirname "$0")" - -BROWSE_DIR="browse" -STAGING_DIR=".browse-staging" -DIST_DIR=".vitepress/dist" -FINAL_DIR=".vitepress/dist-final" -MAX_OLD_SPACE="${MAX_OLD_SPACE:-6144}" -BATCH_SIZE="${BATCH_SIZE:-500}" - -# Generate all formula pages -echo "=== Generating formula pages ===" -node generate.js - -# Save all browse .md files (except index.md) to staging -echo "=== Staging browse pages ===" -rm -rf "$STAGING_DIR" "$FINAL_DIR" -mkdir -p "$STAGING_DIR" - -# Move all .md files except index.md, preserving directory structure -find "$BROWSE_DIR" -name "*.md" ! -name "index.md" | while read -r f; do - relpath="${f#$BROWSE_DIR/}" - mkdir -p "$STAGING_DIR/$(dirname "$relpath")" - mv "$f" "$STAGING_DIR/$relpath" -done - -echo "=== Pass 0: Main site (guide, licenses, browse/index) ===" -NODE_OPTIONS="--max-old-space-size=$MAX_OLD_SPACE" npx vitepress build -mv "$DIST_DIR" "$FINAL_DIR" -main_pages=$(find "$FINAL_DIR" -name "*.html" | wc -l | tr -d ' ') -echo "Main site built: $main_pages pages" - -# Enumerate all staged .md files -mapfile -t all_files < <(find "$STAGING_DIR" -name "*.md" | sort) -total=${#all_files[@]} -num_batches=$(( (total + BATCH_SIZE - 1) / BATCH_SIZE )) -echo "=== $total formula pages in $num_batches batches (batch size: $BATCH_SIZE) ===" - -# Build each batch -for ((batch=0; batch/dev/null || true - done - # Copy any .html files directly in browse/ - cp "$DIST_DIR/browse/"*.html "$FINAL_DIR/browse/" 2>/dev/null || true - batch_pages=$(find "$DIST_DIR/browse" -name "*.html" ! -name "index.html" | wc -l | tr -d ' ') - echo " Added $batch_pages browse pages" - fi - - # Move files back to staging - for ((i=start; i/dev/null || true - done - # Clean empty directories - find "$BROWSE_DIR" -type d -empty -delete 2>/dev/null || true - - rm -rf "$DIST_DIR" -done - -# Clean up staging -rm -rf "$STAGING_DIR" - -# Final dist -rm -rf "$DIST_DIR" -mv "$FINAL_DIR" "$DIST_DIR" - -total_pages=$(find "$DIST_DIR" -name "*.html" | wc -l | tr -d ' ') -echo "" -echo "=== Build complete: $total_pages total HTML pages ===" diff --git a/docs/package.json b/docs/package.json index d6e49e30..1d9a762a 100644 --- a/docs/package.json +++ b/docs/package.json @@ -3,7 +3,7 @@ "predev": "node generate.js", "dev": "vitepress dev", "prebuild": "node generate.js", - "build": "bash build.sh", + "build": "node build.js", "preview": "vitepress preview", "format": "prettier -w ." },