From 8b1e98b337330286c35d1218c5b3b0a64d7499a4 Mon Sep 17 00:00:00 2001
From: Phil Scott <phil.scott@gmail.com>
Date: Fri, 24 Jul 2026 17:31:53 -0400
Subject: [PATCH] perf(pipeline): stop retaining a parsed DOM per page in
 SiteProjection

RenderedPage carried an AngleSharp IElement for every page in the corpus, and
the projection held it for the life of the build. An IElement keeps its whole
owning document reachable via .Owner, so this retained each page's entire
parsed DOM -- chrome, navigation rail and all -- not just the selector-matched
content. Measured at ~10x the cost of the page's own HTML.

At docs-site scale that is invisible. On a large site it is not: a 22k-page
corpus drove private memory past 26 GB at a quarter of the crawl and failed
with OutOfMemoryException out of OutputGenerationService.FetchPagesAsync,
which reports it as a per-page error rather than the systemic failure it is.

Only one consumer actually read the DOM. BookComposer already re-parsed from
page.Html into its own document; SearchArtifactService needs Sections, which
is pure value data; LinkAuditor only ever wanted the HTML string. So:

- RenderedPage exposes `bool HasContent` and an eager `IReadOnlyList<HeadingSection>`
  in place of `IElement? Content` and `Lazy<...> Sections`.
- SiteProjection.RenderOneAsync extracts sections while the element is still
  live, then returns -- the owning document is unreachable immediately after.
  Eagerness is load-bearing here, not an oversight, and is commented as such.
- LlmsTxtService re-parses page.Html per iteration through an HtmlParser on
  BuildContext, so the document is collectable straight after conversion and
  peak cost tracks concurrency rather than corpus size.

Verified on examples/ScaleStressExample (5000 pages, `-- build`):
peak private memory 1638 MB -> 565 MB (2.9x), with byte-identical output
across all 30,224 generated files.

BREAKING: RenderedPage.Content is removed from the public API. Consumers
matching on it should test HasContent and re-parse Html where they need a DOM.
---
 src/Pennington.Book/BookArtifactService.cs    | 18 +++--
 .../Composition/BookComposer.cs               | 36 +++++-----
 src/Pennington/LlmsTxt/LlmsTxtService.cs      | 66 +++++++++++++------
 src/Pennington/Pipeline/RenderedPage.cs       | 23 ++++---
 src/Pennington/Pipeline/SiteProjection.cs     | 54 ++++++++++-----
 .../Search/SearchArtifactService.cs           | 10 +--
 .../Pennington.Book.Tests/BookTestBuilders.cs | 27 ++------
 .../Pipeline/SiteProjectionTests.cs           | 38 ++++++-----
 8 files changed, 164 insertions(+), 108 deletions(-)

diff --git a/src/Pennington.Book/BookArtifactService.cs b/src/Pennington.Book/BookArtifactService.cs
index 25f1ec8e..72e5f340 100644
--- a/src/Pennington.Book/BookArtifactService.cs
+++ b/src/Pennington.Book/BookArtifactService.cs
@@ -38,7 +38,10 @@ public sealed class BookArtifactService : IFileWatchAware
     private readonly ILogger<BookArtifactService> _logger;
 
     private readonly AsyncLazy<ProjectionData> _projectionLazy;
-    private readonly ConcurrentDictionary<string, AsyncLazy<ComposedBook>> _composed = new(StringComparer.OrdinalIgnoreCase);
+
+    private readonly ConcurrentDictionary<string, AsyncLazy<ComposedBook>> _composed =
+        new(StringComparer.OrdinalIgnoreCase);
+
     private readonly ConcurrentDictionary<string, AsyncLazy<byte[]>> _pdfs = new(StringComparer.OrdinalIgnoreCase);
 
     /// <inheritdoc/>
@@ -104,7 +107,8 @@ internal IReadOnlyList<BookArtifact> EnumerateArtifacts()
     public async Task<byte[]?> GetPdfAsync(string pdfPath)
     {
         var key = pdfPath.Trim('/');
-        var artifact = EnumerateArtifacts().FirstOrDefault(a => a.PdfPath.Equals(key, StringComparison.OrdinalIgnoreCase));
+        var artifact = EnumerateArtifacts()
+            .FirstOrDefault(a => a.PdfPath.Equals(key, StringComparison.OrdinalIgnoreCase));
         if (artifact is null)
         {
             return null;
@@ -139,7 +143,8 @@ private AsyncLazy<ComposedBook> GetComposedLazy(BookArtifact artifact)
     private async Task<ComposedBook> ComposeAsync(BookArtifact artifact)
     {
         var data = await _projectionLazy;
-        var scoped = BookScoping.ScopeToc(data.TocItems, artifact.Book.NormalizedRoutePrefix, _localization, artifact.Locale);
+        var scoped = BookScoping.ScopeToc(data.TocItems, artifact.Book.NormalizedRoutePrefix, _localization,
+            artifact.Locale);
         var tree = await _navigationBuilder.BuildTreeAsync(scoped, currentPath: null, locale: artifact.Locale);
 
         // Version auto-detection lives here (not in the composer) so composer tests stay
@@ -164,7 +169,7 @@ private async Task<ProjectionData> BuildProjectionAsync()
         await foreach (var page in _projection.GetPagesAsync())
         {
             // Reuse the existing "don't extract me" opt-out, and skip pages with no body to compose.
-            if (page.Toc.ExcludeFromLlms || page.Content is null)
+            if (page.Toc.ExcludeFromLlms || !page.HasContent)
             {
                 continue;
             }
@@ -179,7 +184,8 @@ private async Task<ProjectionData> BuildProjectionAsync()
 
     private static string NormalizePreview(string path) => path.Trim('/');
 
-    private static int CountPages(ImmutableList<NavigationTreeItem> tree, IReadOnlyDictionary<string, RenderedPage> pages)
+    private static int CountPages(ImmutableList<NavigationTreeItem> tree,
+        IReadOnlyDictionary<string, RenderedPage> pages)
     {
         var count = 0;
         Walk(tree);
@@ -219,4 +225,4 @@ internal sealed record BookArtifact(
     string? Locale,
     string Slug,
     string PdfPath,
-    string PreviewPath);
+    string PreviewPath);
\ No newline at end of file
diff --git a/src/Pennington.Book/Composition/BookComposer.cs b/src/Pennington.Book/Composition/BookComposer.cs
index 8682be80..0f6cd5c9 100644
--- a/src/Pennington.Book/Composition/BookComposer.cs
+++ b/src/Pennington.Book/Composition/BookComposer.cs
@@ -118,7 +118,8 @@ public string Compose(
             : $"{siteTitle} — {book.Title}";
 
         var sb = new StringBuilder();
-        sb.Append("<!DOCTYPE html>\n<html lang=\"").Append(Encode(ResolveLang(stamp?.Locale))).Append("\">\n<head>\n<meta charset=\"utf-8\">\n");
+        sb.Append("<!DOCTYPE html>\n<html lang=\"").Append(Encode(ResolveLang(stamp?.Locale)))
+            .Append("\">\n<head>\n<meta charset=\"utf-8\">\n");
         sb.Append("<title>").Append(Encode(documentTitle)).Append("</title>\n");
         sb.Append("<style>\n").Append(BookCss);
         if (monochrome)
@@ -133,7 +134,8 @@ public string Compose(
 
         sb.Append("\n</style>\n");
         // PagedConfig must precede the polyfill so the auto-run picks up the readiness callback.
-        sb.Append("<script>window.PagedConfig = { auto: true, after: () => { window.__pagedDone = true; } };</script>\n");
+        sb.Append(
+            "<script>window.PagedConfig = { auto: true, after: () => { window.__pagedDone = true; } };</script>\n");
         sb.Append("<script>\n").Append(PagedPolyfill).Append("\n</script>\n");
         sb.Append("</head>\n<body>\n");
 
@@ -219,7 +221,7 @@ private void RenderNode(
         var key = NormalizePath(node.Route.CanonicalPath.Value);
         if (!string.IsNullOrEmpty(key)
             && pageByPath.TryGetValue(key, out var page)
-            && page.Content is not null
+            && page.HasContent
             && !string.IsNullOrEmpty(page.Html))
         {
             var content = ProcessPageContent(page, depth, slug, rewriteHref, resolveImageSrc);
@@ -471,7 +473,8 @@ private void AppendColophon(StringBuilder sb, BookStamp stamp)
         }
 
         var pennington = BookVersion.Pennington();
-        AppendColophonLine(sb, pennington is null ? "Produced with Pennington" : $"Produced with Pennington {pennington}");
+        AppendColophonLine(sb,
+            pennington is null ? "Produced with Pennington" : $"Produced with Pennington {pennington}");
 
         sb.Append("</section>\n");
     }
@@ -487,7 +490,8 @@ private void AppendToc(
         string? locale)
     {
         sb.Append("<nav class=\"book-toc\">\n");
-        sb.Append("<div class=\"book-toc-heading\">").Append(Encode(Translate(locale, ContentsKey, DefaultContents))).Append("</div>\n");
+        sb.Append("<div class=\"book-toc-heading\">").Append(Encode(Translate(locale, ContentsKey, DefaultContents)))
+            .Append("</div>\n");
         sb.Append("<ol>\n");
 
         // The unwrapped area landing reads as an unnumbered introduction entry.
@@ -545,20 +549,21 @@ private static void AppendTocLevel(
     }
 
     /// <summary>True when <paramref name="node"/> maps to a projected page with a body to compose — the same test <see cref="RenderNode"/> applies before emitting page content.</summary>
-    private static bool HasComposableContent(NavigationTreeItem node, IReadOnlyDictionary<string, RenderedPage> pageByPath)
+    private static bool HasComposableContent(NavigationTreeItem node,
+        IReadOnlyDictionary<string, RenderedPage> pageByPath)
     {
         var key = NormalizePath(node.Route.CanonicalPath.Value);
         return !string.IsNullOrEmpty(key)
-            && pageByPath.TryGetValue(key, out var page)
-            && page.Content is not null
-            && !string.IsNullOrEmpty(page.Html);
+               && pageByPath.TryGetValue(key, out var page)
+               && page.HasContent
+               && !string.IsNullOrEmpty(page.Html);
     }
 
     /// <summary>Resolves a book chrome string: the locale's translation, then the default locale's, then <paramref name="fallback"/> — the same chain as <see cref="BookCatalog"/>.</summary>
     private string Translate(string? locale, string key, string fallback)
         => _translations.Get(locale ?? _localization.DefaultLocale, key)
-            ?? _translations.Get(_localization.DefaultLocale, key)
-            ?? fallback;
+           ?? _translations.Get(_localization.DefaultLocale, key)
+           ?? fallback;
 
     /// <summary>The <c>lang</c> attribute value for <paramref name="locale"/>: the locale's configured <see cref="LocaleInfo.HtmlLang"/> when present, else the locale code itself.</summary>
     private string ResolveLang(string? locale)
@@ -619,11 +624,12 @@ private static string LoadResource(string fileName)
     {
         var assembly = typeof(BookComposer).Assembly;
         var name = Array.Find(
-            assembly.GetManifestResourceNames(),
-            n => n.EndsWith("." + fileName, StringComparison.OrdinalIgnoreCase))
-            ?? throw new InvalidOperationException($"Embedded resource '{fileName}' not found in {assembly.GetName().Name}.");
+                       assembly.GetManifestResourceNames(),
+                       n => n.EndsWith("." + fileName, StringComparison.OrdinalIgnoreCase))
+                   ?? throw new InvalidOperationException(
+                       $"Embedded resource '{fileName}' not found in {assembly.GetName().Name}.");
         using var stream = assembly.GetManifestResourceStream(name)!;
         using var reader = new StreamReader(stream);
         return reader.ReadToEnd();
     }
-}
+}
\ No newline at end of file
diff --git a/src/Pennington/LlmsTxt/LlmsTxtService.cs b/src/Pennington/LlmsTxt/LlmsTxtService.cs
index 5ac17931..6d519c5e 100644
--- a/src/Pennington/LlmsTxt/LlmsTxtService.cs
+++ b/src/Pennington/LlmsTxt/LlmsTxtService.cs
@@ -4,7 +4,7 @@ namespace Pennington.LlmsTxt;
 using System.IO.Abstractions;
 using System.Security.Cryptography;
 using System.Text;
-using AngleSharp.Dom;
+using AngleSharp.Html.Parser;
 using Content;
 using FrontMatter;
 using Infrastructure;
@@ -47,12 +47,11 @@ public LlmsTxtService(
         NavigationBuilder navigationBuilder,
         ILogger<LlmsTxtService> logger)
     {
-        _dataLazy = new AsyncLazy<LlmsTxtData>(
-            () => BuildAsync(
-                projection, contentServices, subtrees,
-                fileSystem, hostingEnvironment,
-                pennOptions, llmsTxtOptions, canonicalBase, navigationBuilder,
-                logger));
+        _dataLazy = new AsyncLazy<LlmsTxtData>(() => BuildAsync(
+            projection, contentServices, subtrees,
+            fileSystem, hostingEnvironment,
+            pennOptions, llmsTxtOptions, canonicalBase, navigationBuilder,
+            logger));
     }
 
     /// <summary>Returns the generated llms.txt index content.</summary>
@@ -122,7 +121,8 @@ private static async Task<LlmsTxtData> BuildAsync(
             RewriteHref: BuildLinkRewriter(linkablePaths, canonicalBase),
             Nodes: new List<RenderedNode>(),
             MarkdownFiles: ImmutableList.CreateBuilder<MarkdownFile>(),
-            FullContent: llmsTxtOptions.GenerateFullFile ? new StringBuilder() : null);
+            FullContent: llmsTxtOptions.GenerateFullFile ? new StringBuilder() : null,
+            Parser: new HtmlParser());
 
         await CollectAsync(tree, depth: 0, ctx);
         await CollectAsync(subtreeOnlyTree, depth: 0, ctx);
@@ -147,7 +147,8 @@ private static async Task<LlmsTxtData> BuildAsync(
             }
 
             var (entryCount, totalTokens) = SummarizeSubtree(ctx.Nodes, subtree, subtrees);
-            var canonicalSelf = canonicalBase.Combine(new UrlPath($"/{subtree.RoutePrefix.TrimStart('/')}llms.txt")).Value;
+            var canonicalSelf = canonicalBase.Combine(new UrlPath($"/{subtree.RoutePrefix.TrimStart('/')}llms.txt"))
+                .Value;
             sb.AppendLine($"canonical: {canonicalSelf}");
             sb.AppendLine($"entries: {entryCount}");
             sb.AppendLine($"tokens: ~{FormatTokenEstimate(totalTokens)}");
@@ -210,10 +211,12 @@ private static async Task<ImmutableList<LlmsSubtree>> CollectSubtreesAsync(
                 return s;
             }
         }
+
         return null;
     }
 
-    private static void AppendFrontDoorPreamble(StringBuilder sb, string? userHeader, PenningtonOptions pennOptions, CanonicalBaseUrl canonicalBase)
+    private static void AppendFrontDoorPreamble(StringBuilder sb, string? userHeader, PenningtonOptions pennOptions,
+        CanonicalBaseUrl canonicalBase)
     {
         if (userHeader is not null)
         {
@@ -243,7 +246,8 @@ private static void AppendFrontDoorPreamble(StringBuilder sb, string? userHeader
         sb.AppendLine();
     }
 
-    private static void AppendMapBlock(StringBuilder sb, ImmutableList<LlmsSubtree> subtrees, List<RenderedNode> renderedNodes, CanonicalBaseUrl canonicalBase)
+    private static void AppendMapBlock(StringBuilder sb, ImmutableList<LlmsSubtree> subtrees,
+        List<RenderedNode> renderedNodes, CanonicalBaseUrl canonicalBase)
     {
         if (subtrees.Count == 0)
         {
@@ -263,11 +267,13 @@ private static void AppendMapBlock(StringBuilder sb, ImmutableList<LlmsSubtree>
             var desc = string.IsNullOrWhiteSpace(s.Description) ? "" : $" — {s.Description}";
             sb.AppendLine($"- [{s.Title}]({url}) ({entryLabel}, ~{tokenLabel} tokens){desc}");
         }
+
         sb.AppendLine();
     }
 
     /// <summary>Counts entries and sums token estimates for leaves whose nearest matching subtree is <paramref name="target"/>.</summary>
-    private static (int Count, int Tokens) SummarizeSubtree(List<RenderedNode> renderedNodes, LlmsSubtree target, ImmutableList<LlmsSubtree> allSubtrees)
+    private static (int Count, int Tokens) SummarizeSubtree(List<RenderedNode> renderedNodes, LlmsSubtree target,
+        ImmutableList<LlmsSubtree> allSubtrees)
     {
         var count = 0;
         var tokens = 0;
@@ -279,6 +285,7 @@ private static (int Count, int Tokens) SummarizeSubtree(List<RenderedNode> rende
                 tokens += leaf.Tokens;
             }
         }
+
         return (count, tokens);
     }
 
@@ -290,6 +297,7 @@ private static string FormatTokenEstimate(int tokens)
             var thousands = tokens / 1000.0;
             return thousands >= 10 ? $"{(int)thousands}k" : $"{thousands:0.#}k";
         }
+
         return tokens.ToString();
     }
 
@@ -324,7 +332,7 @@ private static async Task CollectAsync(ImmutableList<NavigationTreeItem> items,
                 continue;
             }
 
-            if (page.Content is null)
+            if (!page.HasContent)
             {
                 continue;
             }
@@ -337,7 +345,16 @@ private static async Task CollectAsync(ImmutableList<NavigationTreeItem> items,
                 continue;
             }
 
-            var markdown = HtmlToMarkdownConverter.Convert(page.Content, ctx.RewriteHref).Trim();
+            // The projection retains only Html, so re-parse here. The document is local to this
+            // iteration and collectable straight after, which keeps peak cost proportional to
+            // concurrency rather than to corpus size.
+            var content = ctx.Parser.ParseDocument(page.Html).Body;
+            if (content is null)
+            {
+                continue;
+            }
+
+            var markdown = HtmlToMarkdownConverter.Convert(content, ctx.RewriteHref).Trim();
             if (string.IsNullOrWhiteSpace(markdown))
             {
                 continue;
@@ -353,7 +370,8 @@ private static async Task CollectAsync(ImmutableList<NavigationTreeItem> items,
             var linkUrl = BuildCoLocatedMarkdownUrl(ctx.CanonicalBase, key);
             var description = frontMatter?.Description ?? page.Toc.Description;
             var derived = page.Origin?.Value is MarkdownOrigin md2 ? md2.Parsed.Derived : null;
-            var sidecarHeader = BuildSidecarHeader(item, frontMatter, description, ctx.CanonicalBase, linkUrl, rendition, derived);
+            var sidecarHeader = BuildSidecarHeader(item, frontMatter, description, ctx.CanonicalBase, linkUrl,
+                rendition, derived);
             var sidecarContent = sidecarHeader + body;
 
             ctx.MarkdownFiles.Add(new MarkdownFile(new FilePath(mdPath), Encoding.UTF8.GetBytes(sidecarContent)));
@@ -407,6 +425,7 @@ private static void RenderBucket(List<RenderedNode> nodes, StringBuilder sb, Fun
                         sb.AppendLine();
                         anyLeafEmittedSinceFlush = false;
                     }
+
                     break;
 
                 case LeafNode leaf when include(leaf):
@@ -418,6 +437,7 @@ private static void RenderBucket(List<RenderedNode> nodes, StringBuilder sb, Fun
                         sb.AppendLine($"{new string('#', level)} {s.Title}");
                         sb.AppendLine();
                     }
+
                     pendingSections.Clear();
                     var desc = leaf.Description is { Length: > 0 } d ? $": {d}" : "";
                     sb.AppendLine($"- [{leaf.Title}]({leaf.SidecarUrl}){desc}");
@@ -448,6 +468,7 @@ private static string BuildSidecarHeader(
         {
             sb.AppendLine($"description: {YamlScalar(description)}");
         }
+
         // URLs and hashes contain `:` but never `: ` (colon-space) or whitespace, so they
         // parse correctly as bare YAML scalars. Bare emission keeps them readable.
         sb.AppendLine($"canonical_url: {canonicalBase.Combine(new UrlPath(item.Route.CanonicalPath.Value)).Value}");
@@ -487,16 +508,19 @@ private static string YamlScalar(string s)
         {
             return "\"\"";
         }
+
         // Conservative: quote anything that isn't plain printable text to avoid YAML edge cases.
         var needsQuote = false;
         foreach (var c in s)
         {
-            if (c is ':' or '#' or '\n' or '\r' or '\t' or '"' or '\'' or '\\' or '{' or '}' or '[' or ']' or ',' or '&' or '*' or '!' or '|' or '>' or '%' or '@' or '`')
+            if (c is ':' or '#' or '\n' or '\r' or '\t' or '"' or '\'' or '\\' or '{' or '}' or '[' or ']' or ',' or '&'
+                or '*' or '!' or '|' or '>' or '%' or '@' or '`')
             {
                 needsQuote = true;
                 break;
             }
         }
+
         if (!needsQuote && (s.StartsWith(' ') || s.EndsWith(' ')))
         {
             needsQuote = true;
@@ -589,7 +613,8 @@ private static string BuildCoLocatedMarkdownUrl(CanonicalBaseUrl canonicalBase,
         IWebHostEnvironment hostingEnvironment,
         PenningtonOptions pennOptions)
     {
-        var contentRoot = FilePath.ResolveAgainstRoot(pennOptions.ContentRootPath.Value, hostingEnvironment.ContentRootPath);
+        var contentRoot =
+            FilePath.ResolveAgainstRoot(pennOptions.ContentRootPath.Value, hostingEnvironment.ContentRootPath);
 
         var headerPath = fileSystem.Path.Combine(contentRoot, "llms-header.txt");
         if (!fileSystem.File.Exists(headerPath))
@@ -615,7 +640,9 @@ internal record LlmsTxtData(
     public record MarkdownFile(FilePath OutputPath, byte[] Content);
 
     private abstract record RenderedNode;
+
     private sealed record SectionNode(int Depth, string Title) : RenderedNode;
+
     private sealed record LeafNode(
         string Title,
         string CanonicalPath,
@@ -632,5 +659,6 @@ private sealed record BuildContext(
         Func<string, string> RewriteHref,
         List<RenderedNode> Nodes,
         ImmutableList<MarkdownFile>.Builder MarkdownFiles,
-        StringBuilder? FullContent);
-}
+        StringBuilder? FullContent,
+        HtmlParser Parser);
+}
\ No newline at end of file
diff --git a/src/Pennington/Pipeline/RenderedPage.cs b/src/Pennington/Pipeline/RenderedPage.cs
index dd89706e..9bc64964 100644
--- a/src/Pennington/Pipeline/RenderedPage.cs
+++ b/src/Pennington/Pipeline/RenderedPage.cs
@@ -1,6 +1,5 @@
 namespace Pennington.Pipeline;
 
-using AngleSharp.Dom;
 using Content;
 using Routing;
 using Search;
@@ -29,12 +28,16 @@ public record EndpointOrigin(string DirectUrl);
 {
     /// <summary>Wrapped case instance; inspect via pattern matching on the case types.</summary>
     public object? Value { get; }
+
     /// <summary>Wraps a <see cref="MarkdownOrigin"/>.</summary>
     public PageOrigin(MarkdownOrigin value) { Value = value; }
+
     /// <summary>Wraps an <see cref="EndpointOrigin"/>.</summary>
     public PageOrigin(EndpointOrigin value) { Value = value; }
+
     /// <summary>Implicit conversion from <see cref="MarkdownOrigin"/>.</summary>
     public static implicit operator PageOrigin(MarkdownOrigin value) => new(value);
+
     /// <summary>Implicit conversion from <see cref="EndpointOrigin"/>.</summary>
     public static implicit operator PageOrigin(EndpointOrigin value) => new(value);
 }
@@ -52,22 +55,24 @@ public record EndpointOrigin(string DirectUrl);
 /// item — the HTML is still available, just without front-matter context.
 /// </para>
 /// <para>
-/// <see cref="Content"/> is owned by a per-page AngleSharp browsing context —
-/// the projection holds the context for the page's lifetime. Treat
-/// <see cref="Content"/> as read-only: mutating it corrupts other consumers'
-/// views.
+/// No parsed DOM is retained. Holding an AngleSharp <c>IElement</c> keeps its whole
+/// owning document alive, so a corpus-sized cache of them costs roughly ten times
+/// what <see cref="Html"/> costs — enough to exhaust memory on large sites. Anything
+/// derived from the DOM is either captured here as value data
+/// (<see cref="Sections"/>) or re-parsed from <see cref="Html"/> on demand by the
+/// consumer that needs it, which keeps the parse transient.
 /// </para>
 /// </summary>
 /// <param name="Route">Canonical route for the page.</param>
 /// <param name="Toc">TOC entry that drove the page's inclusion.</param>
 /// <param name="Origin">Origin information (markdown / endpoint), or <c>null</c> for Razor / programmatic pages.</param>
 /// <param name="Html">Post-pipeline HTML of the selector-matched content element; empty for endpoint pages.</param>
-/// <param name="Content">Parsed content element from the post-pipeline HTML; null for endpoint pages.</param>
-/// <param name="Sections">Lazy heading-section split of <see cref="Content"/>; empty for endpoint pages.</param>
+/// <param name="HasContent">Whether the page produced a content element; <c>false</c> for endpoint pages.</param>
+/// <param name="Sections">Heading-section split of the page content; empty for endpoint pages.</param>
 public sealed record RenderedPage(
     ContentRoute Route,
     ContentTocItem Toc,
     PageOrigin? Origin,
     string Html,
-    IElement? Content,
-    Lazy<IReadOnlyList<HeadingSection>> Sections);
+    bool HasContent,
+    IReadOnlyList<HeadingSection> Sections);
\ No newline at end of file
diff --git a/src/Pennington/Pipeline/SiteProjection.cs b/src/Pennington/Pipeline/SiteProjection.cs
index 8a8d3c8c..c45a175e 100644
--- a/src/Pennington/Pipeline/SiteProjection.cs
+++ b/src/Pennington/Pipeline/SiteProjection.cs
@@ -101,6 +101,7 @@ public FileWatchResponse OnFileChanged(FileChangeNotification change)
                 wildcard = true;
                 break;
             }
+
             foreach (var route in routes.Value)
             {
                 (affected ??= []).Add(Normalize(route.CanonicalPath.Value));
@@ -127,11 +128,13 @@ public FileWatchResponse OnFileChanged(FileChangeNotification change)
                 }
             }
         }
+
         return FileWatchResponse.Refreshed;
     }
 
     /// <inheritdoc/>
-    public async IAsyncEnumerable<RenderedPage> GetPagesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
+    public async IAsyncEnumerable<RenderedPage> GetPagesAsync(
+        [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
         ThrowIfReentrant();
         await _seededLazy;
@@ -142,6 +145,7 @@ public async IAsyncEnumerable<RenderedPage> GetPagesAsync([EnumeratorCancellatio
         {
             snapshot = _pages;
         }
+
         foreach (var page in snapshot)
         {
             cancellationToken.ThrowIfCancellationRequested();
@@ -164,6 +168,7 @@ public async IAsyncEnumerable<RenderedPage> GetPagesAsync([EnumeratorCancellatio
             snapshot = _pages;
             index = _routeIndex;
         }
+
         return index.TryGetValue(key, out var i) ? snapshot[i] : null;
     }
 
@@ -180,6 +185,7 @@ private async Task<bool> SeedAsync()
             _pages = pages;
             _routeIndex = routeIndex;
         }
+
         return true;
     }
 
@@ -191,6 +197,7 @@ private async Task RefreshIfStaleAsync(CancellationToken cancellationToken)
         {
             hasWork = _staleAll || _staleRoutes.Count > 0;
         }
+
         if (!hasWork) return;
 
         await _refreshGate.WaitAsync(cancellationToken);
@@ -209,12 +216,14 @@ private async Task RefreshIfStaleAsync(CancellationToken cancellationToken)
                 {
                     return;
                 }
+
                 // Snapshot the current cache so the build can reuse unchanged entries.
                 var builder = ImmutableDictionary.CreateBuilder<string, RenderedPage>(StringComparer.OrdinalIgnoreCase);
                 foreach (var (key, idx) in _routeIndex)
                 {
                     builder[key] = _pages[idx];
                 }
+
                 reusable = builder.ToImmutable();
             }
 
@@ -295,9 +304,11 @@ private static void ThrowIfReentrant()
                 {
                     continue;
                 }
+
                 _parsedByPath[key] = await _enrichment.EnrichAsync(parsed);
                 seen.Add(key);
             }
+
             foreach (var key in staleRoutes)
             {
                 if (!seen.Contains(key))
@@ -306,6 +317,7 @@ private static void ThrowIfReentrant()
                 }
             }
         }
+
         var parseElapsed = Stopwatch.GetElapsedTime(parseStart);
         var parsedByPath = _parsedByPath;
 
@@ -326,6 +338,7 @@ private static void ThrowIfReentrant()
                 {
                     continue;
                 }
+
                 _sourceByPath[key] = discovered.Source;
             }
         }
@@ -339,6 +352,7 @@ private static void ThrowIfReentrant()
                 {
                     continue;
                 }
+
                 // First fresh write this pass replaces the stale value; later collisions keep an
                 // already-written LlmsOnlySource but otherwise let the last non-llms source win.
                 if (seen.Add(key) || _sourceByPath[key] is not LlmsOnlySource)
@@ -346,6 +360,7 @@ private static void ThrowIfReentrant()
                     _sourceByPath[key] = discovered.Source;
                 }
             }
+
             foreach (var key in staleRoutes)
             {
                 if (!seen.Contains(key))
@@ -354,6 +369,7 @@ private static void ThrowIfReentrant()
                 }
             }
         }
+
         var discoverElapsed = Stopwatch.GetElapsedTime(discoverStart);
         var sourceByPath = _sourceByPath;
 
@@ -412,8 +428,8 @@ await Parallel.ForEachAsync(
                 Toc: toc,
                 Origin: new EndpointOrigin(url),
                 Html: "",
-                Content: null,
-                Sections: new Lazy<IReadOnlyList<HeadingSection>>(() => []));
+                HasContent: false,
+                Sections: []);
         }
 
         var renderElapsed = Stopwatch.GetElapsedTime(renderStart);
@@ -442,6 +458,7 @@ await Parallel.ForEachAsync(
                     // Defensive: deduplicate if two TOC items end up at the same canonical path.
                     continue;
                 }
+
                 indexBuilder[key] = pagesBuilder.Count;
                 pagesBuilder.Add(page);
             }
@@ -466,17 +483,19 @@ await Parallel.ForEachAsync(
                 var rendered = await RenderInProcessAsync(_renderer, _xrefResolver, llmsParsed);
                 if (rendered is null)
                 {
-                    _logger.LogWarning("SiteProjection: failed to render llms-only item {Path}", toc.Route.CanonicalPath.Value);
+                    _logger.LogWarning("SiteProjection: failed to render llms-only item {Path}",
+                        toc.Route.CanonicalPath.Value);
                     return null;
                 }
+
                 var (html, element) = rendered.Value;
                 return new RenderedPage(
                     Route: toc.Route,
                     Toc: toc,
                     Origin: new MarkdownOrigin(llmsParsed),
                     Html: html,
-                    Content: element,
-                    Sections: BuildSectionsLazy(_extractor, element));
+                    HasContent: true,
+                    Sections: ExtractSections(_extractor, element));
             }
 
             var fetched = await _fetcher.FetchContentAsync(toc.Route.CanonicalPath.Value, _options.ContentSelector, ct);
@@ -496,8 +515,8 @@ await Parallel.ForEachAsync(
                 Toc: toc,
                 Origin: origin,
                 Html: fetched.OuterHtml,
-                Content: fetched,
-                Sections: BuildSectionsLazy(_extractor, fetched));
+                HasContent: true,
+                Sections: ExtractSections(_extractor, fetched));
         }
         catch (Exception ex) when (ex is not SelfFetchUnavailableException)
         {
@@ -511,9 +530,11 @@ await Parallel.ForEachAsync(
         // the search index / llms.txt would silently ship with zero pages.
     }
 
-    private static Lazy<IReadOnlyList<HeadingSection>> BuildSectionsLazy(
+    // Eager, not lazy: the element's owning document is released the moment RenderOneAsync
+    // returns, so anything derived from the DOM has to be captured as value data here.
+    private static IReadOnlyList<HeadingSection> ExtractSections(
         HeadingSectionExtractor extractor, IElement element)
-        => new(() => extractor.Extract(element, excludeCodeBlocks: true));
+        => extractor.Extract(element, excludeCodeBlocks: true);
 
     private static async Task<(string Html, IElement Element)?> RenderInProcessAsync(
         IContentRenderer renderer, XrefResolvingService xrefResolver, ParsedItem parsed)
@@ -545,7 +566,8 @@ private static Lazy<IReadOnlyList<HeadingSection>> BuildSectionsLazy(
     /// endpoints whose URL contains a route parameter — the projection needs a
     /// concrete URL, not a pattern.
     /// </summary>
-    private static IEnumerable<(ContentTocItem Toc, string Url)> CollectEndpointEntries(EndpointDataSource endpointDataSource)
+    private static IEnumerable<(ContentTocItem Toc, string Url)> CollectEndpointEntries(
+        EndpointDataSource endpointDataSource)
     {
         foreach (var endpoint in endpointDataSource.Endpoints)
         {
@@ -570,8 +592,7 @@ private static Lazy<IReadOnlyList<HeadingSection>> BuildSectionsLazy(
             var canonicalPath = new UrlPath(url);
             var contentRoute = new ContentRoute
             {
-                CanonicalPath = canonicalPath,
-                OutputFile = new FilePath(url.TrimStart('/')),
+                CanonicalPath = canonicalPath, OutputFile = new FilePath(url.TrimStart('/')),
             };
             var hierarchyParts = url.Trim('/').Split('/', StringSplitOptions.RemoveEmptyEntries);
             yield return (
@@ -581,13 +602,10 @@ private static Lazy<IReadOnlyList<HeadingSection>> BuildSectionsLazy(
                     Order: int.MaxValue,
                     HierarchyParts: hierarchyParts,
                     SectionLabel: null,
-                    Locale: null)
-                {
-                    Description = meta.Description,
-                },
+                    Locale: null) { Description = meta.Description, },
                 url);
         }
     }
 
     private static string Normalize(string canonicalPath) => canonicalPath.Trim('/');
-}
+}
\ No newline at end of file
diff --git a/src/Pennington/Search/SearchArtifactService.cs b/src/Pennington/Search/SearchArtifactService.cs
index 20e3e6aa..70e63031 100644
--- a/src/Pennington/Search/SearchArtifactService.cs
+++ b/src/Pennington/Search/SearchArtifactService.cs
@@ -36,8 +36,8 @@ public SearchArtifactService(
         LocalizationOptions localization,
         ContentRecordRegistry recordRegistry)
     {
-        _filesLazy = new AsyncLazy<IReadOnlyDictionary<string, byte[]>>(
-            () => BuildAllAsync(projection, corpusBuilder, indexBuilder, localization, recordRegistry));
+        _filesLazy = new AsyncLazy<IReadOnlyDictionary<string, byte[]>>(() =>
+            BuildAllAsync(projection, corpusBuilder, indexBuilder, localization, recordRegistry));
     }
 
     /// <summary>Returns every artifact keyed by its relative output path (e.g. <c>search/en/index.json</c>).</summary>
@@ -77,7 +77,7 @@ private static async Task<IReadOnlyDictionary<string, byte[]>> BuildAllAsync(
         {
             // Endpoint entries and llms-only pages have no HTML body to index. The projection
             // already produced empty content for them; skip rather than emitting blank records.
-            if (page.Toc.ExcludeFromSearch || page.Content is null)
+            if (page.Toc.ExcludeFromSearch || !page.HasContent)
             {
                 continue;
             }
@@ -95,7 +95,7 @@ private static async Task<IReadOnlyDictionary<string, byte[]>> BuildAllAsync(
 
             // One record per heading section (plus a page-lead record) so results are
             // heading-level and deep-link to anchors.
-            foreach (var section in page.Sections.Value)
+            foreach (var section in page.Sections)
             {
                 list.Add(corpusBuilder.BuildSection(page.Toc, section, record?.Metadata));
             }
@@ -115,4 +115,4 @@ private static async Task<IReadOnlyDictionary<string, byte[]>> BuildAllAsync(
 
         return files;
     }
-}
+}
\ No newline at end of file
diff --git a/tests/Pennington.Book.Tests/BookTestBuilders.cs b/tests/Pennington.Book.Tests/BookTestBuilders.cs
index 430b6a6a..bfb70097 100644
--- a/tests/Pennington.Book.Tests/BookTestBuilders.cs
+++ b/tests/Pennington.Book.Tests/BookTestBuilders.cs
@@ -7,7 +7,6 @@ namespace Pennington.Book.Tests;
 using Pennington.Navigation;
 using Pennington.Pipeline;
 using Pennington.Routing;
-using Pennington.Search;
 
 /// <summary>Shared builders for fake navigation trees and inline-HTML rendered pages.</summary>
 internal static class BookTestBuilders
@@ -16,22 +15,14 @@ internal static class BookTestBuilders
 
     public static NavigationTreeItem Node(string title, string path, params NavigationTreeItem[] children)
     {
-        var route = new ContentRoute
-        {
-            CanonicalPath = new UrlPath(path),
-            OutputFile = new FilePath(path.Trim('/')),
-        };
+        var route = new ContentRoute { CanonicalPath = new UrlPath(path), OutputFile = new FilePath(path.Trim('/')), };
         return new NavigationTreeItem(title, route, 0, null, false, false, [.. children]);
     }
 
     /// <summary>A section node with no page of its own (empty route), like an auto-created folder header.</summary>
     public static NavigationTreeItem Section(string title, params NavigationTreeItem[] children)
     {
-        var route = new ContentRoute
-        {
-            CanonicalPath = new UrlPath(""),
-            OutputFile = new FilePath(""),
-        };
+        var route = new ContentRoute { CanonicalPath = new UrlPath(""), OutputFile = new FilePath(""), };
         return new NavigationTreeItem(title, route, 0, null, false, false, [.. children]);
     }
 
@@ -42,14 +33,10 @@ public static NavigationTreeItem Section(string title, params NavigationTreeItem
     public static RenderedPage Page(string path, string bodyHtml, string title = "Page Title")
     {
         var html = $"<article id=\"main-content\"><h1>{title}</h1>{bodyHtml}</article>";
-        var content = Parser.ParseDocument(html).Body!;
-        var route = new ContentRoute
-        {
-            CanonicalPath = new UrlPath(path),
-            OutputFile = new FilePath(path.Trim('/')),
-        };
-        var toc = new ContentTocItem(title, route, 0, path.Trim('/').Split('/', StringSplitOptions.RemoveEmptyEntries), null, null);
-        return new RenderedPage(route, toc, null, html, content, new Lazy<IReadOnlyList<HeadingSection>>(() => []));
+        var route = new ContentRoute { CanonicalPath = new UrlPath(path), OutputFile = new FilePath(path.Trim('/')), };
+        var toc = new ContentTocItem(title, route, 0, path.Trim('/').Split('/', StringSplitOptions.RemoveEmptyEntries),
+            null, null);
+        return new RenderedPage(route, toc, null, html, HasContent: true, Sections: []);
     }
 
     public static IReadOnlyDictionary<string, RenderedPage> PageMap(params RenderedPage[] pages)
@@ -65,4 +52,4 @@ public static IReadOnlyDictionary<string, RenderedPage> PageMap(params RenderedP
 
     /// <summary>Parses composed book HTML and returns the document for querying.</summary>
     public static IDocument Parse(string html) => Parser.ParseDocument(html);
-}
+}
\ No newline at end of file
diff --git a/tests/Pennington.Tests/Pipeline/SiteProjectionTests.cs b/tests/Pennington.Tests/Pipeline/SiteProjectionTests.cs
index 4512b51e..06a92d71 100644
--- a/tests/Pennington.Tests/Pipeline/SiteProjectionTests.cs
+++ b/tests/Pennington.Tests/Pipeline/SiteProjectionTests.cs
@@ -53,15 +53,15 @@ public async Task EndpointEntry_YieldsEndpointOriginWithoutFetch()
         only.Toc.Title.ShouldBe("Agent Context");
         only.Toc.Description.ShouldBe("Build context for agents");
         only.Html.ShouldBe("");
-        only.Content.ShouldBeNull();
+        only.HasContent.ShouldBeFalse();
 
         // Endpoint entries route to the user-defined URL — no fetch happens, so the
         // origin carries the DirectUrl the projection's downstream consumers point at.
-        only.Origin.ShouldNotBeNull().Value.ShouldBeOfType<EndpointOrigin>().DirectUrl.ShouldBe("/_llms/agent-context.md");
+        only.Origin.ShouldNotBeNull().Value.ShouldBeOfType<EndpointOrigin>().DirectUrl
+            .ShouldBe("/_llms/agent-context.md");
 
-        // Lazy sections for endpoint entries materialize to an empty list rather than
-        // throwing on null content.
-        only.Sections.Value.ShouldBeEmpty();
+        // Endpoint entries carry an empty section list rather than throwing on absent content.
+        only.Sections.ShouldBeEmpty();
     }
 
     [Fact]
@@ -89,7 +89,8 @@ public async Task GetPageAsync_KnownPath_ReturnsMatchingEntry()
             new LlmsTxtEntryMetadata("Agent Context"));
         var projection = CreateProjection(endpointDataSource: new StubEndpointDataSource(endpoint));
 
-        var page = await projection.GetPageAsync(new UrlPath("/_llms/agent-context.md"), TestContext.Current.CancellationToken);
+        var page = await projection.GetPageAsync(new UrlPath("/_llms/agent-context.md"),
+            TestContext.Current.CancellationToken);
 
         page.ShouldNotBeNull();
         page.Toc.Title.ShouldBe("Agent Context");
@@ -208,8 +209,8 @@ public async Task GetPageAsync_InsideMaterialization_FailsFastInsteadOfDeadlocki
         var projection = CreateProjection();
         using var scope = CorpusFetchScope.EnterMaterialization();
 
-        var ex = await Should.ThrowAsync<InvalidOperationException>(
-            () => projection.GetPageAsync(new UrlPath("/page/"), TestContext.Current.CancellationToken));
+        var ex = await Should.ThrowAsync<InvalidOperationException>(() =>
+            projection.GetPageAsync(new UrlPath("/page/"), TestContext.Current.CancellationToken));
 
         ex.Message.ShouldContain("self-deadlock");
     }
@@ -242,7 +243,8 @@ private static SiteProjection CreateProjection(
             enrichment: new MetadataEnrichmentService([]),
             renderer: renderer ?? new StubRenderer(),
             xrefResolver: new XrefResolvingService(new XrefResolver([])),
-            fetcher: new RenderedHtmlFetcher(dispatcher ?? new StubDispatcher(), NullLogger<RenderedHtmlFetcher>.Instance),
+            fetcher: new RenderedHtmlFetcher(dispatcher ?? new StubDispatcher(),
+                NullLogger<RenderedHtmlFetcher>.Instance),
             extractor: new HeadingSectionExtractor(),
             options: new SiteProjectionOptions(),
             endpointDataSource: endpointDataSource ?? new StubEndpointDataSource(),
@@ -281,7 +283,8 @@ private sealed class StubDispatcher : IInProcessHttpDispatcher
 
         private sealed class StubHandler : HttpMessageHandler
         {
-            protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+            protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request,
+                CancellationToken cancellationToken)
                 => Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.NotFound));
         }
     }
@@ -306,7 +309,8 @@ public HttpClient CreateClient()
 
         private sealed class OkHandler : HttpMessageHandler
         {
-            protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+            protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request,
+                CancellationToken cancellationToken)
                 => Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.OK)
                 {
                     Content = new StringContent(
@@ -343,7 +347,8 @@ public HttpClient CreateClient() =>
 
         private sealed class Handler(string html) : HttpMessageHandler
         {
-            protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+            protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request,
+                CancellationToken cancellationToken)
                 => Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.OK)
                 {
                     Content = new StringContent(html, System.Text.Encoding.UTF8, "text/html"),
@@ -353,7 +358,8 @@ protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage reques
 
     // A service that claims a single route with one source, one indexable entry, and an optional
     // parsed body — enough to stage a same-route collision between two services in the projection.
-    private sealed class CollidingContentService(ContentSource source, ContentTocItem toc, ParsedItem? parsed) : IContentService
+    private sealed class CollidingContentService(ContentSource source, ContentTocItem toc, ParsedItem? parsed)
+        : IContentService
     {
         public async IAsyncEnumerable<DiscoveredItem> DiscoverAsync()
         {
@@ -367,6 +373,7 @@ public async IAsyncEnumerable<ParsedItem> ParseContentAsync()
             {
                 yield return parsed;
             }
+
             await Task.CompletedTask;
         }
 
@@ -389,8 +396,7 @@ private sealed class SinglePageContentService : IContentService
     {
         private static readonly ContentRoute Route = new()
         {
-            CanonicalPath = new UrlPath("/page/"),
-            OutputFile = new FilePath("page/index.html"),
+            CanonicalPath = new UrlPath("/page/"), OutputFile = new FilePath("page/index.html"),
         };
 
         public IAsyncEnumerable<DiscoveredItem> DiscoverAsync() => AsyncEnumerable.Empty<DiscoveredItem>();
@@ -408,4 +414,4 @@ private sealed class SinglePageContentService : IContentService
         public string DefaultSectionLabel => "";
         public int SearchPriority => 0;
     }
-}
+}
\ No newline at end of file