Skip to content

perf(pipeline): stop retaining a parsed DOM per page in SiteProjection #69

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
phil-scott-78 merged 1 commit into from
Jul 24, 2026
Merged

perf(pipeline): stop retaining a parsed DOM per page in SiteProjection #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions src/Pennington.Book/BookArtifactService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,10 @@ public sealed class BookArtifactService : IFileWatchAware
private readonly ILogger<BookArtifactService> _logger;

private readonly AsyncLazy<ProjectionData> _projectionLazy;
private readonly ConcurrentDictionary<string, AsyncLazy<ComposedBook>> _composed = new(StringComparer.OrdinalIgnoreCase);

private readonly ConcurrentDictionary<string, AsyncLazy<ComposedBook>> _composed =
new(StringComparer.OrdinalIgnoreCase);

private readonly ConcurrentDictionary<string, AsyncLazy<byte[]>> _pdfs = new(StringComparer.OrdinalIgnoreCase);

/// <inheritdoc/>
Expand Down Expand Up @@ -104,7 +107,8 @@ internal IReadOnlyList<BookArtifact> EnumerateArtifacts()
public async Task<byte[]?> GetPdfAsync(string pdfPath)
{
var key = pdfPath.Trim('/');
var artifact = EnumerateArtifacts().FirstOrDefault(a => a.PdfPath.Equals(key, StringComparison.OrdinalIgnoreCase));
var artifact = EnumerateArtifacts()
.FirstOrDefault(a => a.PdfPath.Equals(key, StringComparison.OrdinalIgnoreCase));
if (artifact is null)
{
return null;
Expand Down Expand Up @@ -139,7 +143,8 @@ private AsyncLazy<ComposedBook> GetComposedLazy(BookArtifact artifact)
private async Task<ComposedBook> ComposeAsync(BookArtifact artifact)
{
var data = await _projectionLazy;
var scoped = BookScoping.ScopeToc(data.TocItems, artifact.Book.NormalizedRoutePrefix, _localization, artifact.Locale);
var scoped = BookScoping.ScopeToc(data.TocItems, artifact.Book.NormalizedRoutePrefix, _localization,
artifact.Locale);
var tree = await _navigationBuilder.BuildTreeAsync(scoped, currentPath: null, locale: artifact.Locale);

// Version auto-detection lives here (not in the composer) so composer tests stay
Expand All @@ -164,7 +169,7 @@ private async Task<ProjectionData> BuildProjectionAsync()
await foreach (var page in _projection.GetPagesAsync())
{
// Reuse the existing "don't extract me" opt-out, and skip pages with no body to compose.
if (page.Toc.ExcludeFromLlms || page.Content is null)
if (page.Toc.ExcludeFromLlms || !page.HasContent)
{
continue;
}
Expand All @@ -179,7 +184,8 @@ private async Task<ProjectionData> BuildProjectionAsync()

private static string NormalizePreview(string path) => path.Trim('/');

private static int CountPages(ImmutableList<NavigationTreeItem> tree, IReadOnlyDictionary<string, RenderedPage> pages)
private static int CountPages(ImmutableList<NavigationTreeItem> tree,
IReadOnlyDictionary<string, RenderedPage> pages)
{
var count = 0;
Walk(tree);
Expand Down Expand Up @@ -219,4 +225,4 @@ internal sealed record BookArtifact(
string? Locale,
string Slug,
string PdfPath,
string PreviewPath);
string PreviewPath);
36 changes: 21 additions & 15 deletions src/Pennington.Book/Composition/BookComposer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,8 @@ public string Compose(
: $"{siteTitle} — {book.Title}";

var sb = new StringBuilder();
sb.Append("<!DOCTYPE html>\n<html lang=\"").Append(Encode(ResolveLang(stamp?.Locale))).Append("\">\n<head>\n<meta charset=\"utf-8\">\n");
sb.Append("<!DOCTYPE html>\n<html lang=\"").Append(Encode(ResolveLang(stamp?.Locale)))
.Append("\">\n<head>\n<meta charset=\"utf-8\">\n");
sb.Append("<title>").Append(Encode(documentTitle)).Append("</title>\n");
sb.Append("<style>\n").Append(BookCss);
if (monochrome)
Expand All @@ -133,7 +134,8 @@ public string Compose(

sb.Append("\n</style>\n");
// PagedConfig must precede the polyfill so the auto-run picks up the readiness callback.
sb.Append("<script>window.PagedConfig = { auto: true, after: () => { window.__pagedDone = true; } };</script>\n");
sb.Append(
"<script>window.PagedConfig = { auto: true, after: () => { window.__pagedDone = true; } };</script>\n");
sb.Append("<script>\n").Append(PagedPolyfill).Append("\n</script>\n");
sb.Append("</head>\n<body>\n");

Expand Down Expand Up @@ -219,7 +221,7 @@ private void RenderNode(
var key = NormalizePath(node.Route.CanonicalPath.Value);
if (!string.IsNullOrEmpty(key)
&& pageByPath.TryGetValue(key, out var page)
&& page.Content is not null
&& page.HasContent
&& !string.IsNullOrEmpty(page.Html))
{
var content = ProcessPageContent(page, depth, slug, rewriteHref, resolveImageSrc);
Expand Down Expand Up @@ -471,7 +473,8 @@ private void AppendColophon(StringBuilder sb, BookStamp stamp)
}

var pennington = BookVersion.Pennington();
AppendColophonLine(sb, pennington is null ? "Produced with Pennington" : $"Produced with Pennington {pennington}");
AppendColophonLine(sb,
pennington is null ? "Produced with Pennington" : $"Produced with Pennington {pennington}");

sb.Append("</section>\n");
}
Expand All @@ -487,7 +490,8 @@ private void AppendToc(
string? locale)
{
sb.Append("<nav class=\"book-toc\">\n");
sb.Append("<div class=\"book-toc-heading\">").Append(Encode(Translate(locale, ContentsKey, DefaultContents))).Append("</div>\n");
sb.Append("<div class=\"book-toc-heading\">").Append(Encode(Translate(locale, ContentsKey, DefaultContents)))
.Append("</div>\n");
sb.Append("<ol>\n");

// The unwrapped area landing reads as an unnumbered introduction entry.
Expand Down Expand Up @@ -545,20 +549,21 @@ private static void AppendTocLevel(
}

/// <summary>True when <paramref name="node"/> maps to a projected page with a body to compose — the same test <see cref="RenderNode"/> applies before emitting page content.</summary>
private static bool HasComposableContent(NavigationTreeItem node, IReadOnlyDictionary<string, RenderedPage> pageByPath)
private static bool HasComposableContent(NavigationTreeItem node,
IReadOnlyDictionary<string, RenderedPage> pageByPath)
{
var key = NormalizePath(node.Route.CanonicalPath.Value);
return !string.IsNullOrEmpty(key)
&& pageByPath.TryGetValue(key, out var page)
&& page.Content is not null
&& !string.IsNullOrEmpty(page.Html);
&& pageByPath.TryGetValue(key, out var page)
&& page.HasContent
&& !string.IsNullOrEmpty(page.Html);
}

/// <summary>Resolves a book chrome string: the locale's translation, then the default locale's, then <paramref name="fallback"/> — the same chain as <see cref="BookCatalog"/>.</summary>
private string Translate(string? locale, string key, string fallback)
=> _translations.Get(locale ?? _localization.DefaultLocale, key)
?? _translations.Get(_localization.DefaultLocale, key)
?? fallback;
?? _translations.Get(_localization.DefaultLocale, key)
?? fallback;

/// <summary>The <c>lang</c> attribute value for <paramref name="locale"/>: the locale's configured <see cref="LocaleInfo.HtmlLang"/> when present, else the locale code itself.</summary>
private string ResolveLang(string? locale)
Expand Down Expand Up @@ -619,11 +624,12 @@ private static string LoadResource(string fileName)
{
var assembly = typeof(BookComposer).Assembly;
var name = Array.Find(
assembly.GetManifestResourceNames(),
n => n.EndsWith("." + fileName, StringComparison.OrdinalIgnoreCase))
?? throw new InvalidOperationException($"Embedded resource '{fileName}' not found in {assembly.GetName().Name}.");
assembly.GetManifestResourceNames(),
n => n.EndsWith("." + fileName, StringComparison.OrdinalIgnoreCase))
?? throw new InvalidOperationException(
$"Embedded resource '{fileName}' not found in {assembly.GetName().Name}.");
using var stream = assembly.GetManifestResourceStream(name)!;
using var reader = new StreamReader(stream);
return reader.ReadToEnd();
}
}
}
66 changes: 47 additions & 19 deletions src/Pennington/LlmsTxt/LlmsTxtService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ namespace Pennington.LlmsTxt;
using System.IO.Abstractions;
using System.Security.Cryptography;
using System.Text;
using AngleSharp.Dom;
using AngleSharp.Html.Parser;
using Content;
using FrontMatter;
using Infrastructure;
Expand Down Expand Up @@ -47,12 +47,11 @@ public LlmsTxtService(
NavigationBuilder navigationBuilder,
ILogger<LlmsTxtService> logger)
{
_dataLazy = new AsyncLazy<LlmsTxtData>(
() => BuildAsync(
projection, contentServices, subtrees,
fileSystem, hostingEnvironment,
pennOptions, llmsTxtOptions, canonicalBase, navigationBuilder,
logger));
_dataLazy = new AsyncLazy<LlmsTxtData>(() => BuildAsync(
projection, contentServices, subtrees,
fileSystem, hostingEnvironment,
pennOptions, llmsTxtOptions, canonicalBase, navigationBuilder,
logger));
}

/// <summary>Returns the generated llms.txt index content.</summary>
Expand Down Expand Up @@ -122,7 +121,8 @@ private static async Task<LlmsTxtData> BuildAsync(
RewriteHref: BuildLinkRewriter(linkablePaths, canonicalBase),
Nodes: new List<RenderedNode>(),
MarkdownFiles: ImmutableList.CreateBuilder<MarkdownFile>(),
FullContent: llmsTxtOptions.GenerateFullFile ? new StringBuilder() : null);
FullContent: llmsTxtOptions.GenerateFullFile ? new StringBuilder() : null,
Parser: new HtmlParser());

await CollectAsync(tree, depth: 0, ctx);
await CollectAsync(subtreeOnlyTree, depth: 0, ctx);
Expand All @@ -147,7 +147,8 @@ private static async Task<LlmsTxtData> BuildAsync(
}

var (entryCount, totalTokens) = SummarizeSubtree(ctx.Nodes, subtree, subtrees);
var canonicalSelf = canonicalBase.Combine(new UrlPath($"/{subtree.RoutePrefix.TrimStart('/')}llms.txt")).Value;
var canonicalSelf = canonicalBase.Combine(new UrlPath($"/{subtree.RoutePrefix.TrimStart('/')}llms.txt"))
.Value;
sb.AppendLine($"canonical: {canonicalSelf}");
sb.AppendLine($"entries: {entryCount}");
sb.AppendLine($"tokens: ~{FormatTokenEstimate(totalTokens)}");
Expand Down Expand Up @@ -210,10 +211,12 @@ private static async Task<ImmutableList<LlmsSubtree>> CollectSubtreesAsync(
return s;
}
}

return null;
}

private static void AppendFrontDoorPreamble(StringBuilder sb, string? userHeader, PenningtonOptions pennOptions, CanonicalBaseUrl canonicalBase)
private static void AppendFrontDoorPreamble(StringBuilder sb, string? userHeader, PenningtonOptions pennOptions,
CanonicalBaseUrl canonicalBase)
{
if (userHeader is not null)
{
Expand Down Expand Up @@ -243,7 +246,8 @@ private static void AppendFrontDoorPreamble(StringBuilder sb, string? userHeader
sb.AppendLine();
}

private static void AppendMapBlock(StringBuilder sb, ImmutableList<LlmsSubtree> subtrees, List<RenderedNode> renderedNodes, CanonicalBaseUrl canonicalBase)
private static void AppendMapBlock(StringBuilder sb, ImmutableList<LlmsSubtree> subtrees,
List<RenderedNode> renderedNodes, CanonicalBaseUrl canonicalBase)
{
if (subtrees.Count == 0)
{
Expand All @@ -263,11 +267,13 @@ private static void AppendMapBlock(StringBuilder sb, ImmutableList<LlmsSubtree>
var desc = string.IsNullOrWhiteSpace(s.Description) ? "" : $" — {s.Description}";
sb.AppendLine($"- [{s.Title}]({url}) ({entryLabel}, ~{tokenLabel} tokens){desc}");
}

sb.AppendLine();
}

/// <summary>Counts entries and sums token estimates for leaves whose nearest matching subtree is <paramref name="target"/>.</summary>
private static (int Count, int Tokens) SummarizeSubtree(List<RenderedNode> renderedNodes, LlmsSubtree target, ImmutableList<LlmsSubtree> allSubtrees)
private static (int Count, int Tokens) SummarizeSubtree(List<RenderedNode> renderedNodes, LlmsSubtree target,
ImmutableList<LlmsSubtree> allSubtrees)
{
var count = 0;
var tokens = 0;
Expand All @@ -279,6 +285,7 @@ private static (int Count, int Tokens) SummarizeSubtree(List<RenderedNode> rende
tokens += leaf.Tokens;
}
}

return (count, tokens);
}

Expand All @@ -290,6 +297,7 @@ private static string FormatTokenEstimate(int tokens)
var thousands = tokens / 1000.0;
return thousands >= 10 ? $"{(int)thousands}k" : $"{thousands:0.#}k";
}

return tokens.ToString();
}

Expand Down Expand Up @@ -324,7 +332,7 @@ private static async Task CollectAsync(ImmutableList<NavigationTreeItem> items,
continue;
}

if (page.Content is null)
if (!page.HasContent)
{
continue;
}
Expand All @@ -337,7 +345,16 @@ private static async Task CollectAsync(ImmutableList<NavigationTreeItem> items,
continue;
}

var markdown = HtmlToMarkdownConverter.Convert(page.Content, ctx.RewriteHref).Trim();
// The projection retains only Html, so re-parse here. The document is local to this
// iteration and collectable straight after, which keeps peak cost proportional to
// concurrency rather than to corpus size.
var content = ctx.Parser.ParseDocument(page.Html).Body;
if (content is null)
{
continue;
}

var markdown = HtmlToMarkdownConverter.Convert(content, ctx.RewriteHref).Trim();
if (string.IsNullOrWhiteSpace(markdown))
{
continue;
Expand All @@ -353,7 +370,8 @@ private static async Task CollectAsync(ImmutableList<NavigationTreeItem> items,
var linkUrl = BuildCoLocatedMarkdownUrl(ctx.CanonicalBase, key);
var description = frontMatter?.Description ?? page.Toc.Description;
var derived = page.Origin?.Value is MarkdownOrigin md2 ? md2.Parsed.Derived : null;
var sidecarHeader = BuildSidecarHeader(item, frontMatter, description, ctx.CanonicalBase, linkUrl, rendition, derived);
var sidecarHeader = BuildSidecarHeader(item, frontMatter, description, ctx.CanonicalBase, linkUrl,
rendition, derived);
var sidecarContent = sidecarHeader + body;

ctx.MarkdownFiles.Add(new MarkdownFile(new FilePath(mdPath), Encoding.UTF8.GetBytes(sidecarContent)));
Expand Down Expand Up @@ -407,6 +425,7 @@ private static void RenderBucket(List<RenderedNode> nodes, StringBuilder sb, Fun
sb.AppendLine();
anyLeafEmittedSinceFlush = false;
}

break;

case LeafNode leaf when include(leaf):
Expand All @@ -418,6 +437,7 @@ private static void RenderBucket(List<RenderedNode> nodes, StringBuilder sb, Fun
sb.AppendLine($"{new string('#', level)} {s.Title}");
sb.AppendLine();
}

pendingSections.Clear();
var desc = leaf.Description is { Length: > 0 } d ? $": {d}" : "";
sb.AppendLine($"- [{leaf.Title}]({leaf.SidecarUrl}){desc}");
Expand Down Expand Up @@ -448,6 +468,7 @@ private static string BuildSidecarHeader(
{
sb.AppendLine($"description: {YamlScalar(description)}");
}

// URLs and hashes contain `:` but never `: ` (colon-space) or whitespace, so they
// parse correctly as bare YAML scalars. Bare emission keeps them readable.
sb.AppendLine($"canonical_url: {canonicalBase.Combine(new UrlPath(item.Route.CanonicalPath.Value)).Value}");
Expand Down Expand Up @@ -487,16 +508,19 @@ private static string YamlScalar(string s)
{
return "\"\"";
}

// Conservative: quote anything that isn't plain printable text to avoid YAML edge cases.
var needsQuote = false;
foreach (var c in s)
{
if (c is ':' or '#' or '\n' or '\r' or '\t' or '"' or '\'' or '\\' or '{' or '}' or '[' or ']' or ',' or '&' or '*' or '!' or '|' or '>' or '%' or '@' or '`')
if (c is ':' or '#' or '\n' or '\r' or '\t' or '"' or '\'' or '\\' or '{' or '}' or '[' or ']' or ',' or '&'
or '*' or '!' or '|' or '>' or '%' or '@' or '`')
{
needsQuote = true;
break;
}
}

if (!needsQuote && (s.StartsWith(' ') || s.EndsWith(' ')))
{
needsQuote = true;
Expand Down Expand Up @@ -589,7 +613,8 @@ private static string BuildCoLocatedMarkdownUrl(CanonicalBaseUrl canonicalBase,
IWebHostEnvironment hostingEnvironment,
PenningtonOptions pennOptions)
{
var contentRoot = FilePath.ResolveAgainstRoot(pennOptions.ContentRootPath.Value, hostingEnvironment.ContentRootPath);
var contentRoot =
FilePath.ResolveAgainstRoot(pennOptions.ContentRootPath.Value, hostingEnvironment.ContentRootPath);

var headerPath = fileSystem.Path.Combine(contentRoot, "llms-header.txt");
if (!fileSystem.File.Exists(headerPath))
Expand All @@ -615,7 +640,9 @@ internal record LlmsTxtData(
public record MarkdownFile(FilePath OutputPath, byte[] Content);

private abstract record RenderedNode;

private sealed record SectionNode(int Depth, string Title) : RenderedNode;

private sealed record LeafNode(
string Title,
string CanonicalPath,
Expand All @@ -632,5 +659,6 @@ private sealed record BuildContext(
Func<string, string> RewriteHref,
List<RenderedNode> Nodes,
ImmutableList<MarkdownFile>.Builder MarkdownFiles,
StringBuilder? FullContent);
}
StringBuilder? FullContent,
HtmlParser Parser);
}
Loading
Loading