fix: pool self-fetch connections; upgrade DeweySearch to 0.2.0

Directory.Packages.props

@@ -4,8 +4,8 @@
     <PackageVersion Include="Ashcroft" Version="0.5.1" />
     <PackageVersion Include="Beck" Version="0.6.0" />
     <PackageVersion Include="CooklangSharp" Version="0.0.0-alpha.0.9" />
-    <PackageVersion Include="DeweySearch" Version="0.1.2" />
-    <PackageVersion Include="DeweySearch.Web" Version="0.1.2" />
+    <PackageVersion Include="DeweySearch" Version="0.2.0" />
+    <PackageVersion Include="DeweySearch.Web" Version="0.2.0" />
     <PackageVersion Include="DiffPlex" Version="1.9.0" />
     <!-- Linux/CI native binaries for Ashcroft's SkiaSharp + HarfBuzz render path; versions track
          Ashcroft 0.5.1's SkiaSharp 4.150.0 / HarfBuzzSharp 14.2.1 dependencies

docs/Pennington.Docs/Content/explanation/discovery/search.md

@@ -27,7 +27,7 @@ nodes:
 edges:
   - { from: render, to: extractor, label: render fold }
   - { from: extractor, to: builder, label: sections }
-  - { from: builder, to: shards, label: "index.json · t-* · f-*" }
+  - { from: builder, to: shards, label: "index.json · d-* · t-* · f-*" }
   - { from: shards, to: client, label: fetched on demand }
 ```
 
@@ -45,11 +45,14 @@ A site of any size produces an index too large to ship as one file and download
 
 Each locale gets a tree under `/search/{locale}/`:
 
-- `index.json` — the entrypoint: the document table (one row per record: URL, title, length, priority, facet ids), the facet label vocabularies, ranking statistics, and the stemmed synonym map.
+- `index.json` — the entrypoint, carrying only what ranking needs before any result is displayed: the ranking statistics, each record's priority and quantized length, the facet label vocabularies, the breadcrumb label dictionary, the stemmed synonym map, and the list of term-shard keys.
+- `d-*.json` — document-table shards: the URL, title, and breadcrumb ids for a run of records. URLs and titles are front-coded against the previous row, which collapses the long shared prefixes a sorted route list produces.
 - `t-*.json` — term shards. Terms are bucketed by the first few characters of their stemmed form, so a query fetches only the shards for the terms it contains.
 - `f-*.json` — per-page fragments holding the indexed body text, fetched only when a page surfaces in results.
 
-The client downloads `index.json` once, then pulls term shards and fragments on demand. Typing a query fetches a handful of small files rather than one large one; opening a result fetches that page's fragment and nothing else. The shard granularity is tunable, but the default keeps shards small enough that no single fetch dominates.
+The split is deliberate: ranking a query needs statistics for every record, but titles and URLs are needed only for the handful of records actually shown. Holding the document table back shrinks the first-keystroke download to the part that ranking actually uses — this site's 2,695 records ship a 43 KB entrypoint while the 183 KB document table stays cold until a query produces hits. The entrypoint still carries a few bytes per record, so it grows with the corpus; it just grows far more slowly than a catalogue of every URL and title would.
+
+The client downloads `index.json` once, then pulls term shards, the document shards covering its hits, and fragments on demand. Typing a query fetches a handful of small files rather than one large one; opening a result fetches that page's fragment and nothing else. The shard granularity is tunable, but the default keeps shards small enough that no single fetch dominates.
 
 ## The build is a fold over the render
 

src/Pennington/Infrastructure/HttpDispatcher.cs

@@ -9,12 +9,27 @@ namespace Pennington.Infrastructure;
 /// <see cref="IServer"/> and returns an in-memory client when it's a
 /// <see cref="TestServer"/>, or a socket-bound client pointing at Kestrel's
 /// listening address otherwise.
+/// <para>
+/// The handler chain is built once and shared by every client this dispatcher hands out
+/// — clients are cheap wrappers created with <c>disposeHandler: false</c>, so a caller's
+/// <c>using</c> releases the wrapper without tearing down the connection pool. This
+/// matters on the Kestrel path: <see cref="Pipeline.SiteProjection"/> issues one fetch per
+/// page, and a per-fetch handler cannot pool, so every fetch opened a fresh loopback socket
+/// that then sat in TIME_WAIT. A corpus larger than the ~16k Windows ephemeral port range
+/// exhausted it partway through and the remaining fetches failed with
+/// <c>SocketError.AddressAlreadyInUse</c> — silently dropping those pages from the search
+/// index and llms.txt, since <c>RenderOneAsync</c> treats a failed fetch as a per-page error.
+/// </para>
 /// </summary>
-public sealed class HttpDispatcher : IInProcessHttpDispatcher
+public sealed class HttpDispatcher : IInProcessHttpDispatcher, IDisposable
 {
     private readonly IServer _server;
     private readonly BuildHtmlCache _cache;
 
+    private readonly Lock _handlerLock = new();
+    private HttpMessageHandler? _handler;
+    private bool _disposed;
+
     /// <summary>Initializes the dispatcher with the host's registered <see cref="IServer"/> and the shared render cache.</summary>
     public HttpDispatcher(IServer server, BuildHtmlCache cache)
     {
@@ -25,13 +40,71 @@ public HttpDispatcher(IServer server, BuildHtmlCache cache)
     /// <inheritdoc/>
     public HttpClient CreateClient()
     {
-        if (_server is TestServer testServer)
+        var (handler, baseAddress) = GetOrCreateHandler();
+
+        // disposeHandler: false — the chain outlives every client so connections stay pooled.
+        return new HttpClient(handler, disposeHandler: false) { BaseAddress = baseAddress };
+    }
+
+    /// <summary>Disposes the shared handler chain and its pooled connections.</summary>
+    public void Dispose()
+    {
+        lock (_handlerLock)
+        {
+            if (_disposed)
+            {
+                return;
+            }
+
+            _disposed = true;
+            _handler?.Dispose();
+            _handler = null;
+        }
+    }
+
+    private (HttpMessageHandler Handler, Uri BaseAddress) GetOrCreateHandler()
+    {
+        lock (_handlerLock)
+        {
+            ObjectDisposedException.ThrowIf(_disposed, this);
+
+            // The base address is recomputed per call: it is cheap, and on the Kestrel path
+            // it is only known once the server has bound. Only the handler is cached.
+            var baseAddress = ResolveBaseAddress();
+            _handler ??= BuildHandler();
+            return (_handler, baseAddress);
+        }
+    }
+
+    private Uri ResolveBaseAddress()
+    {
+        if (_server is TestServer)
         {
             // TestServer.CreateClient() returns BaseAddress = http://localhost/.
             // Path-relative URLs ("/foo/bar") resolve against that and are dispatched
-            // to the same RequestDelegate Kestrel would have invoked. Wrap its handler
-            // with the cache so repeat self-fetches replay one render.
-            HttpMessageHandler innerHandler;
+            // to the same RequestDelegate Kestrel would have invoked.
+            return new Uri("http://localhost/");
+        }
+
+        var addresses = _server.Features.Get<IServerAddressesFeature>()?.Addresses;
+        if (addresses is null || addresses.Count == 0)
+        {
+            throw new SelfFetchUnavailableException(
+                "HttpDispatcher requires either a started TestServer or a listening Kestrel host. " +
+                "IServerAddressesFeature has no addresses — is the app started yet?");
+        }
+
+        // Prefer http:// to avoid dev-cert trust issues when self-fetching from Kestrel.
+        var baseAddress = addresses.FirstOrDefault(a => a.StartsWith("http://", StringComparison.OrdinalIgnoreCase))
+            ?? addresses.First();
+        return new Uri(baseAddress);
+    }
+
+    private HttpMessageHandler BuildHandler()
+    {
+        HttpMessageHandler innerHandler;
+        if (_server is TestServer testServer)
+        {
             try
             {
                 innerHandler = testServer.CreateHandler();
@@ -42,37 +115,26 @@ public HttpClient CreateClient()
                 // self-fetch issued before that — e.g. a startup hosted service racing the
                 // server start — is an infrastructure failure, not a per-page content error.
                 // Surface it as such so callers retry once the host is up instead of baking
-                // an empty corpus.
+                // an empty corpus. Nothing is cached, so the retry rebuilds.
                 throw new SelfFetchUnavailableException(
                     "The in-process TestServer has not started yet; a self-fetch was issued before " +
                     "the host's server was ready.",
                     ex);
             }
-
-            var testHandler = new CachingHttpHandler(_cache) { InnerHandler = innerHandler };
-            return new HttpClient(testHandler) { BaseAddress = new Uri("http://localhost/") };
         }
-
-        var addresses = _server.Features.Get<IServerAddressesFeature>()?.Addresses;
-        if (addresses is null || addresses.Count == 0)
+        else
         {
-            throw new SelfFetchUnavailableException(
-                "HttpDispatcher requires either a started TestServer or a listening Kestrel host. " +
-                "IServerAddressesFeature has no addresses — is the app started yet?");
+            // Bounded pool: the projection renders pages with Parallel.ForEachAsync at the
+            // default degree of parallelism, so a handful of pooled connections serves the
+            // whole corpus. Left unbounded, a burst opens far more sockets than it reuses.
+            innerHandler = new SocketsHttpHandler
+            {
+                AllowAutoRedirect = false,
+                MaxConnectionsPerServer = Environment.ProcessorCount * 2,
+            };
         }
 
-        // Prefer http:// to avoid dev-cert trust issues when self-fetching from Kestrel.
-        var baseAddress = addresses.FirstOrDefault(a => a.StartsWith("http://", StringComparison.OrdinalIgnoreCase))
-            ?? addresses.First();
-
-        var handler = new CachingHttpHandler(_cache)
-        {
-            InnerHandler = new HttpClientHandler { AllowAutoRedirect = false },
-        };
-        var client = new HttpClient(handler)
-        {
-            BaseAddress = new Uri(baseAddress),
-        };
-        return client;
+        // Wrap with the cache so repeat self-fetches replay one render.
+        return new CachingHttpHandler(_cache) { InnerHandler = innerHandler };
     }
 }