From 257a9585237935817ae199257657a6a308014dcd Mon Sep 17 00:00:00 2001 From: Phil Scott Date: Sun, 26 Jul 2026 12:20:46 -0400 Subject: [PATCH 1/2] fix(infrastructure): pool self-fetch connections instead of a handler per fetch HttpDispatcher.CreateClient() built a fresh HttpClientHandler on every call, and both callers (RenderedHtmlFetcher, OutputGenerationService) wrap the result in `using`. Nothing could pool: every self-fetch opened a new loopback socket that then sat in TIME_WAIT for four minutes. On a corpus larger than the ~16k Windows ephemeral port range that exhausts the range partway through the crawl, and the remaining fetches fail with SocketError.AddressAlreadyInUse. SiteProjection.RenderOneAsync treats a failed fetch as a per-page content error, so those pages are dropped from the search index and llms.txt with only a warning. Measured on a 22,178-page site: 6,641 routes silently missing, ~30% of the corpus. Dev-serve only. Build and diag swap Kestrel for TestServer, which dispatches in-memory with no sockets, so static output was unaffected. Build the handler chain once and share it, handing out HttpClient(handler, disposeHandler: false) wrappers so a caller's `using` releases the wrapper and leaves the pool intact. MaxConnectionsPerServer is bounded at ProcessorCount * 2 so a parallel burst reuses connections rather than opening more than it needs. The base address is still resolved per call, since on the Kestrel path it is not known until the server binds. After the fix the same 22,178-page crawl completes with zero socket failures and a complete index. --- .../Infrastructure/HttpDispatcher.cs | 118 +++++++++++++----- .../Infrastructure/HttpDispatcherTests.cs | 52 ++++++++ 2 files changed, 142 insertions(+), 28 deletions(-) diff --git a/src/Pennington/Infrastructure/HttpDispatcher.cs b/src/Pennington/Infrastructure/HttpDispatcher.cs index cc28a1f8..5a5e29a8 100644 --- a/src/Pennington/Infrastructure/HttpDispatcher.cs +++ b/src/Pennington/Infrastructure/HttpDispatcher.cs @@ -9,12 +9,27 @@ namespace Pennington.Infrastructure; /// and returns an in-memory client when it's a /// , or a socket-bound client pointing at Kestrel's /// listening address otherwise. +/// +/// The handler chain is built once and shared by every client this dispatcher hands out +/// — clients are cheap wrappers created with disposeHandler: false, so a caller's +/// using releases the wrapper without tearing down the connection pool. This +/// matters on the Kestrel path: issues one fetch per +/// page, and a per-fetch handler cannot pool, so every fetch opened a fresh loopback socket +/// that then sat in TIME_WAIT. A corpus larger than the ~16k Windows ephemeral port range +/// exhausted it partway through and the remaining fetches failed with +/// SocketError.AddressAlreadyInUse — silently dropping those pages from the search +/// index and llms.txt, since RenderOneAsync treats a failed fetch as a per-page error. +/// /// -public sealed class HttpDispatcher : IInProcessHttpDispatcher +public sealed class HttpDispatcher : IInProcessHttpDispatcher, IDisposable { private readonly IServer _server; private readonly BuildHtmlCache _cache; + private readonly Lock _handlerLock = new(); + private HttpMessageHandler? _handler; + private bool _disposed; + /// Initializes the dispatcher with the host's registered and the shared render cache. public HttpDispatcher(IServer server, BuildHtmlCache cache) { @@ -25,13 +40,71 @@ public HttpDispatcher(IServer server, BuildHtmlCache cache) /// public HttpClient CreateClient() { - if (_server is TestServer testServer) + var (handler, baseAddress) = GetOrCreateHandler(); + + // disposeHandler: false — the chain outlives every client so connections stay pooled. + return new HttpClient(handler, disposeHandler: false) { BaseAddress = baseAddress }; + } + + /// Disposes the shared handler chain and its pooled connections. + public void Dispose() + { + lock (_handlerLock) + { + if (_disposed) + { + return; + } + + _disposed = true; + _handler?.Dispose(); + _handler = null; + } + } + + private (HttpMessageHandler Handler, Uri BaseAddress) GetOrCreateHandler() + { + lock (_handlerLock) + { + ObjectDisposedException.ThrowIf(_disposed, this); + + // The base address is recomputed per call: it is cheap, and on the Kestrel path + // it is only known once the server has bound. Only the handler is cached. + var baseAddress = ResolveBaseAddress(); + _handler ??= BuildHandler(); + return (_handler, baseAddress); + } + } + + private Uri ResolveBaseAddress() + { + if (_server is TestServer) { // TestServer.CreateClient() returns BaseAddress = http://localhost/. // Path-relative URLs ("/foo/bar") resolve against that and are dispatched - // to the same RequestDelegate Kestrel would have invoked. Wrap its handler - // with the cache so repeat self-fetches replay one render. - HttpMessageHandler innerHandler; + // to the same RequestDelegate Kestrel would have invoked. + return new Uri("http://localhost/"); + } + + var addresses = _server.Features.Get()?.Addresses; + if (addresses is null || addresses.Count == 0) + { + throw new SelfFetchUnavailableException( + "HttpDispatcher requires either a started TestServer or a listening Kestrel host. " + + "IServerAddressesFeature has no addresses — is the app started yet?"); + } + + // Prefer http:// to avoid dev-cert trust issues when self-fetching from Kestrel. + var baseAddress = addresses.FirstOrDefault(a => a.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + ?? addresses.First(); + return new Uri(baseAddress); + } + + private HttpMessageHandler BuildHandler() + { + HttpMessageHandler innerHandler; + if (_server is TestServer testServer) + { try { innerHandler = testServer.CreateHandler(); @@ -42,37 +115,26 @@ public HttpClient CreateClient() // self-fetch issued before that — e.g. a startup hosted service racing the // server start — is an infrastructure failure, not a per-page content error. // Surface it as such so callers retry once the host is up instead of baking - // an empty corpus. + // an empty corpus. Nothing is cached, so the retry rebuilds. throw new SelfFetchUnavailableException( "The in-process TestServer has not started yet; a self-fetch was issued before " + "the host's server was ready.", ex); } - - var testHandler = new CachingHttpHandler(_cache) { InnerHandler = innerHandler }; - return new HttpClient(testHandler) { BaseAddress = new Uri("http://localhost/") }; } - - var addresses = _server.Features.Get()?.Addresses; - if (addresses is null || addresses.Count == 0) + else { - throw new SelfFetchUnavailableException( - "HttpDispatcher requires either a started TestServer or a listening Kestrel host. " + - "IServerAddressesFeature has no addresses — is the app started yet?"); + // Bounded pool: the projection renders pages with Parallel.ForEachAsync at the + // default degree of parallelism, so a handful of pooled connections serves the + // whole corpus. Left unbounded, a burst opens far more sockets than it reuses. + innerHandler = new SocketsHttpHandler + { + AllowAutoRedirect = false, + MaxConnectionsPerServer = Environment.ProcessorCount * 2, + }; } - // Prefer http:// to avoid dev-cert trust issues when self-fetching from Kestrel. - var baseAddress = addresses.FirstOrDefault(a => a.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) - ?? addresses.First(); - - var handler = new CachingHttpHandler(_cache) - { - InnerHandler = new HttpClientHandler { AllowAutoRedirect = false }, - }; - var client = new HttpClient(handler) - { - BaseAddress = new Uri(baseAddress), - }; - return client; + // Wrap with the cache so repeat self-fetches replay one render. + return new CachingHttpHandler(_cache) { InnerHandler = innerHandler }; } } \ No newline at end of file diff --git a/tests/Pennington.Tests/Infrastructure/HttpDispatcherTests.cs b/tests/Pennington.Tests/Infrastructure/HttpDispatcherTests.cs index 19aae1b6..83098e2b 100644 --- a/tests/Pennington.Tests/Infrastructure/HttpDispatcherTests.cs +++ b/tests/Pennington.Tests/Infrastructure/HttpDispatcherTests.cs @@ -1,13 +1,65 @@ +using System.Collections.Concurrent; +using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Pennington.Infrastructure; namespace Pennington.Tests.Infrastructure; public class HttpDispatcherTests { + [Fact] + public async Task CreateClient_KestrelPath_PoolsConnectionsAcrossClients() + { + // Regression: the dispatcher used to build a fresh HttpClientHandler per CreateClient(), + // and both callers wrap the result in `using`. Every self-fetch therefore opened a new + // loopback socket that then sat in TIME_WAIT — on a 22k-page corpus that exhausted the + // ~16k Windows ephemeral port range partway through, and the remaining fetches failed + // with SocketError.AddressAlreadyInUse. Those pages were silently dropped from the search + // index, since SiteProjection treats a failed fetch as a per-page error. Sharing one + // handler chain keeps the connections pooled, so a whole corpus costs a handful of ports. + var clientPorts = new ConcurrentBag(); + + var builder = WebApplication.CreateSlimBuilder(); + builder.Logging.ClearProviders(); + builder.WebHost.UseUrls("http://127.0.0.1:0"); + await using var app = builder.Build(); + + app.Map("/{**slug}", (HttpContext ctx) => + { + clientPorts.Add(ctx.Connection.RemotePort); + return Results.Text("ok", "text/html"); + }); + + await app.StartAsync(TestContext.Current.CancellationToken); + + var dispatcher = new HttpDispatcher( + app.Services.GetRequiredService(), + new BuildHtmlCache([])); + + // Distinct paths on purpose: CachingHttpHandler replays a cached response per path, so + // repeating one URL would never reach the socket and the test would pass vacuously. + const int requests = 200; + for (var i = 0; i < requests; i++) + { + using var client = dispatcher.CreateClient(); + var response = await client.GetAsync($"/page-{i}", TestContext.Current.CancellationToken); + response.EnsureSuccessStatusCode(); + } + + clientPorts.Count.ShouldBe(requests); + + // Before the fix this was `requests` (one socket per fetch, none reused). + clientPorts.Distinct().Count().ShouldBeLessThanOrEqualTo(Environment.ProcessorCount * 2); + + await app.StopAsync(TestContext.Current.CancellationToken); + } + [Fact] public void CreateClient_UnstartedTestServer_ThrowsSelfFetchUnavailable() { From 880e65662e34659a59481ca8e474450081ded462 Mon Sep 17 00:00:00 2001 From: Phil Scott Date: Sun, 26 Jul 2026 12:21:04 -0400 Subject: [PATCH 2/2] chore(deps): upgrade DeweySearch to 0.2.0 (hot/cold index split) 0.2.0 keeps the C# API identical (IndexBuilder.Build, SearchIndex.ToFiles, SearchDocument), so no library source changes. What moves is the emitted wire format: the document table leaves index.json for cold d-{n}.json shards, so the entrypoint carries only what ranking needs before a result is displayed. Four integration tests asserted on index.json's `docs` array and broke. Rather than repeat the decode in each, add SearchIndexReader: one test-side reader that loads the manifest plus doc shards and yields decoded IndexedDoc rows. Urls and titles inside a shard are front-coded against the previous row (leading char is the shared-prefix length as c - '0') and breadcrumbs are indices into the manifest's label dictionary. The reader deserializes into DeweySearch's own IndexManifest/DocShard types so the field-name contract stays the package's to define, and it is now the only place in the suite that knows the layout. Update the search explanation page, which described the document table as living in index.json. Entrypoint vs. cold table on this docs site: 2,695 records ship a 43 KB index.json with the 183 KB document table held back until a query hits. --- Directory.Packages.props | 4 +- .../Content/explanation/discovery/search.md | 9 +- .../DocsSite/LlmsTxtAndSearchEndpointTests.cs | 82 ++++++++--------- .../DocsSite/SearchIndexReader.cs | 87 +++++++++++++++++++ 4 files changed, 132 insertions(+), 50 deletions(-) create mode 100644 tests/Pennington.IntegrationTests/DocsSite/SearchIndexReader.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 2bb1a528..e9993841 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,8 +4,8 @@ - - + +