From 557400876a59a1dd7ea7db2e785336f252d847e9 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 12 Oct 2026 21:38:19 +0100 Subject: [PATCH] feat(doctor): probe both front door listeners instead of trusting the supervisor pid A door whose event loop is blocked is alive and accepts nothing, and doctor and frontdoor status reported one healthy supervisor while every connection to both listeners timed out. Both now ask the listeners themselves: a TLS handshake against the CA on the provider listener and a plain connect on the CONNECT surface, which plain OAuth sessions use, each timed. A listener that does not accept fails the check naming the sessions it cuts off and the restart that replaces it. Refs #428 --- README.md | 2 +- docs/architecture.md | 2 +- docs/cli-reference.md | 2 +- docs/testing.md | 2 +- src/doctorFrontDoor.test.ts | 30 +++++++++++ src/doctorFrontDoor.ts | 30 +++++++++++ src/doctorReport.ts | 19 +++++-- src/frontdoor/commands.ts | 9 ++-- src/frontdoor/listenerReadings.test.ts | 69 ++++++++++++++++++++++++++ src/frontdoor/listenerReadings.ts | 54 ++++++++++++++++++++ src/frontdoor/probe.test.ts | 17 ++++++- src/frontdoor/probe.ts | 42 +++++++++++++++- src/frontdoor/status.ts | 11 ++++ 13 files changed, 276 insertions(+), 13 deletions(-) create mode 100644 src/frontdoor/listenerReadings.test.ts create mode 100644 src/frontdoor/listenerReadings.ts diff --git a/README.md b/README.md index 67561d3e..190f476b 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ agent-shim headroom enable|disable [--session ] [--now] [--json] # which s agent-shim codex status [--json] agent-shim codex login [--no-open] [--json] # Sign in with ChatGPT, for providers with `codex.login: chatgpt-sign-in` agent-shim codex logout [--json] -agent-shim frontdoor status [--json] +agent-shim frontdoor status [--json] # also probes the serving door's two listeners and reports whether each accepted agent-shim frontdoor restart [--json] # replace the serving door in place; running sessions reach it on their next request agent-shim frontdoor rc list [--json] agent-shim frontdoor rc status [] [--json] diff --git a/docs/architecture.md b/docs/architecture.md index bfc9ab78..bd7b10c5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -202,7 +202,7 @@ When a launch resolves `headroom` on, the launcher (synchronous end to end, righ The front door, not the headroom supervisor, hosts the second server: the CONNECT surface OAuth launches point `HTTPS_PROXY` at. It terminates TLS for the intercept hosts, each with its own leaf signed by a local CA persisted under `/frontdoor/ca/` (the key mode 0600): `api.anthropic.com`, whose `/v1/` paths run through the same ordered pipeline the provider listener serves, and `platform.claude.com`, the claude.ai control plane, which is never HTTP-parsed and is pumped byte for byte; every path on the API host the pipeline does not serve, and every host outside the intercept set, is piped untouched. The transparent surface, a second listener for traffic that ignores proxies, terminates redirected connections the same way, keyed on the TLS SNI name the client presents. None of that happens for a client that has not authenticated: the CONNECT request must carry a live launch's capability as its `Proxy-Authorization` credential (the launcher puts it in the `HTTPS_PROXY` URL, from which clients derive that header), checked against the session registry with the same constant-time comparison the pipeline's admission uses, or it is answered 407 before the target is parsed or dialled. A connection ledger inside the surface enforces `ConnectLimits`: a head deadline, a pending-head cap that evicts the oldest waiting connection, a cap on authenticated connections, and a periodic revalidation that closes every connection whose launch's capability has since been pruned. Authenticated tunnels deliberately have no idle timeout, since Remote Control streams may sit silent indefinitely. That shape is what keeps Claude Code's Remote Control and connectors working under compression: they refuse any `ANTHROPIC_BASE_URL` the child can be given, but honour the proxy layer, so the base URL stays pointing at the real API. All of the surface's effects (TCP listening, TLS termination, HTTP parsing, forwarding) sit behind a `ConnectEffects` port in the same style as `SupervisorPorts`, and its routing decisions are pure functions over strings; the TLS round-trip tests run the real server against certificates this module issues with only the upstream target redirected. -The same CA signs the front door's provider listener, which serves HTTPS with a leaf for `127.0.0.1` and `localhost`. Provider sessions send their credential and launch capability on every request, so the child must be able to tell the door from anything else holding the port; a TLS leaf whose CA key only this user can read is that proof, and every routed child trusts the CA through `NODE_EXTRA_CA_CERTS` (combined with any bundle the parent named, since Node reads only one file). The launcher checks the same thing before trusting a door it finds: `ensureFrontDoor` probes the listener's health endpoint over TLS against the CA, from a worker thread that the synchronous launcher waits on with `Atomics.wait`, and a listener that keeps failing is distrusted and replaced, or the launch refused, before any capability is written. This tool's own supervisor that misses its probes is retired first, so the replacement keeps the sticky ports, but only when no live session is registered against it: a miss is what a busy door looks like, so a door with live sessions is waited on until the launch's own deadline and reported, never signalled, and `frontdoor restart` is the deliberate way to replace it: an explicit replace retires the door even when it is not answering, asking it to exit and killing it if it has not exited after a grace period, because a door whose event loop is blocked never runs its signal handler. A launch planned inside the serving door itself (the usage refresh, the window opener, a job) registers its session and returns the door's own addresses without probing, because the thread that would wait on the probe is the one event loop that has to answer it. The door enters its process role (`src/processRole.ts`) before it binds anything, and the primitives that block the calling thread (the synchronous sleep and the synchronous probe) refuse to run while a role is held, so a path that would stall the door throws `BlockingInEventLoopError` at its first wait instead of freezing every session. The door also carries a watchdog on a thread of its own (`frontdoor/watchdog.ts`): the supervisor beats a shared counter every tick and the thread, whose event loop a blocked main loop cannot stall, ends the process when the counter has not moved for twice the longest a launcher waits on a door before calling it unresponsive, after recording the stall in the state file's `lastError`. It is started once the listeners are bound and stopped before any retirement begins, and it restarts its clock when it was itself descheduled for longer than its cadence, so a suspended or starved machine is not taken for a hung door. The next launch starts a fresh door on the same sticky ports with the session registry intact. The direct listener headroom forwards back to stays plain HTTP, because headroom is what connects to it; what keeps that leg harmless is that no credential travels on it. The hop parks the provider's credential headers in an in-memory custody record and sends headroom placeholders and a hop id, and the direct listener's admission step redeems a live id (for the same provider, with the generation's hop secret) for the real headers before routing. Admission is a pipeline stage in its own right, returning the headers a request is routed with, so restoring credentials is admission's job and no route knows about it. +The same CA signs the front door's provider listener, which serves HTTPS with a leaf for `127.0.0.1` and `localhost`. Provider sessions send their credential and launch capability on every request, so the child must be able to tell the door from anything else holding the port; a TLS leaf whose CA key only this user can read is that proof, and every routed child trusts the CA through `NODE_EXTRA_CA_CERTS` (combined with any bundle the parent named, since Node reads only one file). The launcher checks the same thing before trusting a door it finds: `ensureFrontDoor` probes the listener's health endpoint over TLS against the CA, from a worker thread that the synchronous launcher waits on with `Atomics.wait`, and a listener that keeps failing is distrusted and replaced, or the launch refused, before any capability is written. This tool's own supervisor that misses its probes is retired first, so the replacement keeps the sticky ports, but only when no live session is registered against it: a miss is what a busy door looks like, so a door with live sessions is waited on until the launch's own deadline and reported, never signalled, and `frontdoor restart` is the deliberate way to replace it: an explicit replace retires the door even when it is not answering, asking it to exit and killing it if it has not exited after a grace period, because a door whose event loop is blocked never runs its signal handler. A launch planned inside the serving door itself (the usage refresh, the window opener, a job) registers its session and returns the door's own addresses without probing, because the thread that would wait on the probe is the one event loop that has to answer it. The door enters its process role (`src/processRole.ts`) before it binds anything, and the primitives that block the calling thread (the synchronous sleep and the synchronous probe) refuse to run while a role is held, so a path that would stall the door throws `BlockingInEventLoopError` at its first wait instead of freezing every session. The door also carries a watchdog on a thread of its own (`frontdoor/watchdog.ts`): the supervisor beats a shared counter every tick and the thread, whose event loop a blocked main loop cannot stall, ends the process when the counter has not moved for twice the longest a launcher waits on a door before calling it unresponsive, after recording the stall in the state file's `lastError`. It is started once the listeners are bound and stopped before any retirement begins, and it restarts its clock when it was itself descheduled for longer than its cadence, so a suspended or starved machine is not taken for a hung door. The next launch starts a fresh door on the same sticky ports with the session registry intact. `doctor` and `frontdoor status` ask the listeners themselves whether they accept (`frontdoor/listenerReadings.ts`): a TLS handshake against the CA on the provider listener and a plain TCP accept on the CONNECT surface, each timed, because a live supervisor pid cannot tell a door that serves from one whose event loop is blocked, and the two listeners carry different sessions (provider sessions on one, every plain OAuth session on the other). The direct listener headroom forwards back to stays plain HTTP, because headroom is what connects to it; what keeps that leg harmless is that no credential travels on it. The hop parks the provider's credential headers in an in-memory custody record and sends headroom placeholders and a hop id, and the direct listener's admission step redeems a live id (for the same provider, with the generation's hop secret) for the real headers before routing. Admission is a pipeline stage in its own right, returning the headers a request is routed with, so restoring credentials is admission's job and no route knows about it. The front door records the wire protocol it speaks (`FRONT_DOOR_PROTOCOL` in `src/frontdoor/state.ts`) in its state once it is serving. A launcher that finds a live door whose protocol is lower than its own (a door from before the field counts as 0) stops it, after its listener has authenticated as this tool's own, and spawns a replacement that rebinds the same sticky ports. That is safe because the capability registry is on disk and sessions already running keep reaching the addresses they froze at launch. A door of the same or a newer protocol is always joined, so an older launcher never downgrades a newer door. The rule for a bump: a door accepts every lower protocol's launches (protocol 1 accepts the former `x-claude-use-*` header names). diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 3167df51..638aa938 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -109,7 +109,7 @@ agent-shim headroom enable|disable [--session ] [--now] [--json] # which s agent-shim codex status [--json] agent-shim codex login [--no-open] [--json] # Sign in with ChatGPT, for providers with `codex.login: chatgpt-sign-in` agent-shim codex logout [--json] -agent-shim frontdoor status [--json] +agent-shim frontdoor status [--json] # also probes the serving door's two listeners and reports whether each accepted agent-shim frontdoor restart [--json] # replace the serving door in place; running sessions reach it on their next request agent-shim frontdoor rc list [--json] agent-shim frontdoor rc status [] [--json] diff --git a/docs/testing.md b/docs/testing.md index 6a456a71..8fde5b5e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -29,7 +29,7 @@ The credential cache's reported state is tested as a pure function over a fake c `claudeShim.test.ts` follows `identityManager.test.ts`'s real-temp-directory convention (a fake "own executable" file standing in for the running `agent-shim` binary), rather than `doctor.ts`'s pure-function style, since `enableClaudeShim`/`disableClaudeShim` are themselves real filesystem operations, not something to keep separate from a thin wiring layer. Coverage includes the version-drift case that motivates persisting `claude-shim.json` at all (the source file overwritten in place between two `shim enable` runs, proving the marker — not the inode — is what lets the second run refresh cleanly instead of refusing), a foreign file at the target being refused without `--force` and accepted with it, and the cross-device (`EXDEV`) copy-fallback path via a small injectable `LinkFs` seam (mirroring `config/store.ts`'s own `StoreFs`/`nodeStoreFs` pattern), since a real cross-filesystem rig isn't practical in CI. One test also reproduces Homebrew's actual layout (a symlink into a separate "Cellar" directory) to confirm the shim lands next to the symlink users invoke, not buried in the directory its realpath resolves to. `resolveOwnBinaryCheck`, which feeds `discoverClaudeBinary`'s PATH fallback, is covered in the same file over the same temp layout: a different package's claude beside agent-shim (one npm global prefix) reads as foreign, a `shim enable` hardlink and a symlink to the running binary read as this tool with or without a marker, the recorded target still reads as this tool after version drift, and a `--dir` marker elsewhere does not tar a foreign claude next to agent-shim. The fallback's own loop is covered purely in `versionDiscovery.test.ts`, including the co-location case (a claude in agent-shim's own directory must be found) and the self-reference case (a candidate that is this tool must be skipped, and loud failure when it is the only one), and `doctor.test.ts` runs the real wiring once end to end over a temp prefix to prove `collectDoctorReport` reports the co-located claude. -The codex translation and the front-door daemon that serves it keep the same discipline: no real network (beyond loopback listeners the test itself starts), file, clock or process in a test. `codex/translate.test.ts` covers the pure core against the script it replaces (text, images both ways, tool calls and results linked by call id, long tool names shortened and restored, system turns folded into user turns, the tier mapping's order and overrides, effort levels including omitting reasoning for `none`, the prompt cache key from `metadata.user_id`). `codex/auth.test.ts` drives the `~/.codex/auth.json` store over an injected filesystem and token endpoint: one refresh for any number of concurrent 401s, a re-read before every refresh that reuses a token the Codex CLI already refreshed, rotated tokens merged into the file as it stands, the rotated refresh token on disk before the new access token is handed out, and a refusal that leaves a valid file behind when the persist is killed before the rename. `codex/route.test.ts` covers the translation route itself against a scripted upstream (a stream failure answered as an Anthropic-shaped error, a 429 mapped onto the Anthropic rate-limit surface with retry-after, per-provider translation settings), and `codex/agent.test.ts` pins the keep-alive ceiling at ten seconds against a server whose hint asks for longer. `frontdoor/route.test.ts` covers the identity step (the session split out of the headers, exactly the named internal headers stripped, case-insensitively) and the provider path space; `frontdoor/pipeline.test.ts` covers the ordered pipeline over fakes (backpressure waiting for drain and settling when the client goes away, observers run in order at each response head including unrouted errors, an unauthorized request refused as an Anthropic-shaped 401 before any route is asked, a throwing observer dropped without breaking routing, a route failure answered as an Anthropic-shaped 500 before the head and a destroy after it); `frontdoor/providerRoute.test.ts` covers the per-request provider resolution (a codex provider mounted headroom-eligible with its own address as the hop's upstream, an http provider refused by name until the pass-through route exists, unknown and invalid providers, paths naming no provider), the credential an http provider's route attaches (whatever the child presented replaced by the provider file's own resolved token in the bearer target's header form, an apiKey target answered with x-api-key and the presented bearer dropped, an unresolvable block answered as an Anthropic-shaped 502 naming the provider with nothing forwarded, and a re-resolution after a provider file edit applying to the next request with no restart). `frontdoor/server.test.ts` runs the real listener end to end against a scripted codex upstream: the translation served through the whole stack, the identity and session headers stripped before any route sees the request, a request with no or a wrong launch capability refused 401 without any route being asked, streaming delivered chunk by chunk rather than buffered, the client's disconnect aborting the route, an unrouted target answered as an Anthropic-shaped error, the health probe, and a closed door rebinding the same sticky port so a frozen base URL works again, while a sticky port something else holds is left for a free one without the bind failure reaching the listener's fatal error handler (which used to exit the door, so a replacement could never come up beside a squatter). Its TLS cases use real certificates issued by `x509.ts`: a client that trusts only agent-shim's CA completes a handshake with the provider listener serving the loopback leaf (and the CA-verified health probe passes only against that CA), while the same client cannot talk to a complete door whose leaf a different CA signed, which never sees the request or its credential. `frontdoor/x509.test.ts` checks the certificate encoder against Node's own parser (chain verification, issuer matching, host and IP SAN matching, the validity encoding either side of 2050, positive serials, IPv6 address forms) and mutating the issuer name was confirmed to fail the tests that name it. `frontdoor/headroomHop.test.ts` builds the door from the production `createDoorPipelines`, with the provider listener on TLS, and runs the hop against a scripted daemon serving on a real unix socket inside a real mode 0700 directory under a throwaway root, reached through the production path from the state file through the socket check to `socketPath`, which forwards by `x-headroom-base-url` the way the real one does (its own `x-headroom-*` controls stripped, every other header verbatim): a streamed response relayed event by event, a process squatting on the old TCP port (named by an earlier release's state file, or by a current state file rewritten with a port) never offered a connection whether the daemon is down or serving, a socket directory left world accessible, a symlinked socket directory and a symlinked socket each refused with a 502 naming the reason and nothing sent to the daemon, headroom receiving the Anthropic-shaped request and the translator serving what the daemon forwards back through the direct listener (exactly one daemon request and one upstream call, so the loop cannot double-apply), the project identity re-set on the hop with no session header reaching it, a 502 while the daemon is between restarts, a client disconnect unwinding the hop's own request, and a flag-less session served directly. The credential custody cases: for an http provider, headroom sees only `Bearer agent-shim-sequestered` or `agent-shim-sequestered` and a distinct hop id per request, never the credential, while the provider's upstream receives the door-attached credential from the provider file and neither the client's own presentations nor any of the door's hop headers; with the daemon holding a hop open, the direct listener refuses an unknown hop id, the wrong generation secret, a live id presented for another provider and a missing id, the upstream seeing none of them, and refuses the same id again once the hop has ended; an OAuth session's bearer reaches the daemon untouched with no hop id. `frontdoor/passthrough.test.ts` runs the pass-through against a fake upstream (the provider prefix stripped with the query kept, the host replaced, internal headers absent, an unreachable upstream answered as an Anthropic-shaped 502), `frontdoor/connect.test.ts` keeps the moved TLS round-trip suite (termination, streaming, blind tunnels, port release), every client in it now presenting a live capability the way a real one derives it from `HTTPS_PROXY`, and adds the surface's authentication and limits over real sockets with the tunnel dialler recording every target: a CONNECT with no credential, a wrong capability, a non-Basic scheme, a repeated header or a credential with no password is answered 407 with the Basic challenge, for the intercept host and a blind-tunnel host alike, and nothing is dialled, intercepted or routed for it; a client that has not finished its head by the deadline gets 408 and is closed while a tunnel established earlier still carries bytes past that deadline; an oversized head gets 431; with the pending cap full of half-sent heads, a newcomer evicts the oldest (408) and a live launch's connection still gets through; an authenticated CONNECT beyond the tunnel cap gets 503 without a dial, and is admitted again once a tunnel closes; and a launch whose registry record is pruned has its open tunnel closed at the next revalidation and its capability refused 407 from then on, using the production check over the real registry functions. Each of those guarantees was checked to discriminate by mutating `connect.ts` (skipping the capability check, never arming the deadline, dropping the eviction, the tunnel cap, the revalidation or the size check on a head whose terminator has arrived): every mutant fails the case naming it. `frontdoor/capability.test.ts` covers the credential itself: the proxy URL percent-encodes a token holding every userinfo delimiter and a client's Basic credential carries it back exactly, the password half is read scheme case-insensitively with colons kept, other schemes and empty or missing passwords present nothing, and only an exact live token is accepted, and `frontdoor/latency.test.ts` bounds the added hop: the median through-door request may cost at most two milliseconds over the direct one, a bound two orders of magnitude above a loopback hop's real cost and still small enough to fail if the hop ever grows a request's worth of work. The ordering test discriminates by construction: mutating the pipeline to serve the route before the hop fails three hop tests at once. The authentication and custody tests were checked the same way: letting the probe skip certificate verification, making `ensureFrontDoor` skip the probe, sending the credential to headroom unsequestered, admitting any hop id, and serving the provider listener without TLS each fail the tests that name that guarantee. The launcher suites cover both launch shapes ending at the right place: a provider session keeps the OAuth shape (no base URL, HTTPS_PROXY on the door's CONNECT surface carrying the launch's capability as its proxy credential, the provider named in the session headers, and no credential variable at all) with NODE_EXTRA_CA_CERTS on the door's trust bundle, and an OAuth session keeps its base URL untouched with the same proxy, capability and trust bundle. Both shapes gain NO_PROXY and its lowercase spelling exempting every loopback name, appended to a parent's own exclusions without replacing them and with a repeat entry folded away. The parent's own NODE_EXTRA_CA_CERTS reaches the door port, and a bundle warning is logged. `frontdoor/supervisor.test.ts` runs the listener-host lifecycle over injected ports with a clock advanced only by sleeps: all three sticky ports bound and inherited by a successor generation after a crash, moves off occupied ports logged per listener, failed listener starts recorded as `lastError` with the start lock cleared and the already-bound listeners closed, dead sessions pruned, the idle shutdown closing all three listeners and keeping the sticky addresses (the Remote Control stream hub closed exactly once after the last listener, which a startup failure path performs as well, and never while the door is still serving), the stream hub's close idempotent (a second close persists nothing, asserted in the stream suite's boundary-save case), and the idle clock reset by a new session, the binary turnover at the first empty registry once the executable the door serves as is replaced on disk (before the idle window elapses, through the same graceful shutdown and with the sticky addresses kept), a replaced binary still served while a session is live, the turnover firing at the first empty registry tick after a session live at the replacement ends rather than after fresh idle minutes, and no turnover while the executable is unstatable at startup, gone mid-run, or returning to the exact captured identity after an absent window (the removal window a remove-then-link install leaves), a door superseded by another live supervisor exiting on the first tick after the last session it served ends while a launch made against the owner is still live, leaving the owner's state record exactly as the owner wrote it, a launch registering in the tick after the takeover counted as the superseded door's own, the sessions keyed by either door's own pid ignored so a superseded door with no launches exits at the grace tick, a state file naming a dead supervisor leaving the door serving as the owner, and the supersession and drained exit published on the backbone, with the launch lifecycle published on the event backbone as the registry changes between ticks (a registration, an end by the launcher's own exit hook, a prune of a dead launch, and nothing for the baseline a generation inherits). `frontdoor/ensure.test.ts` covers the launcher bring-up (one supervisor under the lock, a dead holder's lock cleared, joining a serving door, a fatal `lastError` refused immediately, a timeout naming the daemon log) and its listener authentication: the listener is verified before anything is registered, a listener failing verification behind a supervisor this launch spawned is refused with the reason and the log named and no session written, and a recorded live pid whose listener fails is distrusted and replaced by a verified supervisor on another port. `usage/identityClaude.inDoor.test.ts` drives the helper the door's background work plans its launches with (`withIdentityClaude`) from inside the front door's role, over the real launch wiring with a stand-in `claude` on PATH and a throwaway state root, and requires the launch to fail with the named error at its first wait without running anything; with the role guard disabled the same test blocks for the launch's whole wait and fails, which is the stall it exists to catch. `frontdoor/watchdog.test.ts` runs the real watchdog thread with a short cadence and bound (a loop that never beats reported with the stall recorded in `lastError` and the rest of the state record kept, a loop that beats within the bound left alone, the stall recorded and reported while the main thread is blocked in a synchronous wait, and nothing after a stop), and pins its derived constants against the launcher's probe patience; the supervisor suite adds that the watchdog starts after the listeners bind, is beaten each tick, is stopped before the first listener closes, and never starts for a door whose bind failed. `processRole.test.ts` pins the guard that keeps the door's event loop unblocked: a process with no role sleeps and probes as the launcher does, a process holding the front door role has its synchronous sleep and synchronous probe refuse with a named error before any wait or worker starts, and the role is put back when left; the ensure suite adds that a launch which would have to wait fails at its first wait inside the role and that a launch planned inside its own door still takes its shortcut. `frontdoor/probe.test.ts` runs the real synchronous probe against listeners on worker threads (the probe blocks its own thread, so a listener there could never answer): it accepts the loopback leaf from agent-shim's CA and rejects a leaf from another CA for the same names, a leaf from the right CA that does not name 127.0.0.1, a plain-HTTP listener, a closed port, and a squatter that accepts the connection and never answers (failed by the probe's own deadline, not the launcher's outer wait). `frontdoor/trust.test.ts` covers the child's CA bundle: the CA alone when the parent set nothing, a combined bundle that keeps the parent's certificates and is written once per combination, an inherited bundle that already holds the CA reused unchanged, and an unreadable inherited file replaced by the CA alone with a warning naming it. `frontdoor/detach.test.ts` the daemon's independence from the launcher's terminal, and `frontdoor/commands.test.ts` plus `codex/commands.test.ts` the two status commands' output over a fake filesystem, and `frontdoor/commands.test.ts` also runs the real turnover stat against a real temporary directory: the entry's identity reported and followed through a symlink, a staged-then-renamed replacement seen as a different identity (the updater's own install shape), and a missing entry or a directory reported as undefined. `watchedInstallEntry`'s own table lives in the supervisor suite: the resolved entry watched as-is when it is named `agent-shim` (with or without Windows' `.exe`), and the `agent-shim` sibling watched when the door was spawned through the `claude` hardlink, the legacy `claude-use` symlink or any other name, and the own entry watched when no sibling exists (the npm bundle's `dist/cli.cjs` shape). The launcher's own front-door tests inject a fake `FrontDoorPort` to cover the child's base URL, the bring-up order (front door before headroom, since headroom's allowlist needs the door's address), the loud refusal when no port is wired, and that an http provider's launch goes through the door too. `frontdoor/state.test.ts` covers the front-door session registry: a launch whose record carries a capability token is listed (the supervisor's idle decision depends on it, and the supervisor tests register sessions through the real writer for the same reason), the token never appears in a listing, malformed and token-less records are skipped, dead launches are pruned along with their capability, and the registry directory and records are written owner-only; `realPorts.test.ts` proves the real filesystem port produces mode 0700 and 0600 under a permissive umask and narrows entries that already existed wider. `frontdoor/eventHub.test.ts` covers the door's event backbone: delivery to two subscribers of one source and an every-source subscriber with a detached one receiving nothing further, per-source filtering with each source's sequence stamped independently and monotonically (and one name's re-registration publishing through the same sequence), the registered-source listing, the bounded drop-oldest stream bridge whose first survivor's sequence names the gap the dropped events left, a closed stream resolving done and delivering nothing published after the close, and the wrap that puts the Remote Control fan-out on as the backbone's first publisher beside its own unchanged delivery, session filtering included. `frontdoor/quotaExpiringEvents.test.ts` covers the expiring-quota publisher against a fake clock: one event when an unspent window enters its final span and silence on the ticks inside it, nothing for a window outside the span, past its reset or fully used, a re-notice for the next window instance's new reset instant, and the backbone-fed freshness (a written usage event alone moves a window into a span the next check publishes, with no file read). `src/usage/store.test.ts` gains the store's publish dep: every written snapshot is handed to the caller's callback exactly as it stands on disk, a record without an identity writes no snapshot so publishes nothing, and the store runs with no dep wired at all. The supervisor suite gains the door's own sources: the generation and idle-shutdown health events in the order the supervisor lives them, the binary-turnover event before the exit a replaced binary causes, the listener-failure event naming which listener and why, and the expiring-quota check called on every tick. The pool ranking's yielding stickiness is covered in `usage/pick.test.ts`: a member whose seven-day window would run dry at its own observed pace is demoted below one that would not (and a five-hour pace is never converted into seven-day fractions, so a member with only five-hour readings stays unprojected), and the move a resumed conversation makes is named with its reason for the run-dry tail and the extra-usage billing tail, while a kept member and a fresh launch report no move; `launcherPool.test.ts` drives the same move through the real launch and asserts the decision line says which member the conversation moved off and why. `frontdoor/launchEvents.test.ts` covers the launch lifecycle publisher: the silent baseline, the register, end and prune classification carrying the registry's own start times and the observing clock, the departures-before-arrivals ordering with pid order inside each group, and a launch dead on arrival pruned and never registered. `frontdoor/eventsApi.test.ts` mounts the door's whole typed API on the real listener shape over TLS and drives a registry mutation through the real writer, asserting a typed subscriber receives the source-tagged event over the wire, that a caller without the generation's token and a filter naming an unknown source are each refused having delivered no event, and that a source-filtered subscriber receives only the source it named. +The codex translation and the front-door daemon that serves it keep the same discipline: no real network (beyond loopback listeners the test itself starts), file, clock or process in a test. `codex/translate.test.ts` covers the pure core against the script it replaces (text, images both ways, tool calls and results linked by call id, long tool names shortened and restored, system turns folded into user turns, the tier mapping's order and overrides, effort levels including omitting reasoning for `none`, the prompt cache key from `metadata.user_id`). `codex/auth.test.ts` drives the `~/.codex/auth.json` store over an injected filesystem and token endpoint: one refresh for any number of concurrent 401s, a re-read before every refresh that reuses a token the Codex CLI already refreshed, rotated tokens merged into the file as it stands, the rotated refresh token on disk before the new access token is handed out, and a refusal that leaves a valid file behind when the persist is killed before the rename. `codex/route.test.ts` covers the translation route itself against a scripted upstream (a stream failure answered as an Anthropic-shaped error, a 429 mapped onto the Anthropic rate-limit surface with retry-after, per-provider translation settings), and `codex/agent.test.ts` pins the keep-alive ceiling at ten seconds against a server whose hint asks for longer. `frontdoor/route.test.ts` covers the identity step (the session split out of the headers, exactly the named internal headers stripped, case-insensitively) and the provider path space; `frontdoor/pipeline.test.ts` covers the ordered pipeline over fakes (backpressure waiting for drain and settling when the client goes away, observers run in order at each response head including unrouted errors, an unauthorized request refused as an Anthropic-shaped 401 before any route is asked, a throwing observer dropped without breaking routing, a route failure answered as an Anthropic-shaped 500 before the head and a destroy after it); `frontdoor/providerRoute.test.ts` covers the per-request provider resolution (a codex provider mounted headroom-eligible with its own address as the hop's upstream, an http provider refused by name until the pass-through route exists, unknown and invalid providers, paths naming no provider), the credential an http provider's route attaches (whatever the child presented replaced by the provider file's own resolved token in the bearer target's header form, an apiKey target answered with x-api-key and the presented bearer dropped, an unresolvable block answered as an Anthropic-shaped 502 naming the provider with nothing forwarded, and a re-resolution after a provider file edit applying to the next request with no restart). `frontdoor/server.test.ts` runs the real listener end to end against a scripted codex upstream: the translation served through the whole stack, the identity and session headers stripped before any route sees the request, a request with no or a wrong launch capability refused 401 without any route being asked, streaming delivered chunk by chunk rather than buffered, the client's disconnect aborting the route, an unrouted target answered as an Anthropic-shaped error, the health probe, and a closed door rebinding the same sticky port so a frozen base URL works again, while a sticky port something else holds is left for a free one without the bind failure reaching the listener's fatal error handler (which used to exit the door, so a replacement could never come up beside a squatter). Its TLS cases use real certificates issued by `x509.ts`: a client that trusts only agent-shim's CA completes a handshake with the provider listener serving the loopback leaf (and the CA-verified health probe passes only against that CA), while the same client cannot talk to a complete door whose leaf a different CA signed, which never sees the request or its credential. `frontdoor/x509.test.ts` checks the certificate encoder against Node's own parser (chain verification, issuer matching, host and IP SAN matching, the validity encoding either side of 2050, positive serials, IPv6 address forms) and mutating the issuer name was confirmed to fail the tests that name it. `frontdoor/headroomHop.test.ts` builds the door from the production `createDoorPipelines`, with the provider listener on TLS, and runs the hop against a scripted daemon serving on a real unix socket inside a real mode 0700 directory under a throwaway root, reached through the production path from the state file through the socket check to `socketPath`, which forwards by `x-headroom-base-url` the way the real one does (its own `x-headroom-*` controls stripped, every other header verbatim): a streamed response relayed event by event, a process squatting on the old TCP port (named by an earlier release's state file, or by a current state file rewritten with a port) never offered a connection whether the daemon is down or serving, a socket directory left world accessible, a symlinked socket directory and a symlinked socket each refused with a 502 naming the reason and nothing sent to the daemon, headroom receiving the Anthropic-shaped request and the translator serving what the daemon forwards back through the direct listener (exactly one daemon request and one upstream call, so the loop cannot double-apply), the project identity re-set on the hop with no session header reaching it, a 502 while the daemon is between restarts, a client disconnect unwinding the hop's own request, and a flag-less session served directly. The credential custody cases: for an http provider, headroom sees only `Bearer agent-shim-sequestered` or `agent-shim-sequestered` and a distinct hop id per request, never the credential, while the provider's upstream receives the door-attached credential from the provider file and neither the client's own presentations nor any of the door's hop headers; with the daemon holding a hop open, the direct listener refuses an unknown hop id, the wrong generation secret, a live id presented for another provider and a missing id, the upstream seeing none of them, and refuses the same id again once the hop has ended; an OAuth session's bearer reaches the daemon untouched with no hop id. `frontdoor/passthrough.test.ts` runs the pass-through against a fake upstream (the provider prefix stripped with the query kept, the host replaced, internal headers absent, an unreachable upstream answered as an Anthropic-shaped 502), `frontdoor/connect.test.ts` keeps the moved TLS round-trip suite (termination, streaming, blind tunnels, port release), every client in it now presenting a live capability the way a real one derives it from `HTTPS_PROXY`, and adds the surface's authentication and limits over real sockets with the tunnel dialler recording every target: a CONNECT with no credential, a wrong capability, a non-Basic scheme, a repeated header or a credential with no password is answered 407 with the Basic challenge, for the intercept host and a blind-tunnel host alike, and nothing is dialled, intercepted or routed for it; a client that has not finished its head by the deadline gets 408 and is closed while a tunnel established earlier still carries bytes past that deadline; an oversized head gets 431; with the pending cap full of half-sent heads, a newcomer evicts the oldest (408) and a live launch's connection still gets through; an authenticated CONNECT beyond the tunnel cap gets 503 without a dial, and is admitted again once a tunnel closes; and a launch whose registry record is pruned has its open tunnel closed at the next revalidation and its capability refused 407 from then on, using the production check over the real registry functions. Each of those guarantees was checked to discriminate by mutating `connect.ts` (skipping the capability check, never arming the deadline, dropping the eviction, the tunnel cap, the revalidation or the size check on a head whose terminator has arrived): every mutant fails the case naming it. `frontdoor/capability.test.ts` covers the credential itself: the proxy URL percent-encodes a token holding every userinfo delimiter and a client's Basic credential carries it back exactly, the password half is read scheme case-insensitively with colons kept, other schemes and empty or missing passwords present nothing, and only an exact live token is accepted, and `frontdoor/latency.test.ts` bounds the added hop: the median through-door request may cost at most two milliseconds over the direct one, a bound two orders of magnitude above a loopback hop's real cost and still small enough to fail if the hop ever grows a request's worth of work. The ordering test discriminates by construction: mutating the pipeline to serve the route before the hop fails three hop tests at once. The authentication and custody tests were checked the same way: letting the probe skip certificate verification, making `ensureFrontDoor` skip the probe, sending the credential to headroom unsequestered, admitting any hop id, and serving the provider listener without TLS each fail the tests that name that guarantee. The launcher suites cover both launch shapes ending at the right place: a provider session keeps the OAuth shape (no base URL, HTTPS_PROXY on the door's CONNECT surface carrying the launch's capability as its proxy credential, the provider named in the session headers, and no credential variable at all) with NODE_EXTRA_CA_CERTS on the door's trust bundle, and an OAuth session keeps its base URL untouched with the same proxy, capability and trust bundle. Both shapes gain NO_PROXY and its lowercase spelling exempting every loopback name, appended to a parent's own exclusions without replacing them and with a repeat entry folded away. The parent's own NODE_EXTRA_CA_CERTS reaches the door port, and a bundle warning is logged. `frontdoor/supervisor.test.ts` runs the listener-host lifecycle over injected ports with a clock advanced only by sleeps: all three sticky ports bound and inherited by a successor generation after a crash, moves off occupied ports logged per listener, failed listener starts recorded as `lastError` with the start lock cleared and the already-bound listeners closed, dead sessions pruned, the idle shutdown closing all three listeners and keeping the sticky addresses (the Remote Control stream hub closed exactly once after the last listener, which a startup failure path performs as well, and never while the door is still serving), the stream hub's close idempotent (a second close persists nothing, asserted in the stream suite's boundary-save case), and the idle clock reset by a new session, the binary turnover at the first empty registry once the executable the door serves as is replaced on disk (before the idle window elapses, through the same graceful shutdown and with the sticky addresses kept), a replaced binary still served while a session is live, the turnover firing at the first empty registry tick after a session live at the replacement ends rather than after fresh idle minutes, and no turnover while the executable is unstatable at startup, gone mid-run, or returning to the exact captured identity after an absent window (the removal window a remove-then-link install leaves), a door superseded by another live supervisor exiting on the first tick after the last session it served ends while a launch made against the owner is still live, leaving the owner's state record exactly as the owner wrote it, a launch registering in the tick after the takeover counted as the superseded door's own, the sessions keyed by either door's own pid ignored so a superseded door with no launches exits at the grace tick, a state file naming a dead supervisor leaving the door serving as the owner, and the supersession and drained exit published on the backbone, with the launch lifecycle published on the event backbone as the registry changes between ticks (a registration, an end by the launcher's own exit hook, a prune of a dead launch, and nothing for the baseline a generation inherits). `frontdoor/ensure.test.ts` covers the launcher bring-up (one supervisor under the lock, a dead holder's lock cleared, joining a serving door, a fatal `lastError` refused immediately, a timeout naming the daemon log) and its listener authentication: the listener is verified before anything is registered, a listener failing verification behind a supervisor this launch spawned is refused with the reason and the log named and no session written, and a recorded live pid whose listener fails is distrusted and replaced by a verified supervisor on another port. `usage/identityClaude.inDoor.test.ts` drives the helper the door's background work plans its launches with (`withIdentityClaude`) from inside the front door's role, over the real launch wiring with a stand-in `claude` on PATH and a throwaway state root, and requires the launch to fail with the named error at its first wait without running anything; with the role guard disabled the same test blocks for the launch's whole wait and fails, which is the stall it exists to catch. `frontdoor/watchdog.test.ts` runs the real watchdog thread with a short cadence and bound (a loop that never beats reported with the stall recorded in `lastError` and the rest of the state record kept, a loop that beats within the bound left alone, the stall recorded and reported while the main thread is blocked in a synchronous wait, and nothing after a stop), and pins its derived constants against the launcher's probe patience; the supervisor suite adds that the watchdog starts after the listeners bind, is beaten each tick, is stopped before the first listener closes, and never starts for a door whose bind failed. `processRole.test.ts` pins the guard that keeps the door's event loop unblocked: a process with no role sleeps and probes as the launcher does, a process holding the front door role has its synchronous sleep and synchronous probe refuse with a named error before any wait or worker starts, and the role is put back when left; the ensure suite adds that a launch which would have to wait fails at its first wait inside the role and that a launch planned inside its own door still takes its shortcut. `frontdoor/probe.test.ts` runs the real synchronous probe against listeners on worker threads (the probe blocks its own thread, so a listener there could never answer): it accepts the loopback leaf from agent-shim's CA and rejects a leaf from another CA for the same names, a leaf from the right CA that does not name 127.0.0.1, a plain-HTTP listener, a closed port, and a squatter that accepts the connection and never answers (failed by the probe's own deadline, not the launcher's outer wait). `frontdoor/trust.test.ts` covers the child's CA bundle: the CA alone when the parent set nothing, a combined bundle that keeps the parent's certificates and is written once per combination, an inherited bundle that already holds the CA reused unchanged, and an unreadable inherited file replaced by the CA alone with a warning naming it. `frontdoor/detach.test.ts` the daemon's independence from the launcher's terminal, and `frontdoor/commands.test.ts` plus `codex/commands.test.ts` the two status commands' output over a fake filesystem, and `frontdoor/commands.test.ts` also runs the real turnover stat against a real temporary directory: the entry's identity reported and followed through a symlink, a staged-then-renamed replacement seen as a different identity (the updater's own install shape), and a missing entry or a directory reported as undefined. `watchedInstallEntry`'s own table lives in the supervisor suite: the resolved entry watched as-is when it is named `agent-shim` (with or without Windows' `.exe`), and the `agent-shim` sibling watched when the door was spawned through the `claude` hardlink, the legacy `claude-use` symlink or any other name, and the own entry watched when no sibling exists (the npm bundle's `dist/cli.cjs` shape). The launcher's own front-door tests inject a fake `FrontDoorPort` to cover the child's base URL, the bring-up order (front door before headroom, since headroom's allowlist needs the door's address), the loud refusal when no port is wired, and that an http provider's launch goes through the door too. `frontdoor/state.test.ts` covers the front-door session registry: a launch whose record carries a capability token is listed (the supervisor's idle decision depends on it, and the supervisor tests register sessions through the real writer for the same reason), the token never appears in a listing, malformed and token-less records are skipped, dead launches are pruned along with their capability, and the registry directory and records are written owner-only; `realPorts.test.ts` proves the real filesystem port produces mode 0700 and 0600 under a permissive umask and narrows entries that already existed wider. `frontdoor/eventHub.test.ts` covers the door's event backbone: delivery to two subscribers of one source and an every-source subscriber with a detached one receiving nothing further, per-source filtering with each source's sequence stamped independently and monotonically (and one name's re-registration publishing through the same sequence), the registered-source listing, the bounded drop-oldest stream bridge whose first survivor's sequence names the gap the dropped events left, a closed stream resolving done and delivering nothing published after the close, and the wrap that puts the Remote Control fan-out on as the backbone's first publisher beside its own unchanged delivery, session filtering included. `frontdoor/quotaExpiringEvents.test.ts` covers the expiring-quota publisher against a fake clock: one event when an unspent window enters its final span and silence on the ticks inside it, nothing for a window outside the span, past its reset or fully used, a re-notice for the next window instance's new reset instant, and the backbone-fed freshness (a written usage event alone moves a window into a span the next check publishes, with no file read). `src/usage/store.test.ts` gains the store's publish dep: every written snapshot is handed to the caller's callback exactly as it stands on disk, a record without an identity writes no snapshot so publishes nothing, and the store runs with no dep wired at all. The supervisor suite gains the door's own sources: the generation and idle-shutdown health events in the order the supervisor lives them, the binary-turnover event before the exit a replaced binary causes, the listener-failure event naming which listener and why, and the expiring-quota check called on every tick. The pool ranking's yielding stickiness is covered in `usage/pick.test.ts`: a member whose seven-day window would run dry at its own observed pace is demoted below one that would not (and a five-hour pace is never converted into seven-day fractions, so a member with only five-hour readings stays unprojected), and the move a resumed conversation makes is named with its reason for the run-dry tail and the extra-usage billing tail, while a kept member and a fresh launch report no move; `launcherPool.test.ts` drives the same move through the real launch and asserts the decision line says which member the conversation moved off and why. `frontdoor/launchEvents.test.ts` covers the launch lifecycle publisher: the silent baseline, the register, end and prune classification carrying the registry's own start times and the observing clock, the departures-before-arrivals ordering with pid order inside each group, and a launch dead on arrival pruned and never registered. `frontdoor/eventsApi.test.ts` mounts the door's whole typed API on the real listener shape over TLS and drives a registry mutation through the real writer, asserting a typed subscriber receives the source-tagged event over the wire, that a caller without the generation's token and a filter naming an unknown source are each refused having delivered no event, and that a source-filtered subscriber receives only the source it named. `frontdoor/listenerReadings.test.ts` reads both listeners of a live door through scripted probes and a stepped clock (provider first, each timed alone, a failing listener's reason carried while the other is still read, nothing probed when no supervisor is alive, a port the state does not record skipped) and formats them for `frontdoor status`; `frontdoor/probe.test.ts` runs the real accept probe against a listener on a worker thread and a closed port; `doctorFrontDoor.test.ts` gains the listener findings (a pass stating the probe's time, a failure naming the sessions the listener cuts off and the restart that replaces it). The self-hosted Remote Control surface's worker web proxies are covered at two layers. `frontdoor/rcWebFetch.test.ts` drives the fetch's bounds over an injected dial and resolver: the private-address rule itself (every refused block, the v4-mapped form, and the fail-closed non-address), the environment lift's exact-value vocabulary, the redirect walk answering the final hop's facts with the requested url, the scheme gate that refuses without dialling, the private refusal by name, by literal and mid-redirect (and the lift that dials it), the unresolvable and malformed refusals, the redirect bound (a chain exactly at it settling, one past it refused), the byte cap, the deadline against a hanging dial, the non-2xx refusal naming its status, and the content-type gate carrying the readable families as text while refusing binary; its last case runs the production dial itself against a loopback stand-in, fetching through the door's own agents with the lift set and refusing the loopback without it. `frontdoor/rcSelfHost.test.ts` covers the serving of both endpoints: the worker web-fetch handing the request's one url to the injected fetch and answering the fetched facts, the OAuth bearer refused where the bridge's own JWT is required, the refusal shape answered as a 200 error object (the form the CLI surfaces as the tool's own failure), the framed answer that escapes the reader cap refused rather than sent to be dropped, and the web-search family (the clear refusal naming that no backend ships, the query and domain-filter validation, a wired backend's results, and a wired backend's refusal). The e2e in `frontdoor/rcSelfHost.e2e.test.ts` adds the real mount: a redirected TLS client speaking the 2.1.289 request shape (worker JWT bearer, anthropic-version, one url) has its fetch served through the transparent surface, the door's own dial following the stand-in's redirect to the final page, and the scheme gate refusing live on the same served path. Breaking the wiring was confirmed to fail the tests that name it: renaming the served branch fails the three serving cases at once, and disabling the private-address check inside the target gate fails the four refusal cases that name it. diff --git a/src/doctorFrontDoor.test.ts b/src/doctorFrontDoor.test.ts index 01fd496f..70a3db93 100644 --- a/src/doctorFrontDoor.test.ts +++ b/src/doctorFrontDoor.test.ts @@ -199,3 +199,33 @@ describe("runDoctor: headroom allowlist file and the serving door", () => { expect(runDoctor(baseParams(headroomWith(undefined))).ok).toBe(true); }); }); + +describe("runDoctor: front door listeners", () => { + const baseDoor = { supervisorPid: SERVING_DOOR_PID, port: 4100, connectPort: 4200 }; + const serving = { ok: true as const, pids: [SERVING_DOOR_PID] }; + + /** The door's report input with the listener readings a probe of it produced. */ + function withListeners(listeners: NonNullable["listeners"]>): Partial { + const { frontDoor } = frontDoorParams(baseDoor, serving); + if (frontDoor === undefined) { + throw new Error("the helper always supplies a front door"); + } + return { frontDoor: { ...frontDoor, listeners } }; + } + + it("passes a listener that accepted and says how long the probe took", () => { + const report = runDoctor({ ...baseParams(), ...withListeners([{ listener: "connect", port: 4200, ok: true, elapsedMs: 12.4 }]) }); + expect(findingsFor(report, "frontdoor").some((finding) => finding.severity === "pass" && finding.message.includes("connect listener on 127.0.0.1:4200 accepted a probe in 12 ms"))).toBe(true); + }); + + it("fails a listener that did not accept, naming the sessions it cuts off and the way out", () => { + const report = runDoctor({ + ...baseParams(), + ...withListeners([{ listener: "connect", port: 4200, ok: false, reason: "no connection accepted within 2000ms", elapsedMs: 2000 }]), + }); + const failing = findingsFor(report, "frontdoor").find((finding) => finding.severity === "fail"); + expect(failing?.message).toContain("every plain OAuth session"); + expect(failing?.message).toContain("agent-shim frontdoor restart"); + }); +}); + diff --git a/src/doctorFrontDoor.ts b/src/doctorFrontDoor.ts index 2b4d11cc..2238193e 100644 --- a/src/doctorFrontDoor.ts +++ b/src/doctorFrontDoor.ts @@ -1,4 +1,5 @@ import type { DoctorSection, DoctorSeverity } from "./doctorReport"; +import type { FrontDoorListenerReading } from "./frontdoor/listenerReadings"; import type { FrontDoorSessionSummary } from "./frontdoor/state"; import type { FrontDoorSupervisorListing } from "./frontdoor/supervisorProcesses"; @@ -51,3 +52,32 @@ export function pushFrontDoor( ); } } + +/** The sessions each listener carries, named in a finding so a person can tell which of their sessions a failing listener cuts off. */ +const LISTENER_ROUTES: Readonly> = { + provider: "provider sessions (`z`, `m` and the like)", + connect: "every plain OAuth session, which reaches Anthropic through it", +}; + +/** + * Reports whether each listener of the serving door accepts, which a live supervisor pid cannot say: a door whose event loop is blocked is alive and accepts nothing. A listener that does not accept within the probe's bound fails the check and names the sessions it cuts off; one that does is a pass that states how long it took on this machine now, as an observation, not a threshold. + */ +export function pushFrontDoorListeners( + push: (section: DoctorSection, severity: DoctorSeverity, message: string, subject?: string) => void, + readings: readonly FrontDoorListenerReading[], +): void { + for (const reading of readings) { + const where = `127.0.0.1:${String(reading.port)}`; + if (reading.ok) { + push("frontdoor", "pass", `The front door's ${reading.listener} listener on ${where} accepted a probe in ${String(Math.round(reading.elapsedMs))} ms.`, reading.listener); + continue; + } + push( + "frontdoor", + "fail", + `The front door's ${reading.listener} listener on ${where} did not accept a probe${reading.reason === undefined ? "" : ` (${reading.reason})`}, so ${LISTENER_ROUTES[reading.listener]} cannot get through while it stays like this. ` + + "The supervisor is alive but is not serving: `agent-shim frontdoor restart` replaces it.", + reading.listener, + ); + } +} diff --git a/src/doctorReport.ts b/src/doctorReport.ts index b6bc03ba..ee836c15 100644 --- a/src/doctorReport.ts +++ b/src/doctorReport.ts @@ -22,8 +22,9 @@ import { reportableAllowlistFileProblem } from "./headroom/socket"; import { pushHeadroomAllowlist, pushRefusedAllowlistFile, type DoctorHeadroomAllowlist } from "./doctorHeadroomAllowlist"; import { HeadroomStateSchema, resolveServingHeadroomState } from "./headroom/state"; import { currentHeadroomAllowlistInputs, undeterminedFrontDoors } from "./headroom/supervisor"; -import { FrontDoorStateSchema, listFrontDoorSessions, type FrontDoorSessionSummary } from "./frontdoor/state"; -import { pushFrontDoor } from "./doctorFrontDoor"; +import { FrontDoorStateSchema, listFrontDoorSessions, readFrontDoorState, type FrontDoorSessionSummary } from "./frontdoor/state"; +import { pushFrontDoor, pushFrontDoorListeners } from "./doctorFrontDoor"; +import { readFrontDoorListeners, realFrontDoorListenerProbes, type FrontDoorListenerReading } from "./frontdoor/listenerReadings"; import { listFrontDoorSupervisors, type FrontDoorSupervisorListing } from "./frontdoor/supervisorProcesses"; import { ENV_PREFIX, LEGACY_ENV_PREFIX, LEGACY_HOME_DIRNAME } from "./legacy"; import { describeProviderEndpoint, legacyProviderConversion, LegacyProviderFileError } from "./providersStore"; @@ -175,6 +176,8 @@ export interface RunDoctorParams { readonly supervisors: FrontDoorSupervisorListing; /** The launches registered with the front door whose launcher is still running, each naming the door it was sent to: what tells a superseded door still serving its launches from one serving nothing. */ readonly liveSessions: readonly FrontDoorSessionSummary[]; + /** What probing the serving door's listeners found: empty when no door serves or the platform cannot say. Omit to skip the listener checks. */ + readonly listeners?: readonly FrontDoorListenerReading[]; }; } @@ -664,6 +667,9 @@ export function runDoctor(params: RunDoctorParams): DoctorReport { if (params.frontDoor !== undefined) { const validated = params.frontDoor.state.raw === undefined ? undefined : validateJson(FrontDoorStateSchema, params.frontDoor.state); pushFrontDoor(push, params.frontDoor.supervisors, validated?.ok === true ? validated.data.supervisorPid : undefined, params.frontDoor.liveSessions); + if (params.frontDoor.listeners !== undefined) { + pushFrontDoorListeners(push, params.frontDoor.listeners); + } } if (params.activeIdentity.raw === undefined) { @@ -768,6 +774,13 @@ export interface CollectDoctorReportParams { readonly env: NodeJS.ProcessEnv; } +/** Probes the listeners of the door the state file names, or reads nothing when no supervisor of it is alive. */ +function servingDoorListeners(paths: LayoutPaths): readonly FrontDoorListenerReading[] { + const state = readFrontDoorState(realFarmFs, paths.frontdoorStateFile) ?? {}; + const alive = state.supervisorPid !== undefined && realIsProcessRunning(state.supervisorPid); + return readFrontDoorListeners(state, alive, realFrontDoorListenerProbes(paths)); +} + /** * Audits the whole state root against this machine: the edge adapter that enumerates every identity, configuration profile and provider on disk, reads every top-level config file as raw text (never pre-parsing, so one malformed file cannot abort the report), discovers the real Claude Code binary and resolves where a bare command name runs from, and hands the facts to the pure `runDoctor`. */ @@ -901,7 +914,7 @@ export function collectDoctorReport(params: CollectDoctorReportParams): DoctorRe }, ...(uid === undefined ? {} - : { frontDoor: { state: { path: paths.frontdoorStateFile, raw: realFsPort.readFileUtf8(paths.frontdoorStateFile) }, supervisors: listFrontDoorSupervisors(realRunPort, uid), liveSessions: listFrontDoorSessions(realFarmFs, paths.frontdoorSessionsDir).filter((session) => realIsProcessRunning(session.pid)) } }), + : { frontDoor: { state: { path: paths.frontdoorStateFile, raw: realFsPort.readFileUtf8(paths.frontdoorStateFile) }, supervisors: listFrontDoorSupervisors(realRunPort, uid), liveSessions: listFrontDoorSessions(realFarmFs, paths.frontdoorSessionsDir).filter((session) => realIsProcessRunning(session.pid)), listeners: servingDoorListeners(paths) } }), }); return report; } diff --git a/src/frontdoor/commands.ts b/src/frontdoor/commands.ts index e6d47194..b0f38e0c 100644 --- a/src/frontdoor/commands.ts +++ b/src/frontdoor/commands.ts @@ -64,7 +64,8 @@ import { RC_STREAM_BACKOFF_MS, createRcEventFanout, createRcStreamHub, type RcSt import type { RoutedRequest } from "./route"; import { createFrontDoorServer, listenFrontDoor } from "./server"; import { DOOR_STALL_BOUND_MS, DOOR_WATCHDOG_CHECK_MS, startThreadWatchdog } from "./watchdog"; -import { collectFrontDoorStatus, formatFrontDoorStatus, headroomSocketTarget } from "./status"; +import { readFrontDoorListeners, realFrontDoorListenerProbes } from "./listenerReadings"; +import { collectFrontDoorStatus, formatFrontDoorListeners, formatFrontDoorStatus, headroomSocketTarget } from "./status"; import { liveSessionTokens, writeFrontDoorSession } from "./state"; import { declareThisProcessTheFrontDoor } from "./selfIdentity"; import { runFrontDoorSupervisor, watchedInstallEntry, type FrontDoorSupervisorPorts, type OwnExecutableStat } from "./supervisor"; @@ -555,11 +556,13 @@ export function registerFrontDoorCommand(program: Command, deps: CommandDeps): v .option("--json", "Print the status as JSON.") .action((options: Readonly<{ json?: boolean }>) => { const status = collectFrontDoorStatus(realFarmFs, realHeadroomSocketTrust, paths, realIsProcessRunning); + // Probed here, not in the collected status: the collection is read-only and shared with the door's own typed API, while this is the CLI asking the listeners themselves whether they accept. + const listeners = readFrontDoorListeners(status.state, status.supervisorAlive, realFrontDoorListenerProbes(paths)); if (options.json === true) { - printJson(status); + printJson({ ...status, listeners }); return; } - for (const line of formatFrontDoorStatus(status, paths.frontdoorCaCertFile)) { + for (const line of [...formatFrontDoorStatus(status, paths.frontdoorCaCertFile), ...formatFrontDoorListeners(listeners)]) { console.log(line); } }), diff --git a/src/frontdoor/listenerReadings.test.ts b/src/frontdoor/listenerReadings.test.ts new file mode 100644 index 00000000..7f854004 --- /dev/null +++ b/src/frontdoor/listenerReadings.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; + +import { readFrontDoorListeners, type FrontDoorListenerProbes } from "./listenerReadings"; +import type { ListenerVerdict } from "./probe"; +import { formatFrontDoorListeners } from "./status"; + +const PROVIDER_PORT = 4100; +const CONNECT_PORT = 4200; +const MILLISECONDS_PER_PROBE = 7; + +/** Probes that answer from a script and a clock that advances a fixed step per probe, so the readings are deterministic. */ +function probes(script: { readonly provider: ListenerVerdict; readonly connect: ListenerVerdict }): FrontDoorListenerProbes & { readonly asked: string[] } { + let clock = 0; + const asked: string[] = []; + return { + asked, + authenticateProvider: (port) => { + asked.push(`provider:${String(port)}`); + clock += MILLISECONDS_PER_PROBE; + return script.provider; + }, + connect: (port) => { + asked.push(`connect:${String(port)}`); + clock += MILLISECONDS_PER_PROBE; + return script.connect; + }, + now: () => clock, + }; +} + +describe("readFrontDoorListeners", () => { + it("reads both listeners of a live door, provider first, each timed on its own", () => { + const fake = probes({ provider: { ok: true }, connect: { ok: true } }); + const readings = readFrontDoorListeners({ port: PROVIDER_PORT, connectPort: CONNECT_PORT }, true, fake); + expect(fake.asked).toEqual([`provider:${String(PROVIDER_PORT)}`, `connect:${String(CONNECT_PORT)}`]); + expect(readings).toEqual([ + { listener: "provider", port: PROVIDER_PORT, ok: true, elapsedMs: MILLISECONDS_PER_PROBE }, + { listener: "connect", port: CONNECT_PORT, ok: true, elapsedMs: MILLISECONDS_PER_PROBE }, + ]); + }); + + it("carries a failing listener's reason and keeps reading the other", () => { + const fake = probes({ provider: { ok: false, reason: "no answer within 2000ms" }, connect: { ok: true } }); + const readings = readFrontDoorListeners({ port: PROVIDER_PORT, connectPort: CONNECT_PORT }, true, fake); + expect(readings[0]).toMatchObject({ listener: "provider", ok: false, reason: "no answer within 2000ms" }); + expect(readings[1]).toMatchObject({ listener: "connect", ok: true }); + }); + + it("probes nothing when no supervisor of the door is alive, and skips a port the state does not record", () => { + const dead = probes({ provider: { ok: true }, connect: { ok: true } }); + expect(readFrontDoorListeners({ port: PROVIDER_PORT, connectPort: CONNECT_PORT }, false, dead)).toEqual([]); + expect(dead.asked).toEqual([]); + const half = probes({ provider: { ok: true }, connect: { ok: true } }); + expect(readFrontDoorListeners({ connectPort: CONNECT_PORT }, true, half).map((reading) => reading.listener)).toEqual(["connect"]); + }); +}); + +describe("formatFrontDoorListeners", () => { + it("says which listener is not accepting and why", () => { + const lines = formatFrontDoorListeners([ + { listener: "provider", port: PROVIDER_PORT, ok: true, elapsedMs: MILLISECONDS_PER_PROBE }, + { listener: "connect", port: CONNECT_PORT, ok: false, reason: "no connection accepted within 2000ms", elapsedMs: 2_000 }, + ]); + expect(lines).toEqual([ + `provider listener: accepting on 127.0.0.1:${String(PROVIDER_PORT)} (probe took ${String(MILLISECONDS_PER_PROBE)} ms)`, + `connect surface: NOT accepting on 127.0.0.1:${String(CONNECT_PORT)} (no connection accepted within 2000ms)`, + ]); + }); +}); diff --git a/src/frontdoor/listenerReadings.ts b/src/frontdoor/listenerReadings.ts new file mode 100644 index 00000000..7d01e513 --- /dev/null +++ b/src/frontdoor/listenerReadings.ts @@ -0,0 +1,54 @@ +import fs from "node:fs"; + +import type { LayoutPaths } from "../paths"; +import type { FrontDoorState } from "./state"; +import { probeFrontDoorSync, probeListenerAcceptSync, type ListenerVerdict } from "./probe"; + +/** One listener of the front door as `doctor` and `frontdoor status` read it: whether it accepted within the probe's bound, and how long the answer took on this machine now. */ +export interface FrontDoorListenerReading { + /** Which listener, and so which sessions it cuts off when it does not accept: `provider` carries provider sessions, `connect` carries every plain OAuth session. */ + readonly listener: "provider" | "connect"; + readonly port: number; + readonly ok: boolean; + /** The probe's own failure reason, absent when it passed. */ + readonly reason?: string; + /** Milliseconds the probe took, as the local clock saw it: a label on a reading, never a threshold. */ + readonly elapsedMs: number; +} + +/** What reads the listeners: each probe authenticates or connects and answers with a verdict, and the clock times it. Injected so the reading runs against fakes. */ +export interface FrontDoorListenerProbes { + readonly authenticateProvider: (port: number) => ListenerVerdict; + readonly connect: (port: number) => ListenerVerdict; + readonly now: () => number; +} + +/** + * Reads both listeners of the door the state file names, when its supervisor is alive and both ports are recorded; nothing otherwise, since a door that is not running has no listener to accept. The two probes are made in turn, the provider listener first, each timed on its own. + */ +export function readFrontDoorListeners(state: Readonly, supervisorAlive: boolean, probes: Readonly): readonly FrontDoorListenerReading[] { + if (!supervisorAlive) { + return []; + } + const readings: FrontDoorListenerReading[] = []; + const take = (listener: FrontDoorListenerReading["listener"], port: number | undefined, probe: (port: number) => ListenerVerdict): void => { + if (port === undefined) { + return; + } + const started = probes.now(); + const verdict = probe(port); + readings.push({ listener, port, ok: verdict.ok, ...(verdict.ok ? {} : { reason: verdict.reason }), elapsedMs: Math.max(0, probes.now() - started) }); + }; + take("provider", state.port, probes.authenticateProvider); + take("connect", state.connectPort, probes.connect); + return readings; +} + +/** The real probes: the CA-verified health check on the provider listener, a plain accept on the CONNECT surface, and the monotonic clock. */ +export function realFrontDoorListenerProbes(paths: LayoutPaths): FrontDoorListenerProbes { + return { + authenticateProvider: (port) => probeFrontDoorSync(port, fs.readFileSync(paths.frontdoorCaCertFile, "utf8")), + connect: probeListenerAcceptSync, + now: () => performance.now(), + }; +} diff --git a/src/frontdoor/probe.test.ts b/src/frontdoor/probe.test.ts index 26b94a1d..e9f8556f 100644 --- a/src/frontdoor/probe.test.ts +++ b/src/frontdoor/probe.test.ts @@ -3,7 +3,7 @@ import { Worker } from "node:worker_threads"; import { afterEach, beforeAll, describe, expect, it } from "vitest"; import { LOOPBACK_LEAF_NAMES, generateCa, mintLeaf, type CaMaterial, type LeafCert } from "./connect"; -import { probeFrontDoorSync } from "./probe"; +import { probeFrontDoorSync, probeListenerAcceptSync } from "./probe"; /** Enough time for the pure-JS 2048-bit keypairs this file generates. */ const KEYGEN_TIMEOUT_MS = 120_000; @@ -124,3 +124,18 @@ describe("probeFrontDoorSync", () => { expect(nothing.ok ? "" : nothing.reason).toContain("ECONNREFUSED"); }); }); + +describe("probeListenerAcceptSync", () => { + it("accepts a listener that accepts a connection, without asking it anything", async () => { + const port = await listenerThread(undefined); + expect(probeListenerAcceptSync(port)).toEqual({ ok: true }); + }); + + it("fails a port nothing listens on, naming the refusal", async () => { + const port = await closedPort(); + const verdict = probeListenerAcceptSync(port); + expect(verdict.ok).toBe(false); + expect(verdict.ok ? "" : verdict.reason).toMatch(/ECONNREFUSED/); + }); +}); + diff --git a/src/frontdoor/probe.ts b/src/frontdoor/probe.ts index c03af9a7..820def9e 100644 --- a/src/frontdoor/probe.ts +++ b/src/frontdoor/probe.ts @@ -61,11 +61,49 @@ request.on("error", (error) => finish(failed, (error.code ? error.code + ": " : */ export function probeFrontDoorSync(port: number, caPem: string): ListenerVerdict { refuseToBlockInRole("a synchronous probe of the front door"); + return runProbeWorker(PROBE_SOURCE, { port, ca: caPem }); +} + +/** + * The accept probe, run on a worker thread for the same reason: one TCP connect to 127.0.0.1 that succeeds as soon as the listener accepts and is closed again at once, so nothing is sent to the surface it asks about. + */ +const ACCEPT_PROBE_SOURCE = ` +const { workerData } = require("node:worker_threads"); +const net = require("node:net"); +const { port, timeoutMs, shared, ok, failed, headerSlots } = workerData; +const header = new Int32Array(shared, 0, headerSlots); +const reason = new Uint8Array(shared, headerSlots * 4); +let done = false; +const finish = (verdict, message) => { + if (done) return; + done = true; + const bytes = Buffer.from(String(message), "utf8").subarray(0, reason.length); + reason.set(bytes); + Atomics.store(header, 1, bytes.length); + Atomics.store(header, 0, verdict); + Atomics.notify(header, 0); +}; +const socket = net.connect({ host: "127.0.0.1", port }); +socket.once("connect", () => { socket.destroy(); finish(ok, ""); }); +socket.once("error", (error) => finish(failed, (error.code ? error.code + ": " : "") + error.message)); +setTimeout(() => { socket.destroy(); finish(failed, "no connection accepted within " + String(timeoutMs) + "ms"); }, timeoutMs); +`; + +/** + * Whether a listener on 127.0.0.1 accepts a TCP connection within the probe's bound, which is all the CONNECT surface can be asked without presenting a launch's capability. Used by `doctor` and `frontdoor status` to tell a listener that accepts from one whose process is alive but whose event loop is not serving, which a supervisor pid check cannot. + */ +export function probeListenerAcceptSync(port: number): ListenerVerdict { + refuseToBlockInRole("a synchronous probe of a front door listener"); + return runProbeWorker(ACCEPT_PROBE_SOURCE, { port }); +} + +/** Runs one probe source on a worker thread and blocks this thread on its verdict, whatever the probe asks. */ +function runProbeWorker(source: string, data: Readonly>): ListenerVerdict { const shared = new SharedArrayBuffer(HEADER_SLOTS * INT32_BYTES + REASON_BYTES); const header = new Int32Array(shared, 0, HEADER_SLOTS); - const worker = new Worker(PROBE_SOURCE, { + const worker = new Worker(source, { eval: true, - workerData: { port, ca: caPem, timeoutMs: PROBE_TIMEOUT_MS, shared, ok: VERDICT_OK, failed: VERDICT_FAILED, headerSlots: HEADER_SLOTS }, + workerData: { ...data, timeoutMs: PROBE_TIMEOUT_MS, shared, ok: VERDICT_OK, failed: VERDICT_FAILED, headerSlots: HEADER_SLOTS }, }); // The worker must never keep the launcher alive: the verdict is all this thread needs from it. worker.unref(); diff --git a/src/frontdoor/status.ts b/src/frontdoor/status.ts index 7fc64397..c0ddb327 100644 --- a/src/frontdoor/status.ts +++ b/src/frontdoor/status.ts @@ -1,5 +1,6 @@ import { verifyHeadroomSocket, type HeadroomSocketTarget, type HeadroomSocketTrustPorts } from "../headroom/socket"; import { readHeadroomState, type HeadroomFs } from "../headroom/state"; +import type { FrontDoorListenerReading } from "./listenerReadings"; import type { LayoutPaths } from "../paths"; import { listFrontDoorSessions, readFrontDoorState, type FrontDoorSessionSummary, type FrontDoorState } from "./state"; @@ -82,3 +83,13 @@ export function formatFrontDoorStatus(status: FrontDoorStatus, caCertPath: strin lines.push(`daemon log: ${status.logPath}${status.logExists ? "" : " (not created yet)"}`); return lines; } + +/** The lines `frontdoor status` adds when it has probed the serving door: one per listener, saying whether it accepted and how long that took on this machine now, so a door that is alive but not serving reads as such. */ +export function formatFrontDoorListeners(readings: readonly FrontDoorListenerReading[]): string[] { + return readings.map((reading) => { + const what = reading.listener === "provider" ? "provider listener" : "connect surface"; + return reading.ok + ? `${what}: accepting on 127.0.0.1:${String(reading.port)} (probe took ${String(Math.round(reading.elapsedMs))} ms)` + : `${what}: NOT accepting on 127.0.0.1:${String(reading.port)}${reading.reason === undefined ? "" : ` (${reading.reason})`}`; + }); +}