fix(#707): route the remaining test servers through a loopback listen helper

docs/CHANGELOG.md

@@ -18,6 +18,35 @@ entry. See `CONTRIBUTING.md` § Releases & changelog.
 
 ## [Unreleased]
 
+### Fixed — the rest of the test servers now bind the port they dial (#707)
+
+- **The remaining 57 `listen(0)` sites are converted.** #703 fixed the
+  mechanism but could only convert the call sites that already waited for
+  their `listening` callback. The rest read `server.address().port`
+  synchronously on the next line, which stops working the moment a host is
+  passed — `listen` then goes through the `dns.lookup` path even for an IP
+  literal and no longer binds synchronously. They now go through a
+  `listenLoopback()` helper that binds 127.0.0.1 and resolves on `listening`,
+  so no test server is left holding a port it never dials.
+- **A correction to #703's explanation.** That entry said the wildcard socket
+  is `IPV6_V6ONLY`. Measured on macOS, it is not: `[::]` is dual-stack and
+  `http://127.0.0.1:<port>` normally reaches it, which is exactly why the bug
+  presented as intermittent rather than as a hard failure. The real mechanism
+  is that the wildcard bind's port is chosen only against other wildcard
+  binds, while a process that binds `127.0.0.1:<port>` **specifically** may
+  already hold it — and on BSD/macOS the more specific bind coexists with the
+  wildcard and wins for connections to 127.0.0.1. Local dev servers bind
+  127.0.0.1 by default, which is why the observed shadowers were an MCP server
+  and a Flask app. The fix and its rationale are unchanged; only the
+  description of *why* the port was unprotected was wrong.
+- `canvas-core`'s WebSocket stub server had the same shape (`port: 0`, no
+  host, callers dialling `ws://127.0.0.1:<port>`) and now binds the loopback
+  too.
+- Removed 23 now-dead `await once('listening')` waits that followed a
+  converted site. The helper already resolves after `listening`, so a second
+  wait could never fire — it hung 12 files to the 120s test timeout.
+
+
 ### Added — the public MCP endpoint serves MRTR to 2026-07-28 clients (#700)
 
 - **Two SDK generations behind one path, routed by protocol era.** A request

middleware/packages/canvas-core/tools/stubServer.ts

@@ -24,7 +24,11 @@ function stamp(message: Record<string, unknown>, turnId: string, canvasSessionId
 }
 
 export function startStubServer(port = 0): Promise<{ port: number; close: () => Promise<void> }> {
-  const wss = new WebSocketServer({ port, path: '/omadia-ui/canvas' });
+  // `host` is explicit for the same reason the HTTP test helper binds it: with
+  // `port = 0` and no host the socket lands on the wildcard, whose chosen port
+  // is not reserved against a process holding that port on 127.0.0.1 — the
+  // address every caller below actually dials.
+  const wss = new WebSocketServer({ port, host: '127.0.0.1', path: '/omadia-ui/canvas' });
 
   wss.on('connection', (ws: WebSocket) => {
     const handshakeId = `hs-${Math.random().toString(36).slice(2)}`;

middleware/test/_helpers/listenLoopback.ts

@@ -0,0 +1,41 @@
+import type { Server } from 'node:http';
+
+/**
+ * Start a test server on a free port of the IPv4 loopback and resolve once it
+ * is actually listening.
+ *
+ * WHY NOT A BARE `listen(0)`
+ * --------------------------
+ * `listen(0)` with no host binds the wildcard `[::]`. That socket is
+ * dual-stack, so `http://127.0.0.1:<port>` normally reaches it — which is why
+ * the bug this replaces looked intermittent rather than simply broken.
+ *
+ * The port, though, is only chosen against other *wildcard* binds. A process
+ * that binds `127.0.0.1:<port>` specifically may already hold that exact port,
+ * and on BSD/macOS the more specific bind coexists with the wildcard and
+ * **wins** for connections addressed to 127.0.0.1. Local dev servers bind
+ * 127.0.0.1 by default, so this is common: a request meant for the harness is
+ * answered by whatever else is listening. Observed in practice — an MCP server
+ * replying `401 … provide valid authorization token`, a Flask app replying
+ * `404 <!doctype html>`, and a non-HTTP peer that surfaced as
+ * `HTTPParserError: Response does not match the HTTP/1.1 protocol`.
+ *
+ * Binding 127.0.0.1 explicitly makes the reserved port and the dialled port
+ * the same port, so a collision is an honest `EADDRINUSE` instead of a test
+ * silently talking to a stranger.
+ *
+ * WHY THIS IS ASYNC
+ * -----------------
+ * Passing a host sends the call through the `dns.lookup` path even for an IP
+ * literal, so the bind no longer completes synchronously and
+ * `server.address()` is `null` on the next line. Awaiting `listening` is the
+ * whole reason this helper exists rather than one extra argument at each site.
+ */
+export function listenLoopback(target: {
+  listen(port: number, host: string, cb: () => void): Server;
+}): Promise<Server> {
+  return new Promise((resolve, reject) => {
+    const server = target.listen(0, '127.0.0.1', () => { resolve(server); });
+    server.once('error', reject);
+  });
+}

middleware/test/auth/adminAuthProvidersRoute.test.ts

@@ -15,6 +15,7 @@ import {
 } from '../../src/auth/providerRegistry.js';
 import type { AuthProvider } from '../../src/auth/providers/AuthProvider.js';
 import { createAdminAuthRouter } from '../../src/routes/adminAuth.js';
+import { listenLoopback } from '../_helpers/listenLoopback.js';
 
 /**
  * Provider-toggle integration test. Stubs the platform-settings KV +
@@ -98,7 +99,7 @@ describe('/api/v1/admin/auth/providers router', () => {
   let audit: InMemoryAuditLog;
   let session: ForgedSession;
 
-  before(() => {
+  before(async () => {
     catalog = new ProviderCatalog();
     catalog.add(fakeLocal);
     catalog.add(fakeEntra);
@@ -130,7 +131,7 @@ describe('/api/v1/admin/auth/providers router', () => {
         audit: audit as unknown as AdminAuditLog,
       }),
     );
-    server = app.listen(0);
+    server = await listenLoopback(app);
     const addr = server.address() as AddressInfo;
     baseUrl = `http://127.0.0.1:${String(addr.port)}`;
   });

middleware/test/auth/adminUsersRoute.test.ts

@@ -14,6 +14,7 @@ import type {
   UserStore,
 } from '../../src/auth/userStore.js';
 import { createAdminUsersRouter } from '../../src/routes/adminUsers.js';
+import { listenLoopback } from '../_helpers/listenLoopback.js';
 
 /**
  * Postgres-free integration test for the admin-users router. Stubs out
@@ -167,7 +168,7 @@ describe('/api/v1/admin/users router', () => {
         audit: audit as unknown as AdminAuditLog,
       }),
     );
-    server = app.listen(0);
+    server = await listenLoopback(app);
     const addr = server.address() as AddressInfo;
     baseUrl = `http://127.0.0.1:${String(addr.port)}`;
   });

middleware/test/auth/requireApiKey.test.ts

@@ -10,23 +10,24 @@ import { createAuditLog } from '../../packages/harness-api-key-auth/src/auditLog
 import { createRateLimiter } from '../../packages/harness-api-key-auth/src/rateLimiter.js';
 import { requireApiKey } from '../../packages/harness-api-key-auth/src/requireApiKey.js';
 import { createFakeSecrets } from '../channelApi/testSecrets.js';
+import { listenLoopback } from '../_helpers/listenLoopback.js';
 
 /**
  * Issue #439 — the reusable half of the story: any route, kernel or plugin,
  * can mount `requireApiKey` and be authenticated by a server-to-server bearer
  * key instead of the `omadia_session` cookie. Mirrors the router-level
  * fixture style of `test/channelApi/chatRouter.test.ts`.
  */
-function startGuardedServer(opts: {
+async function startGuardedServer(opts: {
   scope?: string;
   withRateLimiter?: boolean;
-}): {
+}): Promise<{
   baseUrl: string;
   apiKeys: ReturnType<typeof createApiKeyStore>;
   auditLog: ReturnType<typeof createAuditLog>;
   secrets: ReturnType<typeof createFakeSecrets>;
   close: () => Promise<void>;
-} {
+}> {
   const secrets = createFakeSecrets();
   const apiKeys = createApiKeyStore(secrets);
   const auditLog = createAuditLog(secrets);
@@ -47,7 +48,7 @@ function startGuardedServer(opts: {
       res.json({ keyId: req.apiKey?.keyId, scopes: req.apiKey?.scopes });
     },
   );
-  const server: Server = app.listen(0);
+  const server: Server = await listenLoopback(app);
   const addr = server.address() as AddressInfo;
   return {
     baseUrl: `http://127.0.0.1:${String(addr.port)}/guarded`,
@@ -59,10 +60,10 @@ function startGuardedServer(opts: {
 }
 
 describe('auth/requireApiKey — authentication', () => {
-  let harness: ReturnType<typeof startGuardedServer>;
+  let harness: Awaited<ReturnType<typeof startGuardedServer>>;
 
-  before(() => {
-    harness = startGuardedServer({});
+  before(async () => {
+    harness = await startGuardedServer({});
   });
   after(async () => {
     await harness.close();
@@ -123,7 +124,7 @@ describe('auth/requireApiKey — authentication', () => {
   });
 
   it('does not audit an unauthenticated call — there is no caller identity to attribute', async () => {
-    const local = startGuardedServer({});
+    const local = await startGuardedServer({});
     await fetch(local.baseUrl);
     await fetch(local.baseUrl, { headers: { authorization: 'Bearer omk_nope' } });
     assert.equal((await local.auditLog.list()).length, 0);
@@ -133,7 +134,7 @@ describe('auth/requireApiKey — authentication', () => {
 
 describe('auth/requireApiKey — scopes', () => {
   it('403s a key that lacks the required scope, and audits it as forbidden', async () => {
-    const local = startGuardedServer({ scope: 'memory:read' });
+    const local = await startGuardedServer({ scope: 'memory:read' });
     const created = await local.apiKeys.create({ label: 'chat-only' });
 
     const res = await fetch(local.baseUrl, {
@@ -153,7 +154,7 @@ describe('auth/requireApiKey — scopes', () => {
   });
 
   it('lets a key with the exact scope through', async () => {
-    const local = startGuardedServer({ scope: 'memory:read' });
+    const local = await startGuardedServer({ scope: 'memory:read' });
     const created = await local.apiKeys.create({ scopes: ['memory:read'] });
     const res = await fetch(local.baseUrl, {
       headers: { authorization: `Bearer ${created.token}` },
@@ -163,7 +164,7 @@ describe('auth/requireApiKey — scopes', () => {
   });
 
   it('lets a wildcard key through any scope gate', async () => {
-    const local = startGuardedServer({ scope: 'memory:read' });
+    const local = await startGuardedServer({ scope: 'memory:read' });
     const created = await local.apiKeys.create({ scopes: ['*'] });
     const res = await fetch(local.baseUrl, {
       headers: { authorization: `Bearer ${created.token}` },
@@ -173,7 +174,7 @@ describe('auth/requireApiKey — scopes', () => {
   });
 
   it('authenticates without any scope gate when `scope` is omitted', async () => {
-    const local = startGuardedServer({});
+    const local = await startGuardedServer({});
     const created = await local.apiKeys.create({ scopes: ['memory:read'] });
     const res = await fetch(local.baseUrl, {
       headers: { authorization: `Bearer ${created.token}` },
@@ -188,7 +189,7 @@ describe('auth/requireApiKey — scopes', () => {
     // used to hydrate to `['chat:write']`, so a key deliberately restricted
     // away from chat authenticated against a `chat:write` route.
     for (const corrupt of ['memory:read', ['Chat:Write'], ['chat:write', 'nonsense'], []]) {
-      const local = startGuardedServer({ scope: 'chat:write' });
+      const local = await startGuardedServer({ scope: 'chat:write' });
       const created = await local.apiKeys.create({ label: 'restricted', scopes: ['memory:read'] });
       const raw = await local.secrets.get(`key:${created.record.id}`);
       assert.ok(raw);
@@ -213,7 +214,7 @@ describe('auth/requireApiKey — scopes', () => {
 
 describe('auth/requireApiKey — rate limiting', () => {
   it('429s past the per-key budget and audits it, without invoking the handler', async () => {
-    const local = startGuardedServer({ withRateLimiter: true });
+    const local = await startGuardedServer({ withRateLimiter: true });
     const created = await local.apiKeys.create({ rateLimitPerMinute: 1 });
 
     const first = await fetch(local.baseUrl, {
@@ -241,7 +242,7 @@ describe('auth/requireApiKey — rate limiting', () => {
   });
 
   it('burns quota before the scope check, so scope probing is not free', async () => {
-    const local = startGuardedServer({ withRateLimiter: true, scope: 'memory:read' });
+    const local = await startGuardedServer({ withRateLimiter: true, scope: 'memory:read' });
     const created = await local.apiKeys.create({ rateLimitPerMinute: 1, scopes: ['chat:write'] });
 
     const first = await fetch(local.baseUrl, {

middleware/test/auth/setupRoute.test.ts

@@ -19,6 +19,7 @@ import {
   providerVerifiedAtVaultKey,
 } from '../../src/platform/providerCredentialVerifier.js';
 import type { SecretVault } from '../../src/secrets/vault.js';
+import { listenLoopback } from '../_helpers/listenLoopback.js';
 
 /**
  * OB-61 — /api/v1/auth/setup integration test. Drives the route via
@@ -203,8 +204,7 @@ async function startHarness(opts: {
     return originalFetch(input as RequestInfo, init);
   }) as typeof fetch;
 
-  const server = app.listen(0);
-  await new Promise<void>((resolve) => server.once('listening', () => resolve()));
+  const server = await listenLoopback(app);
   const port = (server.address() as AddressInfo).port;
 
   return {

middleware/test/builder/builderPreviewRoutes.test.ts

@@ -24,6 +24,7 @@ import type {
 } from '../../src/plugins/builder/previewRuntime.js';
 import { createBuilderRouter } from '../../src/routes/builder.js';
 import type { BuildPipeline } from '../../src/plugins/builder/buildPipeline.js';
+import { listenLoopback } from '../_helpers/listenLoopback.js';
 
 // Augment express Request with the session shape the production code expects.
 declare module 'express-serve-static-core' {
@@ -283,8 +284,7 @@ async function startHarness(opts: {
     }),
   );
 
-  const server = app.listen(0);
-  await new Promise<void>((resolve) => server.once('listening', resolve));
+  const server = await listenLoopback(app);
   const port = (server.address() as AddressInfo).port;
   const baseUrl = `http://127.0.0.1:${String(port)}`;
 
@@ -800,8 +800,7 @@ describe('builder preview routes', () => {
         next();
       });
       app.use('/api/v1/builder', createBuilderRouter({ store, quota }));
-      const server = app.listen(0);
-      await new Promise<void>((resolve) => server.once('listening', resolve));
+      const server = await listenLoopback(app);
       const port = (server.address() as AddressInfo).port;
 
       const res = await fetch(`http://127.0.0.1:${String(port)}/api/v1/builder/models`);

middleware/test/channelRouteRebind.test.ts

@@ -5,6 +5,7 @@ import { after, before, describe, it } from 'node:test';
 import express, { type Express } from 'express';
 
 import { ExpressRouteRegistry } from '../src/channels/routeRegistry.js';
+import { listenLoopback } from './_helpers/listenLoopback.js';
 
 /** Regression coverage for #395: a plugin hot-reinstall must rebind the
  *  inbound handler in place, not serve the stale first-mounted route. */
@@ -21,8 +22,7 @@ describe('ExpressRouteRegistry · hot-reinstall handler rebind (#395)', () => {
     app = express();
     app.use(express.json());
     registry = new ExpressRouteRegistry(app);
-    server = app.listen(0);
-    await new Promise<void>((resolve) => server.once('listening', resolve));
+    server = await listenLoopback(app);
     const { port } = server.address() as AddressInfo;
     baseUrl = `http://127.0.0.1:${port}`;
   });

middleware/test/chatRouterAgentRouting.test.ts

@@ -29,6 +29,7 @@ import type {
   SessionConfigSnapshot,
 } from '@omadia/orchestrator';
 import { createChatRouter } from '../src/routes/chat.js';
+import { listenLoopback } from './_helpers/listenLoopback.js';
 
 function fakeChatAgent(label: string): ChatAgent {
   return {
@@ -83,7 +84,7 @@ describe('createChatRouter (Phase A)', () => {
   let availableAgents: Map<string, ChatAgent>;
   let fallbackSlug: string | undefined;
 
-  function mountApp(): void {
+  async function mountApp(): Promise<void> {
     const app = express();
     app.use(express.json());
     app.use(
@@ -101,18 +102,18 @@ describe('createChatRouter (Phase A)', () => {
         }),
       }),
     );
-    server = app.listen(0);
+    server = await listenLoopback(app);
     const addr = server.address() as AddressInfo;
     baseUrl = `http://127.0.0.1:${String(addr.port)}/api/chat`;
   }
 
-  before(() => {
+  before(async () => {
     store = new FakeStore();
     availableAgents = new Map();
     availableAgents.set(SLUG_PUBLIC, fakeChatAgent(SLUG_PUBLIC));
     availableAgents.set(SLUG_GENERAL, fakeChatAgent(SLUG_GENERAL));
     fallbackSlug = SLUG_PUBLIC;
-    mountApp();
+    await mountApp();
   });
 
   after(async () => {

middleware/test/chatSessionsRouterGraceful.test.ts

@@ -20,6 +20,7 @@ import express from 'express';
 
 import type { ChatSession, ChatSessionStore } from '@omadia/orchestrator';
 import { createChatSessionsRouter } from '../src/routes/chatSessions.js';
+import { listenLoopback } from './_helpers/listenLoopback.js';
 
 class FakeStore implements Pick<ChatSessionStore, 'list'> {
   sessions: ChatSession[] = [];
@@ -35,14 +36,14 @@ describe('createChatSessionsRouter — graceful (getStore)', () => {
   // assigning it simulates the orchestrator publishing it after the wizard.
   let liveStore: ChatSessionStore | undefined;
 
-  before(() => {
+  before(async () => {
     const app = express();
     app.use(express.json());
     app.use(
       '/api/chat',
       createChatSessionsRouter({ getStore: () => liveStore }),
     );
-    server = app.listen(0);
+    server = await listenLoopback(app);
     const addr = server.address() as AddressInfo;
     baseUrl = `http://127.0.0.1:${String(addr.port)}/api/chat`;
   });