diff --git a/CHANGELOG.md b/CHANGELOG.md index 51e0cad8..9574eb84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Breaking Changes +- `HttpServerInstance.stop()` and `Server.stop()` now return `Promise` + and must be awaited. The promise resolves only after the listener is closed + and the port is free. A caller that ignores it can return before in-flight + requests finish, and a restart can fail to bind the port. - **Store path must be absolute and non-root.** `DBInstanceConfig.path` now throws `DBInstanceConfig.path must be an absolute, non-root path` for relative paths, the empty string, and filesystem roots such as `/`. Resolve @@ -42,6 +46,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed +- A server `stop()` that races an in-flight `start()` now waits for that start + to settle before closing the listener, instead of returning early and + leaving a bound server behind. A `start()` after a failed teardown retries + cleanly, and a later `force` stop interrupts an in-progress drain even when + the stop is queued behind a restart. Drain-then-close ordering is unchanged, + and force or timeout still never cancel application handlers. - `goatdb init` now scaffolds `.gitignore` and `.npmrc` again when run from the published npm package. `npm pack` drops dotfile sources, so the templates use plain names (`gitignore`, `npmrc`) that `init` maps back to their dotted @@ -96,6 +106,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), call, bounded by a 180s timeout. Exact pins let a transitive peer resolve a newer `react` than `react-dom`, which crashes at runtime with "Incompatible React versions". +- `stop()` is graceful by default: in-flight requests and responses finish, + new requests receive `503 Service Unavailable` with `connection: close`, and + the listener closes when the drain ends. `force: true` closes immediately + and `timeoutMs` bounds the drain (default 10s, an integer from 0 to + 2147483647; other values reject and leave the server running). Neither + cancels application handlers. +- HTTP health checks from the `awsalb` host now accept `http` and `https` + URLs on any port and ignore the query string; the path must be exactly + `/healthy`. The comparison is now case-sensitive, so a mixed-case probe such + as `/HEALTHY` returns 404; the `/healthy` endpoint itself still matches + case-insensitively. Keep ALB health-check paths lowercase. ## [0.6.1] - 2026-09-01 diff --git a/cli/templates/deno/server/server.ts b/cli/templates/deno/server/server.ts index 30c7179f..00c087b7 100644 --- a/cli/templates/deno/server/server.ts +++ b/cli/templates/deno/server/server.ts @@ -3,6 +3,7 @@ import * as path from '@std/path'; import { prettyJSON } from '@goatdb/goatdb'; import { type BuildInfo, + DEFAULT_STOP_TIMEOUT_MS, Server, staticAssetsFromJS, } from '@goatdb/goatdb/server'; @@ -76,7 +77,8 @@ async function main(): Promise { const shutdown = () => { if (stopping) return; stopping = true; - setTimeout(() => Deno.exit(1), 5000); + // The drain budget plus a margin for database flush. + setTimeout(() => Deno.exit(1), DEFAULT_STOP_TIMEOUT_MS + 1_000); server.stop().then(() => Deno.exit(0)).catch((e) => { console.error(e); Deno.exit(1); diff --git a/cli/templates/node/server/server-sea.ts b/cli/templates/node/server/server-sea.ts index ca16bfc6..365ae93a 100644 --- a/cli/templates/node/server/server-sea.ts +++ b/cli/templates/node/server/server-sea.ts @@ -7,12 +7,14 @@ declare module 'node:sea' { import { type BuildInfo, + DEFAULT_STOP_TIMEOUT_MS, Server, staticAssetsFromJS, } from '@goatdb/goatdb/server'; import { prettyJSON, type ReadonlyJSONObject } from '@goatdb/goatdb'; import { registerSchemas } from '../common/schema.js'; import { basename, join } from 'node:path'; +import process from 'node:process'; interface Arguments { path?: string; @@ -181,7 +183,11 @@ async function main(): Promise { const shutdown = () => { if (stopping) return; stopping = true; - const forceExit = setTimeout(() => process.exit(1), 5_000); + // The drain budget plus a margin for database flush. + const forceExit = setTimeout( + () => process.exit(1), + DEFAULT_STOP_TIMEOUT_MS + 1_000, + ); forceExit.unref(); server.stop().then(() => process.exit(0)).catch((e) => { console.error(e); diff --git a/cli/templates/node/server/server.ts b/cli/templates/node/server/server.ts index a3613460..bfd29717 100644 --- a/cli/templates/node/server/server.ts +++ b/cli/templates/node/server/server.ts @@ -1,5 +1,6 @@ import { type BuildInfo, + DEFAULT_STOP_TIMEOUT_MS, Server, staticAssetsFromJS, } from '@goatdb/goatdb/server'; @@ -9,6 +10,7 @@ import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; import { fileURLToPath } from 'node:url'; import { dirname, join, resolve } from 'node:path'; +import process from 'node:process'; // These imported files will be automatically generated during compilation import encodedStaticAssets from '../build/staticAssets.json' with { type: 'json', @@ -86,7 +88,11 @@ async function main(): Promise { const shutdown = () => { if (stopping) return; stopping = true; - const forceExit = setTimeout(() => process.exit(1), 5000); + // The drain budget plus a margin for database flush. + const forceExit = setTimeout( + () => process.exit(1), + DEFAULT_STOP_TIMEOUT_MS + 1_000, + ); forceExit.unref(); server.stop().then(() => process.exit(0)).catch((e) => { console.error(e); diff --git a/docs/api/classes/server.mdx b/docs/api/classes/server.mdx index c4bc1aba..e0758d90 100644 --- a/docs/api/classes/server.mdx +++ b/docs/api/classes/server.mdx @@ -115,11 +115,13 @@ Starts the server and all associated services. ### stop() -**stop(): Promise<void>** +**stop(options?: HttpServerStopOptions): Promise<void>** Gracefully stops the server and all its services. Safe to call multiple times. +`timeoutMs` bounds the drain, not the final listener close. `force` skips the drain. Direct `processRequest` calls are not tracked; embedded servers must drain them. + ### updateStaticAssets() **updateStaticAssets(assets: StaticAssets): void** diff --git a/logging/errors.ts b/logging/errors.ts index 38abfd85..72f787e7 100644 --- a/logging/errors.ts +++ b/logging/errors.ts @@ -12,6 +12,8 @@ import type { EmailType } from './metrics.ts'; * impossible * - FailedAssertion: Used when an assertion check fails * - UncaughtServerError: Used for unhandled exceptions on the server + * - HttpServerError: Emitted by a listening HTTP server after it started + * - ServerStartFailed: Emitted when a server start aborts and rolls back * - JsonLogWorkerError: Used when the JSON log worker fails or dies * - MissingConfiguration: Used when required configuration options are missing */ @@ -20,6 +22,8 @@ export type DeveloperError = | 'NotReached' | 'FailedAssertion' | 'UncaughtServerError' + | 'HttpServerError' + | 'ServerStartFailed' | 'JsonLogWorkerError'; /** @@ -57,6 +61,8 @@ export interface LogEntryDeveloperError extends BaseErrorLogEntry { * - DuplicateFailed: Failed to handle duplicate message * - SessionError: Error occurred while handling session * - EmailSendFailed: Failed to send email + * - ServerStopFailed: HTTP server stop failed + * - StopTimeout: HTTP server drain timed out */ export type OperationalError = | 'FetchError' @@ -70,7 +76,9 @@ export type OperationalError = | 'SessionError' | 'EmailSendFailed' | 'MissingConfiguration' + | 'ServerStopFailed' | 'StorageError' + | 'StopTimeout' | 'UnknownWorkerResponse' | 'ValidationError'; diff --git a/net/server/health.ts b/net/server/health.ts index efce5d38..f5c7c6d5 100644 --- a/net/server/health.ts +++ b/net/server/health.ts @@ -2,6 +2,14 @@ import type { Endpoint, ServerServices } from './server.ts'; import type { Schema } from '../../cfds/base/schema.ts'; import type { GoatRequest, ServeHandlerInfo } from './http-compat.ts'; +/** + * Canonical health-check path, shared by the endpoint and the ALB short-circuit. + * WHY normalized here: the endpoint lowercases probe paths before comparison, + * so a mixed-case literal would silently stop matching. Normalizing at the + * definition keeps both comparisons on one canonical value. + */ +export const kHealthCheckPath = '/healthy'.toLowerCase(); + /** * Endpoint that handles health check requests. * @@ -21,7 +29,7 @@ export class HealthCheckEndpoint implements Endpoint { return false; } const path = new URL(req.url).pathname.toLowerCase(); - return path === '/healthy'; + return path === kHealthCheckPath; } processRequest( diff --git a/net/server/http-compat.ts b/net/server/http-compat.ts index 2db2d9a7..3c982633 100644 --- a/net/server/http-compat.ts +++ b/net/server/http-compat.ts @@ -1,5 +1,6 @@ import { getRuntime } from '../../base/runtime/index.ts'; import { assert, notReached } from '../../base/error.ts'; +import { log } from '../../logging/log.ts'; /** * A minimal interface for Node.js Buffer functionality used in this module. @@ -301,6 +302,7 @@ export type MinimalNodeServerResponse = { on?(event: string, listener: (...args: unknown[]) => void): void; once?(event: string, listener: (...args: unknown[]) => void): void; end(chunk?: Uint8Array | string): void; + destroy(error?: Error): void; }; /** @@ -460,15 +462,322 @@ export type HttpRemoteAddr = { hostname: string }; */ export type ServeHandlerInfo = { remoteAddr: HttpRemoteAddr; - /** Resolves when the response is fully sent. Natively supported in Deno; resolved on `finish`/`close` event in Node.js. */ + /** Settles when response delivery ends, including a client disconnect. */ completed: Promise; }; +/** Status sent for requests rejected while the server drains. */ +export const SERVICE_UNAVAILABLE_STATUS = 503; +/** Body sent for requests rejected while the server drains. */ +export const SERVICE_UNAVAILABLE_BODY = 'Service Unavailable'; +/** Header telling clients not to reuse a socket that is about to close. */ +export const SERVICE_UNAVAILABLE_CONNECTION = 'close'; + +/** + * The one 503 used by every drain gate: the web handler path and the raw Node + * response path. WHY shared: a client must not be able to tell which runtime + * rejected the request. + */ +export function serviceUnavailableResponse(): Response { + return new Response(SERVICE_UNAVAILABLE_BODY, { + status: SERVICE_UNAVAILABLE_STATUS, + headers: { connection: SERVICE_UNAVAILABLE_CONNECTION }, + }); +} + +/** Default graceful drain budget for {@link MinimalHttpServer.stop}. */ +export const DEFAULT_STOP_TIMEOUT_MS = 10_000; + +/** + * Largest delay `setTimeout` honors. Larger values overflow the internal + * 32-bit timer and fire after about 1 ms, which would turn a long drain budget + * into an immediate force-close. + */ +export const MAX_STOP_TIMEOUT_MS = 2_147_483_647; + +/** + * Rejects a drain budget the timers cannot honor: a non-integer, a negative + * value, or one above {@link MAX_STOP_TIMEOUT_MS}. Callers validate before + * touching any state, so a rejected stop leaves the server running. + */ +export function validateStopTimeout(options?: HttpServerStopOptions): void { + const timeoutMs = options?.timeoutMs; + if (timeoutMs === undefined) return; + if ( + !Number.isInteger(timeoutMs) || timeoutMs < 0 || + timeoutMs > MAX_STOP_TIMEOUT_MS + ) { + throw new RangeError( + `timeoutMs must be an integer between 0 and ${MAX_STOP_TIMEOUT_MS}, ` + + `got ${timeoutMs}`, + ); + } +} + +/** Options for {@link MinimalHttpServer.stop}. */ +export interface HttpServerStopOptions { + /** + * Skip the drain and close open connections immediately. Use on process + * termination paths where waiting is not an option. Does not cancel handlers. + */ + readonly force?: boolean; + /** + * Drain budget, default 10s. Expiry closes connections, not app handlers. + * Bounds the drain, not the final listener close. + */ + readonly timeoutMs?: number; +} + +/** + * Tracks in-flight requests so stop() can drain them before closing sockets. + * + * WHY: neither runtime can force-close a listener after starting its native + * graceful shutdown (Deno takes the server handle; Node's close() has no + * abort). Draining first keeps the force fallback available on both runtimes. + * New requests are rejected with 503 once stop begins, so a busy server still + * reaches zero in-flight work. + */ +class RequestDrain { + private _active = 0; + private _accepting = true; + private _forced = false; + private readonly _waiters = new Set<(drained: boolean) => void>(); + + get accepting(): boolean { + return this._accepting; + } + + get forced(): boolean { + return this._forced; + } + + async track( + completed: Promise, + handler: () => Promise, + ): Promise { + ++this._active; + // Observe rejection immediately; disconnects must still wait for the handler. + const responseDone = completed.then(() => {}, () => {}); + try { + return await handler(); + } finally { + // Capture this tracker so a restart cannot receive an old request's end. + void responseDone.then(() => this.end()); + } + } + + private end(): void { + if (this._active === 0) return; + --this._active; + if (this._active > 0) return; + for (const waiter of this._waiters) waiter(true); + } + + stopAccepting(): void { + this._accepting = false; + } + + /** Escalates an in-flight graceful stop to the force path. */ + force(): void { + this._forced = true; + this._accepting = false; + for (const waiter of this._waiters) waiter(false); + } + + /** Resolves true when drained, false when the timeout expired first. */ + wait(timeoutMs: number): Promise { + if (this._active === 0) return Promise.resolve(true); + return new Promise((resolve) => { + const finish = (drained: boolean) => { + clearTimeout(timer); + this._waiters.delete(finish); + resolve(drained); + }; + this._waiters.add(finish); + const timer = setTimeout(() => finish(false), timeoutMs); + }); + } +} + +/** Logs a drain timeout. The caller still force-closes; this is not fatal. */ +function logDrainTimeout(timeoutMs: number): void { + log({ + severity: 'WARNING', + error: 'StopTimeout', + message: `HTTP server stop drain timed out after ${timeoutMs}ms; ` + + 'closing remaining connections', + }); +} + +/** Closes the drain and waits for active requests within the time budget. */ +async function beginDrain( + drain: RequestDrain, + options: HttpServerStopOptions, +): Promise { + drain.stopAccepting(); + const timeoutMs = options.timeoutMs ?? DEFAULT_STOP_TIMEOUT_MS; + const drained = options.force ? false : await drain.wait(timeoutMs); + if (!options.force && !drained && !drain.forced) { + logDrainTimeout(timeoutMs); + } + return drained; +} + +/** Surfaces a rejected background stop instead of dropping the promise. */ +export function logStopFailure(error: unknown): void { + log({ + severity: 'ERROR', + error: 'ServerStopFailed', + message: `HTTP server stop failed: ${ + error instanceof Error ? error.message : String(error) + }`, + trace: error instanceof Error ? error.stack : undefined, + }); +} + +/** + * Stops the server when `signal` aborts. Callers must handle a signal that is + * already aborted before calling this: firing the stop synchronously would + * queue it behind the start that is wiring it. + */ +function wireStopSignal( + signal: AbortSignal | undefined, + isCurrent: () => boolean, + stop: () => Promise, +): void { + if (!signal) return; + signal.addEventListener('abort', () => { + // A restart may own the server now; ignore this signal for that listener. + if (isCurrent()) void stop().catch(logStopFailure); + }, { once: true }); +} + +/** + * Awaits a previous teardown, then clears it. WHY: a start must not race the + * old teardown. Clears even on rejection so the slot cannot stay poisoned; the + * rejection still propagates to this caller. + */ +export async function awaitPendingStop( + pending: Promise | undefined, + clear: () => void, +): Promise { + if (!pending) return; + try { + await pending; + } finally { + clear(); + } +} + +/** + * Clears a pending-stop slot when a teardown rejects, then rethrows. WHY: the + * stop caller must see the failure, but a later start must be able to retry. + */ +export function clearPendingOnReject( + pending: Promise, + clear: () => void, +): Promise { + return pending.catch((error) => { + clear(); + throw error; + }); +} + +/** + * Interrupts an active drain when a force stop is queued behind a restart. + * + * WHY: the lifecycle gate defers a stop until the in-flight start settles, so + * without this the force waits for the handler or the graceful budget. While a + * teardown is pending, the drain field holds that teardown's drain: a start + * clears the pending stop before it installs a new drain. + */ +function escalatePendingStop( + pending: Promise | undefined, + drain: RequestDrain, +): void { + if (pending) drain.force(); +} + +/** + * FIFO lifecycle gate shared by the server classes. + * + * WHY: every start and stop must run alone, in call order. A stop() that + * skipped ahead of a queued start would observe a not-yet-started server, + * return early, and leave the caller believing the listener is closed while + * the start goes on to bind it. Concurrent start() calls still share one bind, + * because a second bind would only fail on the port. + * + * The first op of an idle gate runs synchronously, so callers observe the + * state change (for example the cleared port) before stop() returns. + */ +export class StartStopGate { + /** Non-rejecting tail of the queue. Undefined when no op is pending. */ + private _tail?: Promise; + /** Identifies the newest chain, so only it may clear the tail. */ + private _chain = 0; + /** The in-flight start, shared by concurrent start() callers. */ + private _starting?: Promise; + + /** Runs `start` once; concurrent callers share the in-flight promise. */ + start(start: () => Promise): Promise { + const inFlight = this._starting; + if (inFlight) return inFlight; + const started = this._push(start); + this._starting = started; + const clear = () => { + if (this._starting === started) this._starting = undefined; + }; + void started.then(clear, clear); + return started; + } + + /** Runs `stop` after every earlier op settled, success or failure. */ + stop(stop: () => Promise): Promise { + return this._push(stop); + } + + /** Queues one op behind the current tail, or runs it now when idle. */ + private _push(op: () => Promise): Promise { + const previous = this._tail; + if (!previous) { + let run: Promise; + try { + run = op(); + } catch (error) { + run = Promise.reject(error); + } + this._track(run); + return run; + } + // WHY the rejection handler: a failed op must not block the ops behind it, + // while its own caller must still see the failure. + const run = previous.then(op, op); + this._track(run); + return run; + } + + private _track(run: Promise): void { + // WHY released in the settle callback itself: the slot must read idle by + // the time the op's own caller resumes, so its next stop() runs at once + // instead of queuing a microtask behind an already finished chain. + const chain = ++this._chain; + this._tail = run.then( + () => this._release(chain), + () => this._release(chain), + ); + } + + private _release(chain: number): void { + // A later op may already own the slot; only the newest chain clears it. + if (this._chain === chain) this._tail = undefined; + } +} + /** * Minimal cross-platform HTTP server abstraction for GoatDB. * * This interface and implementation allow the server to be started and stopped - * in a platform-agnostic way. Currently, only the Deno implementation is provided. + * in a platform-agnostic way. */ export interface MinimalHttpServer { start( @@ -476,124 +785,267 @@ export interface MinimalHttpServer { port: number, signal?: AbortSignal, ): Promise; - stop(): void; + /** + * Stops the server. Idempotent; port and address read as undefined immediately. + * + * Default is graceful: the listener stays open while in-flight requests run, + * new requests receive 503, and open connections close once the drain + * finishes. Pass `force: true` to close immediately, or `timeoutMs` to bound + * the drain. `timeoutMs` does not bound the final listener close. Repeated + * calls share one teardown; a later `force` escalates only while the drain is + * pending, while `timeoutMs` is fixed by the first call. A `force` interrupts + * a pending drain even when the stop is queued behind a restart. Resolves + * once the listener is closed. A stop() racing an in-flight start() resolves only + * after that start settles and the listener is closed. Forced or timed-out + * stops cannot cancel application handlers; these can continue after stop + * resolves. + */ + stop(options?: HttpServerStopOptions): Promise; readonly port?: number; readonly address?: { hostname: string; port: number }; } /** - * Deno implementation of the MinimalHttpServer abstraction. + * Shared start/stop lifecycle for the runtime-specific servers. + * + * WHY: draining, stop idempotency, the 503 gate and signal wiring do not + * depend on the runtime, so one owner keeps Deno and Node shutdown behavior + * from drifting. The runtime differences stay behind three hooks: bind a + * listener, close it, and report port/address from it. + * + * The listener handle is opaque here. Clearing it is what makes `port` and + * `address` read undefined from the moment `stop()` starts. */ -export class DenoHttpServer implements MinimalHttpServer { - private _abortController?: AbortController; - private _server?: Deno.HttpServer; - private _started: boolean = false; - private _options: HttpServerOptions; +export abstract class BaseHttpServer implements MinimalHttpServer { + private _listener?: L; + private _started = false; + /** In-flight stop, so repeated stop() calls share one teardown. */ + private _stopPromise?: Promise; + private readonly _lifecycle = new StartStopGate(); + /** In-flight request tracker; replaced by each start. */ + protected _drain: RequestDrain = new RequestDrain(); - constructor(options: HttpServerOptions = {}) { - this._options = options; + /** The bound listener, or undefined before start and from stop() onward. */ + protected get listener(): L | undefined { + return this._listener; } start( - handler: ( - req: GoatRequest, - info: ServeHandlerInfo, - ) => Promise, + handler: (req: GoatRequest, info: ServeHandlerInfo) => Promise, port: number, signal?: AbortSignal, ): Promise { - if (this._started) { - return Promise.resolve(); + return this._lifecycle.start(() => this._start(handler, port, signal)); + } + + stop(options: HttpServerStopOptions = {}): Promise { + // WHY: validate before escalating or clearing state, so a rejected budget + // leaves the listener untouched. + try { + validateStopTimeout(options); + } catch (error) { + return Promise.reject(error); } - this._abortController = new AbortController(); - if (signal) { - signal.addEventListener( - 'abort', - () => this._abortController!.abort(), - { once: true }, - ); + // WHY: the lifecycle gate may queue this stop behind a restart, so force + // must interrupt the active teardown now; the queued stop still closes the + // restarted listener. + if (options.force) escalatePendingStop(this._stopPromise, this._drain); + return this._lifecycle.stop(() => this._requestStop(options)); + } + + private async _start( + handler: (req: GoatRequest, info: ServeHandlerInfo) => Promise, + port: number, + signal?: AbortSignal, + ): Promise { + // WHY: a restart must not bind the port while the old teardown still + // drains requests and closes databases. + await awaitPendingStop(this._stopPromise, () => { + this._stopPromise = undefined; + }); + if (this._started) return; + this._drain = new RequestDrain(); + const listener = await this._bind(handler, port); + this._listener = listener; + this._started = true; + // WHY: a signal that aborted before or during the bind must leave no + // listener behind by the time start() resolves, so tear down inline + // instead of queuing a stop behind this start. + if (signal?.aborted) { + await this._requestStop({}); + return; + } + wireStopSignal( + signal, + () => this._listener === listener, + () => this.stop(), + ); + } + + private _requestStop(options: HttpServerStopOptions): Promise { + const listener = this._listener; + if (!listener) { + // A later force must be able to escalate the active graceful teardown. + if (options.force) this._drain.force(); + return this._stopPromise ?? Promise.resolve(); } - const combinedSignal = this._abortController.signal; - let resolve: () => void; - const started = new Promise((res) => { - resolve = res; + this._listener = undefined; + this._started = false; + // Capture the drain. A restart must not swap it under the active teardown. + const drain = this._drain; + const stopped = clearPendingOnReject( + this._closeAfterDrain(listener, drain, options), + () => { + this._stopPromise = undefined; + }, + ); + this._stopPromise = stopped; + return stopped; + } + + private async _closeAfterDrain( + listener: L, + drain: RequestDrain, + options: HttpServerStopOptions, + ): Promise { + const drained = await beginDrain(drain, options); + await this._close(listener, options.force === true || !drained); + } + + /** Binds a listener. Must reject on failure and leave nothing behind. */ + protected abstract _bind( + handler: (req: GoatRequest, info: ServeHandlerInfo) => Promise, + port: number, + ): Promise; + + /** + * Closes the listener. `force` means the drain was skipped or timed out, so + * live connections must be cut. Resolves once the port is released. + */ + protected abstract _close(listener: L, force: boolean): Promise; +} + +/** A bound Deno listener and the controller that aborts its connections. */ +type DenoListener = { + readonly server: Deno.HttpServer; + readonly controller: AbortController; +}; + +/** + * Deno implementation of the MinimalHttpServer abstraction. + */ +export class DenoHttpServer extends BaseHttpServer { + private readonly _options: HttpServerOptions; + + constructor(options: HttpServerOptions = {}) { + super(); + this._options = options; + } + + protected async _bind( + handler: (req: GoatRequest, info: ServeHandlerInfo) => Promise, + port: number, + ): Promise { + const controller = new AbortController(); + let onListen!: () => void; + const listening = new Promise((resolve) => { + onListen = resolve; }); - // Use HTTPS or HTTP based on configuration - const serveOptions: any = { + // WHY no explicit Deno.ServeTcpOptions annotation: the npm build + // type-checks against a minimal Deno stub that lacks that member. + const serveOptions = { port, - onListen() { - resolve(); - }, - signal: combinedSignal, + onListen, + signal: controller.signal, + }; + const onRequest = (req: Request, info: Deno.ServeHandlerInfo) => { + if (!this._drain.accepting) { + return Promise.resolve(serviceUnavailableResponse()); + } + // Map Deno's info to the unified abstraction. remoteAddr is + // NetAddr | UnixAddr; only the network variant has a hostname. + const remoteAddr: HttpRemoteAddr = { + hostname: 'hostname' in info.remoteAddr + ? info.remoteAddr.hostname + : 'localhost', + }; + return this._drain.track( + info.completed, + () => + handler(new GoatRequest(req), { + remoteAddr, + completed: info.completed, + }), + ); }; - // Add TLS options if HTTPS is configured - if (this._options.https) { - serveOptions.key = this._options.https.key; - serveOptions.cert = this._options.https.cert; + const server = this._options.https + ? Deno.serve({ ...serveOptions, ...this._options.https }, onRequest) + : Deno.serve(serveOptions, onRequest); + try { + // WHY: a failed bind (for example a port already in use) never calls + // onListen and surfaces only by rejecting `finished`. Without this race + // start() would hang instead of reporting the failure. + await Promise.race([ + listening, + server.finished.then(() => { + throw new Error('HTTP server closed before it started listening'); + }), + ]); + } catch (error) { + await server.shutdown().catch(() => {}); + throw error; } - - this._server = Deno.serve( - serveOptions, - (req: Request, info: Deno.ServeHandlerInfo) => { - // Map Deno's info to the unified abstraction - const remoteAddr: HttpRemoteAddr = { - hostname: (info.remoteAddr as any).hostname ?? 'localhost', - }; - return handler(new GoatRequest(req), { - remoteAddr, - completed: info.completed, - }); - }, - ); - this._started = true; - return started; + return { server, controller }; } - stop(): void { - this._abortController?.abort(); - this._started = false; + protected async _close( + { server, controller }: DenoListener, + force: boolean, + ): Promise { + if (force) { + controller.abort(); + } else { + // WHY: shutdown() lets Deno flush responses and close connections + // gracefully. abort() would cancel them and can drop a response that + // is already queued for the client. + await server.shutdown(); + } + // WHY: `finished` resolves after the OS listener socket is dropped, which + // is the only point at which the port stops accepting connections. + await server.finished; } get port(): number | undefined { - if (!this._server) return undefined; - const addr = this._server.addr; + const addr = this.listener?.server.addr; // Handle both network and Unix addresses - if ('port' in addr) { - return addr.port; - } - return undefined; + return addr && 'port' in addr ? addr.port : undefined; } get address(): { hostname: string; port: number } | undefined { - if (!this._server) return undefined; - const addr = this._server.addr; - // Handle both network and Unix addresses - if ('port' in addr && 'hostname' in addr) { - return { - hostname: addr.hostname, - port: addr.port, - }; - } - return undefined; + const addr = this.listener?.server.addr; + if (!addr || !('port' in addr) || !('hostname' in addr)) return undefined; + return { hostname: addr.hostname, port: addr.port }; } } +/** A bound Node listener and the error logger attached to it after listen. */ +type NodeListener = { + readonly server: import('node:http').Server; + readonly onError: (error: Error) => void; +}; + /** * Node.js implementation of the MinimalHttpServer abstraction. * This class provides a Node.js-specific implementation of the HTTP server. */ -export class NodeHttpServer implements MinimalHttpServer { - /** The underlying Node.js HTTP/HTTPS server instance */ - private _server?: import('node:http').Server | import('node:https').Server; - /** Whether the server has been started */ - private _started: boolean = false; +export class NodeHttpServer extends BaseHttpServer { /** Cached reference to the Node.js Buffer implementation */ private _Buffer?: typeof import('node:buffer').Buffer; - /** HTTPS configuration options */ - private _options: HttpServerOptions; + private readonly _options: HttpServerOptions; constructor(options: HttpServerOptions = {}) { + super(); this._options = options; } @@ -606,6 +1058,10 @@ export class NodeHttpServer implements MinimalHttpServer { res: MinimalNodeServerResponse, handler: (req: GoatRequest, info: ServeHandlerInfo) => Promise, ): Promise { + if (!this._drain.accepting) { + writeServiceUnavailable(res); + return; + } try { const goatReq = new GoatRequest(req); const hostname = req.socket?.remoteAddress ?? ''; @@ -623,7 +1079,10 @@ export class NodeHttpServer implements MinimalHttpServer { remoteAddr: { hostname }, completed, }; - const response = await handler(goatReq, info); + const response = await this._drain.track( + completed, + () => handler(goatReq, info), + ); res.statusCode = response.status; response.headers.forEach((value, key) => res.setHeader(key, value)); if (response.body) { @@ -653,14 +1112,28 @@ export class NodeHttpServer implements MinimalHttpServer { } } catch (err) { const code = (err as { code?: string })?.code; - if ( - code !== 'ECONNRESET' && code !== 'EPIPE' && code !== 'ECONNABORTED' - ) { - console.error('Request handling error:', err); + const connectionError = code === 'ECONNRESET' || code === 'EPIPE' || + code === 'ECONNABORTED'; + if (!connectionError) { + log({ + severity: 'ERROR', + error: 'UncaughtServerError', + message: `Request handling error: ${ + err instanceof Error ? err.message : String(err) + }`, + trace: err instanceof Error ? err.stack : undefined, + }); } try { if (res.headersSent) { - res.end(); + // If the client connection was destroyed (force/timeout stop), do + // not send a terminating chunk — that would make a truncated + // response look complete. Destroy the response instead. + if (connectionError) { + res.destroy(); + } else { + res.end(); + } } else { res.statusCode = 500; res.end('Internal Server Error'); @@ -671,105 +1144,98 @@ export class NodeHttpServer implements MinimalHttpServer { } } - /** - * Starts the HTTP server on the specified port. - * - * @param handler - The request handler function that processes incoming requests - * @param port - The port number to listen on - * @param signal - Optional AbortSignal for server shutdown - * @returns A promise that resolves when the server is ready to accept connections - */ - async start( - handler: ( - req: GoatRequest, - info: ServeHandlerInfo, - ) => Promise, + protected async _bind( + handler: (req: GoatRequest, info: ServeHandlerInfo) => Promise, port: number, - signal?: AbortSignal, - ): Promise { - if (this._started) { - return; - } - + ): Promise { // Load required Node.js modules const { Buffer } = await getNodeBufferModule(); this._Buffer = Buffer; - // Create and configure the HTTP/HTTPS server + const onRequest = ( + req: MinimalNodeIncomingMessage, + res: MinimalNodeServerResponse, + ) => this._handleRequest(req, res, handler); + + let server: import('node:http').Server; if (this._options.https) { const https = await import('node:https'); - this._server = https.createServer( + server = https.createServer( { key: this._options.https.key, cert: this._options.https.cert, }, (req, res) => - this._handleRequest( + onRequest( req as MinimalNodeIncomingMessage, res as unknown as MinimalNodeServerResponse, - handler, ), ); } else { const { createServer } = await getNodeHttp(); - this._server = createServer( - (req, res) => - this._handleRequest( - req as MinimalNodeIncomingMessage, - res as unknown as MinimalNodeServerResponse, - handler, - ), + server = createServer((req, res) => + onRequest( + req as MinimalNodeIncomingMessage, + res as unknown as MinimalNodeServerResponse, + ) ); } - // Start listening on specified port - await new Promise((resolve, reject) => { - this._server!.on('error', reject); - this._server!.listen(port, () => { - // HTTPS extends HTTP; Deno's floor types cannot call their overload union. - (this._server! as import('node:http').Server).removeListener( - 'error', - reject, - ); - this._server!.on('error', (err: Error) => { - console.error('HTTP server error:', err); + try { + await new Promise((resolve, reject) => { + const onBindError = (error: Error) => { + server.removeListener('error', onBindError); + reject(error); + }; + server.on('error', onBindError); + server.listen(port, () => { + server.removeListener('error', onBindError); + resolve(); }); - resolve(); }); - }); - this._started = true; - - // Wire abort signal for graceful shutdown - if (signal) { - if (signal.aborted) { - this.stop(); - } else { - signal.addEventListener('abort', () => this.stop(), { once: true }); - } + } catch (error) { + // WHY: a failed bind leaves the server object and the request handler + // attached. Clear both so a retry starts from a clean state and no + // stray listener can log against a discarded server. + if (server.listening) server.close(); + server.removeAllListeners(); + throw error; } + + const onError = (error: Error) => { + log({ + severity: 'ERROR', + error: 'HttpServerError', + message: `HTTP server error: ${error.message}`, + trace: error.stack, + }); + }; + server.on('error', onError); + return { server, onError }; } - /** - * Stops the HTTP server and cleans up resources. - */ - stop(): void { - if (this._server && 'closeAllConnections' in this._server) { - (this._server as any).closeAllConnections(); - } - this._server?.close(); - this._started = false; + protected async _close( + { server, onError }: NodeListener, + force: boolean, + ): Promise { + server.removeListener('error', onError); + const closed = new Promise((resolve) => + server.close(() => resolve()) + ); + // Node 24 close() already closes idle keep-alive sockets. Only active + // sockets need an explicit destroy on force or timeout. + if (force) server.closeAllConnections?.(); + await closed; } get port(): number | undefined { - if (!this._server) return undefined; - const addr = (this._server as any).address(); - return addr ? addr.port : undefined; + const addr = this.listener?.server.address(); + return addr && typeof addr !== 'string' ? addr.port : undefined; } get address(): { hostname: string; port: number } | undefined { - if (!this._server) return undefined; - const addr = (this._server as any).address(); - if (!addr) return undefined; + const addr = this.listener?.server.address(); + if (!addr || typeof addr === 'string') return undefined; return { hostname: addr.address || 'localhost', port: addr.port, @@ -777,6 +1243,13 @@ export class NodeHttpServer implements MinimalHttpServer { } } +/** Writes the shared 503 to a raw Node response. */ +function writeServiceUnavailable(res: MinimalNodeServerResponse): void { + res.statusCode = SERVICE_UNAVAILABLE_STATUS; + res.setHeader('Connection', SERVICE_UNAVAILABLE_CONNECTION); + res.end(SERVICE_UNAVAILABLE_BODY); +} + /** @group HTTP */ export interface HttpServerOptions { /** HTTPS configuration */ diff --git a/net/server/server.ts b/net/server/server.ts index ff38eb7e..782a8ba5 100644 --- a/net/server/server.ts +++ b/net/server/server.ts @@ -10,7 +10,7 @@ import { import type { HTTPMethod } from '../../logging/metrics.ts'; import { StaticAssetsEndpoint } from './static-assets.ts'; import { AuthEndpoint } from './auth.ts'; -import { HealthCheckEndpoint } from './health.ts'; +import { HealthCheckEndpoint, kHealthCheckPath } from './health.ts'; import { MetricsMiddleware } from './metrics.ts'; import { BaseService } from './service.ts'; import { CORSEndpoint, CORSMiddleware } from './cors.ts'; @@ -25,13 +25,40 @@ import type { StaticAssets } from '../../system-assets/system-assets.ts'; import type { Schema } from '../../cfds/base/schema.ts'; import type { ManagedItem } from '../../db/managed-item.ts'; import { + awaitPendingStop, + clearPendingOnReject, createHttpServer, type GoatRequest, type HttpServerInstance, + type HttpServerStopOptions, + logStopFailure, type ServeHandlerInfo, + serviceUnavailableResponse, + StartStopGate, + validateStopTimeout, } from './http-compat.ts'; import { generateSelfSignedCertificate } from './crypto-utils.ts'; +const kAlbHealthCheckHost = 'awsalb'; + +/** + * AWS ALB sends a fixed host for health checks. Match the parsed host and path + * only, so case normalization does not leak into tenant domain resolution. + * Per URL parsing: host is case-insensitive, path is case-sensitive, and the + * port and query string are ignored. Both http: and https: URLs match because + * ALB supports both probe protocols. Malformed URLs never match. + */ +function isAlbHealthCheck(url: string): boolean { + try { + const parsed = new URL(url); + return (parsed.protocol === 'http:' || parsed.protocol === 'https:') && + parsed.hostname === kAlbHealthCheckHost && + parsed.pathname === kHealthCheckPath; + } catch { + return false; + } +} + /** * Information about a user attempting to log in for the first time. * Used by the autoCreateUser function to determine if a new account should be @@ -343,6 +370,10 @@ export class Server { private _abortController: AbortController | undefined; private _httpServer?: HttpServerInstance; private _stopping = false; + /** In-flight stop, so repeated stop() calls await the same teardown. */ + private _stopPromise?: Promise; + /** Serializes start/stop so a stop racing start() cannot return early. */ + private readonly _lifecycle = new StartStopGate(); constructor(options: ServerOptions) { this._endpoints = []; @@ -537,11 +568,11 @@ export class Server { goatReq: GoatRequest, info: ServeHandlerInfo, ): Promise { - if (goatReq.url === 'http://AWSALB/healthy') { - return new Response(null, { status: 200 }); - } if (this._stopping) { - return new Response(null, { status: 503 }); + return serviceUnavailableResponse(); + } + if (isAlbHealthCheck(goatReq.url)) { + return new Response(null, { status: 200 }); } const orgId = this._domain.resolveDomain(goatReq.url); if (!orgId) { @@ -634,7 +665,16 @@ export class Server { * * @returns Promise that resolves when server is fully started */ - async start(): Promise { + start(): Promise { + return this._lifecycle.start(() => this._start()); + } + + private async _start(): Promise { + // WHY: a restart must not reset lifecycle state while the old teardown + // still drains requests and closes databases. + await awaitPendingStop(this._stopPromise, () => { + this._stopPromise = undefined; + }); this._stopping = false; // Reset from prior stop() // Return early if server is already running @@ -651,30 +691,69 @@ export class Server { this._httpServer = createHttpServer({ https: cert }); } - // Start all services for each org - for (const services of this._servicesByOrg.values()) { - for (const v of Object.values(services)) { - if (v instanceof BaseService) { - await v.start(); + const controller = new AbortController(); + const started: BaseService[] = []; + try { + // Start all services for each org + for (const services of this._servicesByOrg.values()) { + for (const v of Object.values(services)) { + if (v instanceof BaseService) { + // WHY: recorded before start settles, so a throwing start is + // rolled back too and its partial work cannot leak. + started.push(v as BaseService); + await v.start(); + } } } + await this._httpServer!.start( + this.processRequest.bind(this), + this._baseOptions.port!, + controller.signal, + ); + } catch (error) { + this._rollbackStart(started); + log({ + severity: 'ERROR', + error: 'ServerStartFailed', + message: `Server start failed: ${ + error instanceof Error ? error.message : String(error) + }`, + trace: error instanceof Error ? error.stack : undefined, + }); + throw error; } - - // Log server start metric + // WHY: committed only after everything succeeded. processRequest() and + // stop() read the controller as "running", and a half-started server must + // not look ready. A stop() that raced this start is queued behind it and + // still closes the listener. + this._abortController = controller; log({ severity: 'METRIC', name: 'ServerStarted', value: 1, unit: 'Count', }); + } - // Initialize abort controller and start HTTP server - this._abortController = new AbortController(); - return this._httpServer!.start( - this.processRequest.bind(this), - this._baseOptions.port!, - this._abortController.signal, - ); + /** Stops the services a failed start() began, in reverse order. */ + private _rollbackStart(started: readonly BaseService[]): void { + // WHY: a failed start must leave no service active, and must not block a + // retry, so the flag is cleared even when a stop throws. + this._stopping = false; + for (const service of started.toReversed()) { + try { + service.stop(); + } catch (error) { + log({ + severity: 'ERROR', + error: 'ServerStopFailed', + message: `Rolling back a failed start could not stop a service: ${ + error instanceof Error ? error.message : String(error) + }`, + trace: error instanceof Error ? error.stack : undefined, + }); + } + } } /** @@ -698,22 +777,60 @@ export class Server { * Gracefully stops the server and all its services. * Safe to call multiple times. * - * Shutdown sequence: stops accepting new connections, rejects new service - * lookups via `_stopping` flag, then flushes and closes all databases. - * Already-dispatched request handlers may still be running during flush — - * there is no request-draining mechanism. + * Shutdown sequence: rejects new requests, drains active handlers and + * responses on the internal HTTP server, then flushes and closes databases. + * `force` skips the drain; `timeoutMs` bounds the drain, not the final + * listener close. A later `force` escalates only while the drain is pending, + * while `timeoutMs` is fixed by the first call; a `force` interrupts a pending + * drain even when the stop is queued behind a restart. Direct `processRequest` + * calls are not tracked; embedded servers must drain them themselves. A handler + * that calls `stop()` is part of the drain, so shutdown endpoints should pass + * `force: true`. A stop() racing an in-flight start() resolves only after + * that start settles and the listener is closed. Neither force nor timeout + * can cancel application handlers, which may continue after their databases + * close. A failed escalated teardown is logged as `ServerStopFailed`; callers + * still observe the primary stop result. */ - async stop(): Promise { + stop(options?: HttpServerStopOptions): Promise { + // WHY: validate before escalating or clearing state, so a rejected budget + // leaves the server and its databases untouched. + try { + validateStopTimeout(options); + } catch (error) { + return Promise.reject(error); + } + // WHY: the lifecycle gate may queue this stop behind a restart, so force + // must interrupt the active teardown now; the queued stop still closes the + // restarted listener. + if (options?.force && this._stopPromise) { + void this._httpServer?.stop(options).catch(logStopFailure); + } + return this._lifecycle.stop(() => this._requestStop(options)); + } + + private _requestStop(options?: HttpServerStopOptions): Promise { if (!this._abortController) { - return; + // A later force must be able to escalate the active graceful teardown. + if (options?.force) { + void this._httpServer?.stop(options).catch(logStopFailure); + } + return this._stopPromise ?? Promise.resolve(); } - // Capture and clear immediately to prevent concurrent stop() calls - const ac = this._abortController; this._abortController = undefined; this._stopping = true; + this._stopPromise = clearPendingOnReject( + this._stop(options), + () => { + this._stopPromise = undefined; + }, + ); + return this._stopPromise; + } - // Stop HTTP server first so no new requests hit closing DBs - this._httpServer?.stop(); + private async _stop(options?: HttpServerStopOptions): Promise { + // Wait for handlers and responses before closing DBs, unless force or the + // drain timeout permits teardown while application work is still running. + await this._httpServer?.stop(options); // Await any in-flight service creation before cleanup for (const pending of this._pendingServices.values()) { @@ -734,21 +851,34 @@ export class Server { } } - // Flush and close all databases before aborting + // HTTP stop has drained requests or exhausted the caller's drain budget. for (const services of this._servicesByOrg.values()) { try { await services.db.flushAll(); } catch (e) { - console.error('Error flushing database during shutdown:', e); + log({ + severity: 'ERROR', + error: 'StorageError', + message: `Error flushing database during shutdown: ${ + e instanceof Error ? e.message : String(e) + }`, + trace: e instanceof Error ? e.stack : undefined, + }); } try { await services.db.close(); } catch (e) { - console.error('Error closing database during shutdown:', e); + log({ + severity: 'ERROR', + error: 'StorageError', + message: `Error closing database during shutdown: ${ + e instanceof Error ? e.message : String(e) + }`, + trace: e instanceof Error ? e.stack : undefined, + }); } } - ac.abort(); this._servicesByOrg.clear(); } } diff --git a/server.ts b/server.ts index 5be2e6ed..d52c59a5 100644 --- a/server.ts +++ b/server.ts @@ -24,12 +24,17 @@ export type { ServerServices, } from './net/server/server.ts'; -export { createHttpServer } from './net/server/http-compat.ts'; +export { + createHttpServer, + DEFAULT_STOP_TIMEOUT_MS, + MAX_STOP_TIMEOUT_MS, +} from './net/server/http-compat.ts'; export type { GoatHeaders, GoatRequest, HttpServerInstance, HttpServerOptions, + HttpServerStopOptions, ServeHandlerInfo, } from './net/server/http-compat.ts'; diff --git a/tests/cli-init.test.ts b/tests/cli-init.test.ts index 5a52784d..587b0c1b 100644 --- a/tests/cli-init.test.ts +++ b/tests/cli-init.test.ts @@ -266,6 +266,36 @@ export default function setupCliInitTests() { }, ); + TEST( + 'CLI-Init', + 'should align shutdown force-exit with the drain budget', + async (ctx: TestSuite) => { + const testDir = await ctx.tempDir('init-shutdown'); + + const initModule = await import('../cli/init.ts'); + await initModule.bootstrapProject({ + targetDir: testDir, + skipDependencies: true, + }); + + const serverFiles = isDeno() + ? ['server/server.ts'] + : ['server/server.ts', 'server/server-sea.ts']; + for (const file of serverFiles) { + const text = await readTextFile(path.join(testDir, file)); + assertExists(text, `${file} should exist`); + assertTrue( + text.includes('DEFAULT_STOP_TIMEOUT_MS'), + `${file} must use the drain budget`, + ); + assertTrue( + !text.includes('5000'), + `${file} must not hardcode a 5s force exit`, + ); + } + }, + ); + TEST( 'CLI-Init', 'should not overwrite existing files', diff --git a/tests/health-check-endpoint.test.ts b/tests/health-check-endpoint.test.ts index 45a834eb..1a5727cb 100644 --- a/tests/health-check-endpoint.test.ts +++ b/tests/health-check-endpoint.test.ts @@ -1,7 +1,12 @@ -import { HealthCheckEndpoint } from '../net/server/health.ts'; +import { HealthCheckEndpoint, kHealthCheckPath } from '../net/server/health.ts'; import { TEST } from './mod.ts'; import { assertEquals, assertTrue } from './asserts.ts'; import type { Schema } from '../cfds/base/schema.ts'; +import type { ServerServices } from '../net/server/server.ts'; +import { + GoatRequest, + type ServeHandlerInfo, +} from '../net/server/http-compat.ts'; export default function setupHealthCheckEndpointTest(): void { // Minimal test schema @@ -11,7 +16,7 @@ export default function setupHealthCheckEndpointTest(): void { fields: { foo: { type: 'string', required: true } }, }; - function makeServices(overrides: any = {}) { + function makeServices(): ServerServices { return { orgId: 'test-org', domain: { @@ -25,15 +30,15 @@ export default function setupHealthCheckEndpointTest(): void { } }, }, - ...overrides, - }; + // This endpoint does not read database or service state. + } as unknown as ServerServices; } - function makeRequest(path: string, method = 'GET') { - return { method, url: `http://localhost${path}`, headers: new Headers() }; + function makeRequest(path: string, method = 'GET'): GoatRequest { + return new GoatRequest(new Request(`http://localhost${path}`, { method })); } - function makeInfo() { + function makeInfo(): ServeHandlerInfo { return { remoteAddr: { hostname: 'localhost' }, completed: Promise.resolve(), @@ -48,12 +53,8 @@ export default function setupHealthCheckEndpointTest(): void { const services = makeServices(); const req = makeRequest('/healthy', 'GET'); const info = makeInfo(); - assertTrue(endpoint.filter(services as any, req as any, info as any)); - const resp = await endpoint.processRequest( - services as any, - req as any, - info as any, - ); + assertTrue(endpoint.filter(services, req, info)); + const resp = await endpoint.processRequest(services, req, info); assertEquals(resp.status, 200); assertEquals(await resp.text(), 'OK'); assertEquals(resp.headers.get('cache-control'), 'no-store'); @@ -65,9 +66,9 @@ export default function setupHealthCheckEndpointTest(): void { const services = makeServices(); const info = makeInfo(); const postReq = makeRequest('/healthy', 'POST'); - assertTrue(!endpoint.filter(services as any, postReq as any, info as any)); + assertTrue(!endpoint.filter(services, postReq, info)); const putReq = makeRequest('/healthy', 'PUT'); - assertTrue(!endpoint.filter(services as any, putReq as any, info as any)); + assertTrue(!endpoint.filter(services, putReq, info)); }); TEST('HealthCheckEndpoint', 'filter rejects wrong path', () => { @@ -75,7 +76,7 @@ export default function setupHealthCheckEndpointTest(): void { const services = makeServices(); const info = makeInfo(); const req = makeRequest('/not-healthy', 'GET'); - assertTrue(!endpoint.filter(services as any, req as any, info as any)); + assertTrue(!endpoint.filter(services, req, info)); }); TEST('HealthCheckEndpoint', 'filter is case-insensitive for path', () => { @@ -83,6 +84,18 @@ export default function setupHealthCheckEndpointTest(): void { const services = makeServices(); const info = makeInfo(); const req = makeRequest('/HeAlThY', 'GET'); - assertTrue(endpoint.filter(services as any, req as any, info as any)); + assertTrue(endpoint.filter(services, req, info)); }); + + TEST( + 'HealthCheckEndpoint', + 'filter accepts the shared constant regardless of its literal case', + () => { + const endpoint = new HealthCheckEndpoint(); + const services = makeServices(); + const info = makeInfo(); + const req = makeRequest(kHealthCheckPath.toUpperCase(), 'GET'); + assertTrue(endpoint.filter(services, req, info)); + }, + ); } diff --git a/tests/http-server-stop.test.ts b/tests/http-server-stop.test.ts new file mode 100644 index 00000000..c2a467aa --- /dev/null +++ b/tests/http-server-stop.test.ts @@ -0,0 +1,1394 @@ +/** Real-socket shutdown contract for both server runtimes. */ +import { + DEFAULT_STOP_TIMEOUT_MS, + MAX_STOP_TIMEOUT_MS, + Server, +} from '../server.ts'; +import { + awaitPendingStop, + createHttpServer, + GoatRequest, + type HttpServerStopOptions, + type MinimalHttpServer, + StartStopGate, +} from '../net/server/http-compat.ts'; +import { DataRegistry } from '../cfds/base/data-registry.ts'; +import { TEST } from './mod.ts'; +import { assertEquals, assertTrue } from './asserts.ts'; +import * as path from '../base/path.ts'; +import { generateBuildInfo } from '../base/build-info.ts'; +import { getRuntime } from '../base/runtime/index.ts'; + +import { clientOrigin } from './http-test-utils.ts'; + +function deferred(): { + promise: Promise; + resolve: (value: T) => void; +} { + let resolve!: (value: T) => void; + const promise = new Promise((res) => { + resolve = res; + }); + return { promise, resolve }; +} + +async function bounded(pending: Promise, timeoutMs = 5000): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + pending, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error('Socket test timed out')), + timeoutMs, + ); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +function requirePort(server: { readonly port?: number }): number { + const port = server.port; + if (port === undefined) { + throw new Error('Started server must expose its port'); + } + return port; +} + +async function stopTwice(server: MinimalHttpServer): Promise { + const firstStop = server.stop(); + const secondStop = server.stop(); + assertEquals(server.port, undefined, 'port must clear when stop starts'); + assertEquals( + server.address, + undefined, + 'address must clear when stop starts', + ); + await Promise.all([firstStop, secondStop]); +} + +async function startAndRead( + server: MinimalHttpServer, + listenPort = 0, +): Promise { + await server.start(() => Promise.resolve(new Response('ok')), listenPort); + const port = requirePort(server); + const response = await fetch(clientOrigin(server), { + // WHY: Deno < 2.9 reuses pooled keep-alive sockets across a server + // restart and fails the next request (denoland/deno#17754). This test + // checks the shutdown contract, not client-pool recovery. + headers: { connection: 'close' }, + }); + assertEquals(await response.text(), 'ok', 'body should match'); + return port; +} + +/** Fails the fetch; used to prove that a connection was torn down. */ +async function expectRejected(pending: Promise): Promise { + const outcome = await pending.then( + () => 'resolved', + () => 'rejected', + ); + assertEquals(outcome, 'rejected', 'request must be cut off by stop()'); +} + +function request(server: MinimalHttpServer, signal?: AbortSignal) { + return fetch(clientOrigin(server), { + headers: { connection: 'close' }, + signal, + }); +} + +/** True when a bind failed because another listener holds the port. */ +function isAddressInUse(error: unknown): boolean { + return error instanceof Error && + /address already in use|AddrInUse|EADDRINUSE/i.test(String(error)); +} + +/** Starts `server` on `port`, retrying while the old listener drains. */ +async function startWhenFree( + server: MinimalHttpServer, + port: number, +): Promise { + await bounded((async () => { + while (true) { + try { + await server.start(() => Promise.resolve(new Response('ok')), port); + return; + } catch (error) { + if (!isAddressInUse(error)) throw error; + await new Promise((resolve) => setTimeout(resolve, 0)); + } + } + })()); +} + +/** Starts a server on a fixed port, retrying while the old listener drains. */ +async function rebind(port: number): Promise { + const server = createHttpServer(); + try { + await startWhenFree(server, port); + return server; + } catch (error) { + await server.stop({ force: true }); + throw error; + } +} + +function streamingBody() { + let controller!: ReadableStreamDefaultController; + const body = new ReadableStream({ + start(value) { + controller = value; + controller.enqueue(new TextEncoder().encode('first')); + }, + }); + return { + body, + finish() { + controller.enqueue(new TextEncoder().encode('last')); + controller.close(); + }, + }; +} + +async function firstChunk(response: Response) { + const reader = response.body!.getReader(); + const chunk = await bounded(reader.read()); + assertEquals(new TextDecoder().decode(chunk.value), 'first'); + assertEquals(chunk.done, false, 'response must still be streaming'); + return reader; +} + +async function interruptedStream(options: HttpServerStopOptions) { + const server = createHttpServer(); + const stream = streamingBody(); + try { + await server.start(() => Promise.resolve(new Response(stream.body)), 0); + const reader = await firstChunk(await bounded(request(server))); + const remaining = reader.read().then( + (result) => ({ type: 'done', result } as const), + (error) => ({ type: 'error', error } as const), + ); + await bounded(server.stop(options)); + const outcome = await bounded(remaining); + // Node.js resolves done on abrupt close; Deno rejects. Both must settle + // because stop() cut the partial response. + assertTrue( + outcome.type === 'error' || + (outcome.result.done && outcome.result.value === undefined), + 'partial stream must settle after force/timeout stop', + ); + } finally { + await server.stop({ force: true }); + } +} + +async function interruptedRequest(options: HttpServerStopOptions) { + const server = createHttpServer(); + const started = deferred(); + const release = deferred(); + try { + await server.start(async () => { + started.resolve(); + await release.promise; + return new Response('late'); + }, 0); + const pending = request(server); + void pending.catch(() => {}); + await bounded(started.promise); + await bounded(server.stop(options)); + await expectRejected(pending); + } finally { + release.resolve(); + await server.stop({ force: true }); + } +} + +async function createStopTestServer( + serverPath: string, + registry: DataRegistry, + getPort: () => number, + resolveDomain: (url: string) => string = () => 'localhost', + port = 0, +) { + const buildInfo = await generateBuildInfo( + path.join(getRuntime().getCWD(), 'deno.json'), + ); + return new Server({ + registry, + path: serverPath, + port, + buildInfo, + disableDefaultEndpoints: true, + // WHY: port 0 needs a custom domain. The default resolveOrg captures the + // port at construction. + domain: { + resolveOrg: () => `http://localhost:${getPort()}`, + resolveDomain, + }, + }); +} + +export default function setupHttpServerStop() { + TEST( + 'HttpServerStop', + 'disconnect does not finish an active handler', + async () => { + const server = createHttpServer(); + const started = deferred(); + const release = deferred(); + const abort = new AbortController(); + let finished = false; + try { + await server.start(async () => { + started.resolve(); + await release.promise; + finished = true; + return new Response('late'); + }, 0); + const origin = clientOrigin(server); + const pending = request(server, abort.signal); + const disconnected = expectRejected(pending); + await bounded(started.promise); + abort.abort(); + await bounded(disconnected); + const stopping = server.stop().then(() => { + assertTrue(finished, 'stop must wait for the disconnected handler'); + }); + const arrivals = await bounded( + fetch(origin, { + headers: { connection: 'close' }, + }), + ); + assertEquals(arrivals.status, 503); + await arrivals.text(); + release.resolve(); + await bounded(stopping); + } finally { + release.resolve(); + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'graceful stop delivers the complete stream', + async () => { + const server = createHttpServer(); + const stream = streamingBody(); + try { + await server.start(() => Promise.resolve(new Response(stream.body)), 0); + const origin = clientOrigin(server); + const reader = await firstChunk(await bounded(request(server))); + let stopped = false; + const stopping = server.stop().then(() => { + stopped = true; + }); + const rejected = await bounded( + fetch(origin), + ); + assertEquals(rejected.status, 503); + await rejected.text(); + assertEquals(stopped, false, 'stop must wait for the response body'); + stream.finish(); + const tail = await bounded(reader.read()); + assertEquals(new TextDecoder().decode(tail.value), 'last'); + assertEquals((await bounded(reader.read())).done, true); + await bounded(stopping); + } finally { + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'timeout interrupts a partial stream', + () => interruptedStream({ timeoutMs: 50 }), + ); + TEST( + 'HttpServerStop', + 'force interrupts a partial stream', + () => interruptedStream({ force: true }), + ); + + TEST( + 'HttpServerStop', + 'stop is idempotent and releases the port', + async () => { + const server = createHttpServer(); + try { + const port = await startAndRead(server); + await stopTwice(server); + // A closed listener is provable by binding the port again. Do not + // probe a closed port: Windows reports refusal only after its SYN + // retransmission budget (~2s), which turns the test into an OS-timing + // assertion instead of a server-contract assertion. + const rebound = await rebind(port); + try { + assertEquals(rebound.port, port, 'same port must be reusable'); + const response = await bounded(request(rebound)); + assertEquals(await response.text(), 'ok'); + } finally { + await rebound.stop({ force: true }); + } + } finally { + await server.stop(); + } + }, + ); + + TEST( + 'HttpServerStop', + 'graceful stop drains in-flight requests and rejects new ones', + async () => { + const server = createHttpServer(); + const started = deferred(); + const release = deferred(); + try { + await server.start(async () => { + started.resolve(); + await release.promise; + return new Response('ok'); + }, 0); + const origin = clientOrigin(server); + const first = fetch(origin, { + headers: { connection: 'close' }, + }); + void first.catch(() => {}); + await started.promise; + + const stopping = server.stop(); + + const duringDrain = await fetch(origin, { + headers: { connection: 'close' }, + }); + assertEquals(duringDrain.status, 503, 'new requests must be rejected'); + assertEquals(await duringDrain.text(), 'Service Unavailable'); + + release.resolve(); + const response = await first; + assertEquals(response.status, 200, 'in-flight request must finish'); + assertEquals(await response.text(), 'ok'); + await stopping; + } finally { + release.resolve(); + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'graceful stop force-closes requests that exceed the drain timeout', + () => interruptedRequest({ timeoutMs: 50 }), + ); + + TEST( + 'HttpServerStop', + 'force stop skips the drain', + () => interruptedRequest({ force: true }), + ); + + TEST( + 'HttpServerStop', + 'force escalates an in-flight graceful stop', + async () => { + const server = createHttpServer(); + const started = deferred(); + const release = deferred(); + try { + await server.start(async () => { + started.resolve(); + await release.promise; + return new Response('late'); + }, 0); + const pending = request(server); + void pending.catch(() => {}); + await bounded(started.promise); + + const graceful = server.stop(); + const forced = server.stop({ force: true }); + await bounded(Promise.all([graceful, forced])); + await expectRejected(pending); + } finally { + release.resolve(); + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'rejected requests carry connection close', + async () => { + const server = createHttpServer(); + const started = deferred(); + const release = deferred(); + try { + await server.start(async () => { + started.resolve(); + await release.promise; + return new Response('late'); + }, 0); + const origin = clientOrigin(server); + const pending = request(server); + void pending.catch(() => {}); + await bounded(started.promise); + const stopping = server.stop(); + const rejected = await bounded( + fetch(origin, { + headers: { connection: 'close' }, + }), + ); + assertEquals(rejected.status, 503); + assertTrue( + rejected.headers.get('connection')?.includes('close') ?? false, + 'rejection must tell clients not to reuse the socket', + ); + await rejected.text(); + release.resolve(); + await bounded(stopping); + await pending.catch(() => {}); + } finally { + release.resolve(); + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'graceful stop closes idle keep-alive connections', + async () => { + const server = createHttpServer(); + try { + await server.start(() => Promise.resolve(new Response('ok')), 0); + // Deliberately keep-alive: stop() must close idle sockets itself. + const response = await fetch(clientOrigin(server)); + assertEquals(await response.text(), 'ok'); + // Bound below the drain budget: a pass through the timeout path fails. + await bounded(server.stop({ timeoutMs: 10_000 })); + assertEquals(server.port, undefined); + } finally { + await server.stop({ force: true }); + } + }, + ); + + TEST('HttpServerStop', 'stop before start is safe', async () => { + const server = createHttpServer(); + await stopTwice(server); + await server.stop({ force: true }); + }); + + TEST('HttpServerStop', 'default stop timeout is 10s', () => { + assertEquals( + DEFAULT_STOP_TIMEOUT_MS, + 10_000, + 'default drain budget must be 10s', + ); + }); + + TEST( + 'HttpServerStop', + 'same instance can restart after stop', + async () => { + const server = createHttpServer(); + try { + await startAndRead(server); + await server.stop(); + await startAndRead(server); + } finally { + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'start waits for an in-flight stop', + async () => { + const server = createHttpServer(); + const started = deferred(); + const release = deferred(); + try { + await server.start(async () => { + started.resolve(); + await release.promise; + return new Response('first'); + }, 0); + const first = request(server); + await bounded(started.promise); + + const stopping = server.stop({ timeoutMs: 5000 }); + const restarting = server.start( + () => Promise.resolve(new Response('second')), + 0, + ); + release.resolve(); + + const firstResponse = await bounded(first); + assertEquals(await firstResponse.text(), 'first'); + await bounded(stopping); + await bounded(restarting); + const second = await bounded(request(server)); + assertEquals(await second.text(), 'second'); + } finally { + release.resolve(); + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'force stop does not cancel the handler', + async () => { + const server = createHttpServer(); + const started = deferred(); + const release = deferred(); + const handlerDone = deferred(); + let handlerFinished = false; + try { + await server.start(async () => { + started.resolve(); + await release.promise; + handlerFinished = true; + handlerDone.resolve(); + return new Response('late'); + }, 0); + const pending = request(server); + void pending.catch(() => {}); + await bounded(started.promise); + // If force waited for the drain, this would block on the held handler. + await bounded(server.stop({ force: true })); + assertEquals( + handlerFinished, + false, + 'force stop must resolve while the handler still runs', + ); + + release.resolve(); + await bounded(handlerDone.promise); + assertTrue(handlerFinished, 'handler must finish after release'); + await pending.catch(() => {}); + } finally { + release.resolve(); + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'Server.stop rejects new requests while handlers retain database access', + async (ctx) => { + // WHY: ctx.tempDir auto-cleans. Manual ./server-test-* dirs littered + // the repo root on prior runs. + const serverPath = await ctx.tempDir('server-stop'); + let actualPort = 0; + const schema = { + ns: 'shutdown-test', + version: 1, + fields: { value: { type: 'string', required: true } }, + } as const; + const registry = new DataRegistry(); + registry.registerSchema(schema); + const server = await createStopTestServer( + serverPath, + registry, + () => actualPort, + ); + const started = deferred(); + const release = deferred(); + server.registerEndpoint({ + filter() { + return true; + }, + async processRequest({ db }) { + started.resolve(); + await release.promise; + const item = db.create('/data/shutdown/item', schema, { + value: 'ok', + }); + await db.flushAll(); + return new Response(item.get('value')); + }, + }); + try { + await server.start(); + actualPort = requirePort(server); + const base = `${clientOrigin(server)}/x`; + const inFlight = fetch(base, { + headers: { connection: 'close' }, + }); + void inFlight.catch(() => {}); + await started.promise; + + const healthBefore = await bounded(server.processRequest( + new GoatRequest(new Request('http://AWSALB/healthy')), + { + remoteAddr: { hostname: 'localhost' }, + completed: Promise.resolve(), + }, + )); + assertEquals( + healthBefore.status, + 200, + 'health checks must pass before shutdown', + ); + + // The HTTP adapter rejects arrivals while the active handler drains. + const stopping = server.stop({ timeoutMs: 10_000 }); + + const rejected = await fetch(base, { + headers: { connection: 'close' }, + }); + assertEquals( + rejected.status, + 503, + 'new requests must be rejected during shutdown', + ); + await rejected.text(); + + const health = await bounded(server.processRequest( + new GoatRequest(new Request('http://AWSALB/healthy')), + { + remoteAddr: { hostname: 'localhost' }, + completed: Promise.resolve(), + }, + )); + assertEquals( + health.status, + 503, + 'health checks must be rejected during drain', + ); + + // A restart must wait for the old teardown before it opens the DBs. + const restarting = server.start(); + release.resolve(); + const done = await inFlight; + assertEquals(done.status, 200, 'in-flight request must finish'); + assertEquals(await done.text(), 'ok'); + await stopping; + await restarting; + + actualPort = requirePort(server); + const after = await fetch(`${clientOrigin(server)}/x`, { + headers: { connection: 'close' }, + }); + assertEquals(after.status, 200, 'restarted server must serve'); + assertEquals(await after.text(), 'ok'); + } finally { + release.resolve(); + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'stop() racing start() closes the restarted listener', + async () => { + const server = createHttpServer(); + const ok = () => Promise.resolve(new Response('ok')); + const started = deferred(); + const release = deferred(); + const blocking = async () => { + started.resolve(); + await release.promise; + return new Response('ok'); + }; + try { + await server.start(blocking, 0); + requirePort(server); + const inFlight = request(server); + void inFlight.catch(() => {}); + await bounded(started.promise); + + // The graceful stop waits for the handler, so the restart awaits it. + const parking = server.stop({ timeoutMs: 10_000 }); + const restarting = server.start(ok, 0); + // WHY: without a start guard this stop() sees no listener yet, returns + // immediately, and the restart keeps the server running. + const racingStop = server.stop({ force: true }); + release.resolve(); + await inFlight.catch(() => {}); + await bounded(Promise.all([parking, restarting, racingStop])); + + assertEquals( + server.port, + undefined, + 'racing stop must close the restarted listener', + ); + } finally { + release.resolve(); + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'force interrupts a drain queued behind a restart', + async () => { + const server = createHttpServer(); + const ok = () => Promise.resolve(new Response('ok')); + const started = deferred(); + const release = deferred(); + const blocking = async () => { + started.resolve(); + await release.promise; + return new Response('ok'); + }; + try { + await server.start(blocking, 0); + const inFlight = request(server); + void inFlight.catch(() => {}); + await bounded(started.promise); + + // The restart queues the forced stop behind the graceful teardown. + const parking = server.stop({ timeoutMs: 30_000 }); + const restarting = server.start(ok, 0); + // WHY: the handler stays pending, so only immediate escalation can + // free the teardown before the graceful budget expires. + await bounded(server.stop({ force: true }), 2000); + assertEquals( + server.port, + undefined, + 'the forced stop must close the restarted listener', + ); + await expectRejected(inFlight); + await bounded(Promise.all([parking, restarting])); + } finally { + release.resolve(); + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'Server.stop during start leaves no running listener', + async (ctx) => { + const serverPath = await ctx.tempDir('server-stop-race'); + let actualPort = 0; + const server = await createStopTestServer( + serverPath, + new DataRegistry(), + () => actualPort, + ); + const started = deferred(); + const release = deferred(); + server.registerEndpoint({ + filter() { + return true; + }, + async processRequest() { + started.resolve(); + await release.promise; + return new Response('ok'); + }, + }); + try { + await server.start(); + actualPort = requirePort(server); + const inFlight = fetch(`${clientOrigin(server)}/x`, { + headers: { connection: 'close' }, + }); + void inFlight.catch(() => {}); + await bounded(started.promise); + + // The graceful stop waits for the handler, so the restart awaits it. + const parking = server.stop({ timeoutMs: 10_000 }); + const restarting = server.start(); + // WHY: this used to return before start() resumed, leaving the server + // bound while the caller believed it had stopped. + const racingStop = server.stop({ force: true }); + release.resolve(); + await inFlight.catch(() => {}); + await bounded(Promise.all([parking, restarting, racingStop])); + + assertEquals( + server.port, + undefined, + 'racing stop must close the restarted listener', + ); + + // The raced restart must not wedge the services for a later start. + await bounded(server.start()); + actualPort = requirePort(server); + const after = await fetch(`${clientOrigin(server)}/x`, { + headers: { connection: 'close' }, + }); + assertEquals(after.status, 200, 'server must still start after race'); + await after.text(); + } finally { + release.resolve(); + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'Server.stop force interrupts a drain queued behind a restart', + async (ctx) => { + const serverPath = await ctx.tempDir('server-stop-force-race'); + let actualPort = 0; + const server = await createStopTestServer( + serverPath, + new DataRegistry(), + () => actualPort, + ); + const started = deferred(); + const release = deferred(); + server.registerEndpoint({ + filter() { + return true; + }, + async processRequest() { + started.resolve(); + await release.promise; + return new Response('late'); + }, + }); + try { + await server.start(); + actualPort = requirePort(server); + const inFlight = fetch(`${clientOrigin(server)}/x`, { + headers: { connection: 'close' }, + }); + void inFlight.catch(() => {}); + await bounded(started.promise); + + // The restart queues the forced stop behind the graceful teardown. + const parking = server.stop({ timeoutMs: 30_000 }); + const restarting = server.start(); + // WHY: the handler stays pending, so force must escalate now instead + // of waiting for the restart queue to reach it. + await bounded(server.stop({ force: true }), 2000); + assertEquals( + server.port, + undefined, + 'the forced stop must close the restarted listener', + ); + await expectRejected(inFlight); + await bounded(Promise.all([parking, restarting])); + } finally { + release.resolve(); + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'ALB health check matches parsed host and path', + async (ctx) => { + const serverPath = await ctx.tempDir('server-health'); + let resolvedUrl: string | undefined; + const server = await createStopTestServer( + serverPath, + new DataRegistry(), + () => 0, + (url) => { + resolvedUrl = url; + return 'localhost'; + }, + ); + const info = { + remoteAddr: { hostname: 'localhost' }, + completed: Promise.resolve(), + }; + // WHY: Request normalizes host case and drops default ports. Mutate the + // normalized URL so the check sees the string Node builds from Host. + const requestFor = (url: string) => { + const req = new GoatRequest(new Request('http://placeholder.invalid/')); + req.url = url; + return req; + }; + try { + await server.start(); + const cases: Array<{ url: string; status: number; resolved?: string }> = + [ + { url: 'http://AWSALB:80/healthy?probe=1', status: 200 }, + { url: 'http://awsalb:8080/healthy', status: 200 }, + { url: 'https://awsalb/healthy', status: 200 }, + { + url: 'http://awsalb/HEALTHY', + status: 404, + resolved: 'http://awsalb/HEALTHY', + }, + { + url: 'http://otherhost/healthy', + status: 404, + resolved: 'http://otherhost/healthy', + }, + { + url: 'http://awsalb/healthy/', + status: 404, + resolved: 'http://awsalb/healthy/', + }, + { + url: 'http://awsalb/not-healthy', + status: 404, + resolved: 'http://awsalb/not-healthy', + }, + { + url: 'not a valid url', + status: 404, + resolved: 'not a valid url', + }, + ]; + for (const { url, status, resolved } of cases) { + resolvedUrl = undefined; + const response = await server.processRequest(requestFor(url), info); + assertEquals(response.status, status, url); + assertEquals(resolvedUrl, resolved, `resolution for ${url}`); + } + } finally { + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'a pre-aborted start signal leaves no listener', + async () => { + const server = createHttpServer(); + const ac = new AbortController(); + try { + ac.abort(); + await bounded(server.start( + () => Promise.resolve(new Response('ok')), + 0, + ac.signal, + )); + assertEquals( + server.port, + undefined, + 'a pre-aborted signal must release the port at once', + ); + assertEquals( + server.address, + undefined, + 'a pre-aborted signal must release the address at once', + ); + await bounded(server.stop()); + await bounded(server.start( + () => Promise.resolve(new Response('ok')), + 0, + )); + const response = await bounded(request(server)); + assertEquals(await response.text(), 'ok'); + } finally { + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'awaitPendingStop clears the slot when teardown rejects', + async () => { + const failure = new Error('teardown failed'); + let cleared = false; + let outcome: unknown; + try { + await awaitPendingStop(Promise.reject(failure), () => { + cleared = true; + }); + } catch (error) { + outcome = error; + } + assertTrue(cleared, 'a rejected teardown must clear the pending slot'); + assertTrue(outcome === failure, 'the teardown error must propagate'); + }, + ); + + TEST( + 'HttpServerStop', + 'Server.stop force escalates an in-flight stop', + async (ctx) => { + const serverPath = await ctx.tempDir('server-stop-force'); + let actualPort = 0; + const server = await createStopTestServer( + serverPath, + new DataRegistry(), + () => actualPort, + ); + const started = deferred(); + const release = deferred(); + server.registerEndpoint({ + filter() { + return true; + }, + async processRequest() { + started.resolve(); + await release.promise; + return new Response('late'); + }, + }); + try { + await server.start(); + actualPort = requirePort(server); + const pending = fetch(`${clientOrigin(server)}/x`, { + headers: { connection: 'close' }, + }); + void pending.catch(() => {}); + await started.promise; + + const graceful = server.stop(); + const forced = server.stop({ force: true }); + await bounded(Promise.all([graceful, forced])); + await expectRejected(pending); + } finally { + release.resolve(); + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'restart ignores the previous start signal and honors the current one', + async () => { + const server = createHttpServer(); + const stale = new AbortController(); + const fresh = new AbortController(); + try { + await server.start( + () => Promise.resolve(new Response('ok')), + 0, + stale.signal, + ); + await server.stop(); + await server.start( + () => Promise.resolve(new Response('ok')), + 0, + fresh.signal, + ); + const port = requirePort(server); + stale.abort(); + // Abort dispatch is synchronous. The stale listener must not stop the + // restart, so the port stays bound. + assertEquals( + server.port, + port, + 'stale signal must not stop the restart', + ); + const response = await bounded(request(server)); + assertEquals(await response.text(), 'ok'); + fresh.abort(); + assertEquals( + server.port, + undefined, + 'the current signal must stop the restarted listener', + ); + } finally { + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'aborting the start signal alone releases the listener', + async () => { + const server = createHttpServer(); + const ac = new AbortController(); + try { + await server.start( + () => Promise.resolve(new Response('ok')), + 0, + ac.signal, + ); + const port = requirePort(server); + ac.abort(); + const rebound = await rebind(port); + try { + assertEquals(server.port, undefined, 'abort must clear the port'); + } finally { + await rebound.stop({ force: true }); + } + } finally { + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'lifecycle gate runs ops in call order', + async () => { + const gate = new StartStopGate(); + const order: string[] = []; + const releaseStop = deferred(); + const started = gate.start(() => { + order.push('start'); + return Promise.resolve(); + }); + const stopping = gate.stop(async () => { + order.push('stop:begin'); + await releaseStop.promise; + order.push('stop:end'); + }); + await started; + // WHY: this start lands while the stop is pending. It must wait for the + // teardown, otherwise it binds a listener the pending stop never sees. + const restarting = gate.start(() => { + order.push('restart'); + return Promise.resolve(); + }); + releaseStop.resolve(); + await Promise.all([stopping, restarting]); + assertEquals(order.join(','), 'start,stop:begin,stop:end,restart'); + }, + ); + + TEST( + 'HttpServerStop', + 'lifecycle gate runs the next op after a rejection', + async () => { + const gate = new StartStopGate(); + const failure = new Error('teardown failed'); + let ran = false; + const failing = gate.stop(() => Promise.reject(failure)); + const next = gate.stop(() => { + ran = true; + return Promise.resolve(); + }); + const outcome = await failing.then(() => 'resolved', () => 'rejected'); + assertEquals(outcome, 'rejected', 'the failing op must report its error'); + await next; + assertTrue(ran, 'a rejected op must not block the queue'); + }, + ); + + TEST( + 'HttpServerStop', + 'invalid drain budgets are rejected and leave the listener bound', + async () => { + const server = createHttpServer(); + try { + await server.start(() => Promise.resolve(new Response('ok')), 0); + const port = requirePort(server); + const invalid = [ + MAX_STOP_TIMEOUT_MS + 1, + -1, + Number.NaN, + 1.5, + Number.POSITIVE_INFINITY, + ]; + for (const timeoutMs of invalid) { + const outcome = await server.stop({ timeoutMs }).then( + () => 'resolved', + (error: unknown) => error, + ); + assertTrue( + outcome instanceof RangeError, + `timeoutMs ${timeoutMs} must reject with a RangeError`, + ); + } + assertEquals( + server.port, + port, + 'a rejected budget must leave the listener bound', + ); + const response = await bounded(request(server)); + assertEquals(await response.text(), 'ok', 'server must still serve'); + } finally { + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'the maximum drain budget still waits for an active request', + async () => { + const server = createHttpServer(); + const started = deferred(); + const release = deferred(); + try { + await server.start(async () => { + started.resolve(); + await release.promise; + return new Response('ok'); + }, 0); + const pending = request(server); + void pending.catch(() => {}); + await bounded(started.promise); + // WHY the max budget: a 32-bit timer overflow would fire in about a + // millisecond and cut this request instead of waiting for it. + const stopping = server.stop({ timeoutMs: MAX_STOP_TIMEOUT_MS }); + release.resolve(); + const response = await bounded(pending); + assertEquals(await response.text(), 'ok'); + await bounded(stopping); + assertEquals(server.port, undefined, 'stop must close the listener'); + } finally { + release.resolve(); + await server.stop({ force: true }); + } + }, + ); + + TEST('HttpServerStop', 'a zero drain budget is valid', async () => { + const server = createHttpServer(); + try { + await server.start(() => Promise.resolve(new Response('ok')), 0); + await bounded(server.stop({ timeoutMs: 0 })); + assertEquals(server.port, undefined, 'stop must close the listener'); + } finally { + await server.stop({ force: true }); + } + }); + + TEST( + 'HttpServerStop', + 'Server.stop rejects an invalid drain budget before touching state', + async (ctx) => { + const serverPath = await ctx.tempDir('server-stop-budget'); + let actualPort = 0; + const server = await createStopTestServer( + serverPath, + new DataRegistry(), + () => actualPort, + ); + server.registerEndpoint({ + filter: () => true, + processRequest: () => Promise.resolve(new Response('ok')), + }); + try { + await server.start(); + actualPort = requirePort(server); + const outcome = await server.stop({ timeoutMs: -1 }).then( + () => 'resolved', + (error: unknown) => error, + ); + assertTrue( + outcome instanceof RangeError, + 'an invalid budget must reject with a RangeError', + ); + assertEquals( + server.port, + actualPort, + 'a rejected budget must leave the server bound', + ); + const response = await bounded(fetch( + `${clientOrigin(server)}/x`, + { headers: { connection: 'close' } }, + )); + assertEquals(response.status, 200, 'server must still serve'); + await response.text(); + } finally { + await server.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'a bind failure reports the error and leaves the port free to retry', + async () => { + const holder = createHttpServer(); + const contender = createHttpServer(); + const ok = () => Promise.resolve(new Response('ok')); + try { + await holder.start(ok, 0); + const port = requirePort(holder); + const outcome = await contender.start(ok, port).then( + () => 'resolved', + (error: unknown) => error, + ); + assertTrue( + isAddressInUse(outcome), + `the second bind must reject with a busy-port error, got ${outcome}`, + ); + assertEquals( + contender.port, + undefined, + 'a failed bind must expose no port', + ); + assertEquals( + contender.address, + undefined, + 'a failed bind must expose no address', + ); + + await holder.stop({ force: true }); + // WHY: retry on the same instance proves the failed bind left no + // half-open listener or stale error handler behind. + await startWhenFree(contender, port); + assertEquals(contender.port, port, 'the retry must bind the port'); + const response = await bounded(request(contender)); + assertEquals(await response.text(), 'ok'); + } finally { + await holder.stop({ force: true }); + await contender.stop({ force: true }); + } + }, + ); + + TEST( + 'HttpServerStop', + 'Server.start rolls back when the listener cannot bind', + async (ctx) => { + const contenderPath = await ctx.tempDir('server-start-rollback'); + const holder = createHttpServer(); + let contenderPort = 0; + await holder.start(() => Promise.resolve(new Response('busy')), 0); + const port = requirePort(holder); + // WHY: constructed on the busy port, so start() hits the real bind path. + const contender = await createStopTestServer( + contenderPath, + new DataRegistry(), + () => contenderPort, + () => 'localhost', + port, + ); + contenderPort = port; + contender.registerEndpoint({ + filter: () => true, + processRequest: () => Promise.resolve(new Response('ok')), + }); + try { + const outcome = await contender.start().then( + () => 'resolved', + (error: unknown) => error, + ); + assertTrue( + isAddressInUse(outcome), + `start must surface the bind error, got ${outcome}`, + ); + assertEquals( + contender.port, + undefined, + 'a failed start must expose no port', + ); + + await holder.stop({ force: true }); + // WHY: a half-committed start used to make this retry a no-op, so the + // server reported success while nothing listened. + await contender.start(); + const response = await bounded(fetch( + `${clientOrigin(contender)}/x`, + { headers: { connection: 'close' } }, + )); + assertEquals(response.status, 200, 'the retry must serve'); + await response.text(); + } finally { + await holder.stop({ force: true }); + await contender.stop({ force: true }); + } + }, + ); +} diff --git a/tests/http-test-utils.ts b/tests/http-test-utils.ts new file mode 100644 index 00000000..5d7008b8 --- /dev/null +++ b/tests/http-test-utils.ts @@ -0,0 +1,16 @@ +import type { MinimalHttpServer } from '../net/server/http-compat.ts'; +import { assertExists } from './asserts.ts'; + +/** Capture before stop(): the public address clears when shutdown starts. */ +export function clientOrigin( + server: Pick, +): string { + const address = server.address; + assertExists(address, 'Started server must expose its listener address'); + // Match the bound family: IPv6 avoids foreign IPv4 SO_REUSEADDR listeners + // on macOS, while IPv4-only hosts must not be sent to IPv6 loopback. + const host = address.hostname.includes(':') + ? `[${address.hostname === '::' ? '::1' : address.hostname}]` + : '127.0.0.1'; + return `http://${host}:${address.port}`; +} diff --git a/tests/node-http-server.test.ts b/tests/node-http-server.test.ts index f12a41fa..5e7d1b2c 100644 --- a/tests/node-http-server.test.ts +++ b/tests/node-http-server.test.ts @@ -10,28 +10,32 @@ import { TEST } from './mod.ts'; import { assertEquals, assertExists, assertTrue } from './asserts.ts'; import { isNode } from '../base/common.ts'; import { - GoatRequest, + type GoatRequest, NodeHttpServer, type ServeHandlerInfo, } from '../net/server/http-compat.ts'; +import { clientOrigin } from './http-test-utils.ts'; + +function responseHandler(body: string, init?: ResponseInit) { + return () => Promise.resolve(new Response(body, init)); +} + export default function setupNodeHttpServerTests() { TEST('NodeHttpServer', 'GET request returns response body', async () => { if (!isNode()) return; const server = new NodeHttpServer(); try { await server.start( - async (_req: GoatRequest, _info: ServeHandlerInfo) => - new Response('hello', { status: 200 }), + responseHandler('hello', { status: 200 }), 0, ); - const url = `http://localhost:${server.port}`; - const res = await fetch(url); + const res = await fetch(clientOrigin(server)); assertEquals(res.status, 200, 'status should be 200'); const body = await res.text(); assertEquals(body, 'hello', 'body should match'); } finally { - server.stop(); + await server.stop(); } }); @@ -46,15 +50,14 @@ export default function setupNodeHttpServerTests() { }, 0, ); - const url = `http://localhost:${server.port}`; - const res = await fetch(url, { + const res = await fetch(clientOrigin(server), { method: 'POST', body: 'echo this', }); const text = await res.text(); assertEquals(text, 'echo this', 'echoed body should match sent body'); } finally { - server.stop(); + await server.stop(); } }); @@ -63,15 +66,13 @@ export default function setupNodeHttpServerTests() { const server = new NodeHttpServer(); try { await server.start( - async (_req: GoatRequest, _info: ServeHandlerInfo) => - new Response('ok', { - status: 200, - headers: { 'x-custom': 'test-value' }, - }), + responseHandler('ok', { + status: 200, + headers: { 'x-custom': 'test-value' }, + }), 0, ); - const url = `http://localhost:${server.port}`; - const res = await fetch(url); + const res = await fetch(clientOrigin(server)); await res.text(); assertEquals( res.headers.get('x-custom'), @@ -79,7 +80,7 @@ export default function setupNodeHttpServerTests() { 'custom response header should be present', ); } finally { - server.stop(); + await server.stop(); } }); @@ -88,16 +89,14 @@ export default function setupNodeHttpServerTests() { const server = new NodeHttpServer(); try { await server.start( - async (_req: GoatRequest, _info: ServeHandlerInfo) => - new Response('not found', { status: 404 }), + responseHandler('not found', { status: 404 }), 0, ); - const url = `http://localhost:${server.port}`; - const res = await fetch(url); + const res = await fetch(clientOrigin(server)); await res.text(); assertEquals(res.status, 404, 'status should be 404'); } finally { - server.stop(); + await server.stop(); } }); @@ -107,8 +106,7 @@ export default function setupNodeHttpServerTests() { const ac = new AbortController(); try { await server.start( - async (_req: GoatRequest, _info: ServeHandlerInfo) => - new Response('alive'), + responseHandler('alive'), 0, ac.signal, ); @@ -125,7 +123,7 @@ export default function setupNodeHttpServerTests() { 'server.port must be undefined after abort', ); } finally { - server.stop(); // idempotent safety cleanup + await server.stop(); // idempotent safety cleanup } }); @@ -134,8 +132,7 @@ export default function setupNodeHttpServerTests() { const server = new NodeHttpServer(); try { await server.start( - async (_req: GoatRequest, _info: ServeHandlerInfo) => - new Response('ok'), + responseHandler('ok'), 0, ); const addr = server.address; @@ -143,7 +140,7 @@ export default function setupNodeHttpServerTests() { assertTrue(addr.port > 0, 'port should be positive'); assertExists(addr.hostname, 'hostname should be defined'); } finally { - server.stop(); + await server.stop(); } }); } diff --git a/tests/test-registry.ts b/tests/test-registry.ts index 0bc8c15b..61d59fbf 100644 --- a/tests/test-registry.ts +++ b/tests/test-registry.ts @@ -26,6 +26,7 @@ import setupWorkerBoundary from './worker-boundary.test.ts'; import setupJsonLogWorkerIO from './json-log-worker-io.test.ts'; import setupJsonLogFormats from './json-log-formats.test.ts'; import setupNodeHttpServerTests from './node-http-server.test.ts'; +import setupHttpServerStop from './http-server-stop.test.ts'; import setupHealthCheckEndpointTest from './health-check-endpoint.test.ts'; import setupMinimalSync from './minimal-client-server-sync.test.ts'; import setupE2ELatency from './e2e-latency.test.ts'; @@ -145,6 +146,7 @@ export async function registerAllTests(): Promise { setupUntrusted(); // Database operations in untrusted mode await setupServerArchitectureTest(); // Server initialization and configuration setupNodeHttpServerTests(); // Node.js HTTP server integration + setupHttpServerStop(); // MinimalHttpServer stop contract (Deno + Node) setupStaticAssetsEndpointTest(); // File serving and asset management setupCliBuildAssetsTests(); // Client bundling plugins, CSS, and assets setupMergeRichText(); // Rich text merge operations