From 7bf21f44cc3eedfffcb1e4dc05146ad0aed64e25 Mon Sep 17 00:00:00 2001
From: Devinder Singh <devinder.stockholm3219@gmail.com>
Date: Sun, 1 Nov 2026 14:04:14 +0100
Subject: [PATCH 1/5] feat: connect the API to Postgres (wait at startup, 503
 when unreachable)

---
 api/bun.lock                  |   3 +
 api/package.json              |   1 +
 api/src/app.test.ts           |   5 +-
 api/src/config.test.ts        |  38 +++++++---
 api/src/config.ts             |  16 +++++
 api/src/db.test.ts            | 128 ++++++++++++++++++++++++++++++++++
 api/src/db.ts                 | 126 +++++++++++++++++++++++++++++++++
 api/src/error-handler.test.ts |  25 ++++++-
 api/src/error-handler.ts      |   9 +++
 api/src/server.ts             |  33 +++++++--
 10 files changed, 367 insertions(+), 17 deletions(-)
 create mode 100644 api/src/db.test.ts
 create mode 100644 api/src/db.ts

diff --git a/api/bun.lock b/api/bun.lock
index 37459eb..d732ff8 100644
--- a/api/bun.lock
+++ b/api/bun.lock
@@ -6,6 +6,7 @@
       "dependencies": {
         "dbmate": "2.37.0",
         "fastify": "5.12.7",
+        "postgres": "3.4.9",
         "zod": "4.6.7",
       },
       "devDependencies": {
@@ -99,6 +100,8 @@
 
     "pino-std-serializers": ["pino-std-serializers@7.1.0", "", {}, "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw=="],
 
+    "postgres": ["postgres@3.4.9", "", {}, "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw=="],
+
     "process-warning": ["process-warning@5.1.1", "", {}, "sha512-oCkNVj4OKnKEcocVBcDfiIugN1/cE6rgfT3nCtzeqZPA8+rIxMaQ0xXzIkGP8Gj2qUaP1UR7uhJ1VD8lUCpzCg=="],
 
     "quick-format-unescaped": ["quick-format-unescaped@4.0.4", "", {}, "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg=="],
diff --git a/api/package.json b/api/package.json
index ccad527..1c35a3f 100644
--- a/api/package.json
+++ b/api/package.json
@@ -12,6 +12,7 @@
   "dependencies": {
     "dbmate": "2.37.0",
     "fastify": "5.12.7",
+    "postgres": "3.4.9",
     "zod": "4.6.7"
   },
   "devDependencies": {
diff --git a/api/src/app.test.ts b/api/src/app.test.ts
index 7e28ee1..3382718 100644
--- a/api/src/app.test.ts
+++ b/api/src/app.test.ts
@@ -3,7 +3,10 @@ import { buildApp } from "./app.ts";
 import { loadConfig } from "./config.ts";
 
 // A config made from a fixed object, so the test never depends on this machine's environment.
-const app = buildApp(loadConfig({ GIT_SHA: "abc123" }), { logger: false });
+// DATABASE_URL is required but harmless here: these tests never connect to it.
+const app = buildApp(loadConfig({ GIT_SHA: "abc123", DATABASE_URL: "postgres://test@localhost/test" }), {
+  logger: false,
+});
 afterAll(() => app.close());
 
 describe("health routes", () => {
diff --git a/api/src/config.test.ts b/api/src/config.test.ts
index 74ff307..4a71085 100644
--- a/api/src/config.test.ts
+++ b/api/src/config.test.ts
@@ -1,23 +1,45 @@
 import { describe, expect, test } from "bun:test";
 import { loadConfig } from "./config.ts";
 
+// DATABASE_URL is required, so every test passes one. It's never connected to here.
+const DB = { DATABASE_URL: "postgres://test@localhost/test" };
+
 describe("loadConfig", () => {
-  test("uses safe defaults when nothing is set", () => {
-    expect(loadConfig({})).toEqual({ PORT: 8080, GIT_SHA: "dev", LOG_LEVEL: "info" });
+  test("uses safe defaults when nothing else is set", () => {
+    expect(loadConfig(DB)).toEqual({
+      PORT: 8080,
+      GIT_SHA: "dev",
+      LOG_LEVEL: "info",
+      DATABASE_URL: "postgres://test@localhost/test",
+      DB_POOL_MAX: 5,
+    });
   });
 
   test("turns PORT from text into a number", () => {
-    expect(loadConfig({ PORT: "3000" }).PORT).toBe(3000);
+    expect(loadConfig({ ...DB, PORT: "3000" }).PORT).toBe(3000);
   });
 
   test("refuses a PORT that is not a valid port", () => {
-    expect(() => loadConfig({ PORT: "banana" })).toThrow(/PORT/);
-    expect(() => loadConfig({ PORT: "70000" })).toThrow(/PORT/);
-    expect(() => loadConfig({ PORT: "3000abc" })).toThrow(/PORT/);
-    expect(() => loadConfig({ PORT: "80.5" })).toThrow(/PORT/);
+    expect(() => loadConfig({ ...DB, PORT: "banana" })).toThrow(/PORT/);
+    expect(() => loadConfig({ ...DB, PORT: "70000" })).toThrow(/PORT/);
+    expect(() => loadConfig({ ...DB, PORT: "3000abc" })).toThrow(/PORT/);
+    expect(() => loadConfig({ ...DB, PORT: "80.5" })).toThrow(/PORT/);
   });
 
   test("refuses an unknown LOG_LEVEL", () => {
-    expect(() => loadConfig({ LOG_LEVEL: "loud" })).toThrow(/LOG_LEVEL/);
+    expect(() => loadConfig({ ...DB, LOG_LEVEL: "loud" })).toThrow(/LOG_LEVEL/);
+  });
+
+  test("stops at once without DATABASE_URL, or with one that isn't postgres://", () => {
+    expect(() => loadConfig({})).toThrow(/DATABASE_URL/);
+    expect(() => loadConfig({ DATABASE_URL: "mysql://test@localhost/test" })).toThrow(/DATABASE_URL/);
+    expect(loadConfig({ DATABASE_URL: "postgresql://test@localhost/test" }).DATABASE_URL).toStartWith("postgresql://");
+  });
+
+  test("DB_POOL_MAX: a whole number from 1 to 20", () => {
+    expect(loadConfig({ ...DB, DB_POOL_MAX: "8" }).DB_POOL_MAX).toBe(8);
+    for (const bad of ["0", "21", "2.5", "five"]) {
+      expect(() => loadConfig({ ...DB, DB_POOL_MAX: bad })).toThrow(/DB_POOL_MAX/);
+    }
   });
 });
diff --git a/api/src/config.ts b/api/src/config.ts
index 89dd7e0..fe59bc2 100644
--- a/api/src/config.ts
+++ b/api/src/config.ts
@@ -21,6 +21,22 @@ const Env = z.object({
    LOG_LEVEL: z
   .enum(["fatal", "error", "warn", "info", "debug", "trace"]).default("info"),
 
+  // Where Postgres is, with the login and password in it. Required: without a database the
+  // API can't do its job, so a missing value stops it at once instead of at the first request.
+  // In Kubernetes it comes from the Secret relay-db (key url), never from a file in git.
+  DATABASE_URL: z
+    .string()
+    .refine((s) => s.startsWith("postgres://") || s.startsWith("postgresql://"), {
+      message: "must be a postgres:// URL",
+    }),
+
+  // How many connections this pod may hold open. 5 is plenty for one pod with 100m CPU, and
+  // keeps every pod together under Postgres's max_connections (the budget in the Phase 2 plan).
+  DB_POOL_MAX: z
+    .string()
+    .transform((s) => Number(s))
+    .refine((n) => Number.isInteger(n) && n >= 1 && n <= 20, { message: "must be a whole number between 1 and 20" })
+    .default(5),
 
 });
 
diff --git a/api/src/db.test.ts b/api/src/db.test.ts
new file mode 100644
index 0000000..47c9711
--- /dev/null
+++ b/api/src/db.test.ts
@@ -0,0 +1,128 @@
+// Unit tests for db.ts: no real database. A fake "sql" answers from a script (ok, or an error
+// code), and a fake clock lets 60 seconds of retries pass in a few milliseconds.
+import { describe, expect, test } from "bun:test";
+import { isDatabaseUnavailable, waitForDatabase, type Log, type Sql } from "./db.ts";
+
+type Outcome = "ok" | string; // "ok", or the error code the database "throws"
+
+// A fake sql: each call takes the next outcome from the script; the last one repeats forever.
+function fakeDb(script: Outcome[]) {
+  let calls = 0;
+  const sql = (() => {
+    const outcome = script[Math.min(calls, script.length - 1)];
+    calls++;
+    if (outcome === "ok") return Promise.resolve([{ "?column?": 1 }]);
+    return Promise.reject(Object.assign(new Error(`fake ${outcome}`), { code: outcome }));
+  }) as unknown as Sql;
+  return { sql, calls: () => calls };
+}
+
+// A fake clock: sleep() moves the time forward instead of waiting.
+function fakeClock() {
+  let t = 0;
+  const pauses: number[] = [];
+  return {
+    now: () => t,
+    sleep: async (ms: number) => { pauses.push(ms); t += ms; },
+    pauses,
+    elapsed: () => t,
+  };
+}
+
+// A fake logger that remembers the messages.
+function fakeLog() {
+  const info: string[] = [];
+  const warn: string[] = [];
+  const log: Log = { info: (_o, msg) => info.push(msg), warn: (_o, msg) => warn.push(msg) };
+  return { log, info, warn };
+}
+
+describe("waitForDatabase", () => {
+  test("the database answers at once → returns, no pause, logs ✅", async () => {
+    const db = fakeDb(["ok"]), clock = fakeClock(), out = fakeLog();
+    await waitForDatabase(db.sql, out.log, clock);
+    expect(db.calls()).toBe(1);
+    expect(clock.pauses).toEqual([]);
+    expect(out.info.join(" ")).toContain("✅");
+  });
+
+  test("refused twice (Postgres still starting), then answers → 3 tries, 2 pauses of 2 s", async () => {
+    const db = fakeDb(["ECONNREFUSED", "ECONNREFUSED", "ok"]), clock = fakeClock(), out = fakeLog();
+    await waitForDatabase(db.sql, out.log, clock);
+    expect(db.calls()).toBe(3);
+    expect(clock.pauses).toEqual([2000, 2000]);
+    expect(out.warn.length).toBe(2);
+  });
+
+  test('"still starting up" (57P03) is retried, not fatal', async () => {
+    const db = fakeDb(["57P03", "ok"]), clock = fakeClock(), out = fakeLog();
+    await waitForDatabase(db.sql, out.log, clock);
+    expect(db.calls()).toBe(2);
+  });
+
+  test("does not start another attempt when the next pause reaches the deadline", async () => {
+    const db = fakeDb(["ECONNREFUSED", "ok"]), clock = fakeClock(), out = fakeLog();
+    await expect(waitForDatabase(db.sql, out.log, { ...clock, deadlineMs: 2_000 }))
+      .rejects.toThrow("Database not reachable within 2000 ms");
+    expect(db.calls()).toBe(1);
+    expect(clock.pauses).toEqual([]);
+  });
+
+  test("a thrown null is retried safely", async () => {
+    let calls = 0;
+    const sql = (() => ++calls === 1 ? Promise.reject(null) : Promise.resolve([])) as unknown as Sql;
+    const clock = fakeClock(), out = fakeLog();
+    await waitForDatabase(sql, out.log, clock);
+    expect(calls).toBe(2);
+    expect(clock.pauses).toEqual([2000]);
+  });
+
+  test("a stalled query is limited by the remaining startup deadline", async () => {
+    const sql = (() => new Promise(() => {})) as unknown as Sql;
+    const out = fakeLog();
+    await expect(waitForDatabase(sql, out.log, { deadlineMs: 20 }))
+      .rejects.toThrow("Database not reachable within 20 ms");
+    expect(out.warn).toEqual([]);
+  });
+
+  const fatal = [["28P01", "wrong password"], ["28000", "login not allowed"], ["3D000", "no such database"]] as const;
+  for (const [code, meaning] of fatal) {
+    test(`${meaning} (${code}) → throws at once, no retry, no pause`, async () => {
+      const db = fakeDb([code, "ok"]), clock = fakeClock(), out = fakeLog();
+      await expect(waitForDatabase(db.sql, out.log, clock)).rejects.toMatchObject({ code });
+      expect(db.calls()).toBe(1);
+      expect(clock.pauses).toEqual([]);
+    });
+  }
+
+  test("never reachable → gives up after the 60 s deadline, never sleeps past it", async () => {
+    const db = fakeDb(["ECONNREFUSED"]), clock = fakeClock(), out = fakeLog();
+    await expect(waitForDatabase(db.sql, out.log, clock)).rejects.toThrow();
+    expect(clock.elapsed()).toBeLessThanOrEqual(60_000);
+    expect(db.calls()).toBeGreaterThanOrEqual(30); // about one try every 2 s for 60 s
+    expect(db.calls()).toBeLessThanOrEqual(31);
+  });
+
+  test("the deadline and the pause can be set (a shorter wait for tests and the laptop)", async () => {
+    const db = fakeDb(["ETIMEDOUT"]), clock = fakeClock(), out = fakeLog();
+    await expect(waitForDatabase(db.sql, out.log, { ...clock, deadlineMs: 5_000, pauseMs: 1_000 })).rejects.toThrow();
+    expect(clock.elapsed()).toBeLessThanOrEqual(5_000);
+    expect(db.calls()).toBeGreaterThanOrEqual(5); // a try about every second for 5 s
+    expect(clock.pauses.every((ms) => ms === 1_000)).toBe(true);
+  });
+});
+
+describe("isDatabaseUnavailable", () => {
+  test("connection problems → true (they become 503)", () => {
+    for (const code of ["ECONNREFUSED", "CONNECT_TIMEOUT", "CONNECTION_CLOSED", "57P03", "08006"]) {
+      expect(isDatabaseUnavailable({ code })).toBe(true);
+    }
+  });
+
+  test("everything else → false (it stays a 500: our bug, not an outage)", () => {
+    // 23505 duplicate key, 42P01 no such table, 57014 cancelled by statement_timeout (slow SQL)
+    for (const error of [{ code: "23505" }, { code: "42P01" }, { code: "57014" }, new Error("boom"), null, undefined, "text"]) {
+      expect(isDatabaseUnavailable(error)).toBe(false);
+    }
+  });
+});
diff --git a/api/src/db.ts b/api/src/db.ts
new file mode 100644
index 0000000..01d9abb
--- /dev/null
+++ b/api/src/db.ts
@@ -0,0 +1,126 @@
+// The database: one connection pool for the whole app (postgres.js), plus the two things
+// startup and the error handler need to know about it.
+//   createDb()               makes the pool. It doesn't connect yet: the first query does.
+//   waitForDatabase()        at startup, waits until the database answers, or stops the app.
+//   isDatabaseUnavailable()  "is this error the database being unreachable?" (→ 503)
+import postgres from "postgres";
+import type { Config } from "./config.ts";
+
+export type Sql = postgres.Sql;
+
+// The logger calls waitForDatabase uses. Fastify's logger (pino) has them; tests pass a fake.
+export type Log = {
+  info: (obj: object, msg: string) => void;
+  warn: (obj: object, msg: string) => void;
+};
+
+// Step 1: the pool. Every number here is a limit, so nothing can hang forever.
+export function createDb(config: Config): Sql {
+  return postgres(config.DATABASE_URL, {
+    max: config.DB_POOL_MAX, // at most this many open connections (5 by default)
+    idle_timeout: 30, // close a connection nobody used for 30 s
+    connect_timeout: 5, // one attempt to connect may take 5 s
+    connection: {
+      application_name: "relay-api", // shows up in pg_stat_activity: who holds which connection
+      statement_timeout: 5000, // Postgres cancels any query running longer than 5 s
+    },
+  });
+}
+
+// Errors where Postgres has answered, but waiting won't help: the login or the database itself
+// is wrong. Their SQLSTATE codes: https://www.postgresql.org/docs/18/errcodes-appendix.html
+//   28P01  wrong password (also an unknown user: Postgres doesn't say which, on purpose)
+//   28000  login not allowed (no pg_hba.conf rule, or a role without LOGIN)
+//   3D000  no such database
+// Everything else at startup (refused, timed out, DNS not ready, 57P03 "still starting up")
+// means "not there yet", so it's worth trying again.
+export const FATAL_STARTUP_CODES = new Set(["28P01", "28000", "3D000"]);
+
+// One try: `select 1`, cut short after timeoutMs. connect_timeout only limits connecting; this
+// also covers a server that accepted the connection and then stopped answering. On timeout the
+// query is cancelled too, so it doesn't keep running in the background.
+export async function pingDatabase(sql: Sql, timeoutMs = 5_000): Promise<void> {
+  const query = sql`select 1`;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const timeout = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      query.cancel?.(); // postgres.js has cancel(); a test's fake may not
+      reject(Object.assign(new Error(`no answer within ${timeoutMs} ms`), { code: "PING_TIMEOUT" }));
+    }, timeoutMs);
+  });
+  try {
+    await Promise.race([query, timeout]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export type WaitOptions = {
+  deadlineMs?: number; // give up after this long in total (wall clock)
+  pauseMs?: number; // wait this long between two tries
+  now?: () => number; // the clock (tests pass a fake one)
+  sleep?: (ms: number) => Promise<void>; // the pause (tests pass a fake one)
+};
+
+// Step 2: at startup, wait until the database answers a tiny query (select 1), before the app
+// opens its port. A fatal error throws at once; anything else is retried every pauseMs until
+// deadlineMs has passed on the clock, then throws. Logs ✅ on success and ⚠️ on each retry.
+export async function waitForDatabase(sql: Sql, log: Log, options: WaitOptions = {}): Promise<void> {
+  const {
+    deadlineMs = 60_000,
+    pauseMs = 2_000,
+    now = () => Date.now(),
+    sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)),
+  } = options;
+
+  const startTime = now();
+  const deadlineError = () => new Error(`Database not reachable within ${deadlineMs} ms`);
+  while (true) {
+    const remainingMs = deadlineMs - (now() - startTime);
+    if (remainingMs <= 0) throw deadlineError();
+    try {
+      await pingDatabase(sql, Math.min(5_000, remainingMs));
+      log.info({}, "✅ database answered");
+      return;
+    } catch (error: unknown) {
+      const code = (error as { code?: unknown } | null)?.code;
+      if (typeof code === "string" && FATAL_STARTUP_CODES.has(code)) {
+        throw error;
+      }
+      const elapsedTime = now() - startTime;
+      if (elapsedTime + pauseMs >= deadlineMs) {
+        throw deadlineError();
+      }
+      log.warn({ code }, "⚠️ database not reachable yet, retrying");
+      await sleep(pauseMs);
+    }
+  }
+}
+
+// Connection problems, as postgres.js and Node report them. These mean "the database isn't
+// reachable right now", so the error handler answers 503 (try again shortly) instead of 500.
+// Anything else from the database (a broken query, a duplicate key) stays a 500: it's our bug,
+// not an outage, and must never be hidden as one.
+const UNAVAILABLE_CODES = new Set([
+  "CONNECT_TIMEOUT", // postgres.js: connecting took longer than connect_timeout
+  "CONNECTION_CLOSED", // postgres.js: the connection dropped
+  "CONNECTION_DESTROYED", // postgres.js: the pool was closed during shutdown
+  "CONNECTION_ENDED", // postgres.js: a query arrived after the pool was closed (shutdown)
+  "ECONNREFUSED", // Node: nothing listens on the port (Postgres down or restarting)
+  "ECONNRESET", // Node: the connection was cut
+  "ETIMEDOUT", // Node: the network didn't answer
+  "ENOTFOUND", // Node: the host name doesn't resolve (yet)
+  "EAI_AGAIN", // Node: DNS lookup failed for now
+  "57P01", // Postgres: the server is shutting down
+  "57P03", // Postgres: the server is still starting up
+  "53300", // Postgres: too many connections right now
+]);
+// Not in the list on purpose: 57014, "cancelled by statement_timeout". A query that runs over
+// 5 s is far more likely our slow SQL than an outage, so it stays a 500 and shows up as a bug.
+
+export function isDatabaseUnavailable(error: unknown): boolean {
+  const code = (error as { code?: unknown } | null)?.code;
+  if (typeof code !== "string") return false;
+  // SQLSTATE class 08 = "connection exception" (for example 08006: connection failure)
+  return UNAVAILABLE_CODES.has(code) || code.startsWith("08");
+}
diff --git a/api/src/error-handler.test.ts b/api/src/error-handler.test.ts
index 3d1f512..2be6e43 100644
--- a/api/src/error-handler.test.ts
+++ b/api/src/error-handler.test.ts
@@ -4,7 +4,8 @@ import { buildApp } from "./app.ts";
 import { loadConfig } from "./config.ts";
 import { conflict } from "./errors.ts";
 
-const app = buildApp(loadConfig({}), { logger: false });
+// A harmless database URL: these tests never connect to it.
+const app = buildApp(loadConfig({ DATABASE_URL: "postgres://test@localhost/test" }), { logger: false });
 
 // Test-only routes that fail on purpose, one for each kind of error.
 app.get("/test/zod", async () => z.object({ name: z.string() }).parse({}));
@@ -15,6 +16,13 @@ app.get("/test/crash", async () => {
   throw new Error("connection failed: password=hunter2");
 });
 app.post("/test/echo", async (request) => request.body);
+// What postgres.js throws when Postgres is down, and when our own SQL is wrong.
+app.get("/test/db-down", async () => {
+  throw Object.assign(new Error("connect ECONNREFUSED 10.43.0.5:5432"), { code: "ECONNREFUSED" });
+});
+app.get("/test/db-bug", async () => {
+  throw Object.assign(new Error('relation "nope" does not exist'), { code: "42P01" });
+});
 
 afterAll(() => app.close());
 
@@ -78,4 +86,19 @@ describe("error handler", () => {
     expect(res.json().error.code).toBe("internal_error");
     expect(res.body).not.toContain("hunter2");
   });
+
+  test("database unreachable → 503 service_unavailable with Retry-After, no address leaked", async () => {
+    const res = await app.inject({ method: "GET", url: "/test/db-down" });
+    expect(res.statusCode).toBe(503);
+    expect(res.headers["retry-after"]).toBe("5");
+    expect(res.json().error.code).toBe("service_unavailable");
+    expect(res.json().error.request_id).toBe(res.headers["x-request-id"]);
+    expect(res.body).not.toContain("10.43.0.5");
+  });
+
+  test("a database error that is our bug (no such table) → still 500, never hidden as an outage", async () => {
+    const res = await app.inject({ method: "GET", url: "/test/db-bug" });
+    expect(res.statusCode).toBe(500);
+    expect(res.json().error.code).toBe("internal_error");
+  });
 });
diff --git a/api/src/error-handler.ts b/api/src/error-handler.ts
index 7449102..5e9a469 100644
--- a/api/src/error-handler.ts
+++ b/api/src/error-handler.ts
@@ -3,6 +3,7 @@
 // Every answer has the same shape: {"error": {"code", "message", "request_id"}}.
 import type { FastifyError, FastifyReply, FastifyRequest } from "fastify";
 import { ZodError } from "zod";
+import { isDatabaseUnavailable } from "./db.ts";
 import { AppError } from "./errors.ts";
 
 // Fastify's own client errors, by status, with our names for them.
@@ -35,6 +36,14 @@ export function errorHandler(error: FastifyError, request: FastifyRequest, reply
     return send(reply, request, error.status, error.code, error.message);
   }
 
+  // The database can't be reached right now (a restart, a short blip). The pod stays up and
+  // ready; only this request fails, with a clear "try again in 5 s" instead of a vague 500.
+  if (isDatabaseUnavailable(error)) {
+    request.log.warn({ err: error }, "⚠️ database unavailable, answered 503");
+    reply.header("retry-after", "5");
+    return send(reply, request, 503, "service_unavailable", "The service is briefly unavailable, try again shortly");
+  }
+
   if (error.statusCode && error.statusCode >= 400 && error.statusCode < 500) {
     const code = CLIENT_ERROR_CODES[error.statusCode] || "bad_request";
     return send(reply, request, error.statusCode, code, error.message);
diff --git a/api/src/server.ts b/api/src/server.ts
index 5b50744..35a50bc 100644
--- a/api/src/server.ts
+++ b/api/src/server.ts
@@ -1,20 +1,25 @@
-// Starts the API: reads the config, builds the app, listens on a port,
+// Starts the API: reads the config, builds the app, waits for the database, listens on a port,
 // and stops cleanly when Kubernetes (or Ctrl+C) asks it to.
 import { buildApp } from "./app.ts";
 import { loadConfig } from "./config.ts";
+import { createDb, waitForDatabase } from "./db.ts";
 
-const config = loadConfig(); // a bad environment stops the app here, with a clear message
-const app = buildApp(config);
+// Step 1: the config. A bad environment stops the app here, with a clear message.
+const config = loadConfig();
 
-// 0.0.0.0 means "every network interface". Fastify's default (localhost) can't be
-// reached from outside the container, so Kubernetes and Traefik would never reach us.
-await app.listen({ port: config.PORT, host: "0.0.0.0" });
+// Step 2: the app and the database pool. The pool doesn't connect yet; the first query does.
+const app = buildApp(config);
+const sql = createDb(config);
 
-// When Kubernetes sends SIGTERM or Ctrl+C sends SIGINT, we stop the server
+// When Kubernetes sends SIGTERM or Ctrl+C sends SIGINT: stop taking requests and let running
+// ones finish, then close the database connections (queries get 5 s), then exit.
+// Set up BEFORE the wait below: in the container bun is process 1, and Linux ignores SIGTERM
+// for process 1 unless it has a handler, so a pod stopped while still waiting would hang.
 async function shutdown(signal: string) {
   app.log.info({ signal }, "shutting down");
   try {
     await app.close();
+    await sql.end({ timeout: 5 });
     process.exit(0);
   } catch (error) {
     app.log.error(error);
@@ -24,3 +29,17 @@ async function shutdown(signal: string) {
 
 process.once("SIGTERM", () => shutdown("SIGTERM"));
 process.once("SIGINT", () => shutdown("SIGINT"));
+
+// Step 3: wait for the database before opening the port. A pod that can't reach it never
+// listens, so its startupProbe fails, the rollout fails, and relay-deploy rolls back.
+try {
+  await waitForDatabase(sql, app.log);
+} catch (error) {
+  app.log.fatal({ err: error }, "❌ database not reachable, stopping");
+  await sql.end({ timeout: 1 });
+  process.exit(1);
+}
+
+// Step 4: listen. 0.0.0.0 means "every network interface". Fastify's default (localhost)
+// can't be reached from outside the container, so Kubernetes and Traefik would never reach us.
+await app.listen({ port: config.PORT, host: "0.0.0.0" });

From 5c9a369862b5a2bbcad316b2f089b88f618df11e Mon Sep 17 00:00:00 2001
From: Devinder Singh <devinder.stockholm3219@gmail.com>
Date: Sun, 1 Nov 2026 14:18:52 +0100
Subject: [PATCH 2/5] test: run the tests against a real Postgres, logged in as
 relay

---
 .github/workflows/pipeline.yml          |   8 +-
 api/package.json                        |   8 +-
 api/scripts/test-db.sh                  |  97 +++++++++++++++++
 api/test/integration/connection.test.ts |  62 +++++++++++
 api/test/integration/helpers.ts         |  60 +++++++++++
 api/test/integration/schema.test.ts     | 134 ++++++++++++++++++++++++
 6 files changed, 366 insertions(+), 3 deletions(-)
 create mode 100755 api/scripts/test-db.sh
 create mode 100644 api/test/integration/connection.test.ts
 create mode 100644 api/test/integration/helpers.ts
 create mode 100644 api/test/integration/schema.test.ts

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 0f2b4b5..4ed57a3 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -40,7 +40,13 @@ jobs:
           bun-version-file: api/.bun-version
       - run: bun install --frozen-lockfile
       - run: bun run typecheck
-      - run: bun test
+      # A throwaway Postgres: production's image and first-start script, then the migrations.
+      # Started with docker run, not a services: container: those start before checkout, so
+      # they can't mount the first-start script from the repo.
+      - name: Start the test database
+        run: bun run db:up
+      - name: Unit and integration tests (logged in as relay)
+        run: bun run test:db
 
   image:
     name: api-image
diff --git a/api/package.json b/api/package.json
index 1c35a3f..d95cb73 100644
--- a/api/package.json
+++ b/api/package.json
@@ -6,8 +6,12 @@
     "dev": "bun --watch src/server.ts",
     "start": "bun src/server.ts",
     "test": "bun test",
+    "test:unit": "bun test src",
+    "test:db": "bash scripts/test-db.sh test",
     "typecheck": "tsc --noEmit",
-    "db:migrate": "dbmate --wait --wait-timeout 60s --no-dump-schema --migration-lock --migration-lock-timeout 30s migrate"
+    "db:migrate": "dbmate --wait --wait-timeout 60s --no-dump-schema --migration-lock --migration-lock-timeout 30s migrate",
+    "db:up": "bash scripts/test-db.sh up",
+    "db:down": "bash scripts/test-db.sh down"
   },
   "dependencies": {
     "dbmate": "2.37.0",
@@ -19,4 +23,4 @@
     "@types/bun": "1.4.4",
     "typescript": "6.0.3"
   }
-}
\ No newline at end of file
+}
diff --git a/api/scripts/test-db.sh b/api/scripts/test-db.sh
new file mode 100755
index 0000000..7973e8e
--- /dev/null
+++ b/api/scripts/test-db.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# A throwaway Postgres for the tests, the same on the laptop and in CI (step 2.6b).
+# The same image as production (its digest is read from k8s/data/postgres.yaml) and the same
+# first-start script, so the tests log in as "relay", never as the superuser.
+#
+#   up    start it on 127.0.0.1:55432, wait until it answers, then run the migrations
+#   test  run every test (unit + integration) against it; extra arguments go to bun test
+#   down  remove it: only the container this script made, which carries our label
+#
+# Nothing is kept: "down" also removes its storage, so the next "up" starts empty.
+set -euo pipefail
+
+NAME=relay-test-pg
+PORT=55432
+LABEL_KEY=relay.test-db
+# Throwaway values for a container that listens on 127.0.0.1 only. Not secrets.
+PASSWORD="test" # quoted: unquoted, shellcheck reads it as the `test` command
+export DATABASE_URL="postgres://relay:$PASSWORD@127.0.0.1:$PORT/relay?sslmode=disable"
+
+API_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+REPO_DIR="$(cd "$API_DIR/.." && pwd)"
+INITDB_DIR="$REPO_DIR/k8s/data/initdb"
+
+fail() { echo "❌ test-db: $*" >&2; exit 1; }
+
+# Step 1: helpers that look, never change anything.
+exists() { docker container inspect "$NAME" >/dev/null 2>&1; }
+ours() { [ "$(docker container inspect -f "{{index .Config.Labels \"$LABEL_KEY\"}}" "$NAME" 2>/dev/null)" = "1" ]; }
+running() { [ "$(docker container inspect -f '{{.State.Running}}' "$NAME" 2>/dev/null)" = "true" ]; }
+port_busy() { (exec 3<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null; }
+
+# The image production runs, pinned by digest. One source of truth: the StatefulSet.
+prod_image() {
+  local image
+  image=$(sed -n 's/^ *image: *\(postgres:[^ ]*\).*/\1/p' "$REPO_DIR/k8s/data/postgres.yaml" | head -n 1)
+  [[ "$image" =~ ^postgres:[0-9.]+@sha256:[0-9a-f]{64}$ ]] || fail "couldn't read a pinned postgres image from k8s/data/postgres.yaml"
+  echo "$image"
+}
+
+remove_ours() {
+  ours || fail "a container named $NAME exists but this script didn't make it, so it's left alone"
+  docker container rm --force --volumes "$NAME" >/dev/null # --volumes: its storage too
+}
+
+up() {
+  docker info >/dev/null 2>&1 || fail "Docker isn't running"
+
+  # Step 2: reuse ours if it's running; never touch anything else.
+  if exists; then
+    if running && ours; then
+      echo "test-db: $NAME is already running, reusing it"
+    else
+      echo "test-db: $NAME exists but isn't running, removing it first"
+      remove_ours
+    fi
+  fi
+
+  # Step 3: start it, with the first-start script mounted read-only.
+  if ! exists; then
+    port_busy && fail "port $PORT on 127.0.0.1 is already in use by something else"
+    docker run --detach --name "$NAME" --label "$LABEL_KEY=1" \
+      --publish "127.0.0.1:$PORT:5432" \
+      --env POSTGRES_PASSWORD="$PASSWORD" --env RELAY_DB_PASSWORD="$PASSWORD" \
+      --volume "$INITDB_DIR:/docker-entrypoint-initdb.d:ro" \
+      "$(prod_image)" >/dev/null
+    echo "test-db: started $NAME on 127.0.0.1:$PORT"
+  fi
+
+  # Step 4: wait for the real server. During the first start a temporary server listens on
+  # the socket only, so asking over TCP (-h 127.0.0.1) only succeeds once that's finished.
+  for _ in $(seq 1 60); do
+    running || { docker logs --tail 30 "$NAME" >&2 || true; fail "$NAME stopped during its first start (log above)"; }
+    docker exec "$NAME" pg_isready -q -h 127.0.0.1 -U postgres && break
+    sleep 1
+  done
+  docker exec "$NAME" pg_isready -q -h 127.0.0.1 -U postgres || fail "$NAME didn't answer within 60 s"
+
+  # Step 5: the migrations, logged in as relay, exactly as the migrate Job does it.
+  (cd "$API_DIR" && bun run --bun db:migrate)
+  echo "✅ test-db: ready. Run the tests with: bun run test:db"
+}
+
+down() {
+  if exists; then
+    remove_ours
+    echo "✅ test-db: removed $NAME"
+  else
+    echo "test-db: nothing to remove"
+  fi
+}
+
+case "${1:-}" in
+  up) up ;;
+  down) down ;;
+  test) shift; cd "$API_DIR" && exec bun test "$@" ;;
+  *) echo "usage: test-db.sh up | test [bun test arguments] | down" >&2; exit 2 ;;
+esac
diff --git a/api/test/integration/connection.test.ts b/api/test/integration/connection.test.ts
new file mode 100644
index 0000000..7f236d7
--- /dev/null
+++ b/api/test/integration/connection.test.ts
@@ -0,0 +1,62 @@
+// The connection itself, against the real Postgres: who we are, which settings arrive, and what
+// happens when the login is wrong, a query hangs, or a connection is cut.
+import { afterAll, describe, expect, test } from "bun:test";
+import { waitForDatabase } from "../../src/db.ts";
+import { DATABASE_URL, silentLog, testDb } from "./helpers.ts";
+
+const sql = testDb();
+afterAll(() => sql.end({ timeout: 5 }));
+
+describe("the connection, against the real Postgres", () => {
+  test("logs in as relay, which is not a superuser and can't create roles or databases", async () => {
+    const [me] = await sql`
+      select current_user as name, current_database() as database,
+             rolsuper, rolcreaterole, rolcreatedb
+      from pg_roles where rolname = current_user`;
+    expect(me).toEqual({ name: "relay", database: "relay", rolsuper: false, rolcreaterole: false, rolcreatedb: false });
+  });
+
+  test("our settings arrive on every connection: application_name and statement_timeout", async () => {
+    const [settings] = await sql`
+      select current_setting('application_name') as app, current_setting('statement_timeout') as timeout`;
+    expect(settings).toEqual({ app: "relay-api", timeout: "5s" });
+  });
+
+  test("waitForDatabase returns at once when the database is up", async () => {
+    await waitForDatabase(sql, silentLog);
+  });
+
+  test("a wrong password → waitForDatabase gives up at once with 28P01, no 60 s wait", async () => {
+    const wrong = new URL(DATABASE_URL);
+    wrong.password = `wrong-${crypto.randomUUID()}`;
+    const bad = testDb(wrong.toString());
+    const started = performance.now();
+    try {
+      await expect(waitForDatabase(bad, silentLog)).rejects.toMatchObject({ code: "28P01" });
+      expect(performance.now() - started).toBeLessThan(2_000); // shorter than a single pause
+    } finally {
+      await bad.end({ timeout: 1 });
+    }
+  });
+
+  test("a stuck query is cancelled by statement_timeout after 5 s, with 57014", async () => {
+    const started = performance.now();
+    await expect(sql`select pg_sleep(6)`).rejects.toMatchObject({ code: "57014" });
+    expect(performance.now() - started).toBeLessThan(5_900); // stopped at 5 s, not finished at 6
+  }, 10_000);
+
+  test("a dropped connection comes back: the next query opens a new one", async () => {
+    const [before] = await sql`select pg_backend_pid() as pid`;
+    // A separate connection ends ours, as a Postgres restart or a cut network would. relay may
+    // end its own sessions without being a superuser. The 2 s = wait until it's really gone.
+    const other = testDb();
+    try {
+      const [result] = await other`select pg_terminate_backend(${before!.pid}, 2000) as ended`;
+      expect(result!.ended).toBe(true);
+    } finally {
+      await other.end({ timeout: 1 });
+    }
+    const [after] = await sql`select pg_backend_pid() as pid`;
+    expect(after!.pid).not.toBe(before!.pid);
+  });
+});
diff --git a/api/test/integration/helpers.ts b/api/test/integration/helpers.ts
new file mode 100644
index 0000000..ca75706
--- /dev/null
+++ b/api/test/integration/helpers.ts
@@ -0,0 +1,60 @@
+// Shared by the integration tests: the real Postgres from `bun run db:up`, logged in as relay.
+import type postgres from "postgres";
+import { loadConfig } from "../../src/config.ts";
+import { createDb, type Log, type Sql } from "../../src/db.ts";
+
+// Step 1: no database → stop with a clear message. Never skip: a broken CI setup would then
+// skip the most important tests and still show green.
+const url = process.env.DATABASE_URL;
+if (!url) {
+  throw new Error(
+    "DATABASE_URL is missing. The integration tests need the test database: run `bun run db:up`, " +
+      "then `bun run test:db`. (`bun run test:unit` runs only the unit tests, without a database.)",
+  );
+}
+export const DATABASE_URL: string = url;
+
+// A pool with production's own settings (createDb), exactly as the API uses it.
+export function testDb(databaseUrl: string = DATABASE_URL): Sql {
+  return createDb(loadConfig({ DATABASE_URL: databaseUrl }));
+}
+
+export const silentLog: Log = { info: () => {}, warn: () => {} };
+
+export type Tx = postgres.TransactionSql;
+
+class Rollback extends Error {}
+
+// Step 2: run a test inside a transaction that is always undone. Tests leave nothing behind and
+// never see each other's rows, so the order they run in doesn't matter.
+export async function inRollback(sql: Sql, body: (tx: Tx) => Promise<void>): Promise<void> {
+  try {
+    await sql.begin(async (tx) => {
+      await body(tx);
+      throw new Rollback(); // an error inside begin() = ROLLBACK
+    });
+  } catch (error) {
+    if (!(error instanceof Rollback)) throw error;
+  }
+}
+
+// Step 3: the database itself must refuse this query, with exactly this SQLSTATE.
+// It runs in its own savepoint: after an error Postgres refuses everything else in the
+// transaction (25P02) until it rolls back to a savepoint, so later queries still test something.
+export async function expectRefused(tx: Tx, sqlstate: string, query: (sql: Tx) => Promise<unknown>): Promise<void> {
+  let caught: unknown;
+  try {
+    await tx.savepoint(async (sp) => {
+      await query(sp);
+    });
+  } catch (error) {
+    caught = error;
+  }
+  if (caught === undefined) {
+    throw new Error(`expected the database to refuse it with ${sqlstate}, but it was accepted`);
+  }
+  const code = (caught as { code?: unknown } | null)?.code;
+  if (code !== sqlstate) {
+    throw new Error(`expected SQLSTATE ${sqlstate}, got ${String(code)}: ${(caught as Error).message}`);
+  }
+}
diff --git a/api/test/integration/schema.test.ts b/api/test/integration/schema.test.ts
new file mode 100644
index 0000000..fad1c9a
--- /dev/null
+++ b/api/test/integration/schema.test.ts
@@ -0,0 +1,134 @@
+// Schema v1 (migration 001), against the real Postgres: what it must accept (the positive
+// controls) and what it must refuse, each with the right SQLSTATE. Every test runs in a
+// transaction that is rolled back, so nothing is left behind.
+import { afterAll, describe, expect, test } from "bun:test";
+import { expectRefused, inRollback, testDb, type Tx } from "./helpers.ts";
+
+const sql = testDb();
+afterAll(() => sql.end({ timeout: 5 }));
+
+type Seed = { ownerId: string; endpointId: string; eventId: string; deliveryId: string };
+
+// One valid row in every table, so a case can point at rows that exist.
+async function seed(tx: Tx): Promise<Seed> {
+  const [owner] = await tx`insert into owners (name) values ('Test owner') returning id`;
+  const ownerId: string = owner!.id;
+  await tx`
+    insert into api_keys (owner_id, prefix, key_hash)
+    values (${ownerId}, 'relay_live_test', sha256(convert_to('relay_live_test_full_key', 'UTF8')))`;
+  const [endpoint] = await tx`
+    insert into endpoints (owner_id, name, url, signing_secret)
+    values (${ownerId}, 'Test endpoint', 'https://example.com/hook', 'whsec_test')
+    returning id`;
+  const [event] = await tx`
+    insert into events (owner_id, type, payload, idempotency_key)
+    values (${ownerId}, 'order.created', ${tx.json({ order_id: 1 })}, 'key-1')
+    returning id`;
+  const [delivery] = await tx`
+    insert into deliveries (owner_id, event_id, endpoint_id)
+    values (${ownerId}, ${event!.id}, ${endpoint!.id})
+    returning id`;
+  await tx`
+    insert into attempts (delivery_id, started_at, duration_ms, status_code)
+    values (${delivery!.id}, now(), 120, 200)`;
+  return { ownerId, endpointId: endpoint!.id, eventId: event!.id, deliveryId: delivery!.id };
+}
+
+describe("schema v1 accepts valid rows (positive controls)", () => {
+  test("one valid row in every table, with the defaults filled in", async () => {
+    await inRollback(sql, async (tx) => {
+      const ids = await seed(tx);
+      const [delivery] = await tx`select status, attempts from deliveries where id = ${ids.deliveryId}`;
+      expect(delivery).toEqual({ status: "pending", attempts: 0 });
+      const [endpoint] = await tx`select event_types, deleted_at from endpoints where id = ${ids.endpointId}`;
+      expect(endpoint).toEqual({ event_types: ["*"], deleted_at: null });
+    });
+  });
+
+  test("two events without an idempotency key are BOTH accepted (NULLs never collide in a UNIQUE)", async () => {
+    await inRollback(sql, async (tx) => {
+      const { ownerId } = await seed(tx);
+      await tx`insert into events (owner_id, type, payload) values (${ownerId}, 'a.b', ${tx.json({})})`;
+      await tx`insert into events (owner_id, type, payload) values (${ownerId}, 'a.b', ${tx.json({})})`;
+      const [row] = await tx`select count(*)::int as n from events where owner_id = ${ownerId} and idempotency_key is null`;
+      expect(row!.n).toBe(2);
+    });
+  });
+
+  test("the same idempotency key for two different owners is accepted (the UNIQUE is per owner)", async () => {
+    await inRollback(sql, async (tx) => {
+      await seed(tx); // owner 1 already used 'key-1'
+      const [other] = await tx`insert into owners (name) values ('Other owner') returning id`;
+      await tx`
+        insert into events (owner_id, type, payload, idempotency_key)
+        values (${other!.id}, 'order.created', ${tx.json({})}, 'key-1')`;
+    });
+  });
+});
+
+// Each case: what it is, the SQLSTATE the database must answer, and the query. It runs after
+// seed(), in its own rolled-back transaction, so `ids` points at rows that exist.
+//   23514 a CHECK refused it   23505 a duplicate in a UNIQUE   23503 the referenced row doesn't exist
+type Case = { what: string; sqlstate: string; query: (sql: Tx, ids: Seed) => Promise<unknown> };
+
+const refused: Case[] = [
+  // A worked example:
+  {
+    what: "an owner with an empty name",
+    sqlstate: "23514",
+    query: (sql) => sql`insert into owners (name) values ('')`,
+  },
+  {
+    what: "an event with a non-object JSON payload",
+    sqlstate: "23514",
+    query: (sql, ids) => sql`insert into events (owner_id, type, payload) values (${ids.ownerId}, 'a.b', ${sql.json([1, 2])})`,
+  },
+  {
+    what: "an event with a type that breaks the pattern",
+    sqlstate: "23514",
+    query: (sql, ids) => sql`insert into events (owner_id, type, payload) values (${ids.ownerId}, 'Order.created', ${sql.json({})})`,
+  },
+  {
+    what: "an event with an idempotency key this owner already used",
+    sqlstate: "23505",
+    query: (sql, ids) => sql`insert into events (owner_id, type, payload, idempotency_key) values (${ids.ownerId}, 'a.b', ${sql.json({})}, 'key-1')`,
+  },
+  {
+    what: "a delivery with an unknown status",
+    sqlstate: "23514",
+    query: (sql, ids) => sql`update deliveries set status = 'unknown' where id = ${ids.deliveryId}`,
+  },
+  {
+    what: "a delivery for an event that doesn't exist",
+    sqlstate: "23503",
+    query: (sql, ids) => sql`insert into deliveries (owner_id, event_id, endpoint_id) values (${ids.ownerId}, '00000000-0000-0000-0000-000000000000', ${ids.endpointId})`,
+  },
+  {
+    what: "a delivery for an endpoint that doesn't exist",
+    sqlstate: "23503",
+    query: (sql, ids) => sql`insert into deliveries (owner_id, event_id, endpoint_id) values (${ids.ownerId}, ${ids.eventId}, '00000000-0000-0000-0000-000000000000')`,
+  },
+  {
+    what: "an attempt for a delivery that doesn't exist",
+    sqlstate: "23503",
+    query: (sql, ids) => sql`insert into attempts (delivery_id, started_at, duration_ms, status_code) values ('00000000-0000-0000-0000-000000000000', now(), 100, 200)`,
+  },
+];
+
+describe("schema v1 refuses what breaks its rules, each with the right SQLSTATE", () => {
+  for (const c of refused) {
+    test(`${c.what} → ${c.sqlstate}`, async () => {
+      await inRollback(sql, async (tx) => {
+        const ids = await seed(tx);
+        await expectRefused(tx, c.sqlstate, (sp) => c.query(sp, ids));
+      });
+    });
+  }
+
+  test("the list covers every kind of rule: CHECK, UNIQUE and foreign key", () => {
+    const kinds = new Set(refused.map((c) => c.sqlstate));
+    for (const sqlstate of ["23514", "23505", "23503"]) {
+      expect(kinds.has(sqlstate)).toBe(true);
+    }
+  });
+});

From 32308e955c1cd571ea8bd2616256edaa58e675a8 Mon Sep 17 00:00:00 2001
From: Devinder Singh <devinder.stockholm3219@gmail.com>
Date: Sun, 1 Nov 2026 14:23:57 +0100
Subject: [PATCH 3/5] feat: give the API its database URL and a startupProbe

---
 api/src/routes/health.ts                  |  5 +++-
 docs/CONTEXT.md                           |  6 ++---
 docs/api.md                               |  3 ++-
 docs/decisions/0003-api-code-structure.md |  1 +
 k8s/app/deployment.yaml                   | 31 ++++++++++++++++++++---
 5 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/api/src/routes/health.ts b/api/src/routes/health.ts
index 25faf92..ea141d7 100644
--- a/api/src/routes/health.ts
+++ b/api/src/routes/health.ts
@@ -1,7 +1,10 @@
 // Routes that Kubernetes and the pipeline call. No API key needed.
 //   /health/live   "is the process alive?"  Never checks the database: a database outage
 //                  must not make Kubernetes restart every pod.
-//   /health/ready  "can I take traffic right now?"  Phase 2 adds the database check here.
+//   /health/ready  "can I take traffic right now?"  Never checks the database either: one
+//                  database blip would take every pod out of the Service at once. Instead the
+//                  API waits for the database before it listens (server.ts), and a request
+//                  that needs it during a blip gets our own 503 (error-handler.ts).
 //                  While shutting down it answers 503 without any code here: after
 //                  app.close() (server.ts), Fastify answers every new request with 503.
 //   /version       which commit is running. The pipeline checks it after each deploy.
diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md
index 9cb59ad..45c1a0d 100644
--- a/docs/CONTEXT.md
+++ b/docs/CONTEXT.md
@@ -81,7 +81,7 @@ endpoint_id); partial INDEX (next_attempt_at) WHERE status = 'pending') · `atte
 **API v1:** `GET /v1/me` · `POST /v1/endpoints` (returns the signing secret once) ·
 `GET /v1/endpoints` · `DELETE /v1/endpoints/:id` (soft delete) · `POST /v1/events` · `GET /v1/deliveries?status=&endpoint_id=&cursor=` ·
 `GET /v1/deliveries/:id` · `POST /v1/deliveries/:id/replay` · `GET /health/live` (no DB check) ·
-`GET /health/ready` (checks DB, 503 while shutting down) · `GET /version` (git SHA) ·
+`GET /health/ready` (never checks the DB; 503 while shutting down) · `GET /version` (git SHA) ·
 `/metrics` on port 9090 only. Errors: `{"error":{"code","message","request_id"}}`. Another
 owner's id answers 404. Bodies: JSON only, max 256 KiB. CORS: `https://cdn.myrelay.tech` only.
 Owners and API keys are created with a `create-owner` script (no signup page).
@@ -94,10 +94,10 @@ Both the API and the dashboard follow it; changes to it go through a PR first.
 |---|---|---|
 | Runtime | Bun **1.4.4**, pinned everywhere (laptop, CI, Docker). 1.4 is a new rewrite in Rust | Node (more common, but we know Bun; it stays our fallback) |
 | Web framework | **Fastify 5** ([decision 0001](decisions/0001-web-framework-fastify.md)): `buildApp()` + `listen({ host: "0.0.0.0" })`, pino logs, Zod validation, `@fastify/cors`, `inject()` tests | Hono (native on Bun, but new to us), Express, Elysia |
-| Database library | postgres.js behind one file `src/db.ts` | `Bun.sql` (ties the data layer to the new runtime), Drizzle |
+| Database library | postgres.js 3.4.9 behind one file `src/db.ts`: pool 5 per pod, 5 s `statement_timeout`; the API waits for the database before it listens, and answers 503 `service_unavailable` while it can't reach it | `Bun.sql` (ties the data layer to the new runtime), Drizzle |
 | API code structure | **Zod 4** for bodies, queries and env vars ([decision 0003](decisions/0003-api-code-structure.md)); layers routes → services → repositories; one Fastify error handler with one error shape; small files, one job each | Fastify JSON Schema, `fastify-type-provider-zod`, logic in route handlers |
 | Migrations | dbmate (plain SQL, `api/db/migrations/`, inside the API image), run by the migrate Job (`k8s/migrate/`) that `relay-deploy` creates before each rollout. Forward-only, expand/contract ([decision 0006](decisions/0006-migrations-expand-contract.md)) | drizzle-kit, an initContainer (would rerun on every scale-up), `down` migrations on rollback |
-| Unit + integration tests | `bun test`; integration against a real Postgres (Docker locally, service container in GitHub Actions) | Testcontainers (no official Bun support), mocked DB |
+| Unit + integration tests | `bun test`; integration tests against a real Postgres, logged in as `relay`: the same throwaway container (production's image and first-start script) on the laptop and in GitHub Actions (`bun run db:up`, then `bun run test:db`) | Testcontainers (no official Bun support), mocked DB |
 | Dashboard | **Vite + React + TypeScript, Tailwind CSS v4, React Router, axios** (one client in `src/api.ts`), plain React state with a small refresh loop while a delivery is pending, served by nginx ([decision 0002](decisions/0002-dashboard-stack.md)) | Next.js, plain CSS, `fetch`, TanStack Query |
 | E2E tests | Playwright, runs on Node in the dashboard folder | Cypress |
 | Queue | Postgres `SKIP LOCKED` + lease with fencing token | RabbitMQ (stretch goal only, as a "doorbell"; dual-write problem) |
diff --git a/docs/api.md b/docs/api.md
index da344d1..e3d90ea 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -47,6 +47,7 @@ A 422 also has `details`, one entry per problem:
 | 422 | `validation_failed` | Body or query fails the rules below (`details` lists why) |
 | 422 | `invalid_endpoint_url` | Endpoint URL that isn't `https`, or points to a private or internal address |
 | 500 | `internal_error` | Our bug. The message never contains internal details |
+| 503 | `service_unavailable` | The database can't be reached right now (a restart, a short blip). Comes with `Retry-After: 5`: try again after that |
 
 ### Pagination
 
@@ -154,7 +155,7 @@ Deleted endpoints aren't listed at all.
 | Route | Answer |
 |---|---|
 | `GET /health/live` | 200 `{ "status": "ok" }`. Never checks the database |
-| `GET /health/ready` | 200 `{ "status": "ok" }`, or 503 while shutting down or when the database is unreachable. The shutdown 503 comes from Fastify: after `app.close()` every new request gets 503, and a pod that has stopped listening refuses the connection, which also counts as not ready |
+| `GET /health/ready` | 200 `{ "status": "ok" }`, or 503 while shutting down. Never checks the database: one database blip would take every pod out at once. The API waits for the database before it listens instead, and requests that need it during a blip get `503 service_unavailable`. The shutdown 503 comes from Fastify: after `app.close()` every new request gets 503, and a pod that has stopped listening refuses the connection, which also counts as not ready |
 | `GET /version` | 200 `{ "sha": "<git commit>" }` |
 
 ### `GET /v1/me`
diff --git a/docs/decisions/0003-api-code-structure.md b/docs/decisions/0003-api-code-structure.md
index 3fc97ad..9fa6554 100644
--- a/docs/decisions/0003-api-code-structure.md
+++ b/docs/decisions/0003-api-code-structure.md
@@ -59,6 +59,7 @@ Code never builds error responses itself. It throws, and **one** Fastify error h
 | `ZodError` | 422 `validation_failed`, with the fields that were wrong |
 | Our own `AppError` (for example `notFound`, `conflict`) | Its own status and code (404 `not_found`, 409 `already_delivered`, …) |
 | Fastify's own client errors | Their status: 400 bad JSON, 413 body too large, 415 not JSON |
+| The database can't be reached (`isDatabaseUnavailable` in `db.ts`; added in Phase 2, step 2.6) | Logged as a warning, answered as 503 `service_unavailable` with `Retry-After: 5` |
 | Anything else | Logged in full, answered as 500 `internal_error` with no details |
 
 ### Files
diff --git a/k8s/app/deployment.yaml b/k8s/app/deployment.yaml
index 2ac0b6e..d320267 100644
--- a/k8s/app/deployment.yaml
+++ b/k8s/app/deployment.yaml
@@ -38,14 +38,40 @@ spec:
         runAsGroup: 1000
         seccompProfile:
           type: RuntimeDefault            # the container runtime's default syscall filter
+      # After SIGTERM, Kubernetes waits this long, then sends SIGKILL. 30 s is the default,
+      # written out because our shutdown must fit inside it: 5 s preStop (below), then running
+      # requests finish (milliseconds for the API), then the database pool closes (sql.end,
+      # at most 5 s). Phase 4's worker needs more: its webhook calls may take 10 s.
+      terminationGracePeriodSeconds: 30
       containers:
         - name: api
           image: ghcr.io/devinder-dev/relay-api:__GIT_SHA__
           ports:
             - name: http                  # probes, the Service and the Ingress use this name
               containerPort: 8080
+          # Where Postgres is, with the relay login in it: straight from the Secret relay-db
+          # (made on the server by bootstrap-secrets.sh), never written in git. If the Secret
+          # is missing, the pod can't start, so the rollout fails and relay-deploy rolls back.
+          env:
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: relay-db
+                  key: url
+          # startup: "has it finished starting?" Runs first, alone; liveness and readiness wait
+          # until it has passed once. 45 × 2 s = 90 s: more than the API's own 60 s wait for
+          # the database, so the API gives up first, with a clear log line, instead of being
+          # killed by the probe in the middle of a retry.
+          startupProbe:
+            httpGet:
+              path: /health/live
+              port: http
+            periodSeconds: 2
+            failureThreshold: 45
           # readiness: "may this pod get traffic?" No → out of the Service, but kept running.
-          # liveness: "is it alive?" No, 3 times → restarted. Never checks the database.
+          # liveness: "is it alive?" No, 3 times → restarted.
+          # Neither checks the database: one database blip would take every pod out at once.
+          # The API waits for the database before it listens instead (server.ts).
           readinessProbe:
             httpGet:
               path: /health/ready
@@ -56,8 +82,7 @@ spec:
             httpGet:
               path: /health/live
               port: http
-            initialDelaySeconds: 15
-            periodSeconds: 20
+            periodSeconds: 20                 # no initialDelaySeconds: the startupProbe covers the start
             failureThreshold: 3
 
           # Before SIGTERM, wait 5 s, so Traefik stops sending this pod traffic first.

From d7ec69c2c05678e2f82b47a29821b59a3928c94b Mon Sep 17 00:00:00 2001
From: Devinder Singh <devinder.stockholm3219@gmail.com>
Date: Sun, 1 Nov 2026 14:37:23 +0100
Subject: [PATCH 4/5] fix: address the review of #26 (no cancel, 503 codes by
 name, fresh test database)

---
 api/scripts/test-db.sh                  | 30 +++++------
 api/src/db.test.ts                      | 23 +++++++--
 api/src/db.ts                           | 44 +++++++++++------
 api/test/integration/connection.test.ts | 46 ++++++++++-------
 api/test/integration/helpers.ts         |  8 +--
 api/test/integration/schema.test.ts     | 66 ++++++++++++++++++++++++-
 k8s/app/deployment.yaml                 | 10 ++--
 7 files changed, 165 insertions(+), 62 deletions(-)

diff --git a/api/scripts/test-db.sh b/api/scripts/test-db.sh
index 7973e8e..b850225 100755
--- a/api/scripts/test-db.sh
+++ b/api/scripts/test-db.sh
@@ -3,7 +3,7 @@
 # The same image as production (its digest is read from k8s/data/postgres.yaml) and the same
 # first-start script, so the tests log in as "relay", never as the superuser.
 #
-#   up    start it on 127.0.0.1:55432, wait until it answers, then run the migrations
+#   up    start a fresh one on 127.0.0.1:55432, wait until it answers, then run the migrations
 #   test  run every test (unit + integration) against it; extra arguments go to bun test
 #   down  remove it: only the container this script made, which carries our label
 #
@@ -45,26 +45,22 @@ remove_ours() {
 up() {
   docker info >/dev/null 2>&1 || fail "Docker isn't running"
 
-  # Step 2: reuse ours if it's running; never touch anything else.
+  # Step 2: always start fresh. An old container of ours may have an older image, an older
+  # first-start script or other ports, so it's removed first (only ours: remove_ours checks
+  # the label). It costs a few seconds and rules all of that out.
   if exists; then
-    if running && ours; then
-      echo "test-db: $NAME is already running, reusing it"
-    else
-      echo "test-db: $NAME exists but isn't running, removing it first"
-      remove_ours
-    fi
+    echo "test-db: removing the old $NAME first, so the new one matches production's setup"
+    remove_ours
   fi
 
   # Step 3: start it, with the first-start script mounted read-only.
-  if ! exists; then
-    port_busy && fail "port $PORT on 127.0.0.1 is already in use by something else"
-    docker run --detach --name "$NAME" --label "$LABEL_KEY=1" \
-      --publish "127.0.0.1:$PORT:5432" \
-      --env POSTGRES_PASSWORD="$PASSWORD" --env RELAY_DB_PASSWORD="$PASSWORD" \
-      --volume "$INITDB_DIR:/docker-entrypoint-initdb.d:ro" \
-      "$(prod_image)" >/dev/null
-    echo "test-db: started $NAME on 127.0.0.1:$PORT"
-  fi
+  port_busy && fail "port $PORT on 127.0.0.1 is already in use by something else"
+  docker run --detach --name "$NAME" --label "$LABEL_KEY=1" \
+    --publish "127.0.0.1:$PORT:5432" \
+    --env POSTGRES_PASSWORD="$PASSWORD" --env RELAY_DB_PASSWORD="$PASSWORD" \
+    --volume "$INITDB_DIR:/docker-entrypoint-initdb.d:ro" \
+    "$(prod_image)" >/dev/null
+  echo "test-db: started $NAME on 127.0.0.1:$PORT"
 
   # Step 4: wait for the real server. During the first start a temporary server listens on
   # the socket only, so asking over TCP (-h 127.0.0.1) only succeeds once that's finished.
diff --git a/api/src/db.test.ts b/api/src/db.test.ts
index 47c9711..d2f41c8 100644
--- a/api/src/db.test.ts
+++ b/api/src/db.test.ts
@@ -1,7 +1,7 @@
 // Unit tests for db.ts: no real database. A fake "sql" answers from a script (ok, or an error
 // code), and a fake clock lets 60 seconds of retries pass in a few milliseconds.
 import { describe, expect, test } from "bun:test";
-import { isDatabaseUnavailable, waitForDatabase, type Log, type Sql } from "./db.ts";
+import { isDatabaseUnavailable, pingDatabase, waitForDatabase, type Log, type Sql } from "./db.ts";
 
 type Outcome = "ok" | string; // "ok", or the error code the database "throws"
 
@@ -112,16 +112,31 @@ describe("waitForDatabase", () => {
   });
 });
 
+describe("pingDatabase", () => {
+  test("a try that times out is abandoned, and its late error is still handled (no crash)", async () => {
+    // The database answers with an error only after the try has given up.
+    const sql = (() =>
+      new Promise((_, reject) =>
+        setTimeout(() => reject(Object.assign(new Error("late"), { code: "ECONNRESET" })), 30),
+      )) as unknown as Sql;
+    await expect(pingDatabase(sql, 10)).rejects.toMatchObject({ code: "PING_TIMEOUT" });
+    // Let the late error arrive: bun test fails the whole run on an uncaught one.
+    await Bun.sleep(50);
+  });
+});
+
 describe("isDatabaseUnavailable", () => {
   test("connection problems → true (they become 503)", () => {
-    for (const code of ["ECONNREFUSED", "CONNECT_TIMEOUT", "CONNECTION_CLOSED", "57P03", "08006"]) {
+    for (const code of ["ECONNREFUSED", "CONNECT_TIMEOUT", "CONNECTION_CLOSED", "EPIPE", "EHOSTUNREACH", "57P02", "57P03", "08001", "08006"]) {
       expect(isDatabaseUnavailable({ code })).toBe(true);
     }
   });
 
   test("everything else → false (it stays a 500: our bug, not an outage)", () => {
-    // 23505 duplicate key, 42P01 no such table, 57014 cancelled by statement_timeout (slow SQL)
-    for (const error of [{ code: "23505" }, { code: "42P01" }, { code: "57014" }, new Error("boom"), null, undefined, "text"]) {
+    // 23505 duplicate key, 42P01 no such table, 57014 cancelled (slow SQL), 08P01 protocol
+    // violation (a driver bug), 08007 commit result unknown ("try again" could save it twice)
+    const notOutages = [{ code: "23505" }, { code: "42P01" }, { code: "57014" }, { code: "08P01" }, { code: "08007" }];
+    for (const error of [...notOutages, new Error("boom"), null, undefined, "text"]) {
       expect(isDatabaseUnavailable(error)).toBe(false);
     }
   });
diff --git a/api/src/db.ts b/api/src/db.ts
index 01d9abb..f2c05fc 100644
--- a/api/src/db.ts
+++ b/api/src/db.ts
@@ -36,20 +36,23 @@ export function createDb(config: Config): Sql {
 // means "not there yet", so it's worth trying again.
 export const FATAL_STARTUP_CODES = new Set(["28P01", "28000", "3D000"]);
 
-// One try: `select 1`, cut short after timeoutMs. connect_timeout only limits connecting; this
-// also covers a server that accepted the connection and then stopped answering. On timeout the
-// query is cancelled too, so it doesn't keep running in the background.
+// One try: `select 1`, given up after timeoutMs. connect_timeout only limits connecting; this
+// also covers a server that accepted the connection and then stopped answering.
+// On timeout the try is abandoned, NOT cancelled. postgres.js 3.4.9's cancel() sends the cancel
+// over a new connection and drops that request's promise, so when the database is stuck
+// (exactly when we'd cancel) its failure would go uncaught and could crash the API.
+// Promise.race still handles the abandoned query's late answer or error. If the database stays
+// stuck, the API exits at the deadline, and that closes every connection anyway.
 export async function pingDatabase(sql: Sql, timeoutMs = 5_000): Promise<void> {
-  const query = sql`select 1`;
   let timer: ReturnType<typeof setTimeout> | undefined;
   const timeout = new Promise<never>((_, reject) => {
-    timer = setTimeout(() => {
-      query.cancel?.(); // postgres.js has cancel(); a test's fake may not
-      reject(Object.assign(new Error(`no answer within ${timeoutMs} ms`), { code: "PING_TIMEOUT" }));
-    }, timeoutMs);
+    timer = setTimeout(
+      () => reject(Object.assign(new Error(`no answer within ${timeoutMs} ms`), { code: "PING_TIMEOUT" })),
+      timeoutMs,
+    );
   });
   try {
-    await Promise.race([query, timeout]);
+    await Promise.race([sql`select 1`, timeout]);
   } finally {
     clearTimeout(timer);
   }
@@ -69,7 +72,7 @@ export async function waitForDatabase(sql: Sql, log: Log, options: WaitOptions =
   const {
     deadlineMs = 60_000,
     pauseMs = 2_000,
-    now = () => Date.now(),
+    now = () => performance.now(), // only moves forward; Date.now() jumps if the clock is set
     sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)),
   } = options;
 
@@ -108,19 +111,30 @@ const UNAVAILABLE_CODES = new Set([
   "CONNECTION_ENDED", // postgres.js: a query arrived after the pool was closed (shutdown)
   "ECONNREFUSED", // Node: nothing listens on the port (Postgres down or restarting)
   "ECONNRESET", // Node: the connection was cut
+  "EPIPE", // Node: wrote to a connection that was already closed
   "ETIMEDOUT", // Node: the network didn't answer
+  "EHOSTUNREACH", // Node: no route to the database host
+  "ENETUNREACH", // Node: no route to its network
   "ENOTFOUND", // Node: the host name doesn't resolve (yet)
   "EAI_AGAIN", // Node: DNS lookup failed for now
   "57P01", // Postgres: the server is shutting down
+  "57P02", // Postgres: the server crashed and is restarting
   "57P03", // Postgres: the server is still starting up
   "53300", // Postgres: too many connections right now
+  // SQLSTATE class 08, "connection exception", by name:
+  "08000", // connection exception
+  "08001", // the client couldn't establish the connection
+  "08003", // the connection doesn't exist
+  "08004", // the server refused the connection
+  "08006", // connection failure
 ]);
-// Not in the list on purpose: 57014, "cancelled by statement_timeout". A query that runs over
-// 5 s is far more likely our slow SQL than an outage, so it stays a 500 and shows up as a bug.
+// Not in the list on purpose, so they stay a 500:
+//   57014  a query was cancelled (by statement_timeout): far more likely our slow SQL than an outage
+//   08P01  protocol violation: a bug in the driver or in Postgres, not "the database is down"
+//   08007  "did the commit happen? unknown". 503 means "try again", which could save it twice
+// Every one of them must show up as a bug, never hide as an outage.
 
 export function isDatabaseUnavailable(error: unknown): boolean {
   const code = (error as { code?: unknown } | null)?.code;
-  if (typeof code !== "string") return false;
-  // SQLSTATE class 08 = "connection exception" (for example 08006: connection failure)
-  return UNAVAILABLE_CODES.has(code) || code.startsWith("08");
+  return typeof code === "string" && UNAVAILABLE_CODES.has(code);
 }
diff --git a/api/test/integration/connection.test.ts b/api/test/integration/connection.test.ts
index 7f236d7..bcc6275 100644
--- a/api/test/integration/connection.test.ts
+++ b/api/test/integration/connection.test.ts
@@ -1,7 +1,7 @@
 // The connection itself, against the real Postgres: who we are, which settings arrive, and what
 // happens when the login is wrong, a query hangs, or a connection is cut.
 import { afterAll, describe, expect, test } from "bun:test";
-import { waitForDatabase } from "../../src/db.ts";
+import { isDatabaseUnavailable, waitForDatabase } from "../../src/db.ts";
 import { DATABASE_URL, silentLog, testDb } from "./helpers.ts";
 
 const sql = testDb();
@@ -26,37 +26,49 @@ describe("the connection, against the real Postgres", () => {
     await waitForDatabase(sql, silentLog);
   });
 
-  test("a wrong password → waitForDatabase gives up at once with 28P01, no 60 s wait", async () => {
+  test("a wrong password → waitForDatabase gives up at the first answer with 28P01, no retry", async () => {
     const wrong = new URL(DATABASE_URL);
     wrong.password = `wrong-${crypto.randomUUID()}`;
-    const bad = testDb(wrong.toString());
-    const started = performance.now();
+    const bad = testDb({ url: wrong.toString() });
+    // No stopwatch (a slow runner would make it flaky): "no ⚠️ retry line" proves it didn't wait.
+    const warnings: string[] = [];
+    const log = { info: () => {}, warn: (_obj: object, msg: string) => warnings.push(msg) };
     try {
-      await expect(waitForDatabase(bad, silentLog)).rejects.toMatchObject({ code: "28P01" });
-      expect(performance.now() - started).toBeLessThan(2_000); // shorter than a single pause
+      await expect(waitForDatabase(bad, log)).rejects.toMatchObject({ code: "28P01" });
+      expect(warnings).toEqual([]);
     } finally {
       await bad.end({ timeout: 1 });
     }
   });
 
-  test("a stuck query is cancelled by statement_timeout after 5 s, with 57014", async () => {
-    const started = performance.now();
+  test("a stuck query is cancelled by statement_timeout, with 57014", async () => {
+    // pg_sleep(6) would succeed after 6 s; the 57014 error alone proves Postgres stopped it.
     await expect(sql`select pg_sleep(6)`).rejects.toMatchObject({ code: "57014" });
-    expect(performance.now() - started).toBeLessThan(5_900); // stopped at 5 s, not finished at 6
-  }, 10_000);
+  }, 15_000);
 
   test("a dropped connection comes back: the next query opens a new one", async () => {
-    const [before] = await sql`select pg_backend_pid() as pid`;
-    // A separate connection ends ours, as a Postgres restart or a cut network would. relay may
-    // end its own sessions without being a superuser. The 2 s = wait until it's really gone.
+    // A pool of exactly one connection, so a different pid can only mean a new connection.
+    const one = testDb({ poolMax: 1 });
     const other = testDb();
+    const pid = async () => (await one`select pg_backend_pid() as pid`)[0]!.pid as number;
     try {
-      const [result] = await other`select pg_terminate_backend(${before!.pid}, 2000) as ended`;
+      const before = await pid();
+      // The other pool ends ours, as a Postgres restart or a cut network would. relay may end
+      // its own sessions without being a superuser. The 2000 = wait until it's really gone.
+      const [result] = await other`select pg_terminate_backend(${before}, 2000) as ended`;
       expect(result!.ended).toBe(true);
+      // The client may not have noticed the cut yet, so one connection error is allowed (in
+      // production that request gets our 503). The query after it must work.
+      let after: number;
+      try {
+        after = await pid();
+      } catch (error) {
+        expect(isDatabaseUnavailable(error)).toBe(true);
+        after = await pid();
+      }
+      expect(after).not.toBe(before);
     } finally {
-      await other.end({ timeout: 1 });
+      await Promise.all([one.end({ timeout: 1 }), other.end({ timeout: 1 })]);
     }
-    const [after] = await sql`select pg_backend_pid() as pid`;
-    expect(after!.pid).not.toBe(before!.pid);
   });
 });
diff --git a/api/test/integration/helpers.ts b/api/test/integration/helpers.ts
index ca75706..0c396ed 100644
--- a/api/test/integration/helpers.ts
+++ b/api/test/integration/helpers.ts
@@ -14,9 +14,11 @@ if (!url) {
 }
 export const DATABASE_URL: string = url;
 
-// A pool with production's own settings (createDb), exactly as the API uses it.
-export function testDb(databaseUrl: string = DATABASE_URL): Sql {
-  return createDb(loadConfig({ DATABASE_URL: databaseUrl }));
+// A pool with production's own settings (createDb), exactly as the API uses it. A test can
+// change the URL (a wrong password) or the pool size (exactly one connection).
+export function testDb(options: { url?: string; poolMax?: number } = {}): Sql {
+  const { url: databaseUrl = DATABASE_URL, poolMax = 5 } = options;
+  return createDb(loadConfig({ DATABASE_URL: databaseUrl, DB_POOL_MAX: String(poolMax) }));
 }
 
 export const silentLog: Log = { info: () => {}, warn: () => {} };
diff --git a/api/test/integration/schema.test.ts b/api/test/integration/schema.test.ts
index fad1c9a..93385a6 100644
--- a/api/test/integration/schema.test.ts
+++ b/api/test/integration/schema.test.ts
@@ -9,6 +9,9 @@ afterAll(() => sql.end({ timeout: 5 }));
 
 type Seed = { ownerId: string; endpointId: string; eventId: string; deliveryId: string };
 
+// A Postgres text[] literal with n event types, for example '{t1,t2}'.
+const eventTypes = (n: number) => `{${Array.from({ length: n }, (_, i) => `t${i + 1}`).join(",")}}`;
+
 // One valid row in every table, so a case can point at rows that exist.
 async function seed(tx: Tx): Promise<Seed> {
   const [owner] = await tx`insert into owners (name) values ('Test owner') returning id`;
@@ -64,6 +67,34 @@ describe("schema v1 accepts valid rows (positive controls)", () => {
         values (${other!.id}, 'order.created', ${tx.json({})}, 'key-1')`;
     });
   });
+
+  test("the limits themselves are accepted: 20 event types, a 255-character idempotency key", async () => {
+    // The positive controls for the "21" and "256" cases below: they prove those are refused
+    // because of the limit, not because something else in the row is wrong.
+    await inRollback(sql, async (tx) => {
+      const { ownerId } = await seed(tx);
+      await tx`
+        insert into endpoints (owner_id, name, url, signing_secret, event_types)
+        values (${ownerId}, 'x', 'https://example.com/hook', 'whsec_x', ${eventTypes(20)})`;
+      await tx`
+        insert into events (owner_id, type, payload, idempotency_key)
+        values (${ownerId}, 'a.b', ${tx.json({})}, ${"k".repeat(255)})`;
+    });
+  });
+
+  test("deleting an owner deletes everything that belongs to it (ON DELETE CASCADE)", async () => {
+    await inRollback(sql, async (tx) => {
+      const ids = await seed(tx);
+      await tx`delete from owners where id = ${ids.ownerId}`;
+      const [left] = await tx`
+        select (select count(*) from api_keys   where owner_id = ${ids.ownerId})::int as api_keys,
+               (select count(*) from endpoints  where owner_id = ${ids.ownerId})::int as endpoints,
+               (select count(*) from events     where owner_id = ${ids.ownerId})::int as events,
+               (select count(*) from deliveries where owner_id = ${ids.ownerId})::int as deliveries,
+               (select count(*) from attempts   where delivery_id = ${ids.deliveryId})::int as attempts`;
+      expect(left).toEqual({ api_keys: 0, endpoints: 0, events: 0, deliveries: 0, attempts: 0 });
+    });
+  });
 });
 
 // Each case: what it is, the SQLSTATE the database must answer, and the query. It runs after
@@ -113,6 +144,37 @@ const refused: Case[] = [
     sqlstate: "23503",
     query: (sql, ids) => sql`insert into attempts (delivery_id, started_at, duration_ms, status_code) values ('00000000-0000-0000-0000-000000000000', now(), 100, 200)`,
   },
+  // Added after the review of #26: one case per remaining rule in migration 001.
+  {
+    what: "a second delivery of the same event to the same endpoint (the fan-out must not double)",
+    sqlstate: "23505",
+    query: (sql, ids) => sql`insert into deliveries (owner_id, event_id, endpoint_id) values (${ids.ownerId}, ${ids.eventId}, ${ids.endpointId})`,
+  },
+  {
+    what: "an API key with a hash that already exists",
+    sqlstate: "23505",
+    query: (sql, ids) => sql`insert into api_keys (owner_id, prefix, key_hash) values (${ids.ownerId}, 'relay_live_copy', sha256(convert_to('relay_live_test_full_key', 'UTF8')))`,
+  },
+  {
+    what: "an event without a type (a required value missing)",
+    sqlstate: "23502",
+    query: (sql, ids) => sql`insert into events (owner_id, payload) values (${ids.ownerId}, ${sql.json({})})`,
+  },
+  {
+    what: "an endpoint with 0 event types",
+    sqlstate: "23514",
+    query: (sql, ids) => sql`insert into endpoints (owner_id, name, url, signing_secret, event_types) values (${ids.ownerId}, 'x', 'https://example.com/hook', 'whsec_x', '{}')`,
+  },
+  {
+    what: "an endpoint with 21 event types (the limit is 20)",
+    sqlstate: "23514",
+    query: (sql, ids) => sql`insert into endpoints (owner_id, name, url, signing_secret, event_types) values (${ids.ownerId}, 'x', 'https://example.com/hook', 'whsec_x', ${eventTypes(21)})`,
+  },
+  {
+    what: "an idempotency key of 256 characters (the limit is 255)",
+    sqlstate: "23514",
+    query: (sql, ids) => sql`insert into events (owner_id, type, payload, idempotency_key) values (${ids.ownerId}, 'a.b', ${sql.json({})}, ${"k".repeat(256)})`,
+  },
 ];
 
 describe("schema v1 refuses what breaks its rules, each with the right SQLSTATE", () => {
@@ -125,9 +187,9 @@ describe("schema v1 refuses what breaks its rules, each with the right SQLSTATE"
     });
   }
 
-  test("the list covers every kind of rule: CHECK, UNIQUE and foreign key", () => {
+  test("the list covers every kind of rule: CHECK, UNIQUE, foreign key and NOT NULL", () => {
     const kinds = new Set(refused.map((c) => c.sqlstate));
-    for (const sqlstate of ["23514", "23505", "23503"]) {
+    for (const sqlstate of ["23514", "23505", "23503", "23502"]) {
       expect(kinds.has(sqlstate)).toBe(true);
     }
   });
diff --git a/k8s/app/deployment.yaml b/k8s/app/deployment.yaml
index d320267..6552f6e 100644
--- a/k8s/app/deployment.yaml
+++ b/k8s/app/deployment.yaml
@@ -38,10 +38,12 @@ spec:
         runAsGroup: 1000
         seccompProfile:
           type: RuntimeDefault            # the container runtime's default syscall filter
-      # After SIGTERM, Kubernetes waits this long, then sends SIGKILL. 30 s is the default,
-      # written out because our shutdown must fit inside it: 5 s preStop (below), then running
-      # requests finish (milliseconds for the API), then the database pool closes (sql.end,
-      # at most 5 s). Phase 4's worker needs more: its webhook calls may take 10 s.
+      # From the moment a pod is told to stop, Kubernetes allows this long IN TOTAL, then sends
+      # SIGKILL. The 5 s preStop (below) runs first, inside these 30 s; only then comes SIGTERM.
+      # So server.ts has the remaining ~25 s: running requests finish (milliseconds for today's
+      # routes; later routes must fit too), then the database pool closes (sql.end, at most
+      # 5 s). 30 s is the default, written out because of that budget. Phase 4's worker needs
+      # more: its webhook calls may take 10 s.
       terminationGracePeriodSeconds: 30
       containers:
         - name: api

From 81e7b6331b8e63b2e7b72afaf4af099d1a661325 Mon Sep 17 00:00:00 2001
From: Devinder Singh <devinder.stockholm3219@gmail.com>
Date: Sun, 1 Nov 2026 15:45:38 +0100
Subject: [PATCH 5/5] fix: one source for the Postgres image, and the
 RELAY_DB_PASSWORD comment from #22

---
 api/scripts/test-db.sh | 15 ++++++---------
 k8s/data/postgres.yaml |  4 +++-
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/api/scripts/test-db.sh b/api/scripts/test-db.sh
index b850225..b45a6ad 100755
--- a/api/scripts/test-db.sh
+++ b/api/scripts/test-db.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # A throwaway Postgres for the tests, the same on the laptop and in CI (step 2.6b).
-# The same image as production (its digest is read from k8s/data/postgres.yaml) and the same
+# The same image as production (postgres-image.sh reads it from k8s/data/postgres.yaml) and the same
 # first-start script, so the tests log in as "relay", never as the superuser.
 #
 #   up    start a fresh one on 127.0.0.1:55432, wait until it answers, then run the migrations
@@ -29,13 +29,6 @@ ours() { [ "$(docker container inspect -f "{{index .Config.Labels \"$LABEL_KEY\"
 running() { [ "$(docker container inspect -f '{{.State.Running}}' "$NAME" 2>/dev/null)" = "true" ]; }
 port_busy() { (exec 3<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null; }
 
-# The image production runs, pinned by digest. One source of truth: the StatefulSet.
-prod_image() {
-  local image
-  image=$(sed -n 's/^ *image: *\(postgres:[^ ]*\).*/\1/p' "$REPO_DIR/k8s/data/postgres.yaml" | head -n 1)
-  [[ "$image" =~ ^postgres:[0-9.]+@sha256:[0-9a-f]{64}$ ]] || fail "couldn't read a pinned postgres image from k8s/data/postgres.yaml"
-  echo "$image"
-}
 
 remove_ours() {
   ours || fail "a container named $NAME exists but this script didn't make it, so it's left alone"
@@ -55,11 +48,15 @@ up() {
 
   # Step 3: start it, with the first-start script mounted read-only.
   port_busy && fail "port $PORT on 127.0.0.1 is already in use by something else"
+  # Production's image, pinned by digest, from the one script that reads it (the CI smoke test
+  # uses the same one). On its own line, so a missing digest stops here with its own message.
+  local image
+  image=$(bash "$API_DIR/scripts/postgres-image.sh")
   docker run --detach --name "$NAME" --label "$LABEL_KEY=1" \
     --publish "127.0.0.1:$PORT:5432" \
     --env POSTGRES_PASSWORD="$PASSWORD" --env RELAY_DB_PASSWORD="$PASSWORD" \
     --volume "$INITDB_DIR:/docker-entrypoint-initdb.d:ro" \
-    "$(prod_image)" >/dev/null
+    "$image" >/dev/null
   echo "test-db: started $NAME on 127.0.0.1:$PORT"
 
   # Step 4: wait for the real server. During the first start a temporary server listens on
diff --git a/k8s/data/postgres.yaml b/k8s/data/postgres.yaml
index 34976cc..49ac8f8 100644
--- a/k8s/data/postgres.yaml
+++ b/k8s/data/postgres.yaml
@@ -85,7 +85,9 @@ spec:
                 secretKeyRef:
                   name: postgres-auth
                   key: password
-            - name: RELAY_DB_PASSWORD           # read once, by the first-start script
+            # Read by the first-start script, and it stays in the pod's env on purpose: section 6b
+            # of infra/server/README.md reads it again (\getenv) to reset a lost Secret's password.
+            - name: RELAY_DB_PASSWORD
               valueFrom:
                 secretKeyRef:
                   name: relay-db