fix: restore liveness and database readiness probes

AGENTS.md

@@ -728,6 +728,20 @@ in this repo.
 
 ## Phase 10 development rules
 
+- Validate probes in the rendered deployment manifest, not only API tests.
+  Backend readiness must call `/readyz` on port 8000; a static root response
+  cannot establish database availability. Never reuse that dependency probe for
+  liveness: a database outage must not become a backend restart trigger.
+
+- Readiness probes must handle native PostgreSQL connection-establishment
+  errors as well as SQLAlchemy wrappers. In PR #1597, a real isolated cluster
+  raised `asyncpg.InvalidCatalogNameError` before `SELECT 1`; catch the driver's
+  `PostgresError` at the readiness boundary and return only the generic 503
+  response. Preserve cancellation propagation. Validate healthy, primary-failure,
+  and read-only-failure paths with both pools returned before claiming readiness.
+  Use a private test cluster, never a shared customer database. See
+  `docs/doctoring/runtime-image-boundary-verification.md` for the evidence limits.
+
 - **Stepwise execution**: Each phase requires an atomic PR, GitHub PR Tracking, Push, and Robot Review. A phase only ends when merged. Do not proceed without merge.
 - **TDD + DDD**: Practice TDD, micro TDD, nano TDD, Domain Driven Development, and Context Driven Development.
 - **API Wiring**: Always work with API wiring completed.

backend/main.py

@@ -1,11 +1,15 @@
 import os
+from asyncpg import PostgresError
 from contextlib import asynccontextmanager
 from urllib.parse import urlsplit
 
 from fastapi import Depends, FastAPI
 from fastapi import Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
+from sqlalchemy import text
+from sqlalchemy.exc import SQLAlchemyError
+from db import session as database_session
 from api.auth import get_auth_context, preload_oidc_jwks
 from api.search import router as search_router
 from api.llm import router as llm_router
@@ -244,3 +248,24 @@ async def add_security_headers(request: Request, call_next):
 @app.get("/")
 def read_root() -> dict[str, str]:
     return {"status": "ok", "message": "AI Email Client API"}
+
+
+@app.get("/healthz", include_in_schema=False)
+async def process_health() -> JSONResponse:
+    """Report process liveness without touching external dependencies."""
+    return JSONResponse({"status": "ok"}, headers={"Cache-Control": "no-store"})
+
+
+@app.get("/readyz", include_in_schema=False)
+async def database_readiness() -> JSONResponse:
+    """Check both database pools without exposing connection or error details."""
+    try:
+        for database_engine in (database_session.engine, database_session.readonly_engine):
+            async with database_engine.connect() as database_connection:
+                await database_connection.execute(text("SELECT 1"))
+    except (SQLAlchemyError, PostgresError, OSError, TimeoutError):
+        return JSONResponse(
+            {"status": "unavailable"}, status_code=503,
+            headers={"Cache-Control": "no-store"},
+        )
+    return JSONResponse({"status": "ready"}, headers={"Cache-Control": "no-store"})

backend/tests/test_release_manifest_digests.py

@@ -46,6 +46,16 @@ def test_release_renderer_binds_both_images_without_changing_source(tmp_path: Pa
         assert rendered == expected
 
 
+def test_rendered_backend_requires_database_readiness(tmp_path: Path) -> None:
+    """Route deployment readiness to the dependency probe, not the static root."""
+    result = run_renderer(tmp_path, BACKEND_DIGEST, FRONTEND_DIGEST)
+    assert result.returncode == 0, result.stderr
+    manifest = yaml.safe_load((tmp_path / "rendered/backend-deployment.yaml").read_text())
+    backend_container = manifest["spec"]["template"]["spec"]["containers"][0]
+    assert backend_container["readinessProbe"]["httpGet"] == {"path": "/readyz", "port": 8000}
+    assert backend_container.get("livenessProbe", {}).get("httpGet", {}).get("path") != "/readyz"
+
+
 @pytest.mark.parametrize("invalid_digest", ["", "latest", "sha256:" + "a" * 63, "sha256:" + "A" * 64, BACKEND_DIGEST + "\ninjected", "$(touch injected)"])
 @pytest.mark.parametrize("invalid_component", ["backend", "frontend"])
 def test_release_renderer_rejects_either_invalid_digest_before_output(

backend/tests/test_runtime_health.py

@@ -0,0 +1,170 @@
+"""Exercise public probe responses without workers or customer/provider access."""
+
+from contextlib import asynccontextmanager
+from asyncio import CancelledError
+import os
+import secrets
+
+import httpx
+import pytest
+from asyncpg import InvalidCatalogNameError
+from sqlalchemy.exc import OperationalError
+
+os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://unit:unit@127.0.0.1:1/unit_db")
+os.environ.setdefault("AUTH_SESSION_HMAC_SECRET", secrets.token_urlsafe(48))
+os.environ.setdefault("DISABLE_BACKGROUND_WORKERS", "1")
+
+from db import session as database_session  # noqa: E402
+from main import app, database_readiness  # noqa: E402
+
+
+def install_probe_engines(
+    monkeypatch, *, failed_dependency=None, failure_factory=None, fail_before_acquisition=False
+):
+    """Install deterministic primary/read-only probes and return lifecycle evidence."""
+    connection_events = []
+
+    class ProbeConnection:
+        async def execute(self, query_statement):
+            assert str(query_statement) == "SELECT 1"
+            if self.dependency_name == failed_dependency and failure_factory is not None:
+                raise failure_factory()
+
+        def __init__(self, dependency_name):
+            self.dependency_name = dependency_name
+
+    class ProbeEngine:
+        def __init__(self, dependency_name):
+            self.dependency_name = dependency_name
+
+        @asynccontextmanager
+        async def connect(self):
+            if fail_before_acquisition and self.dependency_name == failed_dependency:
+                raise failure_factory()
+            connection_events.append((self.dependency_name, "open"))
+            try:
+                yield ProbeConnection(self.dependency_name)
+            finally:
+                connection_events.append((self.dependency_name, "close"))
+
+    monkeypatch.setattr(database_session, "engine", ProbeEngine("primary"))
+    monkeypatch.setattr(database_session, "readonly_engine", ProbeEngine("readonly"))
+    return connection_events
+
+
+@pytest.mark.asyncio
+async def test_liveness_does_not_touch_databases_and_disables_cache(monkeypatch):
+    """Liveness is process-only and returns a stable non-cacheable contract."""
+    connection_events = install_probe_engines(monkeypatch)
+
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://unit.local") as client:
+        response = await client.get("/healthz")
+
+    assert response.status_code == 200
+    assert response.json() == {"status": "ok"}
+    assert response.headers["cache-control"] == "no-store"
+    assert connection_events == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("failed_dependency", [None, "primary", "readonly"])
+async def test_readiness_checks_both_databases_without_leaking_errors(monkeypatch, failed_dependency):
+    """A failed database must remove readiness, and every acquired connection closes."""
+
+    def operational_failure():
+        return OperationalError("SELECT 1", None, Exception("unit-private-detail"))
+
+    connection_events = install_probe_engines(
+        monkeypatch,
+        failed_dependency=failed_dependency,
+        failure_factory=operational_failure if failed_dependency is not None else None,
+    )
+
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://unit.local") as client:
+        response = await client.get("/readyz")
+
+    assert response.status_code == (503 if failed_dependency else 200)
+    assert response.json() == {"status": "unavailable" if failed_dependency else "ready"}
+    assert response.headers["cache-control"] == "no-store"
+    assert "unit-private-detail" not in response.text
+    expected_events = [("primary", "open"), ("primary", "close")]
+    if failed_dependency != "primary":
+        expected_events += [("readonly", "open"), ("readonly", "close")]
+    assert connection_events == expected_events
+
+
+def os_failure():
+    """Return an OS-level connection failure containing private test detail."""
+    return OSError("os-private-detail")
+
+
+def timeout_failure():
+    """Return a timeout failure containing private test detail."""
+    return TimeoutError("timeout-private-detail")
+
+
+def catalog_failure():
+    """Reproduce an unwrapped asyncpg database selection failure."""
+    return InvalidCatalogNameError("catalog-private-detail")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("failed_dependency", ["primary", "readonly"])
+@pytest.mark.parametrize("fail_before_acquisition", [False, True])
+async def test_readiness_propagates_cancellation_and_returns_acquired_connections(
+    monkeypatch, failed_dependency, fail_before_acquisition
+):
+    """Cancellation must escape the probe while acquired connections are returned."""
+    connection_events = install_probe_engines(
+        monkeypatch,
+        failed_dependency=failed_dependency,
+        failure_factory=CancelledError,
+        fail_before_acquisition=fail_before_acquisition,
+    )
+
+    with pytest.raises(CancelledError):
+        await database_readiness()
+
+    expected_events = []
+    for dependency_name in ("primary", "readonly"):
+        if dependency_name == failed_dependency and fail_before_acquisition:
+            break
+        expected_events.extend([(dependency_name, "open"), (dependency_name, "close")])
+        if dependency_name == failed_dependency:
+            break
+    assert connection_events == expected_events
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("failed_dependency", ["primary", "readonly"])
+@pytest.mark.parametrize(
+    ("failure_factory", "private_detail"),
+    [
+        (os_failure, "os-private-detail"),
+        (timeout_failure, "timeout-private-detail"),
+        (catalog_failure, "catalog-private-detail"),
+    ],
+    ids=["os-error", "timeout", "missing-database"],
+)
+async def test_readiness_sanitizes_supported_connection_failures(
+    monkeypatch, failure_factory, private_detail, failed_dependency
+):
+    """Supported transport failures fail closed without leaking their detail."""
+    connection_events = install_probe_engines(
+        monkeypatch,
+        failed_dependency=failed_dependency,
+        failure_factory=failure_factory,
+        fail_before_acquisition=True,
+    )
+
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://unit.local") as client:
+        response = await client.get("/readyz")
+
+    assert response.status_code == 503
+    assert response.json() == {"status": "unavailable"}
+    assert response.headers["cache-control"] == "no-store"
+    assert private_detail not in response.text
+    assert connection_events == (
+        [("primary", "open"), ("primary", "close")]
+        if failed_dependency == "readonly" else []
+    )

docs/doctoring/runtime-image-boundary-verification.md

@@ -204,7 +204,60 @@ ancestor가 아니다. 기존 2026-05-11 release 계획도 이 branch를 통째
 않고 PR·승계 범위를 먼저 확인한다. 이후에는 liveness와 dependency readiness를
 분리하고, 실제 격리 PostgreSQL에서 성공·장애·연결 정리를 검증해야 한다.
 
-## 이미지 경계 참고 문헌
+## Native PostgreSQL readiness regression (PR #1597)
+
+On 2026-09-08, source `854e313013b4a760a2a8f0f47b27b254932c1724`
+returned 200 against an isolated native PostgreSQL cluster but propagated
+`asyncpg.exceptions.InvalidCatalogNameError` when connecting to a missing
+database (verification process exit 1). SQLAlchemy did not wrap this connection
+establishment error. Mocked query failures alone had missed this boundary.
+
+Commit `853aad7f67f0561a079735cb27acb6bf4b0e60fd` adds `PostgresError`
+to the existing readiness exception boundary, retaining generic response bodies
+and cancellation behavior. The focused regression suite passed seven cases with
+warnings treated as errors; Ruff and whitespace checks also passed.
+
+The real-cluster verification at that commit asserted healthy 200, missing
+primary database 503, missing read-only database 503, `Cache-Control: no-store`,
+and zero checked-out connections after each request, disposing all engines in
+`finally`. Its terminal result was exit 0 (execution session 38101). Per-scenario
+stdout was truncated by the tool and was not recovered; only the terminal exit
+result remains directly available. The PostgreSQL stop command also exited 0.
+
+Isolation used a private Unix socket directory, disabled TCP listening, rejected
+host authentication, and a fresh `readiness_owner` cluster. No application
+lifespan, worker, external provider, customer database, or production deployment
+was exercised. The cluster used the initdb default SQL_ASCII encoding; these
+`SELECT 1` probes do not establish application-schema or Unicode compatibility.
+Both application pools reached the same isolated server, not separate replicas.
+
+This repairs only the readiness slice inherited from PR #126. It does not prove
+complete successor coverage, protected merge, hosted review/check success,
+production readiness, or deployment eligibility. Keep prerequisite PR #1587
+and deployment serialization/recovery evidence separate.
+
+## Deployment probe wiring follow-up
+
+At `8b8ac74079b4d0022878719a93eec57562916995`, the backend manifest had
+no readiness or liveness probe. A regression executing the real release renderer
+failed with `KeyError: readinessProbe`. The repair connects backend readiness to
+`/readyz` on port 8000 and verifies the generated immutable-image manifest, not
+just the checked-in YAML. All 26 release-manifest tests passed in 37.33 seconds
+with warnings treated as errors and terminal exit 0. Ruff and diff checks passed.
+
+Kubernetes readiness failure removes a Pod from matching Service endpoints;
+liveness failure can restart its container. Therefore this change does not use
+database readiness as liveness. It retains the existing absence of a liveness
+restart policy until startup timing and restart behavior are validated. The
+readiness probe uses Kubernetes defaults; this is not a measured latency SLO or
+an application/model timeout. No cluster was contacted or deployment performed.
+Actual endpoint removal/recovery and probe-load effects remain unverified.
+
+Kubernetes Authors. (n.d.). *Liveness, readiness, and startup probes*.
+Retrieved September 8, 2026, from
+https://kubernetes.io/docs/concepts/workloads/pods/probes/
+
+## Image boundary bibliography
 
 Kubernetes Authors. (n.d.-a). *Kubernetes API concepts*. Retrieved September 7,
 2026, from https://kubernetes.io/docs/reference/using-api/api-concepts/#updates-to-existing-resources

docs/product-technical-gap-baseline.md

@@ -1051,6 +1051,26 @@ predecessor-head evidence is never reused.
 
 ## 14. Claim boundary
 
+### Readiness evidence update — 2026-09-08
+
+Deployment wiring follow-up `e6726e8c43203b91a55819d88c5a25bf72bf0dd3`
+connects the backend manifest to `/readyz` on port 8000. A test executing the
+release renderer first failed on the missing probe, then all 26 manifest tests
+passed with terminal exit 0. Runtime tests at `8b8ac740` separately cover 14
+cases, including cancellation before acquisition and during query execution.
+Neither result demonstrates kubelet behavior, endpoint removal/recovery, startup
+timing, or live deployment. Liveness restart policy remains unconfigured pending
+those measurements; dependency failure must not become a restart trigger.
+
+PR #1597 source `853aad7f67f0561a079735cb27acb6bf4b0e60fd` repairs an
+unwrapped native PostgreSQL connection error that escaped the generic readiness
+503 response. Seven focused tests passed, and an isolated real PostgreSQL
+verification exited 0; detailed stdout was truncated. The linked
+[doctoring record](doctoring/runtime-image-boundary-verification.md) distinguishes
+the observed RED, assertions, terminal result, and untested deployment boundaries.
+This reduces the dependency-readiness gap but does not close the protected CI,
+independent review, separate-replica, or live deployment acceptance work.
+
 This baseline is a product and technical decision record, not a certification,
 security attestation, market valuation, or claim that Naruon is already GA.
 

k8s/backend-deployment.yaml

@@ -42,6 +42,10 @@ spec:
             type: RuntimeDefault
         ports:
         - containerPort: 8000
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 8000
         resources:
           requests:
             cpu: 250m