diff --git a/AGENTS.md b/AGENTS.md index 198489546..30d004a9a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -728,6 +728,20 @@ in this repo. ## Phase 10 development rules +- Validate probes in the rendered deployment manifest, not only API tests. + Backend readiness must call `/readyz` on port 8000; a static root response + cannot establish database availability. Never reuse that dependency probe for + liveness: a database outage must not become a backend restart trigger. + +- Readiness probes must handle native PostgreSQL connection-establishment + errors as well as SQLAlchemy wrappers. In PR #1597, a real isolated cluster + raised `asyncpg.InvalidCatalogNameError` before `SELECT 1`; catch the driver's + `PostgresError` at the readiness boundary and return only the generic 503 + response. Preserve cancellation propagation. Validate healthy, primary-failure, + and read-only-failure paths with both pools returned before claiming readiness. + Use a private test cluster, never a shared customer database. See + `docs/doctoring/runtime-image-boundary-verification.md` for the evidence limits. + - **Stepwise execution**: Each phase requires an atomic PR, GitHub PR Tracking, Push, and Robot Review. A phase only ends when merged. Do not proceed without merge. - **TDD + DDD**: Practice TDD, micro TDD, nano TDD, Domain Driven Development, and Context Driven Development. - **API Wiring**: Always work with API wiring completed. diff --git a/backend/main.py b/backend/main.py index 51b054dbf..c067063a0 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,4 +1,5 @@ import os +from asyncpg import PostgresError from contextlib import asynccontextmanager from urllib.parse import urlsplit @@ -6,6 +7,9 @@ from fastapi import Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse +from sqlalchemy import text +from sqlalchemy.exc import SQLAlchemyError +from db import session as database_session from api.auth import get_auth_context, preload_oidc_jwks from api.search import router as search_router from api.llm import router as llm_router @@ -244,3 +248,24 @@ async def add_security_headers(request: Request, call_next): @app.get("/") def read_root() -> dict[str, str]: return {"status": "ok", "message": "AI Email Client API"} + + +@app.get("/healthz", include_in_schema=False) +async def process_health() -> JSONResponse: + """Report process liveness without touching external dependencies.""" + return JSONResponse({"status": "ok"}, headers={"Cache-Control": "no-store"}) + + +@app.get("/readyz", include_in_schema=False) +async def database_readiness() -> JSONResponse: + """Check both database pools without exposing connection or error details.""" + try: + for database_engine in (database_session.engine, database_session.readonly_engine): + async with database_engine.connect() as database_connection: + await database_connection.execute(text("SELECT 1")) + except (SQLAlchemyError, PostgresError, OSError, TimeoutError): + return JSONResponse( + {"status": "unavailable"}, status_code=503, + headers={"Cache-Control": "no-store"}, + ) + return JSONResponse({"status": "ready"}, headers={"Cache-Control": "no-store"}) diff --git a/backend/tests/test_release_manifest_digests.py b/backend/tests/test_release_manifest_digests.py index 8195d3c73..5a6232d21 100644 --- a/backend/tests/test_release_manifest_digests.py +++ b/backend/tests/test_release_manifest_digests.py @@ -46,6 +46,16 @@ def test_release_renderer_binds_both_images_without_changing_source(tmp_path: Pa assert rendered == expected +def test_rendered_backend_requires_database_readiness(tmp_path: Path) -> None: + """Route deployment readiness to the dependency probe, not the static root.""" + result = run_renderer(tmp_path, BACKEND_DIGEST, FRONTEND_DIGEST) + assert result.returncode == 0, result.stderr + manifest = yaml.safe_load((tmp_path / "rendered/backend-deployment.yaml").read_text()) + backend_container = manifest["spec"]["template"]["spec"]["containers"][0] + assert backend_container["readinessProbe"]["httpGet"] == {"path": "/readyz", "port": 8000} + assert backend_container.get("livenessProbe", {}).get("httpGet", {}).get("path") != "/readyz" + + @pytest.mark.parametrize("invalid_digest", ["", "latest", "sha256:" + "a" * 63, "sha256:" + "A" * 64, BACKEND_DIGEST + "\ninjected", "$(touch injected)"]) @pytest.mark.parametrize("invalid_component", ["backend", "frontend"]) def test_release_renderer_rejects_either_invalid_digest_before_output( diff --git a/backend/tests/test_runtime_health.py b/backend/tests/test_runtime_health.py new file mode 100644 index 000000000..075604092 --- /dev/null +++ b/backend/tests/test_runtime_health.py @@ -0,0 +1,170 @@ +"""Exercise public probe responses without workers or customer/provider access.""" + +from contextlib import asynccontextmanager +from asyncio import CancelledError +import os +import secrets + +import httpx +import pytest +from asyncpg import InvalidCatalogNameError +from sqlalchemy.exc import OperationalError + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://unit:unit@127.0.0.1:1/unit_db") +os.environ.setdefault("AUTH_SESSION_HMAC_SECRET", secrets.token_urlsafe(48)) +os.environ.setdefault("DISABLE_BACKGROUND_WORKERS", "1") + +from db import session as database_session # noqa: E402 +from main import app, database_readiness # noqa: E402 + + +def install_probe_engines( + monkeypatch, *, failed_dependency=None, failure_factory=None, fail_before_acquisition=False +): + """Install deterministic primary/read-only probes and return lifecycle evidence.""" + connection_events = [] + + class ProbeConnection: + async def execute(self, query_statement): + assert str(query_statement) == "SELECT 1" + if self.dependency_name == failed_dependency and failure_factory is not None: + raise failure_factory() + + def __init__(self, dependency_name): + self.dependency_name = dependency_name + + class ProbeEngine: + def __init__(self, dependency_name): + self.dependency_name = dependency_name + + @asynccontextmanager + async def connect(self): + if fail_before_acquisition and self.dependency_name == failed_dependency: + raise failure_factory() + connection_events.append((self.dependency_name, "open")) + try: + yield ProbeConnection(self.dependency_name) + finally: + connection_events.append((self.dependency_name, "close")) + + monkeypatch.setattr(database_session, "engine", ProbeEngine("primary")) + monkeypatch.setattr(database_session, "readonly_engine", ProbeEngine("readonly")) + return connection_events + + +@pytest.mark.asyncio +async def test_liveness_does_not_touch_databases_and_disables_cache(monkeypatch): + """Liveness is process-only and returns a stable non-cacheable contract.""" + connection_events = install_probe_engines(monkeypatch) + + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://unit.local") as client: + response = await client.get("/healthz") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + assert response.headers["cache-control"] == "no-store" + assert connection_events == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_dependency", [None, "primary", "readonly"]) +async def test_readiness_checks_both_databases_without_leaking_errors(monkeypatch, failed_dependency): + """A failed database must remove readiness, and every acquired connection closes.""" + + def operational_failure(): + return OperationalError("SELECT 1", None, Exception("unit-private-detail")) + + connection_events = install_probe_engines( + monkeypatch, + failed_dependency=failed_dependency, + failure_factory=operational_failure if failed_dependency is not None else None, + ) + + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://unit.local") as client: + response = await client.get("/readyz") + + assert response.status_code == (503 if failed_dependency else 200) + assert response.json() == {"status": "unavailable" if failed_dependency else "ready"} + assert response.headers["cache-control"] == "no-store" + assert "unit-private-detail" not in response.text + expected_events = [("primary", "open"), ("primary", "close")] + if failed_dependency != "primary": + expected_events += [("readonly", "open"), ("readonly", "close")] + assert connection_events == expected_events + + +def os_failure(): + """Return an OS-level connection failure containing private test detail.""" + return OSError("os-private-detail") + + +def timeout_failure(): + """Return a timeout failure containing private test detail.""" + return TimeoutError("timeout-private-detail") + + +def catalog_failure(): + """Reproduce an unwrapped asyncpg database selection failure.""" + return InvalidCatalogNameError("catalog-private-detail") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_dependency", ["primary", "readonly"]) +@pytest.mark.parametrize("fail_before_acquisition", [False, True]) +async def test_readiness_propagates_cancellation_and_returns_acquired_connections( + monkeypatch, failed_dependency, fail_before_acquisition +): + """Cancellation must escape the probe while acquired connections are returned.""" + connection_events = install_probe_engines( + monkeypatch, + failed_dependency=failed_dependency, + failure_factory=CancelledError, + fail_before_acquisition=fail_before_acquisition, + ) + + with pytest.raises(CancelledError): + await database_readiness() + + expected_events = [] + for dependency_name in ("primary", "readonly"): + if dependency_name == failed_dependency and fail_before_acquisition: + break + expected_events.extend([(dependency_name, "open"), (dependency_name, "close")]) + if dependency_name == failed_dependency: + break + assert connection_events == expected_events + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_dependency", ["primary", "readonly"]) +@pytest.mark.parametrize( + ("failure_factory", "private_detail"), + [ + (os_failure, "os-private-detail"), + (timeout_failure, "timeout-private-detail"), + (catalog_failure, "catalog-private-detail"), + ], + ids=["os-error", "timeout", "missing-database"], +) +async def test_readiness_sanitizes_supported_connection_failures( + monkeypatch, failure_factory, private_detail, failed_dependency +): + """Supported transport failures fail closed without leaking their detail.""" + connection_events = install_probe_engines( + monkeypatch, + failed_dependency=failed_dependency, + failure_factory=failure_factory, + fail_before_acquisition=True, + ) + + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://unit.local") as client: + response = await client.get("/readyz") + + assert response.status_code == 503 + assert response.json() == {"status": "unavailable"} + assert response.headers["cache-control"] == "no-store" + assert private_detail not in response.text + assert connection_events == ( + [("primary", "open"), ("primary", "close")] + if failed_dependency == "readonly" else [] + ) diff --git a/docs/doctoring/runtime-image-boundary-verification.md b/docs/doctoring/runtime-image-boundary-verification.md index 94a23724c..3c6f3b5cb 100644 --- a/docs/doctoring/runtime-image-boundary-verification.md +++ b/docs/doctoring/runtime-image-boundary-verification.md @@ -204,7 +204,60 @@ ancestor가 아니다. 기존 2026-05-11 release 계획도 이 branch를 통째 않고 PR·승계 범위를 먼저 확인한다. 이후에는 liveness와 dependency readiness를 분리하고, 실제 격리 PostgreSQL에서 성공·장애·연결 정리를 검증해야 한다. -## 이미지 경계 참고 문헌 +## Native PostgreSQL readiness regression (PR #1597) + +On 2026-09-08, source `854e313013b4a760a2a8f0f47b27b254932c1724` +returned 200 against an isolated native PostgreSQL cluster but propagated +`asyncpg.exceptions.InvalidCatalogNameError` when connecting to a missing +database (verification process exit 1). SQLAlchemy did not wrap this connection +establishment error. Mocked query failures alone had missed this boundary. + +Commit `853aad7f67f0561a079735cb27acb6bf4b0e60fd` adds `PostgresError` +to the existing readiness exception boundary, retaining generic response bodies +and cancellation behavior. The focused regression suite passed seven cases with +warnings treated as errors; Ruff and whitespace checks also passed. + +The real-cluster verification at that commit asserted healthy 200, missing +primary database 503, missing read-only database 503, `Cache-Control: no-store`, +and zero checked-out connections after each request, disposing all engines in +`finally`. Its terminal result was exit 0 (execution session 38101). Per-scenario +stdout was truncated by the tool and was not recovered; only the terminal exit +result remains directly available. The PostgreSQL stop command also exited 0. + +Isolation used a private Unix socket directory, disabled TCP listening, rejected +host authentication, and a fresh `readiness_owner` cluster. No application +lifespan, worker, external provider, customer database, or production deployment +was exercised. The cluster used the initdb default SQL_ASCII encoding; these +`SELECT 1` probes do not establish application-schema or Unicode compatibility. +Both application pools reached the same isolated server, not separate replicas. + +This repairs only the readiness slice inherited from PR #126. It does not prove +complete successor coverage, protected merge, hosted review/check success, +production readiness, or deployment eligibility. Keep prerequisite PR #1587 +and deployment serialization/recovery evidence separate. + +## Deployment probe wiring follow-up + +At `8b8ac74079b4d0022878719a93eec57562916995`, the backend manifest had +no readiness or liveness probe. A regression executing the real release renderer +failed with `KeyError: readinessProbe`. The repair connects backend readiness to +`/readyz` on port 8000 and verifies the generated immutable-image manifest, not +just the checked-in YAML. All 26 release-manifest tests passed in 37.33 seconds +with warnings treated as errors and terminal exit 0. Ruff and diff checks passed. + +Kubernetes readiness failure removes a Pod from matching Service endpoints; +liveness failure can restart its container. Therefore this change does not use +database readiness as liveness. It retains the existing absence of a liveness +restart policy until startup timing and restart behavior are validated. The +readiness probe uses Kubernetes defaults; this is not a measured latency SLO or +an application/model timeout. No cluster was contacted or deployment performed. +Actual endpoint removal/recovery and probe-load effects remain unverified. + +Kubernetes Authors. (n.d.). *Liveness, readiness, and startup probes*. +Retrieved September 8, 2026, from +https://kubernetes.io/docs/concepts/workloads/pods/probes/ + +## Image boundary bibliography Kubernetes Authors. (n.d.-a). *Kubernetes API concepts*. Retrieved September 7, 2026, from https://kubernetes.io/docs/reference/using-api/api-concepts/#updates-to-existing-resources diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d0e95302f..55c81b7c2 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1051,6 +1051,26 @@ predecessor-head evidence is never reused. ## 14. Claim boundary +### Readiness evidence update — 2026-09-08 + +Deployment wiring follow-up `e6726e8c43203b91a55819d88c5a25bf72bf0dd3` +connects the backend manifest to `/readyz` on port 8000. A test executing the +release renderer first failed on the missing probe, then all 26 manifest tests +passed with terminal exit 0. Runtime tests at `8b8ac740` separately cover 14 +cases, including cancellation before acquisition and during query execution. +Neither result demonstrates kubelet behavior, endpoint removal/recovery, startup +timing, or live deployment. Liveness restart policy remains unconfigured pending +those measurements; dependency failure must not become a restart trigger. + +PR #1597 source `853aad7f67f0561a079735cb27acb6bf4b0e60fd` repairs an +unwrapped native PostgreSQL connection error that escaped the generic readiness +503 response. Seven focused tests passed, and an isolated real PostgreSQL +verification exited 0; detailed stdout was truncated. The linked +[doctoring record](doctoring/runtime-image-boundary-verification.md) distinguishes +the observed RED, assertions, terminal result, and untested deployment boundaries. +This reduces the dependency-readiness gap but does not close the protected CI, +independent review, separate-replica, or live deployment acceptance work. + This baseline is a product and technical decision record, not a certification, security attestation, market valuation, or claim that Naruon is already GA. diff --git a/k8s/backend-deployment.yaml b/k8s/backend-deployment.yaml index 46f449f6c..d241393aa 100644 --- a/k8s/backend-deployment.yaml +++ b/k8s/backend-deployment.yaml @@ -42,6 +42,10 @@ spec: type: RuntimeDefault ports: - containerPort: 8000 + readinessProbe: + httpGet: + path: /readyz + port: 8000 resources: requests: cpu: 250m