From 92da287ba533bb82150598187a035aa5b54d719b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 09:17:23 +0900 Subject: [PATCH 01/12] fix: restore runtime health probes Co-authored-by: Codex Signed-off-by: Seongho Bae --- AGENTS.md | 21 +++++++++ backend/main.py | 24 ++++++++++ backend/tests/test_runtime_health.py | 65 ++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 backend/tests/test_runtime_health.py diff --git a/AGENTS.md b/AGENTS.md index 198489546..efaabf699 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -108,6 +108,27 @@ in this repo. keyword/embedding/LLM result presented as STM. +## Learned operating procedure + +- Treat changelog claims and historical release branches as leads, not runtime + evidence. Reproduce the exact current endpoint contract on the exact PR head + before repairing or declaring a successor. +- Keep liveness and readiness separate: `/healthz` must not touch databases or + customer/provider systems; `/readyz` must probe every authoritative database + pool, close each acquired connection, and return a sanitized `503` on probe + failure. A root `200` or a unit mock does not prove deployed readiness. +- For a non-trivial repair, use the Superpowers sequence RED → smallest shared + fix → focused GREEN → isolated real PostgreSQL verification. Patch only the + canonical owner and preserve concurrent router, worker, CSRF, and auth deltas. +- Visual inspection is separate evidence: capture the rendered document at the + requested desktop/mobile viewport and record clipping, overlap, and the exact + inspected revision. Inline screenshots are evidence; DOM assertions alone are + not. +- Automatic PyPI or Rust publication is permitted only after exact package + ownership, version, protected-branch merge, immutable artifact, required + checks, and rollback evidence are verified. The presence of `PYPI` or Rust + secrets never authorizes reading, printing, or bypassing those gates. + ## Release governance defaults - CHANGELOG나 저장소 요약이 endpoint를 구현·배포됐다고 설명해도 현재 라우트와 diff --git a/backend/main.py b/backend/main.py index 51b054dbf..bcfa8ca79 100644 --- a/backend/main.py +++ b/backend/main.py @@ -6,6 +6,9 @@ from fastapi import Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse +from sqlalchemy import text +from sqlalchemy.exc import SQLAlchemyError +from db import session as database_session from api.auth import get_auth_context, preload_oidc_jwks from api.search import router as search_router from api.llm import router as llm_router @@ -244,3 +247,24 @@ async def add_security_headers(request: Request, call_next): @app.get("/") def read_root() -> dict[str, str]: return {"status": "ok", "message": "AI Email Client API"} + + +@app.get("/healthz", include_in_schema=False) +async def process_health() -> JSONResponse: + """Report process liveness without touching external dependencies.""" + return JSONResponse({"status": "ok"}, headers={"Cache-Control": "no-store"}) + + +@app.get("/readyz", include_in_schema=False) +async def database_readiness() -> JSONResponse: + """Check both database pools without exposing connection or error details.""" + try: + for database_engine in (database_session.engine, database_session.readonly_engine): + async with database_engine.connect() as database_connection: + await database_connection.execute(text("SELECT 1")) + except (SQLAlchemyError, OSError, TimeoutError): + return JSONResponse( + {"status": "unavailable"}, status_code=503, + headers={"Cache-Control": "no-store"}, + ) + return JSONResponse({"status": "ready"}, headers={"Cache-Control": "no-store"}) diff --git a/backend/tests/test_runtime_health.py b/backend/tests/test_runtime_health.py new file mode 100644 index 000000000..a8c52fd1a --- /dev/null +++ b/backend/tests/test_runtime_health.py @@ -0,0 +1,65 @@ +"""Exercise public probe responses without workers or customer/provider access.""" + +from contextlib import asynccontextmanager +import os +import secrets + +import httpx +import pytest +from sqlalchemy.exc import OperationalError + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://unit:unit@127.0.0.1:1/unit_db") +os.environ.setdefault("AUTH_SESSION_HMAC_SECRET", secrets.token_urlsafe(48)) +os.environ.setdefault("DISABLE_BACKGROUND_WORKERS", "1") + +from db import session as database_session # noqa: E402 +from main import app # noqa: E402 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_dependency", [None, "primary", "readonly"]) +async def test_readiness_checks_both_databases_without_leaking_errors(monkeypatch, failed_dependency): + """A failed database must remove readiness, and every acquired connection closes.""" + connection_events = [] + + class ProbeConnection: + """Unit-only SQL connection with a failure at the external boundary.""" + + def __init__(self, dependency_name): + self.dependency_name = dependency_name + + async def execute(self, query_statement): + assert str(query_statement) == "SELECT 1" + if self.dependency_name == failed_dependency: + raise OperationalError("SELECT 1", None, Exception("unit-private-detail")) + + class ProbeEngine: + """Record acquisition and cleanup while the actual endpoint executes.""" + + def __init__(self, dependency_name): + self.dependency_name = dependency_name + + @asynccontextmanager + async def connect(self): + connection_events.append((self.dependency_name, "open")) + try: + yield ProbeConnection(self.dependency_name) + finally: + connection_events.append((self.dependency_name, "close")) + + monkeypatch.setattr(database_session, "engine", ProbeEngine("primary")) + monkeypatch.setattr(database_session, "readonly_engine", ProbeEngine("readonly")) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://unit.local") as client: + health_response = await client.get("/healthz") + assert health_response.status_code == 200 + assert connection_events == [] + readiness_response = await client.get("/readyz") + + assert readiness_response.status_code == (503 if failed_dependency else 200) + assert readiness_response.json() == {"status": "unavailable" if failed_dependency else "ready"} + assert readiness_response.headers["cache-control"] == "no-store" + assert "unit-private-detail" not in readiness_response.text + expected_events = [("primary", "open"), ("primary", "close")] + if failed_dependency != "primary": + expected_events += [("readonly", "open"), ("readonly", "close")] + assert connection_events == expected_events From f0a957456109bf993312a34b2f1dfef351ba8ccf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 09:21:57 +0900 Subject: [PATCH 02/12] chore(agents): keep readiness repair product-scoped --- AGENTS.md | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index efaabf699..198489546 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -108,27 +108,6 @@ in this repo. keyword/embedding/LLM result presented as STM. -## Learned operating procedure - -- Treat changelog claims and historical release branches as leads, not runtime - evidence. Reproduce the exact current endpoint contract on the exact PR head - before repairing or declaring a successor. -- Keep liveness and readiness separate: `/healthz` must not touch databases or - customer/provider systems; `/readyz` must probe every authoritative database - pool, close each acquired connection, and return a sanitized `503` on probe - failure. A root `200` or a unit mock does not prove deployed readiness. -- For a non-trivial repair, use the Superpowers sequence RED → smallest shared - fix → focused GREEN → isolated real PostgreSQL verification. Patch only the - canonical owner and preserve concurrent router, worker, CSRF, and auth deltas. -- Visual inspection is separate evidence: capture the rendered document at the - requested desktop/mobile viewport and record clipping, overlap, and the exact - inspected revision. Inline screenshots are evidence; DOM assertions alone are - not. -- Automatic PyPI or Rust publication is permitted only after exact package - ownership, version, protected-branch merge, immutable artifact, required - checks, and rollback evidence are verified. The presence of `PYPI` or Rust - secrets never authorizes reading, printing, or bypassing those gates. - ## Release governance defaults - CHANGELOG나 저장소 요약이 endpoint를 구현·배포됐다고 설명해도 현재 라우트와 From b0a6e2e5247e263dbc353da4b6e2cccff4522b2f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 09:25:59 +0900 Subject: [PATCH 03/12] docs: retain readiness repair lessons Co-authored-by: Codex Signed-off-by: Seongho Bae --- AGENTS.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 198489546..cc9e74edb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -746,3 +746,19 @@ protection, `require_code_owner_review` in rulesets) are disabled across the Con org: there is a single maintainer (solo developer), so a code-owner approval gate can never be satisfied. This is ON HOLD until the org has multiple maintainers — do NOT re-enable these settings or add CODEOWNERS-based merge gates before then. + +## Readiness repair lessons + +- Reproduce a claimed runtime endpoint on the exact PR head; changelog entries + and historical release branches are leads, not runtime evidence. +- Keep liveness and readiness separate: `/healthz` must not touch external + systems, while `/readyz` probes every authoritative database pool, closes + each connection, and returns only a sanitized `503` on failure. +- For a repair, use RED → smallest canonical-owner fix → focused GREEN → real + isolated PostgreSQL verification. Preserve concurrent auth, router, worker, + and CSRF deltas; a unit mock is not deployed-readiness evidence. +- Visual inspection is separate evidence from DOM tests: record the exact + revision, viewport, rendered state, clipping, and overlap result. +- PyPI or Rust publication requires protected-main merge, package ownership, + version and immutable-artifact evidence, required Checks, and rollback proof. + Secret presence never permits reading values or bypassing those gates. From 7bffb35732831b065d033f659d9eace1cf8be999 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 09:27:34 +0900 Subject: [PATCH 04/12] chore(agents): keep readiness slice product-scoped --- AGENTS.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cc9e74edb..198489546 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -746,19 +746,3 @@ protection, `require_code_owner_review` in rulesets) are disabled across the Con org: there is a single maintainer (solo developer), so a code-owner approval gate can never be satisfied. This is ON HOLD until the org has multiple maintainers — do NOT re-enable these settings or add CODEOWNERS-based merge gates before then. - -## Readiness repair lessons - -- Reproduce a claimed runtime endpoint on the exact PR head; changelog entries - and historical release branches are leads, not runtime evidence. -- Keep liveness and readiness separate: `/healthz` must not touch external - systems, while `/readyz` probes every authoritative database pool, closes - each connection, and returns only a sanitized `503` on failure. -- For a repair, use RED → smallest canonical-owner fix → focused GREEN → real - isolated PostgreSQL verification. Preserve concurrent auth, router, worker, - and CSRF deltas; a unit mock is not deployed-readiness evidence. -- Visual inspection is separate evidence from DOM tests: record the exact - revision, viewport, rendered state, clipping, and overlap result. -- PyPI or Rust publication requires protected-main merge, package ownership, - version and immutable-artifact evidence, required Checks, and rollback proof. - Secret presence never permits reading values or bypassing those gates. From b64b1325128e248e47f516c297cf2ad4f9ecfd62 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 09:41:12 +0900 Subject: [PATCH 05/12] test(runtime): cover probe cache and failure boundaries --- backend/tests/test_runtime_health.py | 90 ++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 19 deletions(-) diff --git a/backend/tests/test_runtime_health.py b/backend/tests/test_runtime_health.py index a8c52fd1a..83fcf1754 100644 --- a/backend/tests/test_runtime_health.py +++ b/backend/tests/test_runtime_health.py @@ -16,26 +16,20 @@ from main import app # noqa: E402 -@pytest.mark.asyncio -@pytest.mark.parametrize("failed_dependency", [None, "primary", "readonly"]) -async def test_readiness_checks_both_databases_without_leaking_errors(monkeypatch, failed_dependency): - """A failed database must remove readiness, and every acquired connection closes.""" +def install_probe_engines(monkeypatch, *, failed_dependency=None, failure_factory=None): + """Install deterministic primary/read-only probes and return lifecycle evidence.""" connection_events = [] class ProbeConnection: - """Unit-only SQL connection with a failure at the external boundary.""" - def __init__(self, dependency_name): self.dependency_name = dependency_name async def execute(self, query_statement): assert str(query_statement) == "SELECT 1" - if self.dependency_name == failed_dependency: - raise OperationalError("SELECT 1", None, Exception("unit-private-detail")) + if self.dependency_name == failed_dependency and failure_factory is not None: + raise failure_factory() class ProbeEngine: - """Record acquisition and cleanup while the actual endpoint executes.""" - def __init__(self, dependency_name): self.dependency_name = dependency_name @@ -49,17 +43,75 @@ async def connect(self): monkeypatch.setattr(database_session, "engine", ProbeEngine("primary")) monkeypatch.setattr(database_session, "readonly_engine", ProbeEngine("readonly")) + return connection_events + + +@pytest.mark.asyncio +async def test_liveness_does_not_touch_databases_and_disables_cache(monkeypatch): + """Liveness is process-only and returns a stable non-cacheable contract.""" + connection_events = install_probe_engines(monkeypatch) + + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://unit.local") as client: + response = await client.get("/healthz") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + assert response.headers["cache-control"] == "no-store" + assert connection_events == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_dependency", [None, "primary", "readonly"]) +async def test_readiness_checks_both_databases_without_leaking_errors(monkeypatch, failed_dependency): + """A failed database must remove readiness, and every acquired connection closes.""" + failure_factory = None + if failed_dependency is not None: + failure_factory = lambda: OperationalError( + "SELECT 1", None, Exception("unit-private-detail") + ) + connection_events = install_probe_engines( + monkeypatch, + failed_dependency=failed_dependency, + failure_factory=failure_factory, + ) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://unit.local") as client: - health_response = await client.get("/healthz") - assert health_response.status_code == 200 - assert connection_events == [] - readiness_response = await client.get("/readyz") - - assert readiness_response.status_code == (503 if failed_dependency else 200) - assert readiness_response.json() == {"status": "unavailable" if failed_dependency else "ready"} - assert readiness_response.headers["cache-control"] == "no-store" - assert "unit-private-detail" not in readiness_response.text + response = await client.get("/readyz") + + assert response.status_code == (503 if failed_dependency else 200) + assert response.json() == {"status": "unavailable" if failed_dependency else "ready"} + assert response.headers["cache-control"] == "no-store" + assert "unit-private-detail" not in response.text expected_events = [("primary", "open"), ("primary", "close")] if failed_dependency != "primary": expected_events += [("readonly", "open"), ("readonly", "close")] assert connection_events == expected_events + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failure_factory", "private_detail"), + [ + (lambda: OSError("os-private-detail"), "os-private-detail"), + (lambda: TimeoutError("timeout-private-detail"), "timeout-private-detail"), + ], + ids=["os-error", "timeout"], +) +async def test_readiness_sanitizes_supported_connection_failures( + monkeypatch, failure_factory, private_detail +): + """Supported transport failures fail closed without leaking their detail.""" + connection_events = install_probe_engines( + monkeypatch, + failed_dependency="primary", + failure_factory=failure_factory, + ) + + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://unit.local") as client: + response = await client.get("/readyz") + + assert response.status_code == 503 + assert response.json() == {"status": "unavailable"} + assert response.headers["cache-control"] == "no-store" + assert private_detail not in response.text + assert connection_events == [("primary", "open"), ("primary", "close")] From 854e313013b4a760a2a8f0f47b27b254932c1724 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 09:42:11 +0900 Subject: [PATCH 06/12] fix(test): avoid lambda assignment in readiness regression --- backend/tests/test_runtime_health.py | 31 ++++++++++++++++++---------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/backend/tests/test_runtime_health.py b/backend/tests/test_runtime_health.py index 83fcf1754..7015f834a 100644 --- a/backend/tests/test_runtime_health.py +++ b/backend/tests/test_runtime_health.py @@ -21,14 +21,14 @@ def install_probe_engines(monkeypatch, *, failed_dependency=None, failure_factor connection_events = [] class ProbeConnection: - def __init__(self, dependency_name): - self.dependency_name = dependency_name - async def execute(self, query_statement): assert str(query_statement) == "SELECT 1" if self.dependency_name == failed_dependency and failure_factory is not None: raise failure_factory() + def __init__(self, dependency_name): + self.dependency_name = dependency_name + class ProbeEngine: def __init__(self, dependency_name): self.dependency_name = dependency_name @@ -64,15 +64,14 @@ async def test_liveness_does_not_touch_databases_and_disables_cache(monkeypatch) @pytest.mark.parametrize("failed_dependency", [None, "primary", "readonly"]) async def test_readiness_checks_both_databases_without_leaking_errors(monkeypatch, failed_dependency): """A failed database must remove readiness, and every acquired connection closes.""" - failure_factory = None - if failed_dependency is not None: - failure_factory = lambda: OperationalError( - "SELECT 1", None, Exception("unit-private-detail") - ) + + def operational_failure(): + return OperationalError("SELECT 1", None, Exception("unit-private-detail")) + connection_events = install_probe_engines( monkeypatch, failed_dependency=failed_dependency, - failure_factory=failure_factory, + failure_factory=operational_failure if failed_dependency is not None else None, ) async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://unit.local") as client: @@ -88,12 +87,22 @@ async def test_readiness_checks_both_databases_without_leaking_errors(monkeypatc assert connection_events == expected_events +def os_failure(): + """Return an OS-level connection failure containing private test detail.""" + return OSError("os-private-detail") + + +def timeout_failure(): + """Return a timeout failure containing private test detail.""" + return TimeoutError("timeout-private-detail") + + @pytest.mark.asyncio @pytest.mark.parametrize( ("failure_factory", "private_detail"), [ - (lambda: OSError("os-private-detail"), "os-private-detail"), - (lambda: TimeoutError("timeout-private-detail"), "timeout-private-detail"), + (os_failure, "os-private-detail"), + (timeout_failure, "timeout-private-detail"), ], ids=["os-error", "timeout"], ) From 853aad7f67f0561a079735cb27acb6bf4b0e60fd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 10:43:37 +0900 Subject: [PATCH 07/12] fix(runtime): sanitize native PostgreSQL readiness failures Real isolated PostgreSQL reproduced an unwrapped InvalidCatalogNameError during connection establishment. Preserve existing probe behavior and handle the driver error at the readiness boundary. Co-authored-by: Codex Signed-off-by: Seongho Bae --- backend/main.py | 3 ++- backend/tests/test_runtime_health.py | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/backend/main.py b/backend/main.py index bcfa8ca79..c067063a0 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,4 +1,5 @@ import os +from asyncpg import PostgresError from contextlib import asynccontextmanager from urllib.parse import urlsplit @@ -262,7 +263,7 @@ async def database_readiness() -> JSONResponse: for database_engine in (database_session.engine, database_session.readonly_engine): async with database_engine.connect() as database_connection: await database_connection.execute(text("SELECT 1")) - except (SQLAlchemyError, OSError, TimeoutError): + except (SQLAlchemyError, PostgresError, OSError, TimeoutError): return JSONResponse( {"status": "unavailable"}, status_code=503, headers={"Cache-Control": "no-store"}, diff --git a/backend/tests/test_runtime_health.py b/backend/tests/test_runtime_health.py index 7015f834a..a60e7df91 100644 --- a/backend/tests/test_runtime_health.py +++ b/backend/tests/test_runtime_health.py @@ -6,6 +6,7 @@ import httpx import pytest +from asyncpg import InvalidCatalogNameError from sqlalchemy.exc import OperationalError os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://unit:unit@127.0.0.1:1/unit_db") @@ -97,14 +98,20 @@ def timeout_failure(): return TimeoutError("timeout-private-detail") +def catalog_failure(): + """Reproduce an unwrapped asyncpg database selection failure.""" + return InvalidCatalogNameError("catalog-private-detail") + + @pytest.mark.asyncio @pytest.mark.parametrize( ("failure_factory", "private_detail"), [ (os_failure, "os-private-detail"), (timeout_failure, "timeout-private-detail"), + (catalog_failure, "catalog-private-detail"), ], - ids=["os-error", "timeout"], + ids=["os-error", "timeout", "missing-database"], ) async def test_readiness_sanitizes_supported_connection_failures( monkeypatch, failure_factory, private_detail From a5859c4900b45fa5ff947404282e1ebcab61bd34 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 11:07:12 +0900 Subject: [PATCH 08/12] docs(runtime): record native readiness failure and verification limits Signed-off-by: Seongho Bae --- AGENTS.md | 9 +++++ .../runtime-image-boundary-verification.md | 34 ++++++++++++++++++- docs/product-technical-gap-baseline.md | 11 ++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 198489546..a6a68e904 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -728,6 +728,15 @@ in this repo. ## Phase 10 development rules +- Readiness probes must handle native PostgreSQL connection-establishment + errors as well as SQLAlchemy wrappers. In PR #1597, a real isolated cluster + raised `asyncpg.InvalidCatalogNameError` before `SELECT 1`; catch the driver's + `PostgresError` at the readiness boundary and return only the generic 503 + response. Preserve cancellation propagation. Validate healthy, primary-failure, + and read-only-failure paths with both pools returned before claiming readiness. + Use a private test cluster, never a shared customer database. See + `docs/doctoring/runtime-image-boundary-verification.md` for the evidence limits. + - **Stepwise execution**: Each phase requires an atomic PR, GitHub PR Tracking, Push, and Robot Review. A phase only ends when merged. Do not proceed without merge. - **TDD + DDD**: Practice TDD, micro TDD, nano TDD, Domain Driven Development, and Context Driven Development. - **API Wiring**: Always work with API wiring completed. diff --git a/docs/doctoring/runtime-image-boundary-verification.md b/docs/doctoring/runtime-image-boundary-verification.md index 94a23724c..d73eaa3c2 100644 --- a/docs/doctoring/runtime-image-boundary-verification.md +++ b/docs/doctoring/runtime-image-boundary-verification.md @@ -204,7 +204,39 @@ ancestor가 아니다. 기존 2026-05-11 release 계획도 이 branch를 통째 않고 PR·승계 범위를 먼저 확인한다. 이후에는 liveness와 dependency readiness를 분리하고, 실제 격리 PostgreSQL에서 성공·장애·연결 정리를 검증해야 한다. -## 이미지 경계 참고 문헌 +## Native PostgreSQL readiness regression (PR #1597) + +On 2026-09-08, source `854e313013b4a760a2a8f0f47b27b254932c1724` +returned 200 against an isolated native PostgreSQL cluster but propagated +`asyncpg.exceptions.InvalidCatalogNameError` when connecting to a missing +database (verification process exit 1). SQLAlchemy did not wrap this connection +establishment error. Mocked query failures alone had missed this boundary. + +Commit `853aad7f67f0561a079735cb27acb6bf4b0e60fd` adds `PostgresError` +to the existing readiness exception boundary, retaining generic response bodies +and cancellation behavior. The focused regression suite passed seven cases with +warnings treated as errors; Ruff and whitespace checks also passed. + +The real-cluster verification at that commit asserted healthy 200, missing +primary database 503, missing read-only database 503, `Cache-Control: no-store`, +and zero checked-out connections after each request, disposing all engines in +`finally`. Its terminal result was exit 0 (execution session 38101). Per-scenario +stdout was truncated by the tool and was not recovered; only the terminal exit +result remains directly available. The PostgreSQL stop command also exited 0. + +Isolation used a private Unix socket directory, disabled TCP listening, rejected +host authentication, and a fresh `readiness_owner` cluster. No application +lifespan, worker, external provider, customer database, or production deployment +was exercised. The cluster used the initdb default SQL_ASCII encoding; these +`SELECT 1` probes do not establish application-schema or Unicode compatibility. +Both application pools reached the same isolated server, not separate replicas. + +This repairs only the readiness slice inherited from PR #126. It does not prove +complete successor coverage, protected merge, hosted review/check success, +production readiness, or deployment eligibility. Keep prerequisite PR #1587 +and deployment serialization/recovery evidence separate. + +## Image boundary references Kubernetes Authors. (n.d.-a). *Kubernetes API concepts*. Retrieved September 7, 2026, from https://kubernetes.io/docs/reference/using-api/api-concepts/#updates-to-existing-resources diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d0e95302f..9e196fbb6 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1051,6 +1051,17 @@ predecessor-head evidence is never reused. ## 14. Claim boundary +### Readiness evidence update — 2026-09-08 + +PR #1597 source `853aad7f67f0561a079735cb27acb6bf4b0e60fd` repairs an +unwrapped native PostgreSQL connection error that escaped the generic readiness +503 response. Seven focused tests passed, and an isolated real PostgreSQL +verification exited 0; detailed stdout was truncated. The linked +[doctoring record](doctoring/runtime-image-boundary-verification.md) distinguishes +the observed RED, assertions, terminal result, and untested deployment boundaries. +This reduces the dependency-readiness gap but does not close the protected CI, +independent review, separate-replica, or live deployment acceptance work. + This baseline is a product and technical decision record, not a certification, security attestation, market valuation, or claim that Naruon is already GA. From 706cd74e30869b2a782d8e666e72285e710f0408 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 11:25:38 +0900 Subject: [PATCH 09/12] test(runtime): cover readiness connection acquisition failures Exercise native driver, OS and timeout failures before acquiring either database connection; preserve query failure coverage and verify prior primary connection cleanup. Signed-off-by: Seongho Bae --- backend/tests/test_runtime_health.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/backend/tests/test_runtime_health.py b/backend/tests/test_runtime_health.py index a60e7df91..a1866efda 100644 --- a/backend/tests/test_runtime_health.py +++ b/backend/tests/test_runtime_health.py @@ -17,7 +17,9 @@ from main import app # noqa: E402 -def install_probe_engines(monkeypatch, *, failed_dependency=None, failure_factory=None): +def install_probe_engines( + monkeypatch, *, failed_dependency=None, failure_factory=None, fail_before_acquisition=False +): """Install deterministic primary/read-only probes and return lifecycle evidence.""" connection_events = [] @@ -36,6 +38,8 @@ def __init__(self, dependency_name): @asynccontextmanager async def connect(self): + if fail_before_acquisition and self.dependency_name == failed_dependency: + raise failure_factory() connection_events.append((self.dependency_name, "open")) try: yield ProbeConnection(self.dependency_name) @@ -104,6 +108,7 @@ def catalog_failure(): @pytest.mark.asyncio +@pytest.mark.parametrize("failed_dependency", ["primary", "readonly"]) @pytest.mark.parametrize( ("failure_factory", "private_detail"), [ @@ -114,13 +119,14 @@ def catalog_failure(): ids=["os-error", "timeout", "missing-database"], ) async def test_readiness_sanitizes_supported_connection_failures( - monkeypatch, failure_factory, private_detail + monkeypatch, failure_factory, private_detail, failed_dependency ): """Supported transport failures fail closed without leaking their detail.""" connection_events = install_probe_engines( monkeypatch, - failed_dependency="primary", + failed_dependency=failed_dependency, failure_factory=failure_factory, + fail_before_acquisition=True, ) async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://unit.local") as client: @@ -130,4 +136,7 @@ async def test_readiness_sanitizes_supported_connection_failures( assert response.json() == {"status": "unavailable"} assert response.headers["cache-control"] == "no-store" assert private_detail not in response.text - assert connection_events == [("primary", "open"), ("primary", "close")] + assert connection_events == ( + [("primary", "open"), ("primary", "close")] + if failed_dependency == "readonly" else [] + ) From 8b8ac74079b4d0022878719a93eec57562916995 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 11:46:45 +0900 Subject: [PATCH 10/12] test(runtime): preserve readiness cancellation cleanup Signed-off-by: Seongho Bae --- backend/tests/test_runtime_health.py | 30 +++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/backend/tests/test_runtime_health.py b/backend/tests/test_runtime_health.py index a1866efda..075604092 100644 --- a/backend/tests/test_runtime_health.py +++ b/backend/tests/test_runtime_health.py @@ -1,6 +1,7 @@ """Exercise public probe responses without workers or customer/provider access.""" from contextlib import asynccontextmanager +from asyncio import CancelledError import os import secrets @@ -14,7 +15,7 @@ os.environ.setdefault("DISABLE_BACKGROUND_WORKERS", "1") from db import session as database_session # noqa: E402 -from main import app # noqa: E402 +from main import app, database_readiness # noqa: E402 def install_probe_engines( @@ -107,6 +108,33 @@ def catalog_failure(): return InvalidCatalogNameError("catalog-private-detail") +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_dependency", ["primary", "readonly"]) +@pytest.mark.parametrize("fail_before_acquisition", [False, True]) +async def test_readiness_propagates_cancellation_and_returns_acquired_connections( + monkeypatch, failed_dependency, fail_before_acquisition +): + """Cancellation must escape the probe while acquired connections are returned.""" + connection_events = install_probe_engines( + monkeypatch, + failed_dependency=failed_dependency, + failure_factory=CancelledError, + fail_before_acquisition=fail_before_acquisition, + ) + + with pytest.raises(CancelledError): + await database_readiness() + + expected_events = [] + for dependency_name in ("primary", "readonly"): + if dependency_name == failed_dependency and fail_before_acquisition: + break + expected_events.extend([(dependency_name, "open"), (dependency_name, "close")]) + if dependency_name == failed_dependency: + break + assert connection_events == expected_events + + @pytest.mark.asyncio @pytest.mark.parametrize("failed_dependency", ["primary", "readonly"]) @pytest.mark.parametrize( From e6726e8c43203b91a55819d88c5a25bf72bf0dd3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:01:59 +0900 Subject: [PATCH 11/12] fix(deploy): wire backend database readiness into rendered manifests Signed-off-by: Seongho Bae --- AGENTS.md | 5 ++++ .../tests/test_release_manifest_digests.py | 10 ++++++++ .../runtime-image-boundary-verification.md | 23 +++++++++++++++++++ k8s/backend-deployment.yaml | 4 ++++ 4 files changed, 42 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index a6a68e904..30d004a9a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -728,6 +728,11 @@ in this repo. ## Phase 10 development rules +- Validate probes in the rendered deployment manifest, not only API tests. + Backend readiness must call `/readyz` on port 8000; a static root response + cannot establish database availability. Never reuse that dependency probe for + liveness: a database outage must not become a backend restart trigger. + - Readiness probes must handle native PostgreSQL connection-establishment errors as well as SQLAlchemy wrappers. In PR #1597, a real isolated cluster raised `asyncpg.InvalidCatalogNameError` before `SELECT 1`; catch the driver's diff --git a/backend/tests/test_release_manifest_digests.py b/backend/tests/test_release_manifest_digests.py index 8195d3c73..5a6232d21 100644 --- a/backend/tests/test_release_manifest_digests.py +++ b/backend/tests/test_release_manifest_digests.py @@ -46,6 +46,16 @@ def test_release_renderer_binds_both_images_without_changing_source(tmp_path: Pa assert rendered == expected +def test_rendered_backend_requires_database_readiness(tmp_path: Path) -> None: + """Route deployment readiness to the dependency probe, not the static root.""" + result = run_renderer(tmp_path, BACKEND_DIGEST, FRONTEND_DIGEST) + assert result.returncode == 0, result.stderr + manifest = yaml.safe_load((tmp_path / "rendered/backend-deployment.yaml").read_text()) + backend_container = manifest["spec"]["template"]["spec"]["containers"][0] + assert backend_container["readinessProbe"]["httpGet"] == {"path": "/readyz", "port": 8000} + assert backend_container.get("livenessProbe", {}).get("httpGet", {}).get("path") != "/readyz" + + @pytest.mark.parametrize("invalid_digest", ["", "latest", "sha256:" + "a" * 63, "sha256:" + "A" * 64, BACKEND_DIGEST + "\ninjected", "$(touch injected)"]) @pytest.mark.parametrize("invalid_component", ["backend", "frontend"]) def test_release_renderer_rejects_either_invalid_digest_before_output( diff --git a/docs/doctoring/runtime-image-boundary-verification.md b/docs/doctoring/runtime-image-boundary-verification.md index d73eaa3c2..a8dfbfea1 100644 --- a/docs/doctoring/runtime-image-boundary-verification.md +++ b/docs/doctoring/runtime-image-boundary-verification.md @@ -238,6 +238,29 @@ and deployment serialization/recovery evidence separate. ## Image boundary references +## Deployment probe wiring follow-up + +At `8b8ac74079b4d0022878719a93eec57562916995`, the backend manifest had +no readiness or liveness probe. A regression executing the real release renderer +failed with `KeyError: readinessProbe`. The repair connects backend readiness to +`/readyz` on port 8000 and verifies the generated immutable-image manifest, not +just the checked-in YAML. All 26 release-manifest tests passed in 37.33 seconds +with warnings treated as errors and terminal exit 0. Ruff and diff checks passed. + +Kubernetes readiness failure removes a Pod from matching Service endpoints; +liveness failure can restart its container. Therefore this change does not use +database readiness as liveness. It retains the existing absence of a liveness +restart policy until startup timing and restart behavior are validated. The +readiness probe uses Kubernetes defaults; this is not a measured latency SLO or +an application/model timeout. No cluster was contacted or deployment performed. +Actual endpoint removal/recovery and probe-load effects remain unverified. + +Kubernetes Authors. (n.d.). *Liveness, readiness, and startup probes*. +Retrieved September 8, 2026, from +https://kubernetes.io/docs/concepts/workloads/pods/probes/ + +## Image boundary bibliography + Kubernetes Authors. (n.d.-a). *Kubernetes API concepts*. Retrieved September 7, 2026, from https://kubernetes.io/docs/reference/using-api/api-concepts/#updates-to-existing-resources diff --git a/k8s/backend-deployment.yaml b/k8s/backend-deployment.yaml index 46f449f6c..d241393aa 100644 --- a/k8s/backend-deployment.yaml +++ b/k8s/backend-deployment.yaml @@ -42,6 +42,10 @@ spec: type: RuntimeDefault ports: - containerPort: 8000 + readinessProbe: + httpGet: + path: /readyz + port: 8000 resources: requests: cpu: 250m From 2c126127eb1a2e9bd77401fb940fac82e140cfc6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:03:44 +0900 Subject: [PATCH 12/12] docs(gap): distinguish manifest readiness from live acceptance Signed-off-by: Seongho Bae --- docs/doctoring/runtime-image-boundary-verification.md | 2 -- docs/product-technical-gap-baseline.md | 9 +++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/runtime-image-boundary-verification.md b/docs/doctoring/runtime-image-boundary-verification.md index a8dfbfea1..3c6f3b5cb 100644 --- a/docs/doctoring/runtime-image-boundary-verification.md +++ b/docs/doctoring/runtime-image-boundary-verification.md @@ -236,8 +236,6 @@ complete successor coverage, protected merge, hosted review/check success, production readiness, or deployment eligibility. Keep prerequisite PR #1587 and deployment serialization/recovery evidence separate. -## Image boundary references - ## Deployment probe wiring follow-up At `8b8ac74079b4d0022878719a93eec57562916995`, the backend manifest had diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 9e196fbb6..55c81b7c2 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1053,6 +1053,15 @@ predecessor-head evidence is never reused. ### Readiness evidence update — 2026-09-08 +Deployment wiring follow-up `e6726e8c43203b91a55819d88c5a25bf72bf0dd3` +connects the backend manifest to `/readyz` on port 8000. A test executing the +release renderer first failed on the missing probe, then all 26 manifest tests +passed with terminal exit 0. Runtime tests at `8b8ac740` separately cover 14 +cases, including cancellation before acquisition and during query execution. +Neither result demonstrates kubelet behavior, endpoint removal/recovery, startup +timing, or live deployment. Liveness restart policy remains unconfigured pending +those measurements; dependency failure must not become a restart trigger. + PR #1597 source `853aad7f67f0561a079735cb27acb6bf4b0e60fd` repairs an unwrapped native PostgreSQL connection error that escaped the generic readiness 503 response. Seven focused tests passed, and an isolated real PostgreSQL