fix(api): harden healthcheck probe failure handling

apps/api/schemas.py

@@ -3,6 +3,16 @@
 from typing import Optional
 
 
+class HealthcheckChecksOut(Schema):
+    database: str
+    redis: str
+
+
+class HealthcheckOut(Schema):
+    status: str
+    checks: HealthcheckChecksOut
+
+
 class AdminMetricsSummaryOut(Schema):
     window_start: datetime
     window_end: datetime

apps/api/tests.py

@@ -1050,3 +1050,39 @@ def test_backfill_command_supports_filters_and_limit(
     def test_backfill_command_rejects_conflicting_modes(self):
         with self.assertRaises(CommandError):
             call_command("backfill_qdrant_vectors", "--questions-only", "--answers-only")
+
+
+class HealthcheckLoggingTestCase(TestCase):
+    @patch("apps.api.views.logger")
+    @patch("apps.api.views.connection.cursor")
+    def test_healthcheck_database_failure_returns_503_without_error_log(
+        self,
+        cursor_mock,
+        logger_mock,
+    ):
+        cursor_mock.side_effect = Exception("database unavailable")
+
+        response = self.client.get("/api/healthcheck")
+
+        self.assertEqual(response.status_code, 503)
+        payload = response.json()
+        self.assertEqual(payload["status"], "unhealthy")
+        self.assertEqual(payload["checks"]["database"], "unhealthy")
+        logger_mock.error.assert_not_called()
+
+    @patch("apps.api.views.logger")
+    @patch("apps.api.views.cache.set")
+    def test_healthcheck_redis_failure_returns_503_without_error_log(
+        self,
+        cache_set_mock,
+        logger_mock,
+    ):
+        cache_set_mock.side_effect = Exception("redis unavailable")
+
+        response = self.client.get("/api/healthcheck")
+
+        self.assertEqual(response.status_code, 503)
+        payload = response.json()
+        self.assertEqual(payload["status"], "unhealthy")
+        self.assertEqual(payload["checks"]["redis"], "unhealthy")
+        logger_mock.error.assert_not_called()

apps/api/views.py

@@ -63,6 +63,7 @@
     ProfileSettingsOut,
     UserSettingsOut,
     AdminMetricsSummaryOut,
+    HealthcheckOut,
 )
 
 from agent_commons.utils import get_agent_commons_logger
@@ -154,7 +155,13 @@ def _enforce_verified_profile(profile) -> None:
             "Email verification required. Ask your human to confirm email, then call /api/agent/setup/status.",
         )
 
-@api.get("/healthcheck", auth=None, include_in_schema=False, tags=["private"])
+@api.get(
+    "/healthcheck",
+    response={200: HealthcheckOut, 503: HealthcheckOut},
+    auth=None,
+    include_in_schema=False,
+    tags=["private"],
+)
 def healthcheck(request: HttpRequest):
     """
     Comprehensive healthcheck endpoint for monitoring and load balancers.
@@ -180,7 +187,7 @@ def healthcheck(request: HttpRequest):
     except Exception as e:
         health_status["checks"]["database"] = "unhealthy"
         all_healthy = False
-        logger.error(
+        logger.warning(
             "Healthcheck failed: Database connection error",
             error=str(e),
             exc_info=True
@@ -198,15 +205,15 @@ def healthcheck(request: HttpRequest):
         else:
             health_status["checks"]["redis"] = "unhealthy"
             all_healthy = False
-            logger.error(
+            logger.warning(
                 "Healthcheck failed: Redis value mismatch",
                 expected=cache_value,
                 retrieved=retrieved_value
             )
     except Exception as e:
         health_status["checks"]["redis"] = "unhealthy"
         all_healthy = False
-        logger.error(
+        logger.warning(
             "Healthcheck failed: Redis connection error",
             error=str(e),
             exc_info=True
@@ -223,7 +230,7 @@ def healthcheck(request: HttpRequest):
         return health_status
     else:
         health_status["status"] = "unhealthy"
-        logger.error(
+        logger.warning(
             "Healthcheck failed: One or more services unhealthy",
             database=health_status["checks"]["database"],
             redis=health_status["checks"]["redis"]