diff --git a/.env.example b/.env.example index 0058042e2..fb934a4b8 100644 --- a/.env.example +++ b/.env.example @@ -141,6 +141,12 @@ OPENCODE_MODEL_ID=big-pickle # Maximum file size in KB to send as document (default: 100) # CODE_FILE_MAX_SIZE_KB=100 +# Health Server (optional) +# Port for the internal health endpoint (default: 3100, 0 to disable) +# Exposes GET http://127.0.0.1:3100/health, /health/live, /health/ready +# Used by Docker HEALTHCHECK and hermes-ops. Binds only to loopback. +# BOT_HEALTH_PORT=3100 + # Speech-to-Text / Voice Recognition (optional) # Enable voice message transcription by setting a Whisper-compatible API URL. # Works with OpenAI, Groq, or any Whisper-compatible endpoint. diff --git a/Dockerfile b/Dockerfile index 6c397ba15..a2a0c4954 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,6 +48,9 @@ ENV OPENCODE_TELEGRAM_HOME=/app/data RUN mkdir -p /app/data/logs /app/data/run && \ chown -R node:node /app +# Health server port (internal, loopback only) +EXPOSE 3100 + # Copy built application and production dependencies from builder COPY --from=builder --chown=node:node /app/dist ./dist COPY --from=builder --chown=node:node /app/node_modules ./node_modules diff --git a/docker-compose.yml b/docker-compose.yml index d05308863..89e85c721 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,7 @@ services: - OPENCODE_API_URL=${OPENCODE_API_URL:-http://127.0.0.1:4096} - OPENCODE_TELEGRAM_HOME=/app/data - OPENCODE_TELEGRAM_CONTAINER=1 + - BOT_HEALTH_PORT=${BOT_HEALTH_PORT:-3100} env_file: - .env volumes: @@ -20,6 +21,12 @@ services: options: max-size: "10m" max-file: "5" + healthcheck: + test: ["CMD-SHELL", "node -e \"const p=process.env.BOT_HEALTH_PORT||'3100'; if(p==='0') process.exit(0); fetch('http://127.0.0.1:'+p+'/health').then(r=>r.json()).then(j=>process.exit(j.status==='healthy'?0:1)).catch(()=>process.exit(1))\""] + interval: 30s + timeout: 10s + retries: 3 + start_period: 45s volumes: opencode-bot-data: \ No newline at end of file diff --git a/src/app/bootstrap/start-bot-app.ts b/src/app/bootstrap/start-bot-app.ts index 1053cdca4..9b6160567 100644 --- a/src/app/bootstrap/start-bot-app.ts +++ b/src/app/bootstrap/start-bot-app.ts @@ -18,6 +18,7 @@ import { clearServiceStateFile } from "../../runtime/service/manager.js"; import { getServiceStateFilePathFromEnv, isServiceChildProcess } from "../../runtime/service/env.js"; import { flushLogger, getLogFilePath, initializeLogger, logger } from "../../utils/logger.js"; import { safeBackgroundTask } from "../../utils/safe-background-task.js"; +import { startHealthServer, stopHealthServer } from "../../health/server.js"; const SHUTDOWN_TIMEOUT_MS = 5000; const SETTINGS_FLUSH_TIMEOUT_MS = 1000; @@ -116,6 +117,7 @@ export async function startBotApp(): Promise { await loadSettings(); await reconcileStoredModelSelection(); registerOpenCodeReadyRefreshHandler(); + await startHealthServer(config.health.port, version); const bot = createBot(); await scheduledTaskRuntime.initialize( bot, @@ -142,6 +144,7 @@ export async function startBotApp(): Promise { cleanupBotRuntime(`app_shutdown_${signal.toLowerCase()}`); opencodeAutoRestartService.stop(); scheduledTaskRuntime.shutdown(); + void stopHealthServer(); shutdownTimeout = setTimeout(() => { logger.warn(`[App] Shutdown did not finish in ${SHUTDOWN_TIMEOUT_MS}ms, forcing exit.`); @@ -200,6 +203,7 @@ export async function startBotApp(): Promise { cleanupBotRuntime("app_shutdown_complete"); opencodeAutoRestartService.stop(); scheduledTaskRuntime.shutdown(); + await stopHealthServer(); await clearManagedServiceState().catch((error) => { logger.warn("[App] Failed to clear managed service state", error); }); diff --git a/src/config.ts b/src/config.ts index 19517cb92..b76a7328e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -206,6 +206,9 @@ export const config = { server: { logLevel: getEnvVar("LOG_LEVEL", false) || "info", }, + health: { + port: getOptionalNonNegativeIntEnvVar("BOT_HEALTH_PORT", 3100), + }, bot: { sessionsListLimit: getOptionalPositiveIntEnvVar("SESSIONS_LIST_LIMIT", 10), messagesListLimit: getOptionalPositiveIntEnvVar("MESSAGES_LIST_LIMIT", 10), diff --git a/src/health/server.ts b/src/health/server.ts new file mode 100644 index 000000000..fba3b6c07 --- /dev/null +++ b/src/health/server.ts @@ -0,0 +1,146 @@ +import http from "node:http"; +import { isOpencodeServerHealthy } from "../opencode/ready-refresh.js"; +import { logger } from "../utils/logger.js"; + +let server: http.Server | null = null; +let startTimeMs: number | null = null; +let botVersion = "unknown"; + +interface HealthPayload { + status: "healthy" | "degraded"; + version: string; + uptimeSeconds: number; + timestamp: string; + checks: { + process: { healthy: boolean }; + opencode: { healthy: boolean; latencyMs: number | null; error?: string }; + }; +} + +function getUptimeSeconds(): number { + if (startTimeMs === null) return 0; + return Math.floor((Date.now() - startTimeMs) / 1000); +} + +async function checkOpencodeWithTimeout(timeoutMs = 3000): Promise<{ healthy: boolean; latencyMs: number | null; error?: string }> { + const start = Date.now(); + try { + const result = await Promise.race([ + isOpencodeServerHealthy(), + new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), timeoutMs)), + ]); + return { healthy: result === true, latencyMs: Date.now() - start }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { healthy: false, latencyMs: Date.now() - start, error: message }; + } +} + +async function buildHealthPayload(): Promise { + const opencode = await checkOpencodeWithTimeout(3000); + + let status: HealthPayload["status"] = "healthy"; + if (!opencode.healthy) { + // Bot process alive but dependency down -> degraded (still running, can recover) + status = "degraded"; + } + + return { + status, + version: botVersion, + uptimeSeconds: getUptimeSeconds(), + timestamp: new Date().toISOString(), + checks: { + process: { healthy: true }, + opencode, + }, + }; +} + +function sendJson(res: http.ServerResponse, statusCode: number, data: unknown): void { + const body = JSON.stringify(data); + res.writeHead(statusCode, { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(body), + "Cache-Control": "no-store", + }); + res.end(body); +} + +export async function startHealthServer(port: number, version: string): Promise { + if (port === 0) { + logger.info("[Health] Health server disabled (BOT_HEALTH_PORT=0)"); + return; + } + if (server) { + logger.warn("[Health] Health server already running"); + return; + } + + botVersion = version; + startTimeMs = Date.now(); + + server = http.createServer(async (req, res) => { + const url = new URL(req.url ?? "/", `http://${req.headers.host ?? "127.0.0.1"}`); + const path = url.pathname; + + if (req.method !== "GET") { + sendJson(res, 405, { error: "Method not allowed" }); + return; + } + + if (path === "/health/live") { + // Liveness: process is alive (no dependency checks) + sendJson(res, 200, { + status: "healthy", + version: botVersion, + uptimeSeconds: getUptimeSeconds(), + timestamp: new Date().toISOString(), + }); + return; + } + + if (path === "/health/ready") { + // Readiness: 200 if healthy, 503 if degraded + const payload = await buildHealthPayload(); + sendJson(res, payload.status === "healthy" ? 200 : 503, payload); + return; + } + + if (path === "/health") { + // Full health: always 200 with status field (Docker healthcheck convention) + const payload = await buildHealthPayload(); + sendJson(res, 200, payload); + return; + } + + sendJson(res, 404, { error: "Not found", path }); + }); + + server.on("error", (error) => { + logger.error("[Health] Health server error", error); + }); + + await new Promise((resolve, reject) => { + server!.listen(port, "127.0.0.1", () => { + logger.info(`[Health] Health server listening on 127.0.0.1:${port} (/health, /health/live, /health/ready)`); + resolve(); + }); + server!.once("error", reject); + }); +} + +export async function stopHealthServer(): Promise { + if (!server) return; + const s = server; + server = null; + startTimeMs = null; + await new Promise((resolve) => { + s.close(() => resolve()); + setTimeout(() => { + s.closeAllConnections?.(); + resolve(); + }, 2000).unref?.(); + }); + logger.info("[Health] Health server stopped"); +} \ No newline at end of file diff --git a/tests/app/start-bot-app.test.ts b/tests/app/start-bot-app.test.ts index dc1824f6e..f020fa681 100644 --- a/tests/app/start-bot-app.test.ts +++ b/tests/app/start-bot-app.test.ts @@ -25,6 +25,8 @@ const mocked = vi.hoisted(() => ({ initializeLoggerMock: vi.fn(), getLogFilePathMock: vi.fn(), flushLoggerMock: vi.fn(), + startHealthServerMock: vi.fn(), + stopHealthServerMock: vi.fn(), config: { opencode: { apiUrl: "http://localhost:4096", @@ -41,7 +43,12 @@ vi.mock("../../src/bot/index.js", () => ({ })); vi.mock("../../src/config.js", () => ({ - config: mocked.config, + config: { + ...mocked.config, + health: { + port: 0, + }, + }, })); vi.mock("../../src/opencode/auto-restart.js", () => ({ @@ -101,6 +108,11 @@ vi.mock("../../src/utils/logger.js", () => ({ }, })); +vi.mock("../../src/health/server.js", () => ({ + startHealthServer: mocked.startHealthServerMock, + stopHealthServer: mocked.stopHealthServerMock, +})); + import { startBotApp } from "../../src/app/bootstrap/start-bot-app.js"; import { defined } from "../helpers/defined.js"; @@ -226,6 +238,8 @@ describe("app/start-bot-app", () => { expect(mocked.registerOpenCodeReadyRefreshHandlerMock).toHaveBeenCalledTimes(1); expect(mocked.notifyOpencodeReadyIfHealthyMock).toHaveBeenCalledWith("startup"); + expect(mocked.startHealthServerMock).toHaveBeenCalledTimes(1); + expect(mocked.startHealthServerMock).toHaveBeenCalledWith(0, expect.any(String)); }); it("runs startup health notification even when auto-restart handled startup", async () => { @@ -235,6 +249,7 @@ describe("app/start-bot-app", () => { await flushBackgroundTasks(); expect(mocked.notifyOpencodeReadyIfHealthyMock).toHaveBeenCalledWith("startup"); + expect(mocked.startHealthServerMock).toHaveBeenCalledTimes(1); }); it("starts Telegram polling without waiting for OpenCode startup checks", async () => { @@ -251,6 +266,7 @@ describe("app/start-bot-app", () => { expect(bot.start).toHaveBeenCalledTimes(1); expect(mocked.notifyOpencodeReadyIfHealthyMock).not.toHaveBeenCalled(); + expect(mocked.startHealthServerMock).toHaveBeenCalledTimes(1); resolveAutoRestart(false); await flushBackgroundTasks(); @@ -404,5 +420,6 @@ describe("app/start-bot-app", () => { expect(defined(mocked.flushSettingsMock.mock.invocationCallOrder[0])).toBeGreaterThan( defined(mocked.scheduledTaskShutdownMock.mock.invocationCallOrder[0]), ); + expect(mocked.stopHealthServerMock).toHaveBeenCalledTimes(1); }); }); diff --git a/tests/health/server.test.ts b/tests/health/server.test.ts new file mode 100644 index 000000000..11bc4bbd9 --- /dev/null +++ b/tests/health/server.test.ts @@ -0,0 +1,186 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import http from "node:http"; + +const mocked = vi.hoisted(() => ({ + isOpencodeServerHealthyMock: vi.fn(), + loggerInfoMock: vi.fn(), + loggerWarnMock: vi.fn(), + loggerErrorMock: vi.fn(), +})); + +vi.mock("../../src/opencode/ready-refresh.js", () => ({ + isOpencodeServerHealthy: mocked.isOpencodeServerHealthyMock, +})); + +vi.mock("../../src/utils/logger.js", () => ({ + logger: { + debug: vi.fn(), + info: mocked.loggerInfoMock, + warn: mocked.loggerWarnMock, + error: mocked.loggerErrorMock, + }, +})); + +import { startHealthServer, stopHealthServer } from "../../src/health/server.js"; + +function fetchJson(port: number, path: string): Promise<{ status: number; body: unknown }> { + return new Promise((resolve, reject) => { + const req = http.request({ hostname: "127.0.0.1", port, path, method: "GET" }, (res) => { + let data = ""; + res.on("data", (chunk) => (data += chunk)); + res.on("end", () => { + try { + resolve({ status: res.statusCode ?? 0, body: JSON.parse(data) }); + } catch { + resolve({ status: res.statusCode ?? 0, body: data }); + } + }); + }); + req.on("error", reject); + req.end(); + }); +} + +async function waitForServer(port: number, timeout = 5000): Promise { + const start = Date.now(); + while (Date.now() - start < timeout) { + try { + await fetchJson(port, "/health/live"); + return; + } catch { + await new Promise((r) => setTimeout(r, 50)); + } + } + throw new Error("Server did not start in time"); +} + +describe("health/server", () => { + let port: number; + + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + // Use random port to avoid conflicts + port = 3100 + Math.floor(Math.random() * 1000); + mocked.isOpencodeServerHealthyMock.mockResolvedValue(true); + }); + + afterEach(async () => { + await stopHealthServer(); + await new Promise((r) => setTimeout(r, 100)); + }); + + it("starts and stops server with port > 0", async () => { + await startHealthServer(port, "test-1.0.0"); + await waitForServer(port); + + const live = await fetchJson(port, "/health/live"); + expect(live.status).toBe(200); + expect(live.body).toMatchObject({ status: "healthy", version: "test-1.0.0" }); + + await stopHealthServer(); + const infoCalls = mocked.loggerInfoMock.mock.calls.map((c) => c[0]); + expect(infoCalls.some((msg) => msg.includes("Health server listening"))).toBe(true); + expect(infoCalls.some((msg) => msg.includes("Health server stopped"))).toBe(true); + }); + + it("does not start server when port is 0", async () => { + await startHealthServer(0, "test-1.0.0"); + expect(mocked.loggerInfoMock).toHaveBeenCalledWith("[Health] Health server disabled (BOT_HEALTH_PORT=0)"); + }); + + it("/health/live returns 200 with basic info", async () => { + await startHealthServer(port, "test-1.0.0"); + await waitForServer(port); + + const res = await fetchJson(port, "/health/live"); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ status: "healthy", version: "test-1.0.0" }); + }); + + it("/health/ready returns 200 when opencode healthy", async () => { + mocked.isOpencodeServerHealthyMock.mockResolvedValue(true); + await startHealthServer(port, "test-1.0.0"); + await waitForServer(port); + + const res = await fetchJson(port, "/health/ready"); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ + status: "healthy", + version: "test-1.0.0", + checks: { process: { healthy: true }, opencode: { healthy: true } }, + }); + }); + + it("/health/ready returns 503 when opencode unhealthy", async () => { + mocked.isOpencodeServerHealthyMock.mockResolvedValue(false); + await startHealthServer(port, "test-1.0.0"); + await waitForServer(port); + + const res = await fetchJson(port, "/health/ready"); + expect(res.status).toBe(503); + expect(res.body).toMatchObject({ + status: "degraded", + checks: { process: { healthy: true }, opencode: { healthy: false } }, + }); + }); + + it("/health returns 200 with full payload", async () => { + mocked.isOpencodeServerHealthyMock.mockResolvedValue(true); + await startHealthServer(port, "test-1.0.0"); + await waitForServer(port); + + const res = await fetchJson(port, "/health"); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ + status: "healthy", + version: "test-1.0.0", + checks: { process: { healthy: true }, opencode: { healthy: true } }, + }); + }); + + it("/health returns 200 (not 503) even when degraded", async () => { + mocked.isOpencodeServerHealthyMock.mockResolvedValue(false); + await startHealthServer(port, "test-1.0.0"); + await waitForServer(port); + + const res = await fetchJson(port, "/health"); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ status: "degraded" }); + }); + + it("port=0 disables server completely", async () => { + await startHealthServer(0, "test"); + // Try to connect - should fail + try { + await fetchJson(port, "/health"); + throw new Error("Should have failed"); + } catch (e) { + expect((e as Error).message).toMatch(/ECONNREFUSED|fetch failed/); + } + }); + + it("lifecycle: multiple start/stop calls are safe", async () => { + await startHealthServer(port, "v1"); + await waitForServer(port); + await startHealthServer(port, "v2"); // Should warn, not start second + await stopHealthServer(); + await stopHealthServer(); // Should not throw + expect(mocked.loggerWarnMock).toHaveBeenCalledWith("[Health] Health server already running"); + }); + + it("unknown path returns 404", async () => { + await startHealthServer(port, "test"); + await waitForServer(port); + const res = await fetchJson(port, "/health/unknown"); + expect(res.status).toBe(404); + }); + + it("non-GET returns 405", async () => { + await startHealthServer(port, "test"); + await waitForServer(port); + // Use native fetch for method test + const res = await fetch(`http://127.0.0.1:${port}/health`, { method: "POST" }); + expect(res.status).toBe(405); + }); +}); \ No newline at end of file