From eedce2d8d4efd5af7dd97eba841069455bfcd6cb Mon Sep 17 00:00:00 2001 From: octane0411 Date: Sat, 7 Mar 2026 16:18:12 +0800 Subject: [PATCH] Refactor dashboard server modules --- src/observability/dashboard-format.ts | 94 ++ src/observability/dashboard-http.ts | 80 ++ src/observability/dashboard-live-updates.ts | 146 +++ src/observability/dashboard-render.ts | 771 +++++++++++++ src/observability/dashboard-server.ts | 1072 +------------------ 5 files changed, 1105 insertions(+), 1058 deletions(-) create mode 100644 src/observability/dashboard-format.ts create mode 100644 src/observability/dashboard-http.ts create mode 100644 src/observability/dashboard-live-updates.ts create mode 100644 src/observability/dashboard-render.ts diff --git a/src/observability/dashboard-format.ts b/src/observability/dashboard-format.ts new file mode 100644 index 00000000..ed78e7e8 --- /dev/null +++ b/src/observability/dashboard-format.ts @@ -0,0 +1,94 @@ +export function formatRuntimeAndTurns( + startedAt: string, + turnCount: number, + generatedAt: string, +): string { + const runtime = formatRuntimeSeconds( + runtimeSecondsFromStartedAt(startedAt, generatedAt), + ); + return Number.isInteger(turnCount) && turnCount > 0 + ? `${runtime} / ${turnCount}` + : runtime; +} + +export function formatRuntimeSeconds(seconds: number): string { + if (!Number.isFinite(seconds) || seconds < 0) { + return "0m 0s"; + } + const wholeSeconds = Math.max(0, Math.trunc(seconds)); + const mins = Math.floor(wholeSeconds / 60); + const secs = wholeSeconds % 60; + return `${mins}m ${secs}s`; +} + +export function runtimeSecondsFromStartedAt( + startedAt: string, + generatedAt: string, +): number { + const start = Date.parse(startedAt); + const generated = Date.parse(generatedAt); + if ( + !Number.isFinite(start) || + !Number.isFinite(generated) || + generated < start + ) { + return 0; + } + return (generated - start) / 1000; +} + +export function formatInteger(value: number): string { + return Number.isFinite(value) + ? Math.trunc(value).toLocaleString("en-US") + : "n/a"; +} + +export function prettyValue(value: unknown): string { + return value === null || value === undefined + ? "n/a" + : JSON.stringify(value, null, 2); +} + +export function stateBadgeClass(state: string): string { + const normalized = state.toLowerCase(); + if ( + normalized.includes("progress") || + normalized.includes("running") || + normalized.includes("active") + ) { + return "state-badge state-badge-active"; + } + if ( + normalized.includes("blocked") || + normalized.includes("error") || + normalized.includes("failed") + ) { + return "state-badge state-badge-danger"; + } + if ( + normalized.includes("todo") || + normalized.includes("queued") || + normalized.includes("pending") || + normalized.includes("retry") + ) { + return "state-badge state-badge-warning"; + } + return "state-badge"; +} + +export function escapeHtml(value: string | number): string { + return String(value) + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +export function toErrorMessage(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + + return String(error); +} diff --git a/src/observability/dashboard-http.ts b/src/observability/dashboard-http.ts new file mode 100644 index 00000000..1cf09324 --- /dev/null +++ b/src/observability/dashboard-http.ts @@ -0,0 +1,80 @@ +import type { IncomingMessage, ServerResponse } from "node:http"; + +import type { RuntimeSnapshot } from "../logging/runtime-snapshot.js"; +import type { DashboardServerHost } from "./dashboard-server.js"; + +export async function readSnapshot( + host: DashboardServerHost, + timeoutMs: number, +): Promise { + return await withTimeout(host.getRuntimeSnapshot(), timeoutMs, () => { + return new Error(`Runtime snapshot timed out after ${timeoutMs}ms.`); + }); +} + +export function writeJson( + response: ServerResponse, + statusCode: number, + payload: unknown, +): void { + const body = JSON.stringify(payload); + response.statusCode = statusCode; + response.setHeader("content-type", "application/json; charset=utf-8"); + response.setHeader("content-length", Buffer.byteLength(body)); + response.end(body); +} + +export function writeHtml( + response: ServerResponse, + statusCode: number, + html: string, +): void { + response.statusCode = statusCode; + response.setHeader("content-type", "text/html; charset=utf-8"); + response.setHeader("content-length", Buffer.byteLength(html)); + response.end(html); +} + +export function writeNotFound(response: ServerResponse, path: string): void { + response.statusCode = 404; + response.setHeader("content-type", "text/plain; charset=utf-8"); + response.end(`Not found: ${path}`); +} + +export async function readRequestBody(request: IncomingMessage): Promise { + await new Promise((resolve, reject) => { + request.on("error", reject); + request.on("end", resolve); + request.resume(); + }); +} + +export function isSnapshotTimeoutError(error: unknown): boolean { + return ( + error instanceof Error && + error.message.startsWith("Runtime snapshot timed out after ") + ); +} + +async function withTimeout( + promise: Promise | T, + timeoutMs: number, + createError: () => Error, +): Promise { + return await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(createError()); + }, timeoutMs); + + Promise.resolve(promise).then( + (value) => { + clearTimeout(timeout); + resolve(value); + }, + (error) => { + clearTimeout(timeout); + reject(error); + }, + ); + }); +} diff --git a/src/observability/dashboard-live-updates.ts b/src/observability/dashboard-live-updates.ts new file mode 100644 index 00000000..a02079ab --- /dev/null +++ b/src/observability/dashboard-live-updates.ts @@ -0,0 +1,146 @@ +import type { IncomingMessage, ServerResponse } from "node:http"; + +import { ERROR_CODES } from "../errors/codes.js"; +import type { RuntimeSnapshot } from "../logging/runtime-snapshot.js"; +import { toErrorMessage } from "./dashboard-format.js"; +import { isSnapshotTimeoutError, readSnapshot } from "./dashboard-http.js"; +import type { DashboardServerHost } from "./dashboard-server.js"; + +export class DashboardLiveUpdatesController { + readonly #host: DashboardServerHost; + readonly #snapshotTimeoutMs: number; + readonly #refreshMs: number; + readonly #renderIntervalMs: number; + readonly #clients = new Set>(); + #flushTimer: NodeJS.Timeout | null = null; + #heartbeatTimer: NodeJS.Timeout | null = null; + #unsubscribeHost: (() => void) | null = null; + #closed = false; + + constructor(options: { + host: DashboardServerHost; + snapshotTimeoutMs: number; + refreshMs: number; + renderIntervalMs: number; + }) { + this.#host = options.host; + this.#snapshotTimeoutMs = options.snapshotTimeoutMs; + this.#refreshMs = options.refreshMs; + this.#renderIntervalMs = options.renderIntervalMs; + } + + start(): void { + if (typeof this.#host.subscribeToSnapshots === "function") { + this.#unsubscribeHost = this.#host.subscribeToSnapshots(() => { + this.scheduleBroadcast(); + }); + } + } + + async close(): Promise { + this.#closed = true; + this.#unsubscribeHost?.(); + this.#unsubscribeHost = null; + this.clearTimers(); + + for (const client of this.#clients) { + client.end(); + } + this.#clients.clear(); + } + + async handleEventsRequest( + request: IncomingMessage, + response: ServerResponse, + ): Promise { + response.statusCode = 200; + response.setHeader("content-type", "text/event-stream; charset=utf-8"); + response.setHeader("cache-control", "no-cache, no-transform"); + response.setHeader("connection", "keep-alive"); + response.setHeader("x-accel-buffering", "no"); + response.write(`retry: ${this.#refreshMs}\n\n`); + + this.#clients.add(response); + this.startHeartbeat(); + + const cleanup = () => { + this.#clients.delete(response); + if (this.#clients.size === 0) { + this.stopHeartbeat(); + } + }; + + request.on("close", cleanup); + response.on("close", cleanup); + + await this.writeSnapshot(response); + } + + scheduleBroadcast(): void { + if (this.#closed || this.#clients.size === 0 || this.#flushTimer !== null) { + return; + } + + this.#flushTimer = setTimeout(() => { + this.#flushTimer = null; + void this.broadcastSnapshot(); + }, this.#renderIntervalMs); + } + + private startHeartbeat(): void { + if (this.#heartbeatTimer !== null) { + return; + } + + this.#heartbeatTimer = setInterval(() => { + this.scheduleBroadcast(); + }, this.#refreshMs); + } + + private stopHeartbeat(): void { + if (this.#heartbeatTimer === null) { + return; + } + + clearInterval(this.#heartbeatTimer); + this.#heartbeatTimer = null; + } + + private clearTimers(): void { + if (this.#flushTimer !== null) { + clearTimeout(this.#flushTimer); + this.#flushTimer = null; + } + this.stopHeartbeat(); + } + + private async broadcastSnapshot(): Promise { + const clients = [...this.#clients]; + if (clients.length === 0) { + return; + } + + await Promise.allSettled( + clients.map((client) => this.writeSnapshot(client)), + ); + } + + private async writeSnapshot(response: ServerResponse): Promise { + try { + const snapshot: RuntimeSnapshot = await readSnapshot( + this.#host, + this.#snapshotTimeoutMs, + ); + response.write(`event: snapshot\ndata: ${JSON.stringify(snapshot)}\n\n`); + } catch (error) { + response.write( + `event: error\ndata: ${JSON.stringify({ + code: isSnapshotTimeoutError(error) + ? ERROR_CODES.snapshotTimedOut + : ERROR_CODES.snapshotUnavailable, + message: toErrorMessage(error), + })}\n\n`, + ); + } + } +} diff --git a/src/observability/dashboard-render.ts b/src/observability/dashboard-render.ts new file mode 100644 index 00000000..bab5b7a0 --- /dev/null +++ b/src/observability/dashboard-render.ts @@ -0,0 +1,771 @@ +import type { RuntimeSnapshot } from "../logging/runtime-snapshot.js"; +import { + escapeHtml, + formatInteger, + formatRuntimeAndTurns, + formatRuntimeSeconds, + prettyValue, + stateBadgeClass, +} from "./dashboard-format.js"; + +export interface DashboardRenderOptions { + liveUpdatesEnabled: boolean; +} + +const DASHBOARD_STYLES = String.raw` + :root { + color-scheme: light; + --page: #f7f7f8; + --page-soft: #fbfbfc; + --page-deep: #ececf1; + --card: rgba(255, 255, 255, 0.94); + --card-muted: #f3f4f6; + --ink: #202123; + --muted: #6e6e80; + --line: #ececf1; + --line-strong: #d9d9e3; + --accent: #10a37f; + --accent-ink: #0f513f; + --accent-soft: #e8faf4; + --danger: #b42318; + --danger-soft: #fef3f2; + --warning: #8a5a00; + --warning-soft: #fff7e8; + --warning-line: #f1d8a6; + --shadow-sm: 0 1px 2px rgba(16, 24, 40, 0.05); + --shadow-lg: 0 20px 50px rgba(15, 23, 42, 0.08); + } + * { + box-sizing: border-box; + } + html { + background: var(--page); + } + body { + margin: 0; + min-height: 100vh; + background: + radial-gradient(circle at top, rgba(16, 163, 127, 0.12) 0%, rgba(16, 163, 127, 0) 30%), + linear-gradient(180deg, var(--page-soft) 0%, var(--page) 24%, #f3f4f6 100%); + color: var(--ink); + font-family: "Sohne", "SF Pro Text", "Helvetica Neue", "Segoe UI", sans-serif; + line-height: 1.5; + } + a { + color: var(--ink); + text-decoration: none; + transition: color 140ms ease; + } + a:hover { + color: var(--accent); + } + button { + appearance: none; + border: 1px solid var(--accent); + background: var(--accent); + color: white; + border-radius: 999px; + padding: 0.72rem 1.08rem; + cursor: pointer; + font: inherit; + font-weight: 600; + letter-spacing: -0.01em; + box-shadow: 0 8px 20px rgba(16, 163, 127, 0.18); + transition: + transform 140ms ease, + box-shadow 140ms ease, + background 140ms ease, + border-color 140ms ease; + } + button:hover { + transform: translateY(-1px); + box-shadow: 0 12px 24px rgba(16, 163, 127, 0.22); + } + .subtle-button { + border: 1px solid var(--line-strong); + background: rgba(255, 255, 255, 0.72); + color: var(--muted); + padding: 0.34rem 0.72rem; + font-size: 0.82rem; + letter-spacing: 0.01em; + box-shadow: none; + } + .subtle-button:hover { + transform: none; + box-shadow: none; + background: white; + border-color: var(--muted); + color: var(--ink); + } + code, + pre, + .mono { + font-family: "Sohne Mono", "SFMono-Regular", "SF Mono", Consolas, "Liberation Mono", monospace; + } + .mono, + .numeric { + font-variant-numeric: tabular-nums slashed-zero; + font-feature-settings: "tnum" 1, "zero" 1; + } + .app-shell { + max-width: 1280px; + margin: 0 auto; + padding: 2rem 1rem 3.5rem; + } + .dashboard-shell { + display: grid; + gap: 1rem; + } + .hero-card, + .section-card, + .metric-card { + background: var(--card); + border: 1px solid rgba(217, 217, 227, 0.82); + box-shadow: var(--shadow-sm); + backdrop-filter: blur(18px); + } + .hero-card { + border-radius: 28px; + padding: clamp(1.25rem, 3vw, 2rem); + box-shadow: var(--shadow-lg); + } + .hero-grid { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 1.25rem; + align-items: start; + } + .eyebrow { + margin: 0; + color: var(--muted); + text-transform: uppercase; + letter-spacing: 0.08em; + font-size: 0.76rem; + font-weight: 600; + } + .hero-title { + margin: 0.35rem 0 0; + font-size: clamp(2rem, 4vw, 3.3rem); + line-height: 0.98; + letter-spacing: -0.04em; + } + .hero-copy { + margin: 0.75rem 0 0; + max-width: 46rem; + color: var(--muted); + font-size: 1rem; + } + .status-stack { + display: grid; + justify-items: end; + align-content: start; + min-width: min(100%, 9rem); + } + .status-badge { + display: inline-flex; + align-items: center; + gap: 0.45rem; + min-height: 2rem; + padding: 0.35rem 0.78rem; + border-radius: 999px; + border: 1px solid var(--line); + background: var(--card-muted); + color: var(--muted); + font-size: 0.82rem; + font-weight: 700; + letter-spacing: 0.01em; + } + .status-badge-dot { + width: 0.52rem; + height: 0.52rem; + border-radius: 999px; + background: currentColor; + opacity: 0.9; + } + .status-badge-live { + background: var(--accent-soft); + border-color: rgba(16, 163, 127, 0.18); + color: var(--accent-ink); + } + .metric-grid { + display: grid; + gap: 0.85rem; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); + } + .metric-card { + border-radius: 22px; + padding: 1rem 1.05rem 1.1rem; + } + .metric-label { + margin: 0; + color: var(--muted); + font-size: 0.82rem; + font-weight: 600; + letter-spacing: 0.01em; + } + .metric-value { + margin: 0.35rem 0 0; + font-size: clamp(1.6rem, 2vw, 2.1rem); + line-height: 1.05; + letter-spacing: -0.03em; + } + .metric-detail { + margin: 0.45rem 0 0; + color: var(--muted); + font-size: 0.88rem; + } + .section-card { + border-radius: 24px; + padding: 1.15rem; + } + .section-header { + display: flex; + justify-content: space-between; + align-items: flex-start; + gap: 1rem; + flex-wrap: wrap; + } + .section-title { + margin: 0; + font-size: 1.08rem; + line-height: 1.2; + letter-spacing: -0.02em; + } + .section-copy { + margin: 0.35rem 0 0; + color: var(--muted); + font-size: 0.94rem; + } + .table-wrap { + overflow-x: auto; + margin-top: 1rem; + } + .data-table { + width: 100%; + min-width: 720px; + border-collapse: collapse; + } + .data-table-running { + table-layout: fixed; + min-width: 980px; + } + .data-table th { + padding: 0 0.5rem 0.75rem 0; + text-align: left; + color: var(--muted); + font-size: 0.78rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; + } + .data-table td { + padding: 0.9rem 0.5rem 0.9rem 0; + border-top: 1px solid var(--line); + vertical-align: top; + font-size: 0.94rem; + } + .issue-stack, + .session-stack, + .detail-stack, + .token-stack { + display: grid; + gap: 0.24rem; + min-width: 0; + } + .event-text { + font-weight: 500; + line-height: 1.45; + max-width: 100%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + .event-meta { + max-width: 100%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + .state-badge { + display: inline-flex; + align-items: center; + min-height: 1.85rem; + padding: 0.3rem 0.68rem; + border-radius: 999px; + border: 1px solid var(--line); + background: var(--card-muted); + color: var(--ink); + font-size: 0.8rem; + font-weight: 600; + line-height: 1; + } + .state-badge-active { + background: var(--accent-soft); + border-color: rgba(16, 163, 127, 0.18); + color: var(--accent-ink); + } + .state-badge-warning { + background: var(--warning-soft); + border-color: var(--warning-line); + color: var(--warning); + } + .state-badge-danger { + background: var(--danger-soft); + border-color: #f6d3cf; + color: var(--danger); + } + .issue-id { + font-weight: 600; + letter-spacing: -0.01em; + } + .issue-link { + color: var(--muted); + font-size: 0.86rem; + } + .muted { + color: var(--muted); + } + .code-panel { + margin-top: 1rem; + padding: 1rem; + border-radius: 18px; + background: #f5f5f7; + border: 1px solid var(--line); + color: #353740; + font-size: 0.9rem; + white-space: pre-wrap; + word-break: break-word; + } + .empty-state { + margin: 1rem 0 0; + color: var(--muted); + } + @media (max-width: 860px) { + .app-shell { + padding: 1rem 0.85rem 2rem; + } + .hero-grid { + grid-template-columns: 1fr; + } + .status-stack { + justify-items: start; + } + .metric-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + } + @media (max-width: 560px) { + .metric-grid { + grid-template-columns: 1fr; + } + .section-card, + .hero-card { + border-radius: 20px; + padding: 1rem; + } + } +`; + +export function renderDashboardHtml( + snapshot: RuntimeSnapshot, + options: DashboardRenderOptions, +): string { + const initialRuntimeLabel = formatRuntimeSeconds( + snapshot.codex_totals.seconds_running, + ); + const totalTokensLabel = formatInteger(snapshot.codex_totals.total_tokens); + const inputTokensLabel = formatInteger(snapshot.codex_totals.input_tokens); + const outputTokensLabel = formatInteger(snapshot.codex_totals.output_tokens); + const initialRateLimits = prettyValue(snapshot.rate_limits); + + return ` + + + + + Symphony Observability + + + +
+
+
+
+
+

Symphony Observability

+

Operations Dashboard

+

+ Current state, retry pressure, token usage, and orchestration health for the active Symphony runtime. +

+
+ +
+ + + ${options.liveUpdatesEnabled ? "Live" : "Offline"} + +
+
+
+ +
+
+

Running

+

${snapshot.counts.running}

+

Active issue sessions in the current runtime.

+
+ +
+

Retrying

+

${snapshot.counts.retrying}

+

Issues waiting for the next retry window.

+
+ +
+

Total tokens

+

${totalTokensLabel}

+

In ${inputTokensLabel} / Out ${outputTokensLabel}

+
+ +
+

Runtime

+

${initialRuntimeLabel}

+

Generated at ${escapeHtml(snapshot.generated_at)}

+
+
+ +
+
+
+

Rate limits

+

Latest upstream rate-limit snapshot, when available.

+
+
+ + 
${escapeHtml(initialRateLimits)}
+
+ +
+
+
+

Running sessions

+

Active issues, last known agent activity, and token usage.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + ${renderRunningRows(snapshot)} +
IssueStateSessionRuntime / turnsCodex updateTokens
+
+
+ +
+
+
+

Retry queue

+

Issues waiting for the next retry window.

+
+
+ +
+ + + + + + + + + + ${renderRetryRows(snapshot)} +
IssueAttemptDue atError
+
+
+
+
+ + +`; +} + +function renderDashboardClientScript( + snapshot: RuntimeSnapshot, + options: DashboardRenderOptions, +): string { + return ` window.__SYMPHONY_SNAPSHOT__ = ${JSON.stringify(snapshot)}; + window.__SYMPHONY_LIVE_UPDATES__ = ${JSON.stringify( + options.liveUpdatesEnabled, + )}; + (function () { + const snapshot = window.__SYMPHONY_SNAPSHOT__; + const liveUpdatesEnabled = window.__SYMPHONY_LIVE_UPDATES__ === true; + + function escapeHtml(value) { + return String(value ?? '') + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); + } + + function formatInteger(value) { + const number = Number(value); + if (!Number.isFinite(number)) { + return 'n/a'; + } + return Math.trunc(number).toLocaleString('en-US'); + } + + function formatRuntimeSeconds(value) { + const number = Number(value); + if (!Number.isFinite(number) || number < 0) { + return '0m 0s'; + } + const wholeSeconds = Math.max(0, Math.trunc(number)); + const mins = Math.floor(wholeSeconds / 60); + const secs = wholeSeconds % 60; + return mins + 'm ' + secs + 's'; + } + + function runtimeSecondsFromStartedAt(startedAt, generatedAt) { + const start = Date.parse(startedAt); + const generated = Date.parse(generatedAt); + if (!Number.isFinite(start) || !Number.isFinite(generated) || generated < start) { + return 0; + } + return (generated - start) / 1000; + } + + function formatRuntimeAndTurns(row, generatedAt) { + const runtime = formatRuntimeSeconds(runtimeSecondsFromStartedAt(row.started_at, generatedAt)); + if (Number.isInteger(row.turn_count) && row.turn_count > 0) { + return runtime + ' / ' + row.turn_count; + } + return runtime; + } + + function stateBadgeClass(state) { + const normalized = String(state || '').toLowerCase(); + if (normalized.includes('progress') || normalized.includes('running') || normalized.includes('active')) { + return 'state-badge state-badge-active'; + } + if (normalized.includes('blocked') || normalized.includes('error') || normalized.includes('failed')) { + return 'state-badge state-badge-danger'; + } + if (normalized.includes('todo') || normalized.includes('queued') || normalized.includes('pending') || normalized.includes('retry')) { + return 'state-badge state-badge-warning'; + } + return 'state-badge'; + } + + function prettyValue(value) { + if (value == null) { + return 'n/a'; + } + try { + return JSON.stringify(value, null, 2); + } catch (_error) { + return String(value); + } + } + + function renderRunningRows(next) { + if (!next.running || next.running.length === 0) { + return '

No active sessions.

'; + } + + return next.running.map(function (row) { + const sessionCell = row.session_id + ? '' + : 'n/a'; + + const message = row.last_message || row.last_event || 'n/a'; + const eventMeta = row.last_event + ? escapeHtml(row.last_event) + (row.last_event_at ? ' · ' + escapeHtml(row.last_event_at) + '' : '') + : 'n/a'; + + return '' + + '
' + escapeHtml(row.issue_identifier) + 'JSON details
' + + '' + escapeHtml(row.state) + '' + + '
' + sessionCell + '
' + + '' + formatRuntimeAndTurns(row, next.generated_at) + '' + + '
' + escapeHtml(message) + '' + eventMeta + '
' + + '
Total: ' + formatInteger(row.tokens?.total_tokens) + 'In ' + formatInteger(row.tokens?.input_tokens) + ' / Out ' + formatInteger(row.tokens?.output_tokens) + '
' + + ''; + }).join(''); + } + + function renderRetryRows(next) { + if (!next.retrying || next.retrying.length === 0) { + return '

No issues are currently backing off.

'; + } + + return next.retrying.map(function (row) { + return '' + + '
' + escapeHtml(row.issue_identifier || row.issue_id) + 'JSON details
' + + '' + escapeHtml(row.attempt) + '' + + '' + escapeHtml(row.due_at || 'n/a') + '' + + '' + escapeHtml(row.error || 'n/a') + '' + + ''; + }).join(''); + } + + function setStatus(text, live) { + const element = document.getElementById('live-status'); + if (!element) return; + element.className = live ? 'status-badge status-badge-live' : 'status-badge'; + const label = element.querySelector('span:last-child'); + if (label) { + label.textContent = text; + } + } + + function render(next) { + document.getElementById('generated-at').textContent = 'Generated at ' + next.generated_at; + document.getElementById('metric-running').textContent = String(next.counts.running); + document.getElementById('metric-retrying').textContent = String(next.counts.retrying); + document.getElementById('metric-total').textContent = formatInteger(next.codex_totals.total_tokens); + document.getElementById('metric-total-detail').textContent = 'In ' + formatInteger(next.codex_totals.input_tokens) + ' / Out ' + formatInteger(next.codex_totals.output_tokens); + document.getElementById('metric-runtime').textContent = formatRuntimeSeconds(next.codex_totals.seconds_running); + document.getElementById('running-rows').innerHTML = renderRunningRows(next); + document.getElementById('retry-rows').innerHTML = renderRetryRows(next); + document.getElementById('rate-limits').textContent = prettyValue(next.rate_limits); + } + + render(snapshot); + if (!liveUpdatesEnabled || typeof window.EventSource !== 'function') { + return; + } + + const source = new window.EventSource('/api/v1/events'); + source.addEventListener('open', function () { + setStatus('Live', true); + }); + source.addEventListener('snapshot', function (event) { + try { + const next = JSON.parse(event.data); + render(next); + setStatus('Live', true); + } catch (_error) { + setStatus('Degraded', false); + } + }); + source.addEventListener('error', function () { + setStatus('Reconnecting', false); + }); + })();`; +} + +function renderRunningRows(snapshot: RuntimeSnapshot): string { + return snapshot.running.length === 0 + ? '

No active sessions.

' + : snapshot.running + .map( + (row) => ` + + +
+ ${escapeHtml(row.issue_identifier)} + JSON details +
+ + + ${escapeHtml(row.state)} + + +
+ ${ + row.session_id === null + ? 'n/a' + : `` + } +
+ + ${formatRuntimeAndTurns( + row.started_at, + row.turn_count, + snapshot.generated_at, + )} + +
+ ${escapeHtml( + row.last_message ?? row.last_event ?? "n/a", + )} + ${escapeHtml( + row.last_event ?? "n/a", + )}${ + row.last_event_at === null + ? "" + : ` · ${escapeHtml( + row.last_event_at, + )}` + } +
+ + +
+ Total: ${formatInteger(row.tokens.total_tokens)} + In ${formatInteger( + row.tokens.input_tokens, + )} / Out ${formatInteger(row.tokens.output_tokens)} +
+ + `, + ) + .join(""); +} + +function renderRetryRows(snapshot: RuntimeSnapshot): string { + return snapshot.retrying.length === 0 + ? '

No issues are currently backing off.

' + : snapshot.retrying + .map( + (row) => ` + + +
+ ${escapeHtml(row.issue_identifier ?? row.issue_id)} + JSON details +
+ + ${row.attempt} + ${escapeHtml(row.due_at)} + ${escapeHtml(row.error ?? "n/a")} + `, + ) + .join(""); +} diff --git a/src/observability/dashboard-server.ts b/src/observability/dashboard-server.ts index 43866d94..bf53c9fd 100644 --- a/src/observability/dashboard-server.ts +++ b/src/observability/dashboard-server.ts @@ -11,6 +11,20 @@ import { } from "../config/defaults.js"; import { ERROR_CODES } from "../errors/codes.js"; import type { RuntimeSnapshot } from "../logging/runtime-snapshot.js"; +import { toErrorMessage } from "./dashboard-format.js"; +import { + isSnapshotTimeoutError, + readRequestBody, + readSnapshot, + writeHtml, + writeJson, + writeNotFound, +} from "./dashboard-http.js"; +import { DashboardLiveUpdatesController } from "./dashboard-live-updates.js"; +import { + type DashboardRenderOptions, + renderDashboardHtml, +} from "./dashboard-render.js"; const DEFAULT_SNAPSHOT_TIMEOUT_MS = 1_000; @@ -96,146 +110,6 @@ export interface DashboardServerInstance { close(): Promise; } -interface DashboardRenderOptions { - liveUpdatesEnabled: boolean; -} - -class DashboardLiveUpdatesController { - readonly #host: DashboardServerHost; - readonly #snapshotTimeoutMs: number; - readonly #refreshMs: number; - readonly #renderIntervalMs: number; - readonly #clients = new Set>(); - #flushTimer: NodeJS.Timeout | null = null; - #heartbeatTimer: NodeJS.Timeout | null = null; - #unsubscribeHost: (() => void) | null = null; - #closed = false; - - constructor(options: { - host: DashboardServerHost; - snapshotTimeoutMs: number; - refreshMs: number; - renderIntervalMs: number; - }) { - this.#host = options.host; - this.#snapshotTimeoutMs = options.snapshotTimeoutMs; - this.#refreshMs = options.refreshMs; - this.#renderIntervalMs = options.renderIntervalMs; - } - - start(): void { - if (typeof this.#host.subscribeToSnapshots === "function") { - this.#unsubscribeHost = this.#host.subscribeToSnapshots(() => { - this.scheduleBroadcast(); - }); - } - } - - async close(): Promise { - this.#closed = true; - this.#unsubscribeHost?.(); - this.#unsubscribeHost = null; - this.clearTimers(); - - for (const client of this.#clients) { - client.end(); - } - this.#clients.clear(); - } - - async handleEventsRequest( - request: IncomingMessage, - response: ServerResponse, - ): Promise { - response.statusCode = 200; - response.setHeader("content-type", "text/event-stream; charset=utf-8"); - response.setHeader("cache-control", "no-cache, no-transform"); - response.setHeader("connection", "keep-alive"); - response.setHeader("x-accel-buffering", "no"); - response.write(`retry: ${this.#refreshMs}\n\n`); - - this.#clients.add(response); - this.startHeartbeat(); - - const cleanup = () => { - this.#clients.delete(response); - if (this.#clients.size === 0) { - this.stopHeartbeat(); - } - }; - - request.on("close", cleanup); - response.on("close", cleanup); - - await this.writeSnapshot(response); - } - - scheduleBroadcast(): void { - if (this.#closed || this.#clients.size === 0 || this.#flushTimer !== null) { - return; - } - - this.#flushTimer = setTimeout(() => { - this.#flushTimer = null; - void this.broadcastSnapshot(); - }, this.#renderIntervalMs); - } - - private startHeartbeat(): void { - if (this.#heartbeatTimer !== null) { - return; - } - - this.#heartbeatTimer = setInterval(() => { - this.scheduleBroadcast(); - }, this.#refreshMs); - } - - private stopHeartbeat(): void { - if (this.#heartbeatTimer === null) { - return; - } - - clearInterval(this.#heartbeatTimer); - this.#heartbeatTimer = null; - } - - private clearTimers(): void { - if (this.#flushTimer !== null) { - clearTimeout(this.#flushTimer); - this.#flushTimer = null; - } - this.stopHeartbeat(); - } - - private async broadcastSnapshot(): Promise { - const clients = [...this.#clients]; - if (clients.length === 0) { - return; - } - - await Promise.allSettled( - clients.map((client) => this.writeSnapshot(client)), - ); - } - - private async writeSnapshot(response: ServerResponse): Promise { - try { - const snapshot = await readSnapshot(this.#host, this.#snapshotTimeoutMs); - response.write(`event: snapshot\ndata: ${JSON.stringify(snapshot)}\n\n`); - } catch (error) { - response.write( - `event: error\ndata: ${JSON.stringify({ - code: isSnapshotTimeoutError(error) - ? ERROR_CODES.snapshotTimedOut - : ERROR_CODES.snapshotUnavailable, - message: toErrorMessage(error), - })}\n\n`, - ); - } - } -} - export function createDashboardServer(options: DashboardServerOptions): Server { const hostname = options.hostname ?? "127.0.0.1"; const snapshotTimeoutMs = @@ -414,27 +288,6 @@ export function createDashboardRequestHandler( }; } -async function readSnapshot( - host: DashboardServerHost, - timeoutMs: number, -): Promise { - return await withTimeout(host.getRuntimeSnapshot(), timeoutMs, () => { - return new Error(`Runtime snapshot timed out after ${timeoutMs}ms.`); - }); -} - -function writeJson( - response: ServerResponse, - statusCode: number, - payload: unknown, -): void { - const body = JSON.stringify(payload); - response.statusCode = statusCode; - response.setHeader("content-type", "application/json; charset=utf-8"); - response.setHeader("content-length", Buffer.byteLength(body)); - response.end(body); -} - function writeJsonError( response: ServerResponse, statusCode: number, @@ -465,900 +318,3 @@ function writeMethodNotAllowed( allow, }); } - -function writeHtml( - response: ServerResponse, - statusCode: number, - html: string, -): void { - response.statusCode = statusCode; - response.setHeader("content-type", "text/html; charset=utf-8"); - response.setHeader("content-length", Buffer.byteLength(html)); - response.end(html); -} - -function writeNotFound(response: ServerResponse, path: string): void { - response.statusCode = 404; - response.setHeader("content-type", "text/plain; charset=utf-8"); - response.end(`Not found: ${path}`); -} - -async function readRequestBody(request: IncomingMessage): Promise { - await new Promise((resolve, reject) => { - request.on("error", reject); - request.on("end", resolve); - request.resume(); - }); -} - -async function withTimeout( - promise: Promise | T, - timeoutMs: number, - createError: () => Error, -): Promise { - return await new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - reject(createError()); - }, timeoutMs); - - Promise.resolve(promise).then( - (value) => { - clearTimeout(timeout); - resolve(value); - }, - (error) => { - clearTimeout(timeout); - reject(error); - }, - ); - }); -} - -function isSnapshotTimeoutError(error: unknown): boolean { - return ( - error instanceof Error && - error.message.startsWith("Runtime snapshot timed out after ") - ); -} - -function renderDashboardHtml( - snapshot: RuntimeSnapshot, - options: DashboardRenderOptions, -): string { - const initialRuntimeLabel = formatRuntimeSeconds( - snapshot.codex_totals.seconds_running, - ); - const totalTokensLabel = formatInteger(snapshot.codex_totals.total_tokens); - const inputTokensLabel = formatInteger(snapshot.codex_totals.input_tokens); - const outputTokensLabel = formatInteger(snapshot.codex_totals.output_tokens); - const initialRateLimits = prettyValue(snapshot.rate_limits); - - return ` - - - - - Symphony Observability - - - -
-
-
-
-
-

Symphony Observability

-

Operations Dashboard

-

- Current state, retry pressure, token usage, and orchestration health for the active Symphony runtime. -

-
- -
- - - ${options.liveUpdatesEnabled ? "Live" : "Offline"} - -
-
-
- -
-
-

Running

-

${snapshot.counts.running}

-

Active issue sessions in the current runtime.

-
- -
-

Retrying

-

${snapshot.counts.retrying}

-

Issues waiting for the next retry window.

-
- -
-

Total tokens

-

${totalTokensLabel}

-

In ${inputTokensLabel} / Out ${outputTokensLabel}

-
- -
-

Runtime

-

${initialRuntimeLabel}

-

Generated at ${escapeHtml(snapshot.generated_at)}

-
-
- -
-
-
-

Rate limits

-

Latest upstream rate-limit snapshot, when available.

-
-
- - 
${escapeHtml(initialRateLimits)}
-
- -
-
-
-

Running sessions

-

Active issues, last known agent activity, and token usage.

-
-
- -
- - - - - - - - - - - - - - - - - - - - ${renderRunningRows(snapshot)} -
IssueStateSessionRuntime / turnsCodex updateTokens
-
-
- -
-
-
-

Retry queue

-

Issues waiting for the next retry window.

-
-
- -
- - - - - - - - - - ${renderRetryRows(snapshot)} -
IssueAttemptDue atError
-
-
-
-
- - -`; -} - -function renderRunningRows(snapshot: RuntimeSnapshot): string { - return snapshot.running.length === 0 - ? '

No active sessions.

' - : snapshot.running - .map( - (row) => ` - - -
- ${escapeHtml(row.issue_identifier)} - JSON details -
- - - ${escapeHtml(row.state)} - - -
- ${ - row.session_id === null - ? 'n/a' - : `` - } -
- - ${formatRuntimeAndTurns( - row.started_at, - row.turn_count, - snapshot.generated_at, - )} - -
- ${escapeHtml( - row.last_message ?? row.last_event ?? "n/a", - )} - ${escapeHtml( - row.last_event ?? "n/a", - )}${ - row.last_event_at === null - ? "" - : ` · ${escapeHtml( - row.last_event_at, - )}` - } -
- - -
- Total: ${formatInteger(row.tokens.total_tokens)} - In ${formatInteger( - row.tokens.input_tokens, - )} / Out ${formatInteger(row.tokens.output_tokens)} -
- - `, - ) - .join(""); -} - -function renderRetryRows(snapshot: RuntimeSnapshot): string { - return snapshot.retrying.length === 0 - ? '

No issues are currently backing off.

' - : snapshot.retrying - .map( - (row) => ` - - -
- ${escapeHtml(row.issue_identifier ?? row.issue_id)} - JSON details -
- - ${row.attempt} - ${escapeHtml(row.due_at)} - ${escapeHtml(row.error ?? "n/a")} - `, - ) - .join(""); -} - -function formatRuntimeAndTurns( - startedAt: string, - turnCount: number, - generatedAt: string, -): string { - const runtime = formatRuntimeSeconds( - runtimeSecondsFromStartedAt(startedAt, generatedAt), - ); - return Number.isInteger(turnCount) && turnCount > 0 - ? `${runtime} / ${turnCount}` - : runtime; -} - -function formatRuntimeSeconds(seconds: number): string { - if (!Number.isFinite(seconds) || seconds < 0) { - return "0m 0s"; - } - const wholeSeconds = Math.max(0, Math.trunc(seconds)); - const mins = Math.floor(wholeSeconds / 60); - const secs = wholeSeconds % 60; - return `${mins}m ${secs}s`; -} - -function runtimeSecondsFromStartedAt( - startedAt: string, - generatedAt: string, -): number { - const start = Date.parse(startedAt); - const generated = Date.parse(generatedAt); - if ( - !Number.isFinite(start) || - !Number.isFinite(generated) || - generated < start - ) { - return 0; - } - return (generated - start) / 1000; -} - -function formatInteger(value: number): string { - return Number.isFinite(value) - ? Math.trunc(value).toLocaleString("en-US") - : "n/a"; -} - -function prettyValue(value: unknown): string { - return value === null || value === undefined - ? "n/a" - : JSON.stringify(value, null, 2); -} - -function stateBadgeClass(state: string): string { - const normalized = state.toLowerCase(); - if ( - normalized.includes("progress") || - normalized.includes("running") || - normalized.includes("active") - ) { - return "state-badge state-badge-active"; - } - if ( - normalized.includes("blocked") || - normalized.includes("error") || - normalized.includes("failed") - ) { - return "state-badge state-badge-danger"; - } - if ( - normalized.includes("todo") || - normalized.includes("queued") || - normalized.includes("pending") || - normalized.includes("retry") - ) { - return "state-badge state-badge-warning"; - } - return "state-badge"; -} - -function escapeHtml(value: string | number): string { - return String(value) - .replaceAll("&", "&") - .replaceAll("<", "<") - .replaceAll(">", ">") - .replaceAll('"', """) - .replaceAll("'", "'"); -} - -function toErrorMessage(error: unknown): string { - if (error instanceof Error) { - return error.message; - } - - return String(error); -}