From 57dd455d5a7f68e6e763af8e0e97f95ca6cb12e9 Mon Sep 17 00:00:00 2001 From: Danathar Date: Thu, 19 Nov 2026 01:49:17 +0000 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20metrics:=20authenticated=20GET=20/m?= =?UTF-8?q?etrics=20for=20tool=20calls,=20upstreams=20and=20adapters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a dependency-free Prometheus text endpoint on the existing loopback listener, behind the same bearer as /mcp: - quorum_tool_calls_total{tool,outcome} and quorum_tool_call_duration_seconds{tool}; tool is bounded to the contract list ("unknown" otherwise) - quorum_upstream_requests_total{service,outcome} and quorum_upstream_duration_seconds{service} for transcribe, tts and vexa - quorum_adapter_connected{adapter} for discord, vexa and companion No exporter, collector or off-box data flow. Documented in docs/contract.md. Hive-Run: hivecommons/quorum#54 Hive-Plan: issue #54 recommendation, bounded metrics and /metrics endpoint Hive-Spec: docs/contract.md#metrics Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Danathar --- README.md | 2 +- docs/contract.md | 16 ++- src/app.ts | 25 ++++- src/companion/adapter.ts | 5 + src/companion/index.ts | 4 +- src/discord/adapter.ts | 5 + src/discord/index.ts | 4 +- src/discord/transcribe.ts | 7 +- src/discord/tts.ts | 9 +- src/http.ts | 18 ++- src/mcp.ts | 14 ++- src/metrics.ts | 186 ++++++++++++++++++++++++++++++ src/vexa/client.ts | 9 +- test/metrics.test.ts | 230 ++++++++++++++++++++++++++++++++++++++ 14 files changed, 517 insertions(+), 17 deletions(-) create mode 100644 src/metrics.ts create mode 100644 test/metrics.test.ts diff --git a/README.md b/README.md index d95e616..be886f4 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ The interface hive, quorum and spektacular build against is [docs/contract.md](d Quorum is a child process inside the hive container, like the `spektacular` binary. It listens on loopback only; no extra container or pod change is needed. 1. Hive writes a fresh random bearer token to a file at boot, and the tenant's Vexa API key (and, if used, the Discord bot token) from its keystore to files. -2. Hive starts `node quorum.mjs serve` with the `QUORUM_*` variables below and supervises it. `GET /healthz` answers `{"ok":true,...}` once it is listening. +2. Hive starts `node quorum.mjs serve` with the `QUORUM_*` variables below and supervises it. `GET /healthz` answers `{"ok":true,...}` once it is listening. `GET /metrics` (same bearer) serves Prometheus metrics for tool calls, upstream services and adapter connectivity; see [docs/contract.md](docs/contract.md#metrics). 3. Hive is the MCP client: `POST http://127.0.0.1:8765/mcp` with `Authorization: Bearer ` (streamable HTTP, stateless, JSON responses). Tools: `meeting.join`, `meeting.leave`, `meeting.status`, `transcript.read`, `chat.post`, `say`, `say.tone` (the soft listening chime: played by the Vexa bot, by quorum itself on Discord, and by the desktop companion on the host's computer), `say.prewarm`, `meeting.delete`. Every result is `{ok:true,...}` or `{ok:false,error:{code,message}}` in `structuredContent` and as JSON text. 4. Hive follows each meeting on `GET /events?meeting_id=` (Server-Sent Events, same bearer): `segment`, `chat` and `state` events, one JSON object per `data:` line, `id:` = `seq`. Reconnect with `after_seq=` to replay what was missed. `GET /events` without `meeting_id` carries every meeting, including Discord `/hive join` requests. 5. On SIGTERM quorum leaves Discord channels and exits. Vexa bots keep running until hive calls `meeting.leave`; a restarted quorum that is asked to join the same meeting adopts the running bot. diff --git a/docs/contract.md b/docs/contract.md index c01db2d..0adcc75 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -7,7 +7,7 @@ Three boundaries, versioned together. Every component in hive, quorum and spekta Quorum runs **inside the hive container** as a child process that hive starts and supervises, the same way hive runs the `spektacular` binary. It listens on loopback only. No extra container, no pod spec change, no hub change. - Binary: `quorum serve` (Node 22, published as `@hivecommons/quorum` and baked into the hive image). -- Listen: `QUORUM_LISTEN` (default `127.0.0.1:8765`). MCP endpoint `POST /mcp` (streamable HTTP). Health `GET /healthz`. +- Listen: `QUORUM_LISTEN` (default `127.0.0.1:8765`). MCP endpoint `POST /mcp` (streamable HTTP). Health `GET /healthz`. Metrics `GET /metrics` (bearer, Prometheus text; see Metrics below). - Auth: bearer token from `QUORUM_TOKEN_FILE`, generated by hive per boot and passed to the child. - Vexa: `QUORUM_VEXA_URL` (gateway base URL), `QUORUM_VEXA_API_KEY_FILE` (tenant key; hive writes it from its keystore). - Discord: `QUORUM_DISCORD_TOKEN_FILE` (per-hive bot token, optional; Discord tools return `platform_unavailable` without it). @@ -90,3 +90,17 @@ Behaviour targets: act on final segments only; ask at pauses longer than `SPEK_M - `GET /api/meetings/{id}/ws` live panel: transcript, intents, research status. - Research: each `research` intent becomes a kick to the research agent pool with a kick ID; `[FINDING]` and `[COMPLETE]` signals for that kick come back as `research_update` to Spek and a `chat.post` summary (max one per research item) plus a knowledge fact under the meeting's notes. - On join hive posts the consent notice (`meetings.consent_text`) before anything else. `hive leave` typed in chat or said to hive makes it leave. + +## Metrics + +`GET /metrics` serves Prometheus text format on the same loopback listener, with the same bearer as `/mcp` and `/events` (401 without it). Nothing is pushed anywhere; a scraper on the host pulls it. Label values come from fixed sets, never from caller input: no meeting ids, user ids, URLs or error strings. + +| Metric | Type | Labels | +|---|---|---| +| `quorum_tool_calls_total` | counter | `tool` (a tool in section 1; any other name is `unknown`), `outcome` (`ok`, `error`) | +| `quorum_tool_call_duration_seconds` | histogram | `tool` | +| `quorum_upstream_requests_total` | counter | `service` (`transcribe`, `tts`, `vexa`), `outcome` (`ok` for a 2xx answer, `error` otherwise, timeouts and unreachable included) | +| `quorum_upstream_duration_seconds` | histogram | `service` | +| `quorum_adapter_connected` | gauge | `adapter` (`discord`, `vexa`, `companion`): 1 when the adapter can serve meetings | + +Each HTTP request counts, so a transcription call retried three times is three requests. `quorum_adapter_connected` is read at scrape time: Discord is 1 once the gateway login finished; Vexa is 1 while its websocket is open or while no meeting is followed (the socket only exists for live meetings); the companion is 1 when `QUORUM_TRANSCRIBE_URL` is set. An adapter that is not configured has no series. Counters reset when quorum restarts. diff --git a/src/app.ts b/src/app.ts index c453b3b..2e33c68 100644 --- a/src/app.ts +++ b/src/app.ts @@ -4,6 +4,7 @@ import type { PlatformAdapter } from "./adapter.js"; import { readSecretFile, type Config } from "./config.js"; import { createHttpServer } from "./http.js"; import type { Logger } from "./log.js"; +import { Metrics } from "./metrics.js"; import { Registry, type Platform } from "./meeting.js"; import { Quorum } from "./service.js"; import { companionAdapter } from "./companion/index.js"; @@ -28,7 +29,12 @@ export type AdapterFactory = (ctx: { config: Config; registry: Registry; log: Logger; -}) => Promise Promise }>>>; + metrics?: Metrics; +}) => Promise< + Partial< + Record Promise; connected?: () => boolean }> + > +>; export async function startQuorum( config: Config, @@ -44,6 +50,7 @@ export async function startQuorum( }, ); } + const metrics = new Metrics(); const registry = new Registry(config.meetingRetentionS * MS_PER_S); const adapters: Partial Promise }>> = {}; let stream: VexaStream | undefined; @@ -54,6 +61,7 @@ export async function startQuorum( baseUrl: config.vexaUrl, apiKey, timeoutMs: config.vexaTimeoutMs, + metrics, }); stream = new VexaStream({ baseUrl: config.vexaUrl, @@ -78,17 +86,26 @@ export async function startQuorum( adapters.google_meet = vexa; adapters.teams = vexa; adapters.zoom = vexa; + // The gateway socket only exists while a meeting is followed, so idle counts as connected. + const followed = stream; + metrics.adapter("vexa", () => followed.connected || followed.subscriptions === 0); } else { log.info("QUORUM_VEXA_URL is unset: Meet, Teams and Zoom answer platform_unavailable"); } - const companion = companionAdapter(config, log); + const companion = companionAdapter(config, log, metrics); adapters.companion = companion; + metrics.adapter("companion", () => companion.connected()); if (extraAdapters) { - Object.assign(adapters, await extraAdapters({ config, registry, log })); + const extra = await extraAdapters({ config, registry, log, metrics }); + Object.assign(adapters, extra); + const discord = extra.discord; + if (discord?.connected) { + metrics.adapter("discord", () => discord.connected?.() ?? false); + } } const quorum = new Quorum({ config, registry, adapters, log }); - const server = createHttpServer({ quorum, config, log, token, companion }); + const server = createHttpServer({ quorum, config, log, token, companion, metrics }); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(config.port, config.host, () => { diff --git a/src/companion/adapter.ts b/src/companion/adapter.ts index 58fba06..c7fbe8b 100644 --- a/src/companion/adapter.ts +++ b/src/companion/adapter.ts @@ -157,6 +157,11 @@ export class CompanionAdapter implements PlatformAdapter { this.now = opts.now ?? Date.now; } + /** For quorum_adapter_connected: whether a join can succeed (audio can be transcribed). */ + connected(): boolean { + return this.opts.transcriber !== undefined; + } + async join(meeting: Meeting, args: JoinArgs): Promise { if (!this.opts.transcriber) { throw new QuorumError( diff --git a/src/companion/index.ts b/src/companion/index.ts index bb39b6e..8979800 100644 --- a/src/companion/index.ts +++ b/src/companion/index.ts @@ -1,13 +1,14 @@ import { readSecretFile, type Config } from "../config.js"; import { Transcriber } from "../discord/transcribe.js"; import type { Logger } from "../log.js"; +import type { Metrics } from "../metrics.js"; import { CompanionAdapter } from "./adapter.js"; /** * The companion platform is always registered: it only listens on quorum's own loopback HTTP * surface. Without QUORUM_TRANSCRIBE_URL a companion join answers platform_unavailable. */ -export function companionAdapter(config: Config, log: Logger): CompanionAdapter { +export function companionAdapter(config: Config, log: Logger, metrics?: Metrics): CompanionAdapter { const transcriber = config.transcribeUrl ? new Transcriber({ url: config.transcribeUrl, @@ -17,6 +18,7 @@ export function companionAdapter(config: Config, log: Logger): CompanionAdapter : undefined, ...(config.transcribeLanguage ? { language: config.transcribeLanguage } : {}), timeoutMs: config.transcribeTimeoutMs, + ...(metrics ? { metrics } : {}), hotwords: config.transcribeHotwords, retry: { maxAttempts: config.transcribeMaxAttempts, diff --git a/src/discord/adapter.ts b/src/discord/adapter.ts index 5886f8a..564ad36 100644 --- a/src/discord/adapter.ts +++ b/src/discord/adapter.ts @@ -153,6 +153,11 @@ export class DiscordAdapter implements PlatformAdapter { return this.ready; } + /** For quorum_adapter_connected: the gateway login finished. */ + connected(): boolean { + return this.ready; + } + /** * Logs in and registers the /hive command. Resolves once the client is ready. Typed chat needs * the privileged MessageContent intent; when Discord refuses it, quorum logs that once and logs diff --git a/src/discord/index.ts b/src/discord/index.ts index 8bb4c9d..dca0b72 100644 --- a/src/discord/index.ts +++ b/src/discord/index.ts @@ -13,7 +13,7 @@ import type { DiscordRuntime } from "./types.js"; export function discordAdapters( load: () => Promise = loadDiscordRuntime, ): AdapterFactory { - return async ({ config, registry, log }) => { + return async ({ config, registry, log, metrics }) => { if (!config.discordTokenFile) { return {}; } @@ -35,6 +35,7 @@ export function discordAdapters( : undefined, ...(config.transcribeLanguage ? { language: config.transcribeLanguage } : {}), timeoutMs: config.transcribeTimeoutMs, + ...(metrics ? { metrics } : {}), hotwords: config.transcribeHotwords, retry: { maxAttempts: config.transcribeMaxAttempts, @@ -51,6 +52,7 @@ export function discordAdapters( ? readSecretFile(config.ttsTokenFile, "QUORUM_TTS_TOKEN_FILE") : undefined, timeoutMs: config.ttsTimeoutMs, + ...(metrics ? { metrics } : {}), }) : undefined; const adapter = new DiscordAdapter({ runtime, registry, log, config, transcriber, tts }); diff --git a/src/discord/transcribe.ts b/src/discord/transcribe.ts index f5672b1..917984e 100644 --- a/src/discord/transcribe.ts +++ b/src/discord/transcribe.ts @@ -1,4 +1,5 @@ import { QuorumError } from "../errors.js"; +import { timedUpstream, type Metrics } from "../metrics.js"; /** * Client for the shared CPU transcription service (hivecommons/vexa @@ -48,6 +49,8 @@ export interface TranscriberOptions { /** Vocabulary for every request (QUORUM_TRANSCRIBE_HOTWORDS), after the per-call hints. */ hotwords?: string[]; retry?: Partial; + /** Records each request to the service in quorum_upstream_* (service="transcribe"). */ + metrics?: Metrics; fetchImpl?: typeof fetch; sleep?: (ms: number) => Promise; /** Uniform [0, 1); the jitter source. */ @@ -178,7 +181,9 @@ export class Transcriber { const maxAttempts = Math.max(1, Math.floor(call.maxAttempts ?? this.retry.maxAttempts)); for (let attempt = 1; ; attempt++) { try { - return await this.once(wav, prompt, hotwords); + return await timedUpstream(this.opts.metrics, "transcribe", () => + this.once(wav, prompt, hotwords), + ); } catch (err) { if (!(err instanceof Retryable)) { throw err; diff --git a/src/discord/tts.ts b/src/discord/tts.ts index 4e7bdcf..8eb82a0 100644 --- a/src/discord/tts.ts +++ b/src/discord/tts.ts @@ -1,4 +1,5 @@ import { QuorumError } from "../errors.js"; +import { timedUpstream, type Metrics } from "../metrics.js"; /** * Client for the shared Kokoro TTS service: OpenAI-compatible `POST /v1/audio/speech` with @@ -16,6 +17,8 @@ export interface TtsClientOptions { url: string; token: () => string | undefined; timeoutMs: number; + /** Records each request to the service in quorum_upstream_* (service="tts"). */ + metrics?: Metrics; fetchImpl?: typeof fetch; } @@ -32,7 +35,11 @@ export class TtsClient { } /** Synthesizes `text` and returns 24 kHz s16le mono PCM. Failures are `upstream_error`. */ - async synthesize(text: string, voice: string): Promise { + synthesize(text: string, voice: string): Promise { + return timedUpstream(this.opts.metrics, "tts", () => this.request(text, voice)); + } + + private async request(text: string, voice: string): Promise { const headers: Record = { "Content-Type": "application/json" }; const token = this.opts.token(); if (token) { diff --git a/src/http.ts b/src/http.ts index 77c709c..a32bcb2 100644 --- a/src/http.ts +++ b/src/http.ts @@ -5,6 +5,7 @@ import { ChunkGapError, isCompanionChannel, type CompanionAdapter } from "./comp import type { Config } from "./config.js"; import { QuorumError } from "./errors.js"; import type { Logger } from "./log.js"; +import type { Metrics } from "./metrics.js"; import type { QuorumEvent, StateEvent } from "./meeting.js"; import { createMcpServer } from "./mcp.js"; import type { Quorum } from "./service.js"; @@ -12,7 +13,8 @@ import { CONTRACT_VERSION, VERSION } from "./version.js"; /** * The loopback HTTP surface: `GET /healthz` (no auth), `POST /mcp` (MCP streamable HTTP, - * stateless, JSON responses), `GET /events` (Server-Sent Events), and the companion platform's + * stateless, JSON responses), `GET /events` (Server-Sent Events), `GET /metrics` (Prometheus + * text), and the companion platform's * `POST /companion/audio` and `GET /companion/outbox`. Everything except /healthz needs * `Authorization: Bearer `. */ @@ -89,10 +91,11 @@ export interface HttpOptions { log: Logger; token: string; companion?: CompanionAdapter; + metrics?: Metrics; } export function createHttpServer(opts: HttpOptions): Server { - const { quorum, config, log, token, companion } = opts; + const { quorum, config, log, token, companion, metrics } = opts; async function handleMcp(req: IncomingMessage, res: ServerResponse): Promise { if (req.method !== "POST") { @@ -120,7 +123,7 @@ export function createHttpServer(opts: HttpOptions): Server { }); return; } - const server = createMcpServer(quorum, log); + const server = createMcpServer(quorum, log, metrics); const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined, enableJsonResponse: true, @@ -320,6 +323,15 @@ export function createHttpServer(opts: HttpOptions): Server { }); return; } + if (metrics && url.pathname === "/metrics") { + if (req.method !== "GET") { + sendJson(res, HTTP_METHOD_NOT_ALLOWED, { error: "GET only" }, { Allow: "GET" }); + return; + } + res.writeHead(HTTP_OK, { "Content-Type": "text/plain; version=0.0.4; charset=utf-8" }); + res.end(metrics.render()); + return; + } if (url.pathname === "/events") { handleEvents(req, res, url); return; diff --git a/src/mcp.ts b/src/mcp.ts index 400edd4..28e6d6e 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -3,6 +3,7 @@ import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprot import { failure, type ToolResult } from "./errors.js"; import type { Logger } from "./log.js"; import { silentLogger } from "./log.js"; +import type { Metrics } from "./metrics.js"; import type { Quorum } from "./service.js"; import { VERSION } from "./version.js"; @@ -157,6 +158,11 @@ export function callTool(q: Quorum, name: string, args: unknown): Promise = new Set(TOOLS.map((t) => t.name)); +/** The tool name as a bounded label value: a contract tool, or "unknown" for anything else. */ +export function boundedToolName(name: string): string { + return TOOL_NAMES.has(name) ? name : "unknown"; +} + /** * One log line per tool call: tool, outcome, error code and duration. The tool name comes from * the client, so anything outside the contract list is logged as "unknown" to keep the value @@ -169,7 +175,7 @@ export function logToolCall( durationMs: number, ): void { const fields = { - tool: TOOL_NAMES.has(name) ? name : "unknown", + tool: boundedToolName(name), ok: result.ok, ...(result.ok ? {} : { error_code: result.error.code }), duration_ms: Math.round(durationMs), @@ -181,13 +187,15 @@ export function logToolCall( } } -export function createMcpServer(q: Quorum, log: Logger = silentLogger): Server { +export function createMcpServer(q: Quorum, log: Logger = silentLogger, metrics?: Metrics): Server { const server = new Server({ name: "quorum", version: VERSION }, { capabilities: { tools: {} } }); server.setRequestHandler(ListToolsRequestSchema, () => ({ tools: TOOLS.map((t) => ({ ...t })) })); server.setRequestHandler(CallToolRequestSchema, async (req) => { const started = performance.now(); const result = await callTool(q, req.params.name, req.params.arguments); - logToolCall(log, req.params.name, result, performance.now() - started); + const durationMs = performance.now() - started; + logToolCall(log, req.params.name, result, durationMs); + metrics?.toolCall(boundedToolName(req.params.name), result.ok ? "ok" : "error", durationMs); return { content: [{ type: "text", text: JSON.stringify(result) }], structuredContent: result as unknown as Record, diff --git a/src/metrics.ts b/src/metrics.ts new file mode 100644 index 0000000..79ccd54 --- /dev/null +++ b/src/metrics.ts @@ -0,0 +1,186 @@ +/** + * Dependency-free Prometheus metrics, served by `GET /metrics` (docs/contract.md). Every label + * value comes from a fixed set (the contract tool list, three upstream services, three adapters, + * ok/error), never from caller input: no meeting ids, user ids, URLs or error strings. + */ + +export type Outcome = "ok" | "error"; +export type UpstreamService = "transcribe" | "tts" | "vexa"; +export type AdapterName = "discord" | "vexa" | "companion"; + +/** Seconds; spans a cached local call to a transcription request that runs into its timeout. */ +export const DURATION_BUCKETS = [ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, +] as const; + +const MS_PER_S = 1000; + +type Labels = Record; + +interface Histogram { + counts: number[]; + sum: number; + count: number; +} + +function escapeLabel(value: string): string { + return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n"); +} + +function renderLabels(labels: Labels, extra?: Labels): string { + const all = { ...labels, ...extra }; + const parts = Object.entries(all).map(([k, v]) => `${k}="${escapeLabel(v)}"`); + return parts.length ? `{${parts.join(",")}}` : ""; +} + +/** Series key for a label set; label names are always passed in the same order per metric. */ +function keyOf(labels: Labels): string { + return Object.entries(labels) + .map(([k, v]) => `${k}\u0000${v}`) + .join("\u0001"); +} + +class CounterFamily { + private readonly series = new Map(); + + constructor( + private readonly name: string, + private readonly help: string, + ) {} + + inc(labels: Labels): void { + const k = keyOf(labels); + const s = this.series.get(k); + if (s) { + s.value += 1; + } else { + this.series.set(k, { labels, value: 1 }); + } + } + + render(): string[] { + const lines = [`# HELP ${this.name} ${this.help}`, `# TYPE ${this.name} counter`]; + for (const { labels, value } of this.series.values()) { + lines.push(`${this.name}${renderLabels(labels)} ${value}`); + } + return lines; + } +} + +class HistogramFamily { + private readonly series = new Map(); + + constructor( + private readonly name: string, + private readonly help: string, + ) {} + + observe(labels: Labels, value: number): void { + const k = keyOf(labels); + let s = this.series.get(k); + if (!s) { + s = { labels, h: { counts: DURATION_BUCKETS.map(() => 0), sum: 0, count: 0 } }; + this.series.set(k, s); + } + DURATION_BUCKETS.forEach((le, i) => { + if (value <= le) { + s.h.counts[i] = (s.h.counts[i] ?? 0) + 1; + } + }); + s.h.sum += value; + s.h.count += 1; + } + + render(): string[] { + const lines = [`# HELP ${this.name} ${this.help}`, `# TYPE ${this.name} histogram`]; + for (const { labels, h } of this.series.values()) { + DURATION_BUCKETS.forEach((le, i) => { + lines.push(`${this.name}_bucket${renderLabels(labels, { le: String(le) })} ${h.counts[i]}`); + }); + lines.push(`${this.name}_bucket${renderLabels(labels, { le: "+Inf" })} ${h.count}`); + lines.push(`${this.name}_sum${renderLabels(labels)} ${h.sum}`); + lines.push(`${this.name}_count${renderLabels(labels)} ${h.count}`); + } + return lines; + } +} + +export class Metrics { + private readonly toolCalls = new CounterFamily( + "quorum_tool_calls_total", + "MCP tool calls by contract tool and outcome.", + ); + private readonly toolDuration = new HistogramFamily( + "quorum_tool_call_duration_seconds", + "MCP tool call duration by contract tool.", + ); + private readonly upstreamRequests = new CounterFamily( + "quorum_upstream_requests_total", + "Requests to upstream services by service and outcome (ok: a 2xx answer).", + ); + private readonly upstreamDuration = new HistogramFamily( + "quorum_upstream_duration_seconds", + "Upstream request duration by service.", + ); + private readonly adapters = new Map boolean>(); + + /** `tool` must already be bounded to the contract list (see boundedToolName in mcp.ts). */ + toolCall(tool: string, outcome: Outcome, durationMs: number): void { + this.toolCalls.inc({ tool, outcome }); + this.toolDuration.observe({ tool }, durationMs / MS_PER_S); + } + + upstream(service: UpstreamService, outcome: Outcome, durationMs: number): void { + this.upstreamRequests.inc({ service, outcome }); + this.upstreamDuration.observe({ service }, durationMs / MS_PER_S); + } + + /** Registers an adapter's connectivity, read on every scrape. */ + adapter(name: AdapterName, connected: () => boolean): void { + this.adapters.set(name, connected); + } + + render(): string { + const lines = [ + ...this.toolCalls.render(), + ...this.toolDuration.render(), + ...this.upstreamRequests.render(), + ...this.upstreamDuration.render(), + "# HELP quorum_adapter_connected 1 when the adapter can serve meetings, 0 when it cannot.", + "# TYPE quorum_adapter_connected gauge", + ]; + for (const [adapter, connected] of this.adapters) { + let up: boolean; + try { + up = connected(); + } catch { + up = false; + } + lines.push(`quorum_adapter_connected${renderLabels({ adapter })} ${up ? 1 : 0}`); + } + return lines.join("\n") + "\n"; + } +} + +/** + * Runs one upstream request and records its duration. It counts as `ok` when `fn` resolves and as + * `error` when it throws (the error is rethrown unchanged). Without `metrics` it just runs `fn`. + */ +export async function timedUpstream( + metrics: Metrics | undefined, + service: UpstreamService, + fn: () => Promise, +): Promise { + if (!metrics) { + return fn(); + } + const started = performance.now(); + let outcome: Outcome = "error"; + try { + const result = await fn(); + outcome = "ok"; + return result; + } finally { + metrics.upstream(service, outcome, performance.now() - started); + } +} diff --git a/src/vexa/client.ts b/src/vexa/client.ts index ae1e3f2..f45a1fa 100644 --- a/src/vexa/client.ts +++ b/src/vexa/client.ts @@ -1,4 +1,5 @@ import { QuorumError, type ErrorCode } from "../errors.js"; +import { timedUpstream, type Metrics } from "../metrics.js"; /** * REST client for the Vexa gateway (hivecommons/vexa core/gateway/contracts/api.v1). Auth is the @@ -43,6 +44,8 @@ export interface VexaClientOptions { baseUrl: string; apiKey: () => string; timeoutMs: number; + /** Records each request to the gateway in quorum_upstream_* (service="vexa"). */ + metrics?: Metrics; fetchImpl?: typeof fetch; } @@ -113,7 +116,11 @@ export class VexaClient { return this.opts.baseUrl; } - async request( + request(method: string, path: string, body?: unknown): Promise<{ status: number; data: T }> { + return timedUpstream(this.opts.metrics, "vexa", () => this.send(method, path, body)); + } + + private async send( method: string, path: string, body?: unknown, diff --git a/test/metrics.test.ts b/test/metrics.test.ts new file mode 100644 index 0000000..1c2adb7 --- /dev/null +++ b/test/metrics.test.ts @@ -0,0 +1,230 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { TtsClient } from "../src/discord/tts.js"; +import { Transcriber } from "../src/discord/transcribe.js"; +import { TOOLS, boundedToolName } from "../src/mcp.js"; +import { DURATION_BUCKETS, Metrics, timedUpstream } from "../src/metrics.js"; +import { VexaClient } from "../src/vexa/client.js"; +import { TOKEN, startHarness, type Harness } from "./helpers/harness.js"; + +function reply(status: number, body: string): typeof fetch { + return (async () => new Response(body, { status })) as typeof fetch; +} + +const unreachable = (async () => { + throw new TypeError("fetch failed"); +}) as unknown as typeof fetch; + +describe("Metrics", () => { + it("renders counters, histograms and adapter gauges in Prometheus text", () => { + const m = new Metrics(); + m.toolCall("say", "ok", 120); + m.toolCall("say", "ok", 30); + m.toolCall("say", "error", 7000); + m.adapter("discord", () => true); + m.adapter("vexa", () => false); + const text = m.render(); + expect(text).toContain("# TYPE quorum_tool_calls_total counter"); + expect(text).toContain('quorum_tool_calls_total{tool="say",outcome="ok"} 2'); + expect(text).toContain('quorum_tool_calls_total{tool="say",outcome="error"} 1'); + expect(text).toContain("# TYPE quorum_tool_call_duration_seconds histogram"); + // 0.03 s and 0.12 s fall in the 0.05 and 0.25 buckets and above; 7 s only from le=10. + expect(text).toContain('quorum_tool_call_duration_seconds_bucket{tool="say",le="0.025"} 0'); + expect(text).toContain('quorum_tool_call_duration_seconds_bucket{tool="say",le="0.05"} 1'); + expect(text).toContain('quorum_tool_call_duration_seconds_bucket{tool="say",le="5"} 2'); + expect(text).toContain('quorum_tool_call_duration_seconds_bucket{tool="say",le="10"} 3'); + expect(text).toContain('quorum_tool_call_duration_seconds_bucket{tool="say",le="+Inf"} 3'); + expect(text).toContain('quorum_tool_call_duration_seconds_count{tool="say"} 3'); + expect(text).toContain('quorum_tool_call_duration_seconds_sum{tool="say"} 7.15'); + expect(text).toContain('quorum_adapter_connected{adapter="discord"} 1'); + expect(text).toContain('quorum_adapter_connected{adapter="vexa"} 0'); + expect(text.endsWith("\n")).toBe(true); + expect(DURATION_BUCKETS.length).toBeGreaterThan(0); + }); + + it("renders a family header even with no series yet", () => { + const text = new Metrics().render(); + expect(text).toContain("# TYPE quorum_upstream_requests_total counter"); + expect(text).toContain("# TYPE quorum_adapter_connected gauge"); + expect(text).not.toContain("quorum_adapter_connected{"); + }); + + it("escapes label values and treats a throwing adapter probe as down", () => { + const m = new Metrics(); + m.toolCall('a"b\\c\nd', "ok", 1); + m.adapter("companion", () => { + throw new Error("boom"); + }); + const text = m.render(); + expect(text).toContain('tool="a\\"b\\\\c\\nd"'); + expect(text).toContain('quorum_adapter_connected{adapter="companion"} 0'); + }); + + it("times an upstream call as ok or error, and rethrows", async () => { + const m = new Metrics(); + await expect(timedUpstream(m, "tts", async () => 5)).resolves.toBe(5); + await expect( + timedUpstream(m, "tts", async () => { + throw new Error("down"); + }), + ).rejects.toThrow("down"); + await expect(timedUpstream(undefined, "tts", async () => 1)).resolves.toBe(1); + const text = m.render(); + expect(text).toContain('quorum_upstream_requests_total{service="tts",outcome="ok"} 1'); + expect(text).toContain('quorum_upstream_requests_total{service="tts",outcome="error"} 1'); + expect(text).toContain('quorum_upstream_duration_seconds_count{service="tts"} 2'); + }); +}); + +describe("upstream clients record metrics", () => { + it("counts each transcription request, retries included", async () => { + const m = new Metrics(); + const statuses = [503, 200]; + const t = new Transcriber({ + url: "http://stt", + token: () => undefined, + timeoutMs: 1000, + metrics: m, + sleep: async () => undefined, + fetchImpl: (async () => { + const status = statuses.shift() ?? 200; + return new Response(JSON.stringify({ text: "hi" }), { status }); + }) as unknown as typeof fetch, + }); + await expect(t.transcribe(Buffer.alloc(4))).resolves.toBe("hi"); + const text = m.render(); + expect(text).toContain( + 'quorum_upstream_requests_total{service="transcribe",outcome="error"} 1', + ); + expect(text).toContain('quorum_upstream_requests_total{service="transcribe",outcome="ok"} 1'); + }); + + it("counts an unreachable transcription service as an error", async () => { + const m = new Metrics(); + const t = new Transcriber({ + url: "http://stt", + token: () => undefined, + timeoutMs: 100, + metrics: m, + retry: { maxAttempts: 1 }, + fetchImpl: unreachable, + }); + await expect(t.transcribe(Buffer.alloc(4))).rejects.toMatchObject({ code: "upstream_error" }); + expect(m.render()).toContain( + 'quorum_upstream_requests_total{service="transcribe",outcome="error"} 1', + ); + }); + + it("counts TTS requests by outcome", async () => { + const m = new Metrics(); + const opts = { url: "http://tts", token: () => undefined, timeoutMs: 100, metrics: m }; + await new TtsClient({ ...opts, fetchImpl: reply(200, "pcm") }).synthesize("x", "v"); + await expect( + new TtsClient({ ...opts, fetchImpl: reply(500, "") }).synthesize("x", "v"), + ).rejects.toMatchObject({ code: "upstream_error" }); + const text = m.render(); + expect(text).toContain('quorum_upstream_requests_total{service="tts",outcome="ok"} 1'); + expect(text).toContain('quorum_upstream_requests_total{service="tts",outcome="error"} 1'); + }); + + it("counts Vexa requests by outcome without putting paths in labels", async () => { + const m = new Metrics(); + const make = (f: typeof fetch): VexaClient => + new VexaClient({ + baseUrl: "http://vexa", + apiKey: () => "k", + timeoutMs: 100, + metrics: m, + fetchImpl: f, + }); + await make(reply(200, "{}")).request("GET", "/bots/zoom/123456789"); + await expect(make(reply(404, "{}")).request("GET", "/bots/zoom/123456789")).rejects.toThrow(); + await expect(make(unreachable).request("GET", "/bots")).rejects.toThrow(); + const text = m.render(); + expect(text).toContain('quorum_upstream_requests_total{service="vexa",outcome="ok"} 1'); + expect(text).toContain('quorum_upstream_requests_total{service="vexa",outcome="error"} 2'); + expect(text).not.toContain("123456789"); + }); +}); + +describe("GET /metrics", () => { + let h: Harness; + beforeEach(async () => { + h = await startHarness({}); + }); + afterEach(() => h.close()); + + const scrape = (auth: string | undefined): Promise => + fetch(h.q.url + "/metrics", auth ? { headers: { Authorization: auth } } : {}); + + it("rejects a missing or wrong bearer token", async () => { + const missing = await scrape(undefined); + expect(missing.status).toBe(401); + expect(missing.headers.get("www-authenticate")).toBe("Bearer"); + expect((await scrape("Bearer wrong")).status).toBe(401); + }); + + it("answers GET only", async () => { + const res = await fetch(h.q.url + "/metrics", { + method: "POST", + headers: { Authorization: `Bearer ${TOKEN}` }, + }); + expect(res.status).toBe(405); + expect(res.headers.get("allow")).toBe("GET"); + }); + + it("counts tool calls by contract tool and outcome", async () => { + await h.call("meeting.status", { meeting_id: "nope" }); + await h.call("meeting.status", { meeting_id: "nope" }); + await h.call("not.a.tool"); + const res = await scrape(`Bearer ${TOKEN}`); + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toContain("text/plain"); + const text = await res.text(); + expect(text).toContain('quorum_tool_calls_total{tool="meeting.status",outcome="error"} 2'); + expect(text).toContain('quorum_tool_calls_total{tool="unknown",outcome="error"} 1'); + expect(text).toContain('quorum_tool_call_duration_seconds_count{tool="meeting.status"} 2'); + expect(text).not.toContain("not.a.tool"); + expect(text).not.toContain("nope"); + }); + + it("keeps every tool label inside the contract tool list", async () => { + await h.call("meeting.status", { meeting_id: "x" }); + await h.call("x".repeat(300)); + await h.call("../../etc/passwd"); + const text = await (await scrape(`Bearer ${TOKEN}`)).text(); + const allowed = new Set([...TOOLS.map((t) => t.name), "unknown"]); + const tools = [...text.matchAll(/\btool="([^"]*)"/g)].map((m) => m[1] as string); + expect(tools.length).toBeGreaterThan(0); + for (const tool of tools) { + expect(allowed.has(tool)).toBe(true); + } + expect(boundedToolName("say")).toBe("say"); + expect(boundedToolName("anything else")).toBe("unknown"); + }); + + it("reports adapter connectivity", async () => { + // No QUORUM_TRANSCRIBE_URL: a companion join would answer platform_unavailable. + const text = await (await scrape(`Bearer ${TOKEN}`)).text(); + expect(text).toContain('quorum_adapter_connected{adapter="companion"} 0'); + expect(text).not.toContain('adapter="vexa"'); + expect(text).not.toContain('adapter="discord"'); + }); +}); + +describe("GET /metrics with upstreams configured", () => { + it("reports companion up with a transcription URL and vexa idle as connected", async () => { + const h = await startHarness({ + QUORUM_TRANSCRIBE_URL: "http://127.0.0.1:9", + QUORUM_VEXA_URL: "http://127.0.0.1:9", + }); + try { + const text = await ( + await fetch(h.q.url + "/metrics", { headers: { Authorization: `Bearer ${TOKEN}` } }) + ).text(); + expect(text).toContain('quorum_adapter_connected{adapter="companion"} 1'); + expect(text).toContain('quorum_adapter_connected{adapter="vexa"} 1'); + } finally { + await h.close(); + } + }); +});