diff --git a/.changeset/mcp-service-health-tool.md b/.changeset/mcp-service-health-tool.md new file mode 100644 index 0000000000..9df837c658 --- /dev/null +++ b/.changeset/mcp-service-health-tool.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/api': minor +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +--- + +feat(mcp): add `clickstack_service_health`, a one-call per-service health report for a trace source (throughput, error rate, p50/p95/p99, top error groups), compared with yesterday and ranked worst first diff --git a/MCP.md b/MCP.md index 132a7a9d79..a380fb8909 100644 --- a/MCP.md +++ b/MCP.md @@ -126,6 +126,7 @@ with: | `clickstack_delete_source` | Permanently delete a data source by ID | | `clickstack_list_metrics` | Paginated catalog of metric names on a metric source with optional kind / namePattern / time-window filters | | `clickstack_describe_metric` | Per-metric drill-down: kind(s), unit, description, attribute keys per map column, and sampled values | +| `clickstack_service_health` | One-call service health on a trace source: throughput, error rate, p50/p95 and top errors per service, compared with yesterday and ranked worst first | | `clickstack_timeseries` | Plot metrics over time as a line or stacked bar chart (works on log, trace, and metric sources) | | `clickstack_table` | Compute aggregated metrics as a table, single number, or pie chart (works on log, trace, and metric sources) | | `clickstack_search` | Browse individual log, event, or trace rows | diff --git a/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts b/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts new file mode 100644 index 0000000000..be604d69eb --- /dev/null +++ b/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts @@ -0,0 +1,295 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; + +import * as config from '@/config'; +import { + bulkInsertData, + DEFAULT_DATABASE, + DEFAULT_LOGS_TABLE, + DEFAULT_TRACES_TABLE, + getLoggedInAgent, + getServer, +} from '@/fixtures'; +import { McpContext } from '@/mcp/tools/types'; +import Connection from '@/models/connection'; +import { Source, type SourceDocument } from '@/models/source'; + +import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; + +const TOOL = 'clickstack_service_health'; +const HOUR = 60 * 60 * 1000; +// ClickHouse is shared across tests and files, so scope every query to this +// run's services. +const RUN = `sh${Date.now()}`; +const svc = (name: string) => `${RUN}-${name}`; + +type Span = { + ServiceName: string; + SpanKind?: string; + durationMs?: number; + error?: string; +}; + +type ServiceReport = { + service: string; + status: string; + flags: string[]; + current: { + requests: number; + errors: number; + errorRate: number; + p95Ms: number; + }; + comparison?: { requests: number; errors: number } | null; + topErrors: { + group: Record; + count: number; + comparisonCount?: number; + status?: string; + }[]; +}; + +describe('MCP Service Health Tool', () => { + const server = getServer(); + let client: Client; + let traceSource: SourceDocument; + let logSource: SourceDocument; + const now = Date.now(); + + async function insertSpans(windowStart: number, spans: Span[]) { + await bulkInsertData( + `${DEFAULT_DATABASE}.${DEFAULT_TRACES_TABLE}`, + spans.map((s, i) => ({ + // Spread across the first 50 minutes of the window. + Timestamp: new Date(windowStart + ((i * 30_000) % (50 * 60_000))), + TraceId: `trace-${windowStart}-${i}`, + SpanId: `span-${windowStart}-${i}`, + ParentSpanId: '', + SpanName: 'GET /api', + SpanKind: s.SpanKind ?? (i % 2 ? 'Server' : 'SPAN_KIND_SERVER'), + ServiceName: s.ServiceName, + Duration: (s.durationMs ?? 100) * 1e6, + // Both error spellings must count. + StatusCode: s.error ? (i % 2 ? 'Error' : 'STATUS_CODE_ERROR') : 'Ok', + StatusMessage: s.error ?? '', + SpanAttributes: {}, + ResourceAttributes: {}, + })), + ); + } + + const repeat = (n: number, s: Span): Span[] => + Array.from({ length: n }, () => s); + + beforeAll(async () => { + await server.start(); + }); + + beforeEach(async () => { + const { team, user } = await getLoggedInAgent(server); + const connection = await Connection.create({ + team: team._id, + name: 'Default', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + logSource = await Source.create({ + kind: SourceKind.Log, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: DEFAULT_LOGS_TABLE }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Logs', + }); + traceSource = await Source.create({ + kind: SourceKind.Trace, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: DEFAULT_TRACES_TABLE }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Traces', + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + durationExpression: 'Duration', + durationPrecision: 9, + serviceNameExpression: 'ServiceName', + statusCodeExpression: 'StatusCode', + statusMessageExpression: 'StatusMessage', + }); + const context: McpContext = { + teamId: team._id.toString(), + userId: user._id.toString(), + }; + client = await createTestClient(context); + }); + + afterEach(async () => { + await client.close(); + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + const call = (args: Record = {}) => + callTool(client, TOOL, { + sourceId: traceSource._id.toString(), + startTime: new Date(now - HOUR).toISOString(), + endTime: new Date(now).toISOString(), + where: `startsWith(ServiceName, '${RUN}')`, + whereLanguage: 'sql', + ...args, + }); + + const parse = (result: Awaited>) => { + expect(result.isError).toBeFalsy(); + const body: { + summary: Record; + services: ServiceReport[]; + } = JSON.parse(getFirstText(result)); + return body; + }; + + it('is listed with its input schema', async () => { + const { tools } = await client.listTools(); + const tool = tools.find(t => t.name === TOOL); + expect(tool?.inputSchema.required).toEqual(['sourceId']); + expect(Object.keys(tool?.inputSchema.properties ?? {})).toEqual( + expect.arrayContaining([ + 'comparison', + 'spanKind', + 'errorGroupBy', + 'topN', + ]), + ); + }); + + it('rejects non-trace sources', async () => { + const result = await call({ sourceId: logSource._id.toString() }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('kind="trace"'); + }); + + it('points a table name passed as sourceId at the right source ID', async () => { + const result = await call({ sourceId: DEFAULT_TRACES_TABLE }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain( + `sourceId:"${traceSource._id.toString()}"`, + ); + }); + + it('rejects a window longer than the comparison shift', async () => { + const result = await call({ + startTime: new Date(now - 30 * HOUR).toISOString(), + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toMatch(/overlap/); + }); + + describe('with data in both windows', () => { + beforeAll(async () => { + const currentStart = now - HOUR; + const yesterdayStart = currentStart - 24 * HOUR; + await insertSpans(currentStart, [ + ...repeat(99, { ServiceName: svc('steady') }), + ...repeat(1, { ServiceName: svc('steady'), error: 'flaky' }), + // Internal error spans must not count with the default spanKind. + ...repeat(100, { + ServiceName: svc('steady'), + SpanKind: 'Internal', + error: 'internal', + }), + ...repeat(80, { ServiceName: svc('erroring') }), + ...repeat(15, { ServiceName: svc('erroring'), error: 'db timeout' }), + ...repeat(5, { ServiceName: svc('erroring'), error: 'auth failed' }), + ...repeat(100, { ServiceName: svc('slow'), durationMs: 600 }), + ]); + await insertSpans(yesterdayStart, [ + ...repeat(99, { ServiceName: svc('steady') }), + ...repeat(1, { ServiceName: svc('steady'), error: 'flaky' }), + ...repeat(99, { ServiceName: svc('erroring') }), + ...repeat(1, { ServiceName: svc('erroring'), error: 'db timeout' }), + ...repeat(100, { ServiceName: svc('slow'), durationMs: 100 }), + ]); + }); + + it('ranks regressions first and leaves steady services healthy', async () => { + const { summary, services } = parse(await call()); + expect(services.map(s => [s.service, s.status])).toEqual([ + [svc('erroring'), 'critical'], + [svc('slow'), 'degraded'], + [svc('steady'), 'healthy'], + ]); + expect(services[0].flags).toEqual( + expect.arrayContaining(['error_rate_up', 'high_error_rate']), + ); + expect(services[1].flags).toContain('latency_up'); + expect(services[1].current.p95Ms).toBeCloseTo(600, -1); + expect(summary.statusCounts).toEqual({ + critical: 1, + degraded: 1, + healthy: 1, + }); + }); + + it('counts server spans only by default', async () => { + const { services } = parse(await call()); + const steady = services.find(s => s.service === svc('steady')); + expect(steady?.current.requests).toBe(100); + expect(steady?.current.errors).toBe(1); + + const all = parse(await call({ spanKind: 'all' })); + expect( + all.services.find(s => s.service === svc('steady'))?.current.requests, + ).toBe(200); + }); + + it('tags error groups as new or growing against the comparison', async () => { + const { services } = parse(await call()); + expect(services[0].topErrors).toEqual([ + expect.objectContaining({ + group: { SpanName: 'GET /api', StatusMessage: 'db timeout' }, + count: 15, + comparisonCount: 1, + status: 'growing', + }), + expect.objectContaining({ + group: { SpanName: 'GET /api', StatusMessage: 'auth failed' }, + count: 5, + comparisonCount: 0, + status: 'new', + }), + ]); + // Spans are spread over the first 50 minutes, so nothing is ongoing. + const first = services[0].topErrors[0] as Record; + expect(Date.parse(String(first.firstSeen))).toBeLessThan( + Date.parse(String(first.lastSeen)), + ); + expect(first.ongoing).toBe(false); + }); + + it('accepts a custom errorGroupBy', async () => { + const { services } = parse(await call({ errorGroupBy: 'StatusMessage' })); + expect(services[0].topErrors[0].group).toEqual({ + StatusMessage: 'db timeout', + }); + }); + + it('omits comparisons with comparison:none', async () => { + const { summary, services } = parse(await call({ comparison: 'none' })); + expect(summary.comparisonWindow).toBeNull(); + expect(services[0].comparison).toBeUndefined(); + expect(services[0].topErrors[0].status).toBeUndefined(); + }); + + it('filters to the requested services', async () => { + const { services } = parse(await call({ services: [svc('slow')] })); + expect(services.map(s => s.service)).toEqual([svc('slow')]); + }); + }); +}); diff --git a/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts b/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts new file mode 100644 index 0000000000..4a75cdec5b --- /dev/null +++ b/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts @@ -0,0 +1,141 @@ +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; + +import { ClickhouseClient } from '@/clickhouse'; +import { resolveComparison } from '@/mcp/tools/query/serviceHealth/comparison'; +import { findNewOperations } from '@/mcp/tools/query/serviceHealth/operations'; +import type { + HealthSource, + QueryContext, +} from '@/mcp/tools/query/serviceHealth/queries'; +import { fetchServiceStats } from '@/mcp/tools/query/serviceHealth/queries'; +import { + comparisonWindow, + serviceHealthSchema, +} from '@/mcp/tools/query/serviceHealth/schema'; + +jest.mock('@/mcp/tools/query/serviceHealth/queries', () => ({ + ...jest.requireActual('@/mcp/tools/query/serviceHealth/queries'), + fetchServiceStats: jest.fn(), +})); + +const fetchStats = jest.mocked(fetchServiceStats); + +const current = { + start: new Date('2026-10-16T10:00:00Z'), + end: new Date('2026-10-16T11:00:00Z'), +}; +const yesterday = comparisonWindow(current, 'yesterday'); +const stats = (services: string[]) => ({ + stats: new Map( + services.map(s => [ + s, + { requests: 100, errors: 0, p50Ms: 1, p95Ms: 2, p99Ms: 3 }, + ]), + ), +}); + +const source: HealthSource = { + id: 'src', + connection: 'conn', + from: { databaseName: 'db', tableName: 't' }, + timestampValueExpression: 'Timestamp', + serviceNameExpression: 'ServiceName', + statusCodeExpression: 'StatusCode', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + durationExpression: 'Duration', + durationPrecision: 9, + querySettings: undefined, +}; + +const ctxWith = (results: { data: Record[] }[]) => { + const client = new ClickhouseClient({ host: 'http://localhost:1' }); + const query = jest.spyOn(client, 'queryChartConfig'); + for (const r of results) query.mockResolvedValueOnce({ data: r.data }); + const ctx: QueryContext = { + client, + metadata: getMetadata(client), + source, + input: serviceHealthSchema.parse({ sourceId: 'src' }), + signal: new AbortController().signal, + }; + return ctx; +}; + +describe('resolveComparison', () => { + beforeEach(() => fetchStats.mockReset()); + + it('keeps the requested window when it has data', async () => { + const r = await resolveComparison(ctxWith([]), { + current, + requested: yesterday, + requestedMode: 'yesterday', + curHasData: true, + requestedCmp: stats(['a']), + }); + expect(r.comparisonMode).toBe('yesterday'); + expect(r.comparisonNote).toBeUndefined(); + expect(fetchStats).not.toHaveBeenCalled(); + }); + + it('falls back to the previous period when the requested window is empty', async () => { + fetchStats.mockResolvedValue(stats(['a'])); + const r = await resolveComparison(ctxWith([]), { + current, + requested: yesterday, + requestedMode: 'yesterday', + curHasData: true, + requestedCmp: stats([]), + }); + expect(r.comparisonMode).toBe('previous_period'); + expect(r.comparison?.end.toISOString()).toBe(current.start.toISOString()); + expect(r.comparisonNote).toMatch(/previous period/); + }); + + it('drops the comparison when the previous period is empty too', async () => { + fetchStats.mockResolvedValue(stats([])); + const r = await resolveComparison(ctxWith([]), { + current, + requested: yesterday, + requestedMode: 'yesterday', + curHasData: true, + requestedCmp: stats([]), + }); + expect(r).toMatchObject({ + comparison: null, + cmp: null, + comparisonMode: 'none', + }); + expect(r.comparisonNote).toMatch(/nothing is labelled new/); + }); +}); + +describe('findNewOperations', () => { + const row = (service: string, op: string, requests: number) => ({ + __hdx_service: service, + __hdx_operation: op, + requests, + }); + + it('reports operations with no prior traffic', async () => { + const ctx = ctxWith([ + { data: [row('svc', 'GET /old', 900), row('svc', 'GET /new', 100)] }, + { data: [row('svc', 'GET /old', 900)] }, + ]); + const r = await findNewOperations(ctx, current, yesterday!, ['svc'], 50); + expect(r.complete).toBe(true); + expect(r.newOperations.get('svc')).toEqual([ + { operation: 'GET /new', requests: 100, shareOfService: 0.1 }, + ]); + }); + + it('reports nothing when a result is truncated', async () => { + const many = Array.from({ length: 5000 }, (_, i) => + row('svc', `op${i}`, 20), + ); + const ctx = ctxWith([{ data: many }, { data: [row('svc', 'op0', 900)] }]); + const r = await findNewOperations(ctx, current, yesterday!, ['svc'], 50); + expect(r.complete).toBe(false); + expect(r.newOperations.size).toBe(0); + }); +}); diff --git a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts new file mode 100644 index 0000000000..a0a2140c54 --- /dev/null +++ b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts @@ -0,0 +1,383 @@ +import { errorGroupKey } from '@/mcp/tools/query/serviceHealth/errorGroups'; +import type { NewOperation } from '@/mcp/tools/query/serviceHealth/operations'; +import { + assessService, + HEALTH_THRESHOLDS, + type WindowStats, +} from '@/mcp/tools/query/serviceHealth/rank'; +import { + buildReport, + errorGroupStatus, + rankServices, + selectTopErrors, + topGroupsPerService, +} from '@/mcp/tools/query/serviceHealth/report'; +import { comparisonWindow } from '@/mcp/tools/query/serviceHealth/schema'; +import { + errorTiming, + serviceSummary, +} from '@/mcp/tools/query/serviceHealth/summary'; + +const stats = (s: Partial = {}): WindowStats => ({ + requests: 1000, + errors: 5, + p50Ms: 50, + p95Ms: 200, + p99Ms: 300, + ...s, +}); + +describe('assessService', () => { + it('is healthy when nothing moved', () => { + const a = assessService(stats(), stats(), 50); + expect(a.status).toBe('healthy'); + expect(a.flags).toEqual([]); + }); + + it('ignores a doubled error rate below the absolute floor', () => { + // 0.5% -> 1.0%: 2x, but only +0.5pp + const a = assessService(stats({ errors: 10 }), stats({ errors: 5 }), 50); + expect(a.flags).not.toContain('error_rate_up'); + expect(a.status).toBe('healthy'); + }); + + it('flags error_rate_up at exactly the ratio and delta thresholds', () => { + // 1% -> 2%: exactly 2x and +1pp + const a = assessService(stats({ errors: 20 }), stats({ errors: 10 }), 50); + expect(a.flags).toContain('error_rate_up'); + expect(a.status).toBe('degraded'); + expect(a.deltas.errorRateDeltaPp).toBe(1); + }); + + it('is critical when errors rose past the high error rate', () => { + const a = assessService(stats({ errors: 80 }), stats({ errors: 5 }), 50); + expect(a.flags).toEqual( + expect.arrayContaining(['error_rate_up', 'high_error_rate']), + ); + expect(a.status).toBe('critical'); + }); + + it('marks a chronically high error rate degraded, not critical', () => { + const a = assessService(stats({ errors: 80 }), stats({ errors: 80 }), 50); + expect(a.flags).toEqual(['high_error_rate']); + expect(a.status).toBe('degraded'); + }); + + it('needs both the ratio and the ms delta for latency_up', () => { + expect( + assessService(stats({ p95Ms: 12 }), stats({ p95Ms: 4 }), 50).flags, + ).not.toContain('latency_up'); + expect( + assessService( + stats({ p95Ms: 200 * HEALTH_THRESHOLDS.latencyRatio }), + stats(), + 50, + ).flags, + ).toContain('latency_up'); + }); + + it('flags traffic drops and spikes', () => { + const drop = assessService(stats({ requests: 500 }), stats(), 50); + expect(drop.flags).toContain('traffic_drop'); + expect(drop.status).toBe('degraded'); + + const spike = assessService(stats({ requests: 3000 }), stats(), 50); + expect(spike.flags).toContain('traffic_spike'); + expect(spike.status).toBe('healthy'); + }); + + it('flags new and missing services', () => { + const fresh = assessService(stats(), null, 50); + expect(fresh.flags).toEqual(['new_service']); + expect(fresh.status).toBe('healthy'); + + const gone = assessService(null, stats(), 50); + expect(gone.flags).toEqual(['missing_service']); + expect(gone.status).toBe('degraded'); + }); + + it('treats a handful of prior requests as no baseline', () => { + const a = assessService(stats(), stats({ requests: 1, errors: 1 }), 50); + expect(a.flags).toEqual(['new_service']); + expect(a.status).toBe('healthy'); + }); + + it('never flags services under minRequests in both windows', () => { + const a = assessService( + stats({ requests: 10, errors: 9 }), + stats({ requests: 10, errors: 0 }), + 50, + ); + expect(a.flags).toEqual(['low_volume']); + expect(a.score).toBe(0); + expect(a.status).toBe('healthy'); + }); + + it('applies absolute rules only when comparisons are off', () => { + const a = assessService(stats({ errors: 80 }), undefined, 50); + expect(a.flags).toEqual(['high_error_rate']); + expect(a.deltas.errorRateDeltaPp).toBeNull(); + }); + + it('handles a zero comparison p95 without dividing by zero', () => { + const a = assessService(stats(), stats({ p95Ms: 0 }), 50); + expect(a.deltas.p95Ratio).toBeNull(); + }); +}); + +describe('rankServices', () => { + it('sorts by status, then by score with volume as a multiplier', () => { + const cur = new Map([ + ['steady', stats()], + ['small-regression', stats({ requests: 100, errors: 10 })], + ['big-regression', stats({ requests: 100_000, errors: 10_000 })], + ['broken', stats({ errors: 300 })], + ]); + const cmp = new Map([ + ['steady', stats()], + ['small-regression', stats({ requests: 100, errors: 0 })], + ['big-regression', stats({ requests: 100_000, errors: 0 })], + ['broken', stats()], + ]); + expect(rankServices(cur, cmp, 50).map(r => r.service)).toEqual([ + 'broken', + 'big-regression', + 'small-regression', + 'steady', + ]); + }); + + it('includes services seen in only one window', () => { + const ranked = rankServices( + new Map([['a', stats()]]), + new Map([['b', stats()]]), + 50, + ); + expect(ranked.map(r => [r.service, r.assessment.flags])).toEqual([ + ['b', ['missing_service']], + ['a', ['new_service']], + ]); + }); +}); + +describe('topGroupsPerService', () => { + it('keeps the first N groups per service', () => { + const g = (service: string, count: number) => ({ + service, + group: { SpanName: `op-${count}` }, + count, + }); + const out = topGroupsPerService( + [g('a', 9), g('b', 8), g('a', 7), g('a', 6)], + 2, + ); + expect(out.get('a')?.map(x => x.count)).toEqual([9, 7]); + expect(out.get('b')?.map(x => x.count)).toEqual([8]); + }); +}); + +describe('comparisonWindow', () => { + const current = { + start: new Date('2026-10-16T10:00:00Z'), + end: new Date('2026-10-16T11:00:00Z'), + }; + + it.each([ + ['yesterday', '2026-10-15T10:00:00.000Z', '2026-10-15T11:00:00.000Z'], + ['last_week', '2026-10-09T10:00:00.000Z', '2026-10-09T11:00:00.000Z'], + ['previous_period', '2026-10-16T09:00:00.000Z', '2026-10-16T10:00:00.000Z'], + ] as const)('%s', (mode, start, end) => { + const w = comparisonWindow(current, mode); + expect(w?.start.toISOString()).toBe(start); + expect(w?.end.toISOString()).toBe(end); + }); + + it('returns null for none', () => { + expect(comparisonWindow(current, 'none')).toBeNull(); + }); +}); + +describe('error group selection', () => { + const g = (msg: string, count: number) => ({ + service: 'svc', + group: { StatusMessage: msg }, + count, + }); + const counts = (entries: [string, number][]) => + new Map(entries.map(([msg, n]) => [errorGroupKey(g(msg, 0)), n])); + + it('labels growth only when it is beyond random variation', () => { + expect(errorGroupStatus(5, 0)).toBe('new'); + expect(errorGroupStatus(13, 5)).toBe('existing'); + expect(errorGroupStatus(4060, 170)).toBe('growing'); + }); + + it('lists new and growing groups before larger steady ones', () => { + const picked = selectTopErrors( + [g('steady', 500), g('growing', 300), g('new', 40), g('tiny-new', 2)], + counts([ + ['steady', 480], + ['growing', 20], + ]), + 3, + ); + expect(picked.map(x => x.group.StatusMessage)).toEqual([ + 'growing', + 'new', + 'steady', + ]); + }); + + it('does not promote new groups that a new operation explains', () => { + const picked = selectTopErrors( + [g('steady', 500), g('on-new-endpoint', 40), g('new', 30)], + counts([['steady', 480]]), + 2, + x => x.group.StatusMessage === 'on-new-endpoint', + ); + expect(picked.map(x => x.group.StatusMessage)).toEqual(['new', 'steady']); + }); + + it('keeps count order without a comparison', () => { + const picked = selectTopErrors([g('a', 9), g('b', 5), g('c', 1)], null, 2); + expect(picked.map(x => x.group.StatusMessage)).toEqual(['a', 'b']); + }); +}); + +describe('buildReport', () => { + it('lifts a service with a new error group into the top N', () => { + const ranked = rankServices( + new Map([ + ['busy', stats({ requests: 100_000, errors: 600 })], + ['quiet', stats({ requests: 1000, errors: 5 })], + ]), + new Map([ + ['busy', stats({ requests: 100_000, errors: 500 })], + ['quiet', stats({ requests: 1000, errors: 5 })], + ]), + 50, + ); + expect(ranked.map(r => r.service)).toEqual(['busy', 'quiet']); + const novel = { + service: 'quiet', + group: { SpanName: 'POST /pay', StatusMessage: 'payment unavailable' }, + count: 8, + }; + const report = buildReport({ + ranked, + current: { start: new Date(0), end: new Date(600_000) }, + comparison: { start: new Date(-600_000), end: new Date(0) }, + comparisonMode: 'previous_period', + spanKind: 'server', + topN: 1, + errorGroupBy: ['SpanName', 'StatusMessage'], + currentErrors: new Map([['quiet', [novel]]]), + comparisonErrorCounts: new Map(), + newOperations: new Map(), + spanNameKey: 'SpanName', + }); + expect(report.services.map(s => s.service)).toEqual(['quiet']); + expect(report.services[0].flags).toContain('new_errors'); + }); + + const reportFor = ( + cur: WindowStats, + cmp: WindowStats, + newOperations: Map = new Map(), + ) => + buildReport({ + ranked: rankServices( + new Map([['svc', cur]]), + new Map([['svc', cmp]]), + 50, + ), + current: { start: new Date(0), end: new Date(600_000) }, + comparison: { start: new Date(-600_000), end: new Date(0) }, + comparisonMode: 'previous_period', + spanKind: 'server', + topN: 5, + errorGroupBy: ['SpanName', 'StatusMessage'], + currentErrors: new Map([ + [ + 'svc', + [ + { + service: 'svc', + group: { SpanName: 'GET /v2', StatusMessage: 'x' }, + count: 20, + }, + ], + ], + ]), + comparisonErrorCounts: new Map(), + newOperations, + spanNameKey: 'SpanName', + }).services[0]; + + it('does not call errors on a brand-new operation new errors', () => { + const svc = reportFor( + stats(), + stats(), + new Map([ + ['svc', [{ operation: 'GET /v2', requests: 100, shareOfService: 0.1 }]], + ]), + ); + expect(svc.flags).not.toContain('new_errors'); + expect(svc.topErrors[0].status).toBe('new_operation'); + }); + + it('does not call errors new when the service has no baseline', () => { + const svc = reportFor(stats(), stats({ requests: 1, errors: 0 })); + expect(svc.flags).toEqual(['new_service']); + expect(svc.summary).not.toMatch(/New or growing errors/); + expect(svc.topErrors[0]).not.toHaveProperty('status'); + }); +}); + +describe('errorTiming', () => { + const window = { + start: new Date('2026-10-20T17:05:00Z'), + end: new Date('2026-10-20T18:05:00Z'), + }; + const at = (hhmm: string) => Date.parse(`2026-10-20T${hhmm}:00Z`); + const group = (first: string, last: string) => ({ + service: 'svc', + group: { SpanName: 'POST /x' }, + count: 30, + firstSeenMs: at(first), + lastSeenMs: at(last), + }); + + it('calls a group still firing at the window end ongoing', () => { + expect(errorTiming(group('17:30', '18:04'), window).pattern).toBe( + 'ongoing', + ); + }); + + it('calls errors spread across the window spread, not a burst', () => { + // 52 minutes of errors, last one 7 minutes before the window end. + expect(errorTiming(group('17:06', '17:58'), window).pattern).toBe('spread'); + }); + + it('calls a short spike that ended a burst', () => { + expect(errorTiming(group('17:20', '17:21'), window).pattern).toBe('burst'); + }); + + it('only calls bursts transient in the summary', () => { + const summary = (g: ReturnType) => + serviceSummary({ + current: stats(), + comparison: stats(), + assessment: assessService(stats(), stats(), 50), + changedErrors: [g], + newOperations: [], + window, + comparisonMode: 'yesterday', + }); + const trickle = summary(group('17:06', '17:58')); + expect(trickle).toMatch(/spread across 17:06-17:58 UTC, none since 17:58/); + expect(trickle).not.toMatch(/transient/); + expect(trickle).toMatch(/check clickstack_timeseries/); + expect(summary(group('17:20', '17:21'))).toMatch(/transient/); + }); +}); diff --git a/packages/api/src/mcp/tools/query/builderCatalog.ts b/packages/api/src/mcp/tools/query/builderCatalog.ts index c4b1a77a91..ec195b744f 100644 --- a/packages/api/src/mcp/tools/query/builderCatalog.ts +++ b/packages/api/src/mcp/tools/query/builderCatalog.ts @@ -6,6 +6,7 @@ /** Bulleted list of the builder (non-SQL) query tools and what they're for. */ export const BUILDER_TOOLS_LIST = [ + ' • clickstack_service_health — per-service throughput, errors, latency vs yesterday, ranked', ' • clickstack_table — aggregations, top-N, single-value KPIs, breakdowns', ' • clickstack_timeseries — trends / metrics over time', ' • clickstack_search — browsing individual log/trace rows', diff --git a/packages/api/src/mcp/tools/query/index.ts b/packages/api/src/mcp/tools/query/index.ts index 700a8a24ca..1af3984f12 100644 --- a/packages/api/src/mcp/tools/query/index.ts +++ b/packages/api/src/mcp/tools/query/index.ts @@ -4,11 +4,13 @@ import { registerEmergingSignals } from './emergingSignals'; import { registerEventDeltas } from './eventDeltas'; import { registerEventPatterns } from './eventPatterns'; import { registerSearch } from './search'; +import { registerServiceHealth } from './serviceHealth'; import { registerSql } from './sql'; import { registerTable } from './table'; import { registerTimeseries } from './timeseries'; const queryTools: ToolDefinition = (registrar: ToolRegistrar) => { + registerServiceHealth(registrar); registerTimeseries(registrar); registerTable(registrar); registerSearch(registrar); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/collect.ts b/packages/api/src/mcp/tools/query/serviceHealth/collect.ts new file mode 100644 index 0000000000..13216b603b --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/collect.ts @@ -0,0 +1,119 @@ +import { + type ErrorGroup, + errorGroupKey, + fetchErrorGroups, + groupValue, + MAX_ERROR_ROWS, +} from './errorGroups'; +import { findNewOperations, type NewOperation } from './operations'; +import type { QueryContext } from './queries'; +import { + ERROR_CANDIDATES_PER_SERVICE, + hasBaseline, + type RankedService, + selectTopErrors, + topGroupsPerService, +} from './report'; +import type { Window } from './schema'; + +export type CollectedErrors = { + currentErrors: Map; + comparisonErrorCounts: Map | null; + newOperations: Map; + /** Set when a limit kept some services or error groups out of the check. */ + coverageNote: string | undefined; +}; + +/** + * Error groups and new operations for the services that will be shown. + * Errors on a brand-new operation are expected (it had no traffic before), so + * they do not compete with genuinely new failure modes for the top slots. + */ +export async function collectErrors( + ctx: QueryContext, + args: { + current: Window; + comparison: Window | null; + shown: RankedService[]; + perService: number; + groupExprs: string[]; + minRequests: number; + /** Services assessed in total; `shown` may be a prefix of them. */ + totalServices: number; + }, +): Promise { + const { current, comparison, shown, perService, groupExprs, minRequests } = + args; + const withErrors = shown + .filter(r => (r.current?.errors ?? 0) > 0) + .map(r => r.service); + const fetchErrors = perService > 0 && withErrors.length > 0; + + const [rows, ops] = await Promise.all([ + fetchErrors + ? fetchErrorGroups(ctx, current, groupExprs, { services: withErrors }) + : Promise.resolve([]), + comparison + ? findNewOperations( + ctx, + current, + comparison, + shown.filter(r => r.current).map(r => r.service), + minRequests, + ) + : Promise.resolve({ + newOperations: new Map(), + complete: true, + }), + ]); + const { newOperations } = ops; + + const perServiceCap = Math.max(ERROR_CANDIDATES_PER_SERVICE, perService); + const candidates = topGroupsPerService(rows, perServiceCap); + const all = [...candidates.values()].flat(); + const comparisonErrorCounts = + comparison && all.length + ? new Map( + ( + await fetchErrorGroups(ctx, comparison, groupExprs, { match: all }) + ).map(g => [errorGroupKey(g), g.count]), + ) + : null; + + const spanKey = ctx.source.spanNameExpression; + const currentErrors = new Map(); + const baseline = new Map(shown.map(r => [r.service, hasBaseline(r)])); + for (const [service, groups] of candidates) { + const newOps = new Set( + (newOperations.get(service) ?? []).map(o => o.operation), + ); + currentErrors.set( + service, + selectTopErrors( + groups, + comparisonErrorCounts, + perService, + g => !baseline.get(service) || newOps.has(groupValue(g, spanKey) ?? ''), + ), + ); + } + const groupsPerService = new Map(); + for (const g of rows) { + groupsPerService.set(g.service, (groupsPerService.get(g.service) ?? 0) + 1); + } + const limits = [ + args.totalServices > shown.length && + `only the first ${shown.length} of ${args.totalServices} services were checked for new errors`, + rows.length >= MAX_ERROR_ROWS && + `the error query hit its ${MAX_ERROR_ROWS}-row limit`, + [...groupsPerService.values()].some(n => n > perServiceCap) && + `only the ${perServiceCap} largest error groups per service were compared`, + !ops.complete && + 'too many operations to detect new ones reliably, so none are reported', + ].filter((x): x is string => typeof x === 'string'); + const coverageNote = limits.length + ? `Incomplete: ${limits.join('; ')}. Narrow with services or a where filter for a full check.` + : undefined; + + return { currentErrors, comparisonErrorCounts, newOperations, coverageNote }; +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/comparison.ts b/packages/api/src/mcp/tools/query/serviceHealth/comparison.ts new file mode 100644 index 0000000000..224c39c142 --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/comparison.ts @@ -0,0 +1,64 @@ +import { fetchServiceStats, type QueryContext } from './queries'; +import { type ComparisonMode, comparisonWindow, type Window } from './schema'; + +type Stats = Awaited>; + +export type ResolvedComparison = { + comparison: Window | null; + cmp: Stats | null; + comparisonMode: ComparisonMode; + comparisonNote: string | undefined; +}; + +/** + * An empty comparison window usually means retention or a fresh source, not + * that every service is new. Comparing against nothing would label every + * service `new_service` and every error group `new`, which reads as an + * incident. Fall back to the previous period, and to no comparison if that is + * empty too. + */ +export async function resolveComparison( + ctx: QueryContext, + args: { + current: Window; + requested: Window | null; + requestedMode: ComparisonMode; + curHasData: boolean; + requestedCmp: Stats | null; + }, +): Promise { + const { current, requested, requestedMode, curHasData, requestedCmp } = args; + if (!requested || !requestedCmp || !curHasData || requestedCmp.stats.size) { + return { + comparison: requested, + cmp: requestedCmp, + comparisonMode: requestedMode, + comparisonNote: undefined, + }; + } + + const fallback = + requestedMode === 'previous_period' + ? null + : comparisonWindow(current, 'previous_period'); + const fallbackCmp = fallback ? await fetchServiceStats(ctx, fallback) : null; + if (fallback && fallbackCmp?.stats.size) { + return { + comparison: fallback, + cmp: fallbackCmp, + comparisonMode: 'previous_period', + comparisonNote: + `The "${requestedMode}" window has no data (likely outside retention), ` + + 'so this compares with the previous period instead.', + }; + } + return { + comparison: null, + cmp: null, + comparisonMode: 'none', + comparisonNote: + `The "${requestedMode}" window${fallback ? ' and the previous period have' : ' has'} ` + + 'no data, so there is no comparison. Only absolute thresholds apply; ' + + 'nothing is labelled new.', + }; +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/errorGroups.ts b/packages/api/src/mcp/tools/query/serviceHealth/errorGroups.ts new file mode 100644 index 0000000000..c7220bb2fc --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/errorGroups.ts @@ -0,0 +1,124 @@ +import { errorStatusPredicateSql } from '@hyperdx/common-utils/dist/core/spanPredicates'; +import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils'; + +import { + baseConfig, + columns, + type HealthSource, + type QueryContext, + run, + SERVICE_ALIAS, + sqlList, + toNumberOrNull, +} from './queries'; +import type { Window } from './schema'; + +const ERROR_COUNT_ALIAS = '__hdx_error_count'; +const FIRST_SEEN_ALIAS = '__hdx_first_seen'; +const HASH_ALIAS = '__hdx_group_hash'; +const LAST_SEEN_ALIAS = '__hdx_last_seen'; +const errorKeyAlias = (i: number) => `__hdx_error_key_${i}`; + +/** Upper bound on error-group rows fetched, before bucketing per service. */ +export const MAX_ERROR_ROWS = 2500; + +export type ErrorGroup = { + service: string; + group: Record; + count: number; + /** cityHash64 of the group's key, used to look it up in another window. */ + hash?: string; + /** Epoch ms of the first and last error in the window, if known. */ + firstSeenMs?: number; + lastSeenMs?: number; +}; + +/** The value of one grouping expression in an error group. */ +export const groupValue = (g: Pick, key: string) => + new Map(Object.entries(g.group)).get(key); + +/** The expressions that define an error group, in output order. */ +export function errorGroupExpressions( + source: HealthSource, + errorGroupBy: string | undefined, + split: (s: string) => string[], +): string[] { + const exprs = errorGroupBy + ? split(errorGroupBy) + : [source.spanNameExpression, source.statusMessageExpression]; + return exprs.filter( + (e): e is string => !!e && e !== source.serviceNameExpression, + ); +} + +/** + * Error counts per (service, group) in one window. `match` restricts rows to + * exact (service, ...group) tuples, so the comparison window is looked up for + * the groups found in the current window instead of its own top-N. + */ +export async function fetchErrorGroups( + ctx: QueryContext, + window: Window, + groupExprs: string[], + scope: { services: string[] } | { match: ErrorGroup[] }, +): Promise { + const { source } = ctx; + const keyExprs = [source.serviceNameExpression, ...groupExprs]; + // Match groups across windows by a fixed-size hash. Inlining the raw key + // text (status messages can be stack traces) can exceed max_query_size. + const hashExpr = `cityHash64(${keyExprs.map(e => `toString(${e})`).join(', ')})`; + const epochMs = `toUnixTimestamp64Milli(toDateTime64(${getFirstTimestampValueExpression(source.timestampValueExpression)}, 3))`; + const scopeCondition = + 'services' in scope + ? `${source.serviceNameExpression} IN (${sqlList(scope.services)})` + : `${hashExpr} IN (${ + scope.match + .map(g => g.hash) + .filter((h): h is string => !!h && /^\d+$/.test(h)) + .join(', ') || 'NULL' + })`; + + const { rows } = await run(ctx, { + ...baseConfig(ctx, window, [ + { + type: 'sql', + condition: errorStatusPredicateSql(source.statusCodeExpression), + }, + { type: 'sql', condition: scopeCondition }, + ]), + select: [ + { aggFn: 'count', valueExpression: '', alias: ERROR_COUNT_ALIAS }, + // Epoch ms, because min/max cast their argument to Float64. + { aggFn: 'min', valueExpression: epochMs, alias: FIRST_SEEN_ALIAS }, + { aggFn: 'max', valueExpression: epochMs, alias: LAST_SEEN_ALIAS }, + // 'none' passes the expression through on the sample-weighted path too. + { aggFn: 'none', valueExpression: `any(${hashExpr})`, alias: HASH_ALIAS }, + ], + groupBy: [ + { valueExpression: source.serviceNameExpression, alias: SERVICE_ALIAS }, + ...groupExprs.map((e, i) => ({ + valueExpression: `toString(${e})`, + alias: errorKeyAlias(i), + })), + ], + orderBy: `${ERROR_COUNT_ALIAS} DESC`, + limit: { limit: MAX_ERROR_ROWS }, + }); + + return rows.map(row => { + const c = columns(row); + return { + service: String(c.get(SERVICE_ALIAS) ?? ''), + group: Object.fromEntries( + groupExprs.map((e, i) => [e, String(c.get(errorKeyAlias(i)) ?? '')]), + ), + count: Number(c.get(ERROR_COUNT_ALIAS)) || 0, + hash: String(c.get(HASH_ALIAS) ?? '') || undefined, + firstSeenMs: toNumberOrNull(c.get(FIRST_SEEN_ALIAS)) || undefined, + lastSeenMs: toNumberOrNull(c.get(LAST_SEEN_ALIAS)) || undefined, + }; + }); +} + +export const errorGroupKey = (g: Pick) => + JSON.stringify([g.service, ...Object.values(g.group)]); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/index.ts b/packages/api/src/mcp/tools/query/serviceHealth/index.ts new file mode 100644 index 0000000000..d0d9e72128 --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/index.ts @@ -0,0 +1,208 @@ +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; +import { trace } from '@opentelemetry/api'; + +import { READ_ONLY_TOOL } from '@/mcp/tools/annotations'; +import { PREFER_BUILDER_OVER_SQL_NUDGE } from '@/mcp/tools/query/builderCatalog'; +import { + clickHouseErrorResult, + isQueryOutOfTime, + parseTimeRange, +} from '@/mcp/tools/query/helpers'; +import { getMcpClickhouseClient } from '@/mcp/tools/trace/shared'; +import type { ToolRegistrar } from '@/mcp/tools/types'; +import { mcpUserError } from '@/mcp/utils/errors'; +import { trimToolResponse } from '@/utils/trimToolResponse'; + +import { collectErrors } from './collect'; +import { resolveComparison } from './comparison'; +import { errorGroupExpressions } from './errorGroups'; +import { fetchServiceStats, type QueryContext } from './queries'; +import { buildReport, rankServices } from './report'; +import { + comparisonWindow, + DEFAULT_WINDOW_MS, + serviceHealthSchema, +} from './schema'; +import { loadHealthSource } from './source'; + +const TOOL_NAME = 'clickstack_service_health'; +const MAX_ERROR_SERVICES = 100; + +const TIMEOUT_SUFFIX = + 'Service health aggregates every span in both windows. Retry with a ' + + 'shorter startTime/endTime window, list specific services, add a where ' + + 'filter, or set topErrorsPerService:0 to skip the error breakdown.'; + +const DESCRIPTION = + 'One-call health report for every service on a trace source: throughput, ' + + 'error rate, p50/p95/p99 latency and top error groups, compared with the ' + + 'same window yesterday and ranked worst first. Each service gets a status ' + + '(critical / degraded / healthy), flags, and a one-line summary of which ' + + 'changes crossed a threshold. New or growing error groups come first, ' + + 'with first/last-seen times; operations with no prior traffic are listed ' + + 'as likely rollouts.\n\n' + + 'USE THIS FIRST for status checks, "is X healthy", post-deploy checks, and ' + + 'as the entry point for triage.\n\n' + + 'It is per service and per window. To follow up:\n' + + ' - clickstack_timeseries by minute to see when a change happened\n' + + ' - clickstack_table grouped by span name for per-endpoint shifts\n' + + ' - clickstack_emerging_signals on the log source for new log patterns\n' + + 'For a flagged service, check its own runtime metrics (memory, GC, CPU, ' + + 'restarts by pod) on the metric source before blaming dependencies, then ' + + 'use clickstack_event_deltas and ' + + 'clickstack_trace_top_time_consuming_operations.\n\n' + + 'CALIBRATION: flags fire only past fixed thresholds (see `usage` in the ' + + 'response). Small swings on a healthy service are normal variance. An ' + + 'all-healthy report is a valid answer.\n\n' + + PREFER_BUILDER_OVER_SQL_NUDGE; + +export function registerServiceHealth({ + context, + registerTool, +}: ToolRegistrar) { + const teamId = context.teamId.toString(); + + registerTool( + TOOL_NAME, + { + title: 'Service Health', + annotations: READ_ONLY_TOOL, + description: DESCRIPTION, + inputSchema: serviceHealthSchema, + }, + async input => { + const range = parseTimeRange( + input.startTime, + input.endTime, + DEFAULT_WINDOW_MS, + ); + if ('error' in range) return mcpUserError(range.error); + const current = { start: range.startDate, end: range.endDate }; + const requested = comparisonWindow(current, input.comparison); + if (requested && requested.end > current.start) { + return mcpUserError( + `The window is longer than the "${input.comparison}" shift, so the ` + + 'two windows overlap. Shorten the window or use ' + + 'comparison:"previous_period".', + ); + } + + const source = await loadHealthSource(teamId, input.sourceId, TOOL_NAME); + if ('isError' in source) return source; + const client = await getMcpClickhouseClient(teamId, source.connection); + if (!client) { + return mcpUserError( + `Connection not found for source: ${input.sourceId}`, + ); + } + const groupExprs = errorGroupExpressions( + source, + input.errorGroupBy, + splitAndTrimWithBracket, + ); + + const abort = new AbortController(); + const ctx: QueryContext = { + client, + metadata: getMetadata(client), + source, + input, + signal: abort.signal, + }; + + try { + const [cur, requestedCmp] = await Promise.all([ + fetchServiceStats(ctx, current), + requested ? fetchServiceStats(ctx, requested) : null, + ]); + const { comparison, cmp, comparisonMode, comparisonNote } = + await resolveComparison(ctx, { + current, + requested, + requestedMode: input.comparison, + curHasData: cur.stats.size > 0, + requestedCmp, + }); + const ranked = rankServices( + cur.stats, + cmp?.stats ?? null, + input.minRequests, + ); + + const { + currentErrors, + comparisonErrorCounts, + newOperations, + coverageNote, + } = await collectErrors(ctx, { + current, + comparison, + // Novelty can lift a service into the top N, so assess errors + // beyond it rather than only for services already ranked there. + shown: ranked.slice(0, MAX_ERROR_SERVICES), + perService: input.topErrorsPerService, + groupExprs, + minRequests: input.minRequests, + totalServices: ranked.length, + }); + + const report = buildReport({ + ranked, + current, + comparison, + comparisonMode, + comparisonNote, + coverageNote, + spanKind: input.spanKind, + topN: input.topN, + errorGroupBy: groupExprs, + currentErrors, + comparisonErrorCounts, + newOperations, + spanNameKey: source.spanNameExpression, + metricSourceId: source.metricSourceId, + }); + + trace.getActiveSpan()?.setAttributes({ + 'hyperdx.source.id': source.id, + 'mcp.service_health.comparison': input.comparison, + 'mcp.service_health.comparison_used': comparisonMode, + 'mcp.service_health.span_kind': input.spanKind, + 'mcp.service_health.window_ms': + current.end.getTime() - current.start.getTime(), + 'mcp.service_health.services_assessed': ranked.length, + 'mcp.service_health.critical_count': + report.summary.statusCounts.critical, + 'mcp.service_health.degraded_count': + report.summary.statusCounts.degraded, + }); + + const { data, isTrimmed } = trimToolResponse(report); + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + isTrimmed + ? { + ...data, + note: 'Trimmed for context size. Lower topN or topErrorsPerService.', + } + : data, + null, + 2, + ), + }, + ], + }; + } catch (e) { + abort.abort(); + return clickHouseErrorResult(e, { + prefix: 'Failed to compute service health', + suffix: isQueryOutOfTime(e) ? TIMEOUT_SUFFIX : undefined, + }); + } + }, + ); +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/operations.ts b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts new file mode 100644 index 0000000000..c3665267a7 --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts @@ -0,0 +1,94 @@ +import type { QueryContext } from './queries'; +import { baseConfig, columns, run, SERVICE_ALIAS, sqlList } from './queries'; +import type { Window } from './schema'; + +const OP_ALIAS = '__hdx_operation'; +const MAX_OPERATION_ROWS = 5000; +/** A new operation needs this many requests to be reported. */ +const NEW_OPERATION_MIN_REQUESTS = 10; +/** New operations shown per service; all of them still classify errors. */ +export const MAX_NEW_OPERATIONS = 5; + +export type NewOperation = { + operation: string; + requests: number; + shareOfService: number; +}; + +async function operationCounts( + ctx: QueryContext, + window: Window, + services: string[], +): Promise<{ counts: Map>; truncated: boolean }> { + const { source } = ctx; + const { rows } = await run(ctx, { + ...baseConfig(ctx, window, [ + { + type: 'sql', + condition: `${source.serviceNameExpression} IN (${sqlList(services)})`, + }, + ]), + select: [{ aggFn: 'count', valueExpression: '', alias: 'requests' }], + groupBy: [ + { valueExpression: source.serviceNameExpression, alias: SERVICE_ALIAS }, + { valueExpression: source.spanNameExpression, alias: OP_ALIAS }, + ], + limit: { limit: MAX_OPERATION_ROWS }, + }); + const out = new Map>(); + for (const row of rows) { + const c = columns(row); + const svc = String(c.get(SERVICE_ALIAS) ?? ''); + const ops = out.get(svc) ?? new Map(); + ops.set(String(c.get(OP_ALIAS) ?? ''), Number(row.requests) || 0); + out.set(svc, ops); + } + return { counts: out, truncated: rows.length >= MAX_OPERATION_ROWS }; +} + +/** + * Operations (span names) with traffic now and none in the comparison + * window, per service: usually a rollout. Errors on them are explained by the + * operation being new, not by a new failure mode. + */ +export async function findNewOperations( + ctx: QueryContext, + current: Window, + comparison: Window, + services: string[], + minPriorRequests: number, +): Promise<{ newOperations: Map; complete: boolean }> { + const out = new Map(); + if (services.length === 0) return { newOperations: out, complete: true }; + const [curRes, cmpRes] = await Promise.all([ + operationCounts(ctx, current, services), + operationCounts(ctx, comparison, services), + ]); + // A truncated comparison can omit operations that did exist, which would + // make them look new. Report nothing rather than guess. + if (curRes.truncated || cmpRes.truncated) { + return { newOperations: out, complete: false }; + } + const cmp = cmpRes.counts; + for (const [svc, ops] of curRes.counts) { + const total = [...ops.values()].reduce((a, b) => a + b, 0); + const before = cmp.get(svc); + // A service with no real prior traffic is new as a whole (new_service); + // calling each of its operations a rollout would be noise. + const priorTotal = [...(before?.values() ?? [])].reduce((a, b) => a + b, 0); + if (priorTotal < minPriorRequests) continue; + const fresh = [...ops] + .filter( + ([op, n]) => + op !== '' && n >= NEW_OPERATION_MIN_REQUESTS && !before?.get(op), + ) + .sort((a, b) => b[1] - a[1]) + .map(([operation, requests]) => ({ + operation, + requests, + shareOfService: Math.round((requests / total) * 10000) / 10000, + })); + if (fresh.length) out.set(svc, fresh); + } + return { newOperations: out, complete: true }; +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts new file mode 100644 index 0000000000..f71a893670 --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts @@ -0,0 +1,181 @@ +import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { + errorStatusPredicateSql, + serverSpanKindPredicateSql, +} from '@hyperdx/common-utils/dist/core/spanPredicates'; +import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils'; +import { + type BuilderChartConfigWithDateRange, + DisplayType, + type Filter, + pickSampleWeightExpressionProps, + type QuerySettings, + SourceKind, + type TTraceSource, +} from '@hyperdx/common-utils/dist/types'; + +import type { ClickhouseClient } from '@/clickhouse'; +import { durationDivisor, mcpQuerySettings } from '@/mcp/tools/trace/shared'; + +import type { WindowStats } from './rank'; +import type { ServiceHealthInput, Window } from './schema'; + +export const SERVICE_ALIAS = '__hdx_service'; + +/** + * Plain copies of the trace source fields the queries read. The Mongoose + * document's subdocuments (e.g. `from`) don't belong in a chart config, which + * the renderer may clone. + */ +export type HealthSource = Pick< + TTraceSource, + | 'id' + | 'timestampValueExpression' + | 'implicitColumnExpression' + | 'useTextIndexForImplicitColumn' + | 'sampleRateExpression' + | 'spanNameExpression' + | 'spanKindExpression' + | 'statusMessageExpression' + | 'durationExpression' + | 'durationPrecision' + | 'metricSourceId' +> & { + connection: string; + from: { databaseName: string; tableName: string }; + serviceNameExpression: string; + statusCodeExpression: string; + querySettings: QuerySettings | undefined; +}; + +export type QueryContext = { + client: ClickhouseClient; + metadata: Metadata; + source: HealthSource; + input: ServiceHealthInput; + signal: AbortSignal; +}; + +/** A result row's columns by alias. Build once per row, then read with get. */ +export const columns = (row: Record) => + new Map(Object.entries(row)); + +export const sqlList = (values: string[]) => + values.map(v => `'${escapeSqlString(v)}'`).join(', '); + +export function baseConfig( + { source, input }: QueryContext, + window: Window, + extraFilters: Filter[], +): Omit { + const filters: Filter[] = [...extraFilters]; + if (input.spanKind === 'server') { + filters.push({ + type: 'sql', + condition: serverSpanKindPredicateSql(source.spanKindExpression), + }); + } + if (input.services?.length) { + filters.push({ + type: 'sql', + condition: `${source.serviceNameExpression} IN (${sqlList(input.services)})`, + }); + } + return { + displayType: DisplayType.Table, + source: source.id, + connection: source.connection, + from: source.from, + timestampValueExpression: source.timestampValueExpression, + implicitColumnExpression: source.implicitColumnExpression, + useTextIndexForImplicitColumn: source.useTextIndexForImplicitColumn, + ...pickSampleWeightExpressionProps({ + kind: SourceKind.Trace, + sampleRateExpression: source.sampleRateExpression, + }), + where: input.where, + whereLanguage: input.where ? input.whereLanguage : 'sql', + filters, + dateRange: [window.start, window.end], + // Half-open windows: with an inclusive end, a span at exactly the + // boundary lands in both the comparison and the current window. + dateRangeEndInclusive: false, + }; +} + +export async function run( + ctx: QueryContext, + config: BuilderChartConfigWithDateRange, +): Promise<{ rows: Record[] }> { + const { client, metadata, source, signal } = ctx; + // No materialized-view rewrite: it rounds the window to the view's + // granularity, which would break the half-open windows the comparison + // relies on, and the error-group queries can't use a view anyway. + const result = await client.queryChartConfig({ + config, + metadata, + querySettings: source.querySettings, + opts: { + clickhouse_settings: mcpQuerySettings(source.querySettings), + abort_signal: signal, + }, + }); + return { rows: result.data ?? [] }; +} + +export const toNumberOrNull = (v: unknown) => { + const n = Number(v); + return v == null || !Number.isFinite(n) ? null : n; +}; + +/** Requests, errors, and latency quantiles per service in one window. */ +export async function fetchServiceStats( + ctx: QueryContext, + window: Window, +): Promise<{ stats: Map }> { + const { source } = ctx; + const quantile = (level: number, alias: string) => ({ + aggFn: 'quantile' as const, + level, + valueExpression: source.durationExpression, + aggCondition: '', + aggConditionLanguage: 'sql' as const, + alias, + }); + const { rows } = await run(ctx, { + ...baseConfig(ctx, window, []), + select: [ + { aggFn: 'count', valueExpression: '', alias: 'requests' }, + { + aggFn: 'count', + valueExpression: '', + aggCondition: errorStatusPredicateSql(source.statusCodeExpression), + aggConditionLanguage: 'sql', + alias: 'errors', + }, + quantile(0.5, 'p50'), + quantile(0.95, 'p95'), + quantile(0.99, 'p99'), + ], + groupBy: [ + { valueExpression: source.serviceNameExpression, alias: SERVICE_ALIAS }, + ], + }); + + const divisor = durationDivisor(source.durationPrecision); + const toMs = (v: unknown) => { + const n = toNumberOrNull(v); + return n == null ? null : Math.round((n / divisor) * 100) / 100; + }; + const stats = new Map(); + for (const row of rows) { + stats.set(String(columns(row).get(SERVICE_ALIAS) ?? ''), { + requests: Number(row.requests) || 0, + errors: Number(row.errors) || 0, + p50Ms: toMs(row.p50), + p95Ms: toMs(row.p95), + p99Ms: toMs(row.p99), + }); + } + return { stats }; +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/rank.ts b/packages/api/src/mcp/tools/query/serviceHealth/rank.ts new file mode 100644 index 0000000000..3c4ebeb1a8 --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/rank.ts @@ -0,0 +1,210 @@ +/** + * Thresholds that decide when a service is flagged. Each "up" rule needs a + * ratio AND an absolute delta: the ratio alone flags 0.1% → 0.3% errors or + * 4ms → 8ms latency, which is routine variance, not degraded health. + */ +export const HEALTH_THRESHOLDS = { + /** error_rate_up: current error rate at least this multiple of the comparison... */ + errorRateRatio: 2, + /** ...and at least this many percentage points higher. */ + errorRateMinDeltaPp: 1, + /** high_error_rate, and critical when combined with error_rate_up. */ + highErrorRate: 0.05, + /** latency_up: p95 at least this multiple of the comparison... */ + latencyRatio: 1.5, + /** ...and at least this many ms slower. */ + latencyMinDeltaMs: 50, + /** traffic_drop: requests at or below this fraction of the comparison. */ + trafficDropRatio: 0.5, + /** traffic_spike (informational): requests at least this multiple. */ + trafficSpikeRatio: 3, +} as const; + +export type WindowStats = { + requests: number; + errors: number; + p50Ms: number | null; + p95Ms: number | null; + p99Ms: number | null; +}; + +export type HealthFlag = + | 'error_rate_up' + | 'high_error_rate' + | 'latency_up' + | 'traffic_drop' + | 'missing_service' + // Context for the reader; these never change status. + | 'traffic_spike' + | 'new_service' + | 'low_volume' + // Set by the report: a shown error group is new or growing. + | 'new_errors'; + +const DEGRADING_FLAGS: ReadonlySet = new Set([ + 'error_rate_up', + 'high_error_rate', + 'latency_up', + 'traffic_drop', + 'missing_service', +]); +export type HealthStatus = 'critical' | 'degraded' | 'healthy'; + +export type ServiceAssessment = { + status: HealthStatus; + flags: HealthFlag[]; + score: number; + deltas: { + throughputRatio: number | null; + errorRateDeltaPp: number | null; + p95Ratio: number | null; + p95DeltaMs: number | null; + }; +}; + +export const STATUS_ORDER: Record = { + critical: 0, + degraded: 1, + healthy: 2, +}; + +const errorRate = (s: WindowStats) => + s.requests > 0 ? s.errors / s.requests : 0; + +const round = (n: number, places = 4) => { + const f = 10 ** places; + return Math.round(n * f) / f; +}; + +/** + * Assess one service. `comparison` is undefined when comparisons are off, and + * null when they are on but the service had no requests in that window. + */ +export function assessService( + current: WindowStats | null, + comparison: WindowStats | null | undefined, + minRequests: number, +): ServiceAssessment { + const cur = current ?? { + requests: 0, + errors: 0, + p50Ms: null, + p95Ms: null, + p99Ms: null, + }; + const cmp = comparison ?? null; + const comparing = comparison !== undefined; + const flags: HealthFlag[] = []; + + const curRate = errorRate(cur); + const cmpRate = cmp ? errorRate(cmp) : null; + const errorRateDeltaPp = + cmpRate != null ? round((curRate - cmpRate) * 100) : null; + const throughputRatio = + cmp && cmp.requests > 0 ? round(cur.requests / cmp.requests) : null; + const p95Ratio = + cmp?.p95Ms && cur.p95Ms != null ? round(cur.p95Ms / cmp.p95Ms) : null; + const p95DeltaMs = + cmp?.p95Ms != null && cur.p95Ms != null + ? round(cur.p95Ms - cmp.p95Ms, 2) + : null; + + const curEnough = cur.requests >= minRequests; + const cmpEnough = (cmp?.requests ?? 0) >= minRequests; + + if (!curEnough && !cmpEnough) { + flags.push('low_volume'); + } else if (comparing && !cmpEnough && curEnough) { + // A handful of prior requests is not a baseline; treat it like none. + flags.push('new_service'); + } else if (comparing && cmp && cur.requests === 0) { + flags.push('missing_service'); + } + + if (curEnough) { + if (curRate >= HEALTH_THRESHOLDS.highErrorRate) + flags.push('high_error_rate'); + if ( + cmpEnough && + cmpRate != null && + curRate >= cmpRate * HEALTH_THRESHOLDS.errorRateRatio && + (errorRateDeltaPp ?? 0) >= HEALTH_THRESHOLDS.errorRateMinDeltaPp + ) { + flags.push('error_rate_up'); + } + if ( + cmpEnough && + p95Ratio != null && + p95Ratio >= HEALTH_THRESHOLDS.latencyRatio && + (p95DeltaMs ?? 0) >= HEALTH_THRESHOLDS.latencyMinDeltaMs + ) { + flags.push('latency_up'); + } + if ( + cmpEnough && + throughputRatio != null && + throughputRatio >= HEALTH_THRESHOLDS.trafficSpikeRatio + ) { + flags.push('traffic_spike'); + } + } + if ( + cmpEnough && + cur.requests > 0 && + throughputRatio != null && + throughputRatio <= HEALTH_THRESHOLDS.trafficDropRatio + ) { + flags.push('traffic_drop'); + } + + const status: HealthStatus = + flags.includes('error_rate_up') && + curRate >= HEALTH_THRESHOLDS.highErrorRate + ? 'critical' + : flags.some(f => DEGRADING_FLAGS.has(f)) + ? 'degraded' + : 'healthy'; + + return { + status, + flags, + // Too few requests to judge, so it should not outrank real services. + score: flags.includes('low_volume') + ? 0 + : severityScore(cur, cmp, { curRate, cmpRate, p95Ratio }), + deltas: { throughputRatio, errorRateDeltaPp, p95Ratio, p95DeltaMs }, + }; +} + +/** + * Orders services within a status. Sums how far each metric moved the wrong + * way, then scales by log volume so a busy service outranks a quiet one with + * the same shift. Error points are percentage points; latency and traffic are + * log2 ratios, so doubling p95 weighs the same as +2pp errors. + */ +function severityScore( + cur: WindowStats, + cmp: WindowStats | null, + m: { curRate: number; cmpRate: number | null; p95Ratio: number | null }, +): number { + const errorPp = Math.max(0, m.curRate - (m.cmpRate ?? 0)) * 100; + const latency = m.p95Ratio && m.p95Ratio > 1 ? 2 * Math.log2(m.p95Ratio) : 0; + const traffic = + cmp && cmp.requests > 0 && cur.requests < cmp.requests + ? Math.log2(cmp.requests / Math.max(cur.requests, 1)) + : 0; + const volume = Math.log10(Math.max(cur.requests, cmp?.requests ?? 0) + 10); + return round((errorPp + latency + traffic) * volume, 2); +} + +export function compareAssessments( + a: { assessment: ServiceAssessment; errors: number; requests: number }, + b: { assessment: ServiceAssessment; errors: number; requests: number }, +): number { + return ( + STATUS_ORDER[a.assessment.status] - STATUS_ORDER[b.assessment.status] || + b.assessment.score - a.assessment.score || + b.errors - a.errors || + b.requests - a.requests + ); +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/report.ts b/packages/api/src/mcp/tools/query/serviceHealth/report.ts new file mode 100644 index 0000000000..c5f7fc2b2c --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/report.ts @@ -0,0 +1,297 @@ +import { type ErrorGroup, errorGroupKey, groupValue } from './errorGroups'; +import { MAX_NEW_OPERATIONS, type NewOperation } from './operations'; +import { + assessService, + compareAssessments, + HEALTH_THRESHOLDS, + type HealthFlag, + type HealthStatus, + type ServiceAssessment, + STATUS_ORDER, + type WindowStats, +} from './rank'; +import type { Window } from './schema'; +import { errorTiming, serviceSummary } from './summary'; + +export type RankedService = { + service: string; + current: WindowStats | null; + /** undefined when comparisons are off; null when the service was absent. */ + comparison: WindowStats | null | undefined; + assessment: ServiceAssessment; +}; + +/** Assess every service seen in either window and sort worst first. */ +export function rankServices( + current: Map, + comparison: Map | null, + minRequests: number, +): RankedService[] { + const names = new Set([...current.keys(), ...(comparison?.keys() ?? [])]); + return [...names] + .map(service => { + const cur = current.get(service) ?? null; + const cmp = comparison ? (comparison.get(service) ?? null) : undefined; + return { + service, + current: cur, + comparison: cmp, + assessment: assessService(cur, cmp, minRequests), + }; + }) + .sort((a, b) => + compareAssessments( + { + assessment: a.assessment, + errors: a.current?.errors ?? 0, + requests: a.current?.requests ?? 0, + }, + { + assessment: b.assessment, + errors: b.current?.errors ?? 0, + requests: b.current?.requests ?? 0, + }, + ), + ); +} + +/** The first `perService` groups for each service; input is sorted by count. */ +export function topGroupsPerService( + groups: ErrorGroup[], + perService: number, +): Map { + const out = new Map(); + for (const g of groups) { + const list = out.get(g.service) ?? []; + if (list.length < perService) list.push(g); + out.set(g.service, list); + } + return out; +} + +/** Candidate groups fetched per service before choosing the ones to show. */ +export const ERROR_CANDIDATES_PER_SERVICE = 25; +/** A new group needs at least this many errors to be called out. */ +const NEW_ERROR_MIN = 3; +/** + * Growth must clear this z-score, treating both counts as Poisson. Small + * counts swing a lot by chance (5 -> 13 is within noise across dozens of + * groups); 2x alone flags them. + */ +const GROWTH_MIN_Z = 3; + +export type ErrorGroupStatus = 'new' | 'growing' | 'existing' | 'new_operation'; + +export function errorGroupStatus( + count: number, + comparisonCount: number, +): ErrorGroupStatus { + if (comparisonCount === 0) return 'new'; + const z = (count - comparisonCount) / Math.sqrt(count + comparisonCount); + return count >= 2 * comparisonCount && z >= GROWTH_MIN_Z + ? 'growing' + : 'existing'; +} + +const isChanged = (count: number, comparisonCount: number) => { + const status = errorGroupStatus(count, comparisonCount); + return status === 'growing' || (status === 'new' && count >= NEW_ERROR_MIN); +}; + +/** + * Pick the groups to show for one service. With a comparison, new and growing + * groups come first, largest gain first: a burst of a novel error is the + * finding, and ranking by count alone buries it under steady background + * errors. The remaining slots go to the largest groups. + */ +export function selectTopErrors( + candidates: ErrorGroup[], + comparisonCounts: Map | null, + limit: number, + explained: (g: ErrorGroup) => boolean = () => false, +): ErrorGroup[] { + if (!comparisonCounts) return candidates.slice(0, limit); + const prior = (g: ErrorGroup) => comparisonCounts.get(errorGroupKey(g)) ?? 0; + const novel = (g: ErrorGroup) => + isChanged(g.count, prior(g)) && !explained(g); + const changed = candidates + .filter(novel) + .sort((a, b) => b.count - prior(b) - (a.count - prior(a))); + const rest = candidates.filter(g => !novel(g)); + return [...changed, ...rest].slice(0, limit); +} + +function formatStats(s: WindowStats | null, windowMinutes: number) { + const requests = s?.requests ?? 0; + const errors = s?.errors ?? 0; + return { + requests, + throughputPerMin: Math.round((requests / windowMinutes) * 100) / 100, + errors, + errorRate: + requests > 0 ? Math.round((errors / requests) * 10000) / 10000 : 0, + p50Ms: s?.p50Ms ?? null, + p95Ms: s?.p95Ms ?? null, + p99Ms: s?.p99Ms ?? null, + }; +} + +const iso = (w: Window) => ({ + start: w.start.toISOString(), + end: w.end.toISOString(), +}); + +const USAGE = + 'Services are sorted by status (critical, degraded, healthy), then score. ' + + `error_rate_up: error rate >= ${HEALTH_THRESHOLDS.errorRateRatio}x the comparison AND ` + + `>= ${HEALTH_THRESHOLDS.errorRateMinDeltaPp} percentage point higher. ` + + `high_error_rate: error rate >= ${HEALTH_THRESHOLDS.highErrorRate * 100}%. ` + + 'critical = error_rate_up together with high_error_rate. ' + + `latency_up: p95 >= ${HEALTH_THRESHOLDS.latencyRatio}x AND >= ${HEALTH_THRESHOLDS.latencyMinDeltaMs}ms slower. ` + + `traffic_drop: requests <= ${HEALTH_THRESHOLDS.trafficDropRatio * 100}% of the comparison. ` + + 'missing_service: served requests in the comparison window but none now. ' + + 'Any of those makes a service degraded. traffic_spike, new_service and ' + + 'low_volume (under minRequests in both windows, never flagged) and ' + + 'new_errors (a listed error group is new or growing; such services sort ' + + 'first within their status) are context only. Healthy services are not ' + + 'regressions: report them as steady, and treat moves below these ' + + "thresholds as normal variance. Each service's " + + '`summary` states which moves crossed one. topErrors carry firstSeen / ' + + 'lastSeen and a pattern: ongoing (still firing), spread (most of the ' + + 'window, quiet now; not proven resolved) or burst (short, so transient). topErrors lists new and ' + + 'growing error groups first (largest gain first), then the largest steady ' + + 'ones. newOperations = span names with traffic now and none in the ' + + 'comparison window (usually a rollout); errors on them have status ' + + '"new_operation". status "new" = absent from the comparison window, "growing" = at ' + + 'least 2x its comparison count and beyond random variation, "existing" = ' + + 'otherwise.'; + +/** Without real prior traffic every error group looks new; skip novelty. */ +export const hasBaseline = (r: RankedService) => + !r.assessment.flags.some(f => f === 'new_service' || f === 'low_volume'); + +export function buildReport(args: { + ranked: RankedService[]; + current: Window; + comparison: Window | null; + comparisonMode: string; + comparisonNote?: string; + coverageNote?: string; + spanKind: string; + topN: number; + errorGroupBy: string[]; + currentErrors: Map; + comparisonErrorCounts: Map | null; + newOperations: Map; + spanNameKey: string; + metricSourceId?: string | null; +}) { + const onNewOperation = (r: RankedService, g: ErrorGroup) => + (args.newOperations.get(r.service) ?? []).some( + o => o.operation === groupValue(g, args.spanNameKey), + ); + const windowMinutes = + (args.current.end.getTime() - args.current.start.getTime()) / 60_000; + const counts: Record = { + critical: 0, + degraded: 0, + healthy: 0, + }; + for (const r of args.ranked) counts[r.assessment.status]++; + + const prior = (g: ErrorGroup) => + args.comparisonErrorCounts?.get(errorGroupKey(g)) ?? 0; + const changedErrors = (r: RankedService) => + args.comparisonErrorCounts && hasBaseline(r) + ? (args.currentErrors.get(r.service) ?? []).filter( + g => isChanged(g.count, prior(g)) && !onNewOperation(r, g), + ) + : []; + const hasNewErrors = (r: RankedService) => changedErrors(r).length > 0; + // Within a status, a service with a new or growing error group goes first: + // its aggregate rates may be flat, but it has the most novel signal. + const shown = args.ranked + .map((r, i) => ({ r, i, novel: hasNewErrors(r) })) + .sort( + (a, b) => + STATUS_ORDER[a.r.assessment.status] - + STATUS_ORDER[b.r.assessment.status] || + Number(b.novel) - Number(a.novel) || + a.i - b.i, + ) + .slice(0, args.topN); + + const services = shown.map(({ r, novel }, i) => ({ + rank: i + 1, + service: r.service, + status: r.assessment.status, + flags: novel + ? [...r.assessment.flags, 'new_errors' satisfies HealthFlag] + : r.assessment.flags, + summary: serviceSummary({ + current: r.current, + comparison: r.comparison, + assessment: r.assessment, + changedErrors: changedErrors(r), + newOperations: (args.newOperations.get(r.service) ?? []).slice( + 0, + MAX_NEW_OPERATIONS, + ), + window: args.current, + comparisonMode: args.comparisonMode, + metricSourceId: args.metricSourceId, + spanKind: args.spanKind, + }), + score: r.assessment.score, + current: formatStats(r.current, windowMinutes), + ...(r.comparison !== undefined && { + comparison: r.comparison + ? formatStats(r.comparison, windowMinutes) + : null, + deltas: r.assessment.deltas, + }), + ...(args.newOperations.has(r.service) && { + newOperations: args.newOperations + .get(r.service) + ?.slice(0, MAX_NEW_OPERATIONS), + }), + topErrors: (args.currentErrors.get(r.service) ?? []).map(g => { + const comparisonCount = prior(g); + return { + group: g.group, + count: g.count, + ...errorTiming(g, args.current), + ...(args.comparisonErrorCounts && + hasBaseline(r) && { + comparisonCount, + status: onNewOperation(r, g) + ? ('new_operation' satisfies ErrorGroupStatus) + : errorGroupStatus(g.count, comparisonCount), + }), + }; + }), + })); + + return { + summary: { + currentWindow: iso(args.current), + comparisonWindow: args.comparison ? iso(args.comparison) : null, + comparison: args.comparisonMode, + spanKind: args.spanKind, + errorGroupBy: args.errorGroupBy, + servicesAssessed: args.ranked.length, + servicesReturned: services.length, + statusCounts: counts, + ...(args.ranked.length === 0 && { + warning: + 'No spans matched in either window. Check the time window, the ' + + 'where filter, and spanKind ("server" skips sources without server ' + + 'spans; try spanKind:"all").', + }), + ...(args.comparisonNote && { comparisonNote: args.comparisonNote }), + ...(args.coverageNote && { coverageNote: args.coverageNote }), + }, + services, + usage: USAGE, + }; +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/schema.ts b/packages/api/src/mcp/tools/query/serviceHealth/schema.ts new file mode 100644 index 0000000000..def9e23751 --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/schema.ts @@ -0,0 +1,124 @@ +import ms from 'ms'; +import { z } from 'zod'; + +import { + sourceIdSchema, + whereLanguageSchema, + whereSchema, +} from '@/mcp/tools/query/schemas'; + +const COMPARISON_MODES = [ + 'yesterday', + 'previous_period', + 'last_week', + 'none', +] as const; +export type ComparisonMode = (typeof COMPARISON_MODES)[number]; + +export const DEFAULT_WINDOW_MS = ms('1h'); + +export const serviceHealthSchema = z.object({ + sourceId: sourceIdSchema.describe( + 'The "id" of a kind="trace" source from clickstack_list_sources (24-char ' + + 'hex, e.g. "6a4c18f1d64c5f47bb275fbe"). Not the source name or table.', + ), + startTime: z + .string() + .optional() + .describe( + 'Start of the window to assess (ISO 8601). Default: 1 hour before endTime.', + ), + endTime: z + .string() + .optional() + .describe('End of the window to assess (ISO 8601). Defaults to now.'), + comparison: z + .enum(COMPARISON_MODES) + .optional() + .default('yesterday') + .describe( + 'Window to compare against, always the same length as the assessed window:\n' + + ' yesterday (default): shifted back 24h, so daily traffic cycles line up\n' + + ' previous_period: the window immediately before startTime\n' + + ' last_week: shifted back 7 days\n' + + ' none: no comparison; only absolute thresholds apply\n' + + 'If the chosen window has no data, the previous period is used instead.', + ), + where: whereSchema, + whereLanguage: whereLanguageSchema, + spanKind: z + .enum(['server', 'all']) + .optional() + .default('server') + .describe( + 'Which spans count as requests. server (default): entry-point spans only, ' + + 'matching the Services dashboard. all: every span, including internal ' + + 'and client calls, which inflates throughput and dilutes error rate and latency changes.', + ), + services: z + .array(z.string().min(1)) + .max(100) + .optional() + .describe('Only assess these service names. Omit to assess every service.'), + topN: z + .number() + .int() + .min(1) + .max(100) + .optional() + .default(20) + .describe('Max services returned, worst first.'), + topErrorsPerService: z + .number() + .int() + .min(0) + .max(10) + .optional() + .default(3) + .describe( + 'Error groups returned per service. 0 skips the error breakdown, which is faster.', + ), + errorGroupBy: z + .string() + .optional() + .describe( + 'Comma-separated SQL expressions that define an error group. Defaults to the ' + + "source's span name and status message. Example: " + + '"SpanName, SpanAttributes[\'exception.type\']".', + ), + minRequests: z + .number() + .int() + .min(1) + .optional() + .default(50) + .describe( + 'Services with fewer requests than this in both windows are never flagged; ' + + 'their ratios are too noisy.', + ), +}); + +export type ServiceHealthInput = z.infer; + +export type Window = { start: Date; end: Date }; + +/** The comparison window for a mode, or null for `none`. */ +export function comparisonWindow( + current: Window, + mode: ComparisonMode, +): Window | null { + const lengthMs = current.end.getTime() - current.start.getTime(); + const shiftMs = + mode === 'yesterday' + ? ms('1d') + : mode === 'last_week' + ? ms('7d') + : mode === 'previous_period' + ? lengthMs + : null; + if (shiftMs == null) return null; + return { + start: new Date(current.start.getTime() - shiftMs), + end: new Date(current.end.getTime() - shiftMs), + }; +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/source.ts b/packages/api/src/mcp/tools/query/serviceHealth/source.ts new file mode 100644 index 0000000000..4e7553693f --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/source.ts @@ -0,0 +1,92 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { getSource, getSources } from '@/controllers/sources'; +import type { McpErrorResult } from '@/mcp/utils/errors'; +import { mcpUserError } from '@/mcp/utils/errors'; + +import type { HealthSource } from './queries'; + +const MAX_LISTED_SOURCES = 10; + +/** + * Agents often pass a table or source name where the ID belongs. Point them at + * the exact ID when the input names one trace source, else list the options. + */ +async function sourceNotFound( + teamId: string, + sourceId: string, + toolName: string, +): Promise { + const traces = (await getSources(teamId)).filter( + s => s.kind === SourceKind.Trace, + ); + const needle = sourceId.trim().toLowerCase(); + const describe = (s: (typeof traces)[number]) => + `${s._id.toString()} ("${s.name}", table ${s.from.databaseName}.${s.from.tableName})`; + const matches = traces.filter(s => + [ + s.name, + s.from.tableName, + `${s.from.databaseName}.${s.from.tableName}`, + ].some(v => v.toLowerCase() === needle), + ); + // With several matches, naming one would be a guess; list them instead. + const match = matches.length === 1 ? matches[0] : undefined; + if (match) { + return mcpUserError( + `"${sourceId}" is a source name or table, not a source ID. Retry ${toolName} ` + + `with sourceId:"${match._id.toString()}".`, + ); + } + const listed = traces.slice(0, MAX_LISTED_SOURCES).map(describe); + return mcpUserError( + `Source not found: "${sourceId}". sourceId must be the "id" field from ` + + 'clickstack_list_sources, not a name or table. ' + + (listed.length + ? `Trace sources: ${listed.join('; ')}.` + : 'This team has no trace sources.'), + ); +} + +/** Load a trace source and check it has the fields the tool needs. */ +export async function loadHealthSource( + teamId: string, + sourceId: string, + toolName: string, +): Promise { + const doc = await getSource(teamId, sourceId); + if (!doc) return sourceNotFound(teamId, sourceId, toolName); + if (doc.kind !== SourceKind.Trace) { + return mcpUserError( + `Source ${sourceId} is kind="${doc.kind}". ${toolName} requires a source of kind="trace".`, + ); + } + if (!doc.serviceNameExpression || !doc.statusCodeExpression) { + return mcpUserError( + `Source ${sourceId} has no ` + + `${doc.serviceNameExpression ? 'statusCodeExpression' : 'serviceNameExpression'}. ` + + 'Set it with clickstack_save_source, then retry.', + ); + } + return { + id: doc._id.toString(), + connection: doc.connection.toString(), + from: { + databaseName: doc.from.databaseName, + tableName: doc.from.tableName, + }, + timestampValueExpression: doc.timestampValueExpression, + implicitColumnExpression: doc.implicitColumnExpression, + useTextIndexForImplicitColumn: doc.useTextIndexForImplicitColumn, + sampleRateExpression: doc.sampleRateExpression, + serviceNameExpression: doc.serviceNameExpression, + statusCodeExpression: doc.statusCodeExpression, + statusMessageExpression: doc.statusMessageExpression, + spanNameExpression: doc.spanNameExpression, + spanKindExpression: doc.spanKindExpression, + durationExpression: doc.durationExpression, + durationPrecision: doc.durationPrecision, + metricSourceId: doc.metricSourceId, + querySettings: doc.querySettings, + }; +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/summary.ts b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts new file mode 100644 index 0000000000..a7737af20a --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts @@ -0,0 +1,148 @@ +import type { ErrorGroup } from './errorGroups'; +import type { NewOperation } from './operations'; +import { + HEALTH_THRESHOLDS, + type ServiceAssessment, + type WindowStats, +} from './rank'; +import type { Window } from './schema'; + +/** An error group whose last error is this close to the window end is ongoing. */ +const ONGOING_TAIL_MS = 2 * 60_000; +/** + * A group that is not ongoing but spans this share of the window is not a + * short burst. First/last times can't tell a trickle from one long incident, + * so it is only called spread, never transient. + */ +const SPREAD_SHARE = 0.5; + +type ErrorPattern = 'ongoing' | 'spread' | 'burst'; + +const pct = (s: WindowStats | null) => + s && s.requests > 0 ? `${((s.errors / s.requests) * 100).toFixed(2)}%` : '0%'; +const hhmm = (ms: number) => new Date(ms).toISOString().slice(11, 16); + +export function errorTiming(g: ErrorGroup, window: Window) { + if (g.firstSeenMs == null || g.lastSeenMs == null) return {}; + const ongoing = window.end.getTime() - g.lastSeenMs <= ONGOING_TAIL_MS; + const windowMs = window.end.getTime() - window.start.getTime(); + const spread = windowMs > 0 ? (g.lastSeenMs - g.firstSeenMs) / windowMs : 0; + const pattern: ErrorPattern = ongoing + ? 'ongoing' + : spread >= SPREAD_SHARE + ? 'spread' + : 'burst'; + return { + firstSeen: new Date(g.firstSeenMs).toISOString(), + lastSeen: new Date(g.lastSeenMs).toISOString(), + activeMinutes: + Math.round(((g.lastSeenMs - g.firstSeenMs) / 60_000) * 10) / 10, + ongoing, + pattern, + }; +} + +function describeError(g: ErrorGroup, window: Window): string { + const label = Object.values(g.group).filter(Boolean).join(' / ') || 'error'; + const t = errorTiming(g, window); + if (t.pattern == null || g.firstSeenMs == null || g.lastSeenMs == null) + return `"${label}" (${g.count})`; + const [from, to] = [hhmm(g.firstSeenMs), hhmm(g.lastSeenMs)]; + const when = from === to ? `at ${from} UTC` : `${from}-${to} UTC`; + const how = + t.pattern === 'ongoing' + ? `${when}, still occurring` + : t.pattern === 'spread' + ? `spread across ${from}-${to} UTC, none since ${to}` + : `${when}, stopped`; + return `"${label}" (${g.count}, ${how})`; +} + +/** + * A plain verdict per service. Agents otherwise read every nonzero delta as a + * trend ("error rate up 0.08pp"), so this states which moves crossed a + * threshold and which are ordinary variance. + */ +export function serviceSummary(args: { + current: WindowStats | null; + comparison: WindowStats | null | undefined; + assessment: ServiceAssessment; + changedErrors: ErrorGroup[]; + newOperations: NewOperation[]; + window: Window; + comparisonMode: string; + metricSourceId?: string | null; + spanKind?: string; +}): string { + const { current, comparison, assessment, changedErrors, window } = args; + const f = new Set(assessment.flags); + const parts: string[] = []; + if (f.has('missing_service')) + parts.push('no requests now, but had traffic in the comparison window'); + if (f.has('error_rate_up')) + parts.push(`error rate ${pct(current)} vs ${pct(comparison ?? null)}`); + else if (f.has('high_error_rate')) + parts.push( + `error rate ${pct(current)} is at or above ${HEALTH_THRESHOLDS.highErrorRate * 100}%`, + ); + if (f.has('latency_up')) + parts.push(`p95 ${current?.p95Ms}ms vs ${comparison?.p95Ms}ms`); + if (f.has('traffic_drop')) + parts.push(`requests ${current?.requests} vs ${comparison?.requests}`); + + const head = + parts.length > 0 + ? `${assessment.status.toUpperCase()}: ${parts.join('; ')}.` + : comparison === undefined + ? 'Healthy: no absolute threshold crossed (no comparison window).' + : `Healthy: no metric crossed a threshold vs ${args.comparisonMode}; ` + + 'the deltas are normal variance' + + (changedErrors.length > 0 + ? ', but see the new errors below.' + : '; report this service as steady, no action needed.'); + + const context: string[] = []; + if (f.has('new_service')) + context.push('No traffic in the comparison window.'); + if (f.has('traffic_spike')) + context.push('Traffic is well above the comparison window.'); + if (f.has('low_volume')) context.push('Too few requests to judge.'); + if (f.has('latency_up') || f.has('error_rate_up')) + context.push( + "Before blaming dependencies, check this service's runtime metrics " + + '(memory, GC, CPU, restarts) by pod or instance' + + (args.metricSourceId + ? ` on metric source ${args.metricSourceId}` + : ' on a kind="metric" source from clickstack_list_sources') + + '; slow or failing spans are often a symptom of the service itself.', + ); + if (args.newOperations.length > 0) + context.push( + `New operations, likely a rollout: ${args.newOperations + .map( + o => + `${o.operation} (${o.requests} requests, ${(o.shareOfService * 100).toFixed(1)}% of traffic)`, + ) + .join('; ')}.`, + ); + if (changedErrors.length > 0) { + const patterns = new Set( + changedErrors.map(g => errorTiming(g, window).pattern), + ); + context.push( + `New or growing errors: ${changedErrors.map(g => describeError(g, window)).join('; ')}.` + + (patterns.has('burst') + ? ' A short burst that stopped is transient, not an ongoing problem.' + : '') + + (patterns.has('spread') + ? ' An error spread across much of the window may still recur; check clickstack_timeseries by minute before calling it resolved.' + : ''), + ); + } + if (args.spanKind === 'all' && assessment.status === 'healthy') + context.push( + 'Internal and client spans are included, which dilutes server-side ' + + 'changes; check spanKind:"server" before concluding nothing changed.', + ); + return [head, ...context].join(' '); +} diff --git a/packages/app/src/llm/lib/expressions.ts b/packages/app/src/llm/lib/expressions.ts index ae5743b164..27140cde54 100644 --- a/packages/app/src/llm/lib/expressions.ts +++ b/packages/app/src/llm/lib/expressions.ts @@ -1,7 +1,6 @@ +import { errorStatusPredicateSql } from '@hyperdx/common-utils/dist/core/spanPredicates'; import { TLogSource, TTraceSource } from '@hyperdx/common-utils/dist/types'; -import { errorStatusPredicateSql } from '@/source'; - import { generateCostSqlExpression } from './cost'; import { buildAnyKeyExistsSql, buildLLMSpanSqlPredicate } from './detect'; diff --git a/packages/app/src/serviceDashboard.ts b/packages/app/src/serviceDashboard.ts index 51aa0d6fc4..8208a8e544 100644 --- a/packages/app/src/serviceDashboard.ts +++ b/packages/app/src/serviceDashboard.ts @@ -1,11 +1,14 @@ import { useMemo } from 'react'; import { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { + errorStatusPredicateSql, + serverSpanKindPredicateSql, +} from '@hyperdx/common-utils/dist/core/spanPredicates'; import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils'; import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types'; import { useColumns, useJsonColumns } from './hooks/useMetadata'; -import { errorStatusPredicateSql } from './source'; const COALESCE_FIELDS_LIMIT = 100; @@ -227,7 +230,7 @@ export function getExpressions( const filterExpressions = { isEndpointNonEmpty: `NOT empty(${auxExpressions.endpoint})`, isError: errorStatusPredicateSql(fieldExpressions.severityText), - isSpanKindServer: `${fieldExpressions.spanKind} IN ('Server', 'SPAN_KIND_SERVER')`, + isSpanKindServer: serverSpanKindPredicateSql(fieldExpressions.spanKind), isDbSpan: `${fieldExpressions.dbStatement} <> ''`, }; diff --git a/packages/app/src/source.ts b/packages/app/src/source.ts index b98ee52b2b..f235e01786 100644 --- a/packages/app/src/source.ts +++ b/packages/app/src/source.ts @@ -540,20 +540,6 @@ export function getDurationSecondsExpression(source: TTraceSource) { return `(${source.durationExpression})/1e${source.durationPrecision ?? 9}`; } -/** - * The single definition of an errored span, as a SQL predicate over a span - * status-code expression. Matches both the short OTel status ('Error'/'ERROR') - * and the legacy collector form ('STATUS_CODE_ERROR'), which some collectors - * write instead. - * - * Span status only. Log severity text is a different vocabulary (its error - * values are 'fatal', 'crit' and friends, never 'status_code_error'), so log - * callers keep their own predicate rather than reusing this one. - */ -export function errorStatusPredicateSql(statusCodeExpression: string): string { - return `lower(${statusCodeExpression}) IN ('error', 'status_code_error')`; -} - // Aggregate functions whose output preserves the unit of the input value. // count and count_distinct produce dimensionless counts and should not // inherit the duration format. diff --git a/packages/common-utils/src/__tests__/spanPredicates.test.ts b/packages/common-utils/src/__tests__/spanPredicates.test.ts new file mode 100644 index 0000000000..e5ee7c0b96 --- /dev/null +++ b/packages/common-utils/src/__tests__/spanPredicates.test.ts @@ -0,0 +1,20 @@ +import { + errorStatusPredicateSql, + serverSpanKindPredicateSql, +} from '@/core/spanPredicates'; + +describe('errorStatusPredicateSql', () => { + it('matches both short and legacy error statuses case-insensitively', () => { + expect(errorStatusPredicateSql('StatusCode')).toBe( + "lower(StatusCode) IN ('error', 'status_code_error')", + ); + }); +}); + +describe('serverSpanKindPredicateSql', () => { + it('matches both short and legacy server span kinds', () => { + expect(serverSpanKindPredicateSql('SpanKind')).toBe( + "SpanKind IN ('Server', 'SPAN_KIND_SERVER')", + ); + }); +}); diff --git a/packages/common-utils/src/core/spanPredicates.ts b/packages/common-utils/src/core/spanPredicates.ts new file mode 100644 index 0000000000..e2d37549e6 --- /dev/null +++ b/packages/common-utils/src/core/spanPredicates.ts @@ -0,0 +1,18 @@ +/** + * The single definition of an errored span, as a SQL predicate over a span + * status-code expression. Matches both the short OTel status ('Error'/'ERROR') + * and the legacy collector form ('STATUS_CODE_ERROR'), which some collectors + * write instead. + * + * Span status only. Log severity text is a different vocabulary (its error + * values are 'fatal', 'crit' and friends, never 'status_code_error'), so log + * callers keep their own predicate rather than reusing this one. + */ +export function errorStatusPredicateSql(statusCodeExpression: string): string { + return `lower(${statusCodeExpression}) IN ('error', 'status_code_error')`; +} + +/** Server (request entry-point) spans, in both short and legacy enum forms. */ +export function serverSpanKindPredicateSql(spanKindExpression: string): string { + return `${spanKindExpression} IN ('Server', 'SPAN_KIND_SERVER')`; +}