From 4250e5ca0a5b2d956dbc7c868dc15433aaae1845 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 19 Oct 2026 13:22:22 -0600 Subject: [PATCH 01/15] feat(mcp): add clickstack_service_health tool One-call per-service health for a trace source: throughput, error rate, p50/p95/p99 and top error groups, compared with yesterday (falling back to the previous period when that window is empty) and ranked worst first. New and growing error groups are listed first with first/last-seen times, and operations with no prior traffic are reported as likely rollouts. Moves the span error/server-kind predicates to common-utils so the API and app share one definition. --- .changeset/mcp-service-health-tool.md | 7 + MCP.md | 1 + .../__tests__/serviceHealthTool.int.test.ts | 286 ++++++++++++++++++ .../query/__tests__/serviceHealthRank.test.ts | 232 ++++++++++++++ .../api/src/mcp/tools/query/builderCatalog.ts | 1 + packages/api/src/mcp/tools/query/index.ts | 2 + .../mcp/tools/query/serviceHealth/collect.ts | 85 ++++++ .../tools/query/serviceHealth/comparison.ts | 64 ++++ .../mcp/tools/query/serviceHealth/index.ts | 218 +++++++++++++ .../tools/query/serviceHealth/operations.ts | 99 ++++++ .../mcp/tools/query/serviceHealth/queries.ts | 281 +++++++++++++++++ .../src/mcp/tools/query/serviceHealth/rank.ts | 206 +++++++++++++ .../mcp/tools/query/serviceHealth/report.ts | 285 +++++++++++++++++ .../mcp/tools/query/serviceHealth/schema.ts | 123 ++++++++ .../mcp/tools/query/serviceHealth/source.ts | 91 ++++++ .../mcp/tools/query/serviceHealth/summary.ts | 114 +++++++ packages/app/src/llm/lib/expressions.ts | 3 +- packages/app/src/serviceDashboard.ts | 7 +- packages/app/src/source.ts | 14 - .../src/__tests__/spanPredicates.test.ts | 20 ++ .../common-utils/src/core/spanPredicates.ts | 18 ++ 21 files changed, 2139 insertions(+), 18 deletions(-) create mode 100644 .changeset/mcp-service-health-tool.md create mode 100644 packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts create mode 100644 packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts create mode 100644 packages/api/src/mcp/tools/query/serviceHealth/collect.ts create mode 100644 packages/api/src/mcp/tools/query/serviceHealth/comparison.ts create mode 100644 packages/api/src/mcp/tools/query/serviceHealth/index.ts create mode 100644 packages/api/src/mcp/tools/query/serviceHealth/operations.ts create mode 100644 packages/api/src/mcp/tools/query/serviceHealth/queries.ts create mode 100644 packages/api/src/mcp/tools/query/serviceHealth/rank.ts create mode 100644 packages/api/src/mcp/tools/query/serviceHealth/report.ts create mode 100644 packages/api/src/mcp/tools/query/serviceHealth/schema.ts create mode 100644 packages/api/src/mcp/tools/query/serviceHealth/source.ts create mode 100644 packages/api/src/mcp/tools/query/serviceHealth/summary.ts create mode 100644 packages/common-utils/src/__tests__/spanPredicates.test.ts create mode 100644 packages/common-utils/src/core/spanPredicates.ts diff --git a/.changeset/mcp-service-health-tool.md b/.changeset/mcp-service-health-tool.md new file mode 100644 index 0000000000..bd5b4aa9ac --- /dev/null +++ b/.changeset/mcp-service-health-tool.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/api': minor +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +--- + +feat(mcp): add `clickstack_service_health`, a one-call per-service health report (throughput, error rate, p50/p95, top errors) compared with yesterday and ranked worst first diff --git a/MCP.md b/MCP.md index 132a7a9d79..a380fb8909 100644 --- a/MCP.md +++ b/MCP.md @@ -126,6 +126,7 @@ with: | `clickstack_delete_source` | Permanently delete a data source by ID | | `clickstack_list_metrics` | Paginated catalog of metric names on a metric source with optional kind / namePattern / time-window filters | | `clickstack_describe_metric` | Per-metric drill-down: kind(s), unit, description, attribute keys per map column, and sampled values | +| `clickstack_service_health` | One-call service health on a trace source: throughput, error rate, p50/p95 and top errors per service, compared with yesterday and ranked worst first | | `clickstack_timeseries` | Plot metrics over time as a line or stacked bar chart (works on log, trace, and metric sources) | | `clickstack_table` | Compute aggregated metrics as a table, single number, or pie chart (works on log, trace, and metric sources) | | `clickstack_search` | Browse individual log, event, or trace rows | diff --git a/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts b/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts new file mode 100644 index 0000000000..1578f4a85b --- /dev/null +++ b/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts @@ -0,0 +1,286 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; + +import * as config from '@/config'; +import { + bulkInsertData, + DEFAULT_DATABASE, + DEFAULT_LOGS_TABLE, + DEFAULT_TRACES_TABLE, + getLoggedInAgent, + getServer, +} from '@/fixtures'; +import { McpContext } from '@/mcp/tools/types'; +import Connection from '@/models/connection'; +import { Source, type SourceDocument } from '@/models/source'; + +import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; + +const TOOL = 'clickstack_service_health'; +const HOUR = 60 * 60 * 1000; +// ClickHouse is shared across tests and files, so scope every query to this +// run's services. +const RUN = `sh${Date.now()}`; +const svc = (name: string) => `${RUN}-${name}`; + +type Span = { + ServiceName: string; + SpanKind?: string; + durationMs?: number; + error?: string; +}; + +type ServiceReport = { + service: string; + status: string; + flags: string[]; + current: { + requests: number; + errors: number; + errorRate: number; + p95Ms: number; + }; + comparison?: { requests: number; errors: number } | null; + topErrors: { + group: Record; + count: number; + comparisonCount?: number; + status?: string; + }[]; +}; + +describe('MCP Service Health Tool', () => { + const server = getServer(); + let client: Client; + let traceSource: SourceDocument; + let logSource: SourceDocument; + const now = Date.now(); + + async function insertSpans(windowStart: number, spans: Span[]) { + await bulkInsertData( + `${DEFAULT_DATABASE}.${DEFAULT_TRACES_TABLE}`, + spans.map((s, i) => ({ + // Spread across the first 50 minutes of the window. + Timestamp: new Date(windowStart + ((i * 30_000) % (50 * 60_000))), + TraceId: `trace-${windowStart}-${i}`, + SpanId: `span-${windowStart}-${i}`, + ParentSpanId: '', + SpanName: 'GET /api', + SpanKind: s.SpanKind ?? (i % 2 ? 'Server' : 'SPAN_KIND_SERVER'), + ServiceName: s.ServiceName, + Duration: (s.durationMs ?? 100) * 1e6, + // Both error spellings must count. + StatusCode: s.error ? (i % 2 ? 'Error' : 'STATUS_CODE_ERROR') : 'Ok', + StatusMessage: s.error ?? '', + SpanAttributes: {}, + ResourceAttributes: {}, + })), + ); + } + + const repeat = (n: number, s: Span): Span[] => + Array.from({ length: n }, () => s); + + beforeAll(async () => { + await server.start(); + }); + + beforeEach(async () => { + const { team, user } = await getLoggedInAgent(server); + const connection = await Connection.create({ + team: team._id, + name: 'Default', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + logSource = await Source.create({ + kind: SourceKind.Log, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: DEFAULT_LOGS_TABLE }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Logs', + }); + traceSource = await Source.create({ + kind: SourceKind.Trace, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: DEFAULT_TRACES_TABLE }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Traces', + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + durationExpression: 'Duration', + durationPrecision: 9, + serviceNameExpression: 'ServiceName', + statusCodeExpression: 'StatusCode', + statusMessageExpression: 'StatusMessage', + }); + const context: McpContext = { + teamId: team._id.toString(), + userId: user._id.toString(), + }; + client = await createTestClient(context); + }); + + afterEach(async () => { + await client.close(); + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + const call = (args: Record = {}) => + callTool(client, TOOL, { + sourceId: traceSource._id.toString(), + startTime: new Date(now - HOUR).toISOString(), + endTime: new Date(now).toISOString(), + where: `startsWith(ServiceName, '${RUN}')`, + whereLanguage: 'sql', + ...args, + }); + + const parse = (result: Awaited>) => { + expect(result.isError).toBeFalsy(); + return JSON.parse(getFirstText(result)) as { + summary: Record; + services: ServiceReport[]; + }; + }; + + it('is listed with its input schema', async () => { + const { tools } = await client.listTools(); + const tool = tools.find(t => t.name === TOOL); + expect(tool?.inputSchema.required).toEqual(['sourceId']); + expect(Object.keys(tool?.inputSchema.properties ?? {})).toEqual( + expect.arrayContaining([ + 'comparison', + 'spanKind', + 'errorGroupBy', + 'topN', + ]), + ); + }); + + it('rejects non-trace sources', async () => { + const result = await call({ sourceId: logSource._id.toString() }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('kind="trace"'); + }); + + it('rejects a window longer than the comparison shift', async () => { + const result = await call({ + startTime: new Date(now - 30 * HOUR).toISOString(), + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toMatch(/overlap/); + }); + + describe('with data in both windows', () => { + beforeAll(async () => { + const currentStart = now - HOUR; + const yesterdayStart = currentStart - 24 * HOUR; + await insertSpans(currentStart, [ + ...repeat(99, { ServiceName: svc('steady') }), + ...repeat(1, { ServiceName: svc('steady'), error: 'flaky' }), + // Internal error spans must not count with the default spanKind. + ...repeat(100, { + ServiceName: svc('steady'), + SpanKind: 'Internal', + error: 'internal', + }), + ...repeat(80, { ServiceName: svc('erroring') }), + ...repeat(15, { ServiceName: svc('erroring'), error: 'db timeout' }), + ...repeat(5, { ServiceName: svc('erroring'), error: 'auth failed' }), + ...repeat(100, { ServiceName: svc('slow'), durationMs: 600 }), + ]); + await insertSpans(yesterdayStart, [ + ...repeat(99, { ServiceName: svc('steady') }), + ...repeat(1, { ServiceName: svc('steady'), error: 'flaky' }), + ...repeat(99, { ServiceName: svc('erroring') }), + ...repeat(1, { ServiceName: svc('erroring'), error: 'db timeout' }), + ...repeat(100, { ServiceName: svc('slow'), durationMs: 100 }), + ]); + }); + + it('ranks regressions first and leaves steady services healthy', async () => { + const { summary, services } = parse(await call()); + expect(services.map(s => [s.service, s.status])).toEqual([ + [svc('erroring'), 'critical'], + [svc('slow'), 'degraded'], + [svc('steady'), 'healthy'], + ]); + expect(services[0].flags).toEqual( + expect.arrayContaining(['error_rate_up', 'high_error_rate']), + ); + expect(services[1].flags).toContain('latency_up'); + expect(services[1].current.p95Ms).toBeCloseTo(600, -1); + expect(summary.statusCounts).toEqual({ + critical: 1, + degraded: 1, + healthy: 1, + }); + }); + + it('counts server spans only by default', async () => { + const { services } = parse(await call()); + const steady = services.find(s => s.service === svc('steady')); + expect(steady?.current.requests).toBe(100); + expect(steady?.current.errors).toBe(1); + + const all = parse(await call({ spanKind: 'all' })); + expect( + all.services.find(s => s.service === svc('steady'))?.current.requests, + ).toBe(200); + }); + + it('tags error groups as new or growing against the comparison', async () => { + const { services } = parse(await call()); + expect(services[0].topErrors).toEqual([ + expect.objectContaining({ + group: { SpanName: 'GET /api', StatusMessage: 'db timeout' }, + count: 15, + comparisonCount: 1, + status: 'growing', + }), + expect.objectContaining({ + group: { SpanName: 'GET /api', StatusMessage: 'auth failed' }, + count: 5, + comparisonCount: 0, + status: 'new', + }), + ]); + // Spans are spread over the first 50 minutes, so nothing is ongoing. + const first = services[0].topErrors[0] as Record; + expect(Date.parse(String(first.firstSeen))).toBeLessThan( + Date.parse(String(first.lastSeen)), + ); + expect(first.ongoing).toBe(false); + }); + + it('accepts a custom errorGroupBy', async () => { + const { services } = parse(await call({ errorGroupBy: 'StatusMessage' })); + expect(services[0].topErrors[0].group).toEqual({ + StatusMessage: 'db timeout', + }); + }); + + it('omits comparisons with comparison:none', async () => { + const { summary, services } = parse(await call({ comparison: 'none' })); + expect(summary.comparisonWindow).toBeNull(); + expect(services[0].comparison).toBeUndefined(); + expect(services[0].topErrors[0].status).toBeUndefined(); + }); + + it('filters to the requested services', async () => { + const { services } = parse(await call({ services: [svc('slow')] })); + expect(services.map(s => s.service)).toEqual([svc('slow')]); + }); + }); +}); diff --git a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts new file mode 100644 index 0000000000..4b526291bf --- /dev/null +++ b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts @@ -0,0 +1,232 @@ +import { errorGroupKey } from '@/mcp/tools/query/serviceHealth/queries'; +import { + assessService, + HEALTH_THRESHOLDS, + type WindowStats, +} from '@/mcp/tools/query/serviceHealth/rank'; +import { + errorGroupStatus, + rankServices, + selectTopErrors, + topGroupsPerService, +} from '@/mcp/tools/query/serviceHealth/report'; +import { comparisonWindow } from '@/mcp/tools/query/serviceHealth/schema'; + +const stats = (s: Partial = {}): WindowStats => ({ + requests: 1000, + errors: 5, + p50Ms: 50, + p95Ms: 200, + p99Ms: 300, + ...s, +}); + +describe('assessService', () => { + it('is healthy when nothing moved', () => { + const a = assessService(stats(), stats(), 50); + expect(a.status).toBe('healthy'); + expect(a.flags).toEqual([]); + }); + + it('ignores a doubled error rate below the absolute floor', () => { + // 0.5% -> 1.0%: 2x, but only +0.5pp + const a = assessService(stats({ errors: 10 }), stats({ errors: 5 }), 50); + expect(a.flags).not.toContain('error_rate_up'); + expect(a.status).toBe('healthy'); + }); + + it('flags error_rate_up at exactly the ratio and delta thresholds', () => { + // 1% -> 2%: exactly 2x and +1pp + const a = assessService(stats({ errors: 20 }), stats({ errors: 10 }), 50); + expect(a.flags).toContain('error_rate_up'); + expect(a.status).toBe('degraded'); + expect(a.deltas.errorRateDeltaPp).toBe(1); + }); + + it('is critical when errors rose past the high error rate', () => { + const a = assessService(stats({ errors: 80 }), stats({ errors: 5 }), 50); + expect(a.flags).toEqual( + expect.arrayContaining(['error_rate_up', 'high_error_rate']), + ); + expect(a.status).toBe('critical'); + }); + + it('marks a chronically high error rate degraded, not critical', () => { + const a = assessService(stats({ errors: 80 }), stats({ errors: 80 }), 50); + expect(a.flags).toEqual(['high_error_rate']); + expect(a.status).toBe('degraded'); + }); + + it('needs both the ratio and the ms delta for latency_up', () => { + expect( + assessService(stats({ p95Ms: 12 }), stats({ p95Ms: 4 }), 50).flags, + ).not.toContain('latency_up'); + expect( + assessService( + stats({ p95Ms: 200 * HEALTH_THRESHOLDS.latencyRatio }), + stats(), + 50, + ).flags, + ).toContain('latency_up'); + }); + + it('flags traffic drops and spikes', () => { + const drop = assessService(stats({ requests: 500 }), stats(), 50); + expect(drop.flags).toContain('traffic_drop'); + expect(drop.status).toBe('degraded'); + + const spike = assessService(stats({ requests: 3000 }), stats(), 50); + expect(spike.flags).toContain('traffic_spike'); + expect(spike.status).toBe('healthy'); + }); + + it('flags new and missing services', () => { + const fresh = assessService(stats(), null, 50); + expect(fresh.flags).toEqual(['new_service']); + expect(fresh.status).toBe('healthy'); + + const gone = assessService(null, stats(), 50); + expect(gone.flags).toEqual(['missing_service']); + expect(gone.status).toBe('degraded'); + }); + + it('never flags services under minRequests in both windows', () => { + const a = assessService( + stats({ requests: 10, errors: 9 }), + stats({ requests: 10, errors: 0 }), + 50, + ); + expect(a.flags).toEqual(['low_volume']); + expect(a.status).toBe('healthy'); + }); + + it('applies absolute rules only when comparisons are off', () => { + const a = assessService(stats({ errors: 80 }), undefined, 50); + expect(a.flags).toEqual(['high_error_rate']); + expect(a.deltas.errorRateDeltaPp).toBeNull(); + }); + + it('handles a zero comparison p95 without dividing by zero', () => { + const a = assessService(stats(), stats({ p95Ms: 0 }), 50); + expect(a.deltas.p95Ratio).toBeNull(); + }); +}); + +describe('rankServices', () => { + it('sorts by status, then by score with volume as a multiplier', () => { + const cur = new Map([ + ['steady', stats()], + ['small-regression', stats({ requests: 100, errors: 10 })], + ['big-regression', stats({ requests: 100_000, errors: 10_000 })], + ['broken', stats({ errors: 300 })], + ]); + const cmp = new Map([ + ['steady', stats()], + ['small-regression', stats({ requests: 100, errors: 0 })], + ['big-regression', stats({ requests: 100_000, errors: 0 })], + ['broken', stats()], + ]); + expect(rankServices(cur, cmp, 50).map(r => r.service)).toEqual([ + 'broken', + 'big-regression', + 'small-regression', + 'steady', + ]); + }); + + it('includes services seen in only one window', () => { + const ranked = rankServices( + new Map([['a', stats()]]), + new Map([['b', stats()]]), + 50, + ); + expect(ranked.map(r => [r.service, r.assessment.flags])).toEqual([ + ['b', ['missing_service']], + ['a', ['new_service']], + ]); + }); +}); + +describe('topGroupsPerService', () => { + it('keeps the first N groups per service', () => { + const g = (service: string, count: number) => ({ + service, + group: { SpanName: `op-${count}` }, + count, + }); + const out = topGroupsPerService( + [g('a', 9), g('b', 8), g('a', 7), g('a', 6)], + 2, + ); + expect(out.get('a')?.map(x => x.count)).toEqual([9, 7]); + expect(out.get('b')?.map(x => x.count)).toEqual([8]); + }); +}); + +describe('comparisonWindow', () => { + const current = { + start: new Date('2026-10-16T10:00:00Z'), + end: new Date('2026-10-16T11:00:00Z'), + }; + + it.each([ + ['yesterday', '2026-10-15T10:00:00.000Z', '2026-10-15T11:00:00.000Z'], + ['last_week', '2026-10-09T10:00:00.000Z', '2026-10-09T11:00:00.000Z'], + ['previous_period', '2026-10-16T09:00:00.000Z', '2026-10-16T10:00:00.000Z'], + ] as const)('%s', (mode, start, end) => { + const w = comparisonWindow(current, mode); + expect(w?.start.toISOString()).toBe(start); + expect(w?.end.toISOString()).toBe(end); + }); + + it('returns null for none', () => { + expect(comparisonWindow(current, 'none')).toBeNull(); + }); +}); + +describe('error group selection', () => { + const g = (msg: string, count: number) => ({ + service: 'svc', + group: { StatusMessage: msg }, + count, + }); + const counts = (entries: [string, number][]) => + new Map(entries.map(([msg, n]) => [errorGroupKey(g(msg, 0)), n])); + + it('labels growth only when it is beyond random variation', () => { + expect(errorGroupStatus(5, 0)).toBe('new'); + expect(errorGroupStatus(13, 5)).toBe('existing'); + expect(errorGroupStatus(4060, 170)).toBe('growing'); + }); + + it('lists new and growing groups before larger steady ones', () => { + const picked = selectTopErrors( + [g('steady', 500), g('growing', 300), g('new', 40), g('tiny-new', 2)], + counts([ + ['steady', 480], + ['growing', 20], + ]), + 3, + ); + expect(picked.map(x => x.group.StatusMessage)).toEqual([ + 'growing', + 'new', + 'steady', + ]); + }); + + it('does not promote new groups that a new operation explains', () => { + const picked = selectTopErrors( + [g('steady', 500), g('on-new-endpoint', 40), g('new', 30)], + counts([['steady', 480]]), + 2, + x => x.group.StatusMessage === 'on-new-endpoint', + ); + expect(picked.map(x => x.group.StatusMessage)).toEqual(['new', 'steady']); + }); + + it('keeps count order without a comparison', () => { + const picked = selectTopErrors([g('a', 9), g('b', 5), g('c', 1)], null, 2); + expect(picked.map(x => x.group.StatusMessage)).toEqual(['a', 'b']); + }); +}); diff --git a/packages/api/src/mcp/tools/query/builderCatalog.ts b/packages/api/src/mcp/tools/query/builderCatalog.ts index c4b1a77a91..ec195b744f 100644 --- a/packages/api/src/mcp/tools/query/builderCatalog.ts +++ b/packages/api/src/mcp/tools/query/builderCatalog.ts @@ -6,6 +6,7 @@ /** Bulleted list of the builder (non-SQL) query tools and what they're for. */ export const BUILDER_TOOLS_LIST = [ + ' • clickstack_service_health — per-service throughput, errors, latency vs yesterday, ranked', ' • clickstack_table — aggregations, top-N, single-value KPIs, breakdowns', ' • clickstack_timeseries — trends / metrics over time', ' • clickstack_search — browsing individual log/trace rows', diff --git a/packages/api/src/mcp/tools/query/index.ts b/packages/api/src/mcp/tools/query/index.ts index 700a8a24ca..1af3984f12 100644 --- a/packages/api/src/mcp/tools/query/index.ts +++ b/packages/api/src/mcp/tools/query/index.ts @@ -4,11 +4,13 @@ import { registerEmergingSignals } from './emergingSignals'; import { registerEventDeltas } from './eventDeltas'; import { registerEventPatterns } from './eventPatterns'; import { registerSearch } from './search'; +import { registerServiceHealth } from './serviceHealth'; import { registerSql } from './sql'; import { registerTable } from './table'; import { registerTimeseries } from './timeseries'; const queryTools: ToolDefinition = (registrar: ToolRegistrar) => { + registerServiceHealth(registrar); registerTimeseries(registrar); registerTable(registrar); registerSearch(registrar); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/collect.ts b/packages/api/src/mcp/tools/query/serviceHealth/collect.ts new file mode 100644 index 0000000000..aecd3ce568 --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/collect.ts @@ -0,0 +1,85 @@ +import { findNewOperations, type NewOperation } from './operations'; +import { + type ErrorGroup, + errorGroupKey, + fetchErrorGroups, + type QueryContext, +} from './queries'; +import { + ERROR_CANDIDATES_PER_SERVICE, + type RankedService, + selectTopErrors, + topGroupsPerService, +} from './report'; +import type { Window } from './schema'; + +export type CollectedErrors = { + currentErrors: Map; + comparisonErrorCounts: Map | null; + newOperations: Map; +}; + +/** + * Error groups and new operations for the services that will be shown. + * Errors on a brand-new operation are expected (it had no traffic before), so + * they do not compete with genuinely new failure modes for the top slots. + */ +export async function collectErrors( + ctx: QueryContext, + args: { + current: Window; + comparison: Window | null; + shown: RankedService[]; + perService: number; + groupExprs: string[]; + }, +): Promise { + const { current, comparison, shown, perService, groupExprs } = args; + const withErrors = shown + .filter(r => (r.current?.errors ?? 0) > 0) + .map(r => r.service); + const fetchErrors = perService > 0 && withErrors.length > 0; + + const [rows, newOperations] = await Promise.all([ + fetchErrors + ? fetchErrorGroups(ctx, current, groupExprs, { services: withErrors }) + : Promise.resolve([]), + comparison + ? findNewOperations( + ctx, + current, + comparison, + shown.filter(r => r.current).map(r => r.service), + ) + : Promise.resolve(new Map()), + ]); + + const candidates = topGroupsPerService( + rows, + Math.max(ERROR_CANDIDATES_PER_SERVICE, perService), + ); + const all = [...candidates.values()].flat(); + const comparisonErrorCounts = + comparison && all.length + ? new Map( + ( + await fetchErrorGroups(ctx, comparison, groupExprs, { match: all }) + ).map(g => [errorGroupKey(g), g.count]), + ) + : null; + + const spanKey = ctx.source.spanNameExpression; + const currentErrors = new Map(); + for (const [service, groups] of candidates) { + const newOps = new Set( + (newOperations.get(service) ?? []).map(o => o.operation), + ); + currentErrors.set( + service, + selectTopErrors(groups, comparisonErrorCounts, perService, g => + newOps.has(g.group[spanKey] ?? ''), + ), + ); + } + return { currentErrors, comparisonErrorCounts, newOperations }; +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/comparison.ts b/packages/api/src/mcp/tools/query/serviceHealth/comparison.ts new file mode 100644 index 0000000000..224c39c142 --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/comparison.ts @@ -0,0 +1,64 @@ +import { fetchServiceStats, type QueryContext } from './queries'; +import { type ComparisonMode, comparisonWindow, type Window } from './schema'; + +type Stats = Awaited>; + +export type ResolvedComparison = { + comparison: Window | null; + cmp: Stats | null; + comparisonMode: ComparisonMode; + comparisonNote: string | undefined; +}; + +/** + * An empty comparison window usually means retention or a fresh source, not + * that every service is new. Comparing against nothing would label every + * service `new_service` and every error group `new`, which reads as an + * incident. Fall back to the previous period, and to no comparison if that is + * empty too. + */ +export async function resolveComparison( + ctx: QueryContext, + args: { + current: Window; + requested: Window | null; + requestedMode: ComparisonMode; + curHasData: boolean; + requestedCmp: Stats | null; + }, +): Promise { + const { current, requested, requestedMode, curHasData, requestedCmp } = args; + if (!requested || !requestedCmp || !curHasData || requestedCmp.stats.size) { + return { + comparison: requested, + cmp: requestedCmp, + comparisonMode: requestedMode, + comparisonNote: undefined, + }; + } + + const fallback = + requestedMode === 'previous_period' + ? null + : comparisonWindow(current, 'previous_period'); + const fallbackCmp = fallback ? await fetchServiceStats(ctx, fallback) : null; + if (fallback && fallbackCmp?.stats.size) { + return { + comparison: fallback, + cmp: fallbackCmp, + comparisonMode: 'previous_period', + comparisonNote: + `The "${requestedMode}" window has no data (likely outside retention), ` + + 'so this compares with the previous period instead.', + }; + } + return { + comparison: null, + cmp: null, + comparisonMode: 'none', + comparisonNote: + `The "${requestedMode}" window${fallback ? ' and the previous period have' : ' has'} ` + + 'no data, so there is no comparison. Only absolute thresholds apply; ' + + 'nothing is labelled new.', + }; +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/index.ts b/packages/api/src/mcp/tools/query/serviceHealth/index.ts new file mode 100644 index 0000000000..c7376844c6 --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/index.ts @@ -0,0 +1,218 @@ +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; +import { trace } from '@opentelemetry/api'; + +import { PREFER_BUILDER_OVER_SQL_NUDGE } from '@/mcp/tools/query/builderCatalog'; +import { + clickHouseErrorResult, + isQueryOutOfTime, + parseTimeRange, +} from '@/mcp/tools/query/helpers'; +import { getMcpClickhouseClient } from '@/mcp/tools/trace/shared'; +import type { ToolRegistrar } from '@/mcp/tools/types'; +import { mcpUserError } from '@/mcp/utils/errors'; +import { trimToolResponse } from '@/utils/trimToolResponse'; + +import { collectErrors } from './collect'; +import { resolveComparison } from './comparison'; +import { + errorGroupExpressions, + fetchServiceStats, + type QueryContext, +} from './queries'; +import { buildReport, rankServices } from './report'; +import { + comparisonWindow, + DEFAULT_WINDOW_MS, + serviceHealthSchema, +} from './schema'; +import { loadHealthSource } from './source'; + +const TOOL_NAME = 'clickstack_service_health'; + +const TIMEOUT_SUFFIX = + 'Service health aggregates every span in both windows. Retry with a ' + + 'shorter startTime/endTime window, list specific services, add a where ' + + 'filter, or set topErrorsPerService:0 to skip the error breakdown.'; + +const DESCRIPTION = + 'One-call health report for every service on a trace source: throughput, ' + + 'error rate, p50/p95/p99 latency and top error groups, each compared with ' + + 'the same window yesterday (configurable), ranked worst first with a ' + + 'status (critical / degraded / healthy) and the flags that explain it. ' + + 'Error groups that are new or growing versus the comparison are listed ' + + 'first.\n\n' + + 'USE THIS FIRST for status checks, "how are my services doing", "is X ' + + 'healthy", post-deploy checks, and as the entry point for triage. It ' + + 'replaces several clickstack_table / clickstack_timeseries calls.\n\n' + + 'sourceId is the "id" of a kind="trace" source from ' + + 'clickstack_list_sources (a 24-character hex string), not its name or ' + + 'table.\n\n' + + 'This report is per service and per window, so it cannot show WHEN ' + + 'something happened inside the window, or per-endpoint shifts. For a ' + + 'status report, follow up with:\n' + + ' - clickstack_timeseries (error count and p99 by minute) to place a ' + + 'new error group or latency change in time: a one-minute burst that ' + + 'recovered is not an ongoing problem\n' + + ' - clickstack_table grouped by span name to spot new or shifting ' + + 'endpoints\n' + + ' - clickstack_emerging_signals on the related log source for new log ' + + 'patterns, which span data does not show\n' + + 'For triage of a flagged service, check its own runtime metrics first ' + + '(memory, GC, CPU, restarts: clickstack_list_metrics then ' + + 'clickstack_timeseries on the metric source, grouped by pod or instance). ' + + 'Slow or failing spans are often a symptom of the service itself, not ' + + 'its dependencies. Then use clickstack_event_deltas to find what ' + + 'separates its failing or slow spans, and ' + + 'clickstack_trace_top_time_consuming_operations for slow downstream ' + + 'calls.\n\n' + + 'Requires serviceName and statusCode expressions on the source (see ' + + 'clickstack_describe_source). Counts server spans by default.\n\n' + + 'CALIBRATION: flags only fire past fixed thresholds (see `usage` in the ' + + 'response). A healthy service with small swings is steady, not a ' + + 'regression and needs no action. An all-healthy report is a ' + + 'valid answer.\n\n' + + PREFER_BUILDER_OVER_SQL_NUDGE; + +export function registerServiceHealth({ + context, + registerTool, +}: ToolRegistrar) { + const teamId = context.teamId.toString(); + + registerTool( + TOOL_NAME, + { + title: 'Service Health', + annotations: { readOnlyHint: true }, + description: DESCRIPTION, + inputSchema: serviceHealthSchema, + }, + async input => { + const range = parseTimeRange( + input.startTime, + input.endTime, + DEFAULT_WINDOW_MS, + ); + if ('error' in range) return mcpUserError(range.error); + const current = { start: range.startDate, end: range.endDate }; + const requested = comparisonWindow(current, input.comparison); + if (requested && requested.end > current.start) { + return mcpUserError( + `The window is longer than the "${input.comparison}" shift, so the ` + + 'two windows overlap. Shorten the window or use ' + + 'comparison:"previous_period".', + ); + } + + const source = await loadHealthSource(teamId, input.sourceId, TOOL_NAME); + if ('isError' in source) return source; + const client = await getMcpClickhouseClient(teamId, source.connection); + if (!client) { + return mcpUserError( + `Connection not found for source: ${input.sourceId}`, + ); + } + const groupExprs = errorGroupExpressions( + source, + input.errorGroupBy, + splitAndTrimWithBracket, + ); + + const abort = new AbortController(); + const ctx: QueryContext = { + client, + metadata: getMetadata(client), + source, + input, + signal: abort.signal, + }; + + try { + const [cur, requestedCmp] = await Promise.all([ + fetchServiceStats(ctx, current), + requested ? fetchServiceStats(ctx, requested) : null, + ]); + const { comparison, cmp, comparisonMode, comparisonNote } = + await resolveComparison(ctx, { + current, + requested, + requestedMode: input.comparison, + curHasData: cur.stats.size > 0, + requestedCmp, + }); + const ranked = rankServices( + cur.stats, + cmp?.stats ?? null, + input.minRequests, + ); + + const { currentErrors, comparisonErrorCounts, newOperations } = + await collectErrors(ctx, { + current, + comparison, + shown: ranked.slice(0, input.topN), + perService: input.topErrorsPerService, + groupExprs, + }); + + const report = buildReport({ + ranked, + current, + comparison, + comparisonMode, + comparisonNote, + spanKind: input.spanKind, + topN: input.topN, + errorGroupBy: groupExprs, + currentErrors, + comparisonErrorCounts, + newOperations, + spanNameKey: source.spanNameExpression, + metricSourceId: source.metricSourceId, + mvUsed: cur.mvUsed || !!cmp?.mvUsed, + }); + + trace.getActiveSpan()?.setAttributes({ + 'hyperdx.source.id': source.id, + 'mcp.service_health.comparison': input.comparison, + 'mcp.service_health.comparison_used': comparisonMode, + 'mcp.service_health.span_kind': input.spanKind, + 'mcp.service_health.window_ms': + current.end.getTime() - current.start.getTime(), + 'mcp.service_health.services_assessed': ranked.length, + 'mcp.service_health.critical_count': + report.summary.statusCounts.critical, + 'mcp.service_health.degraded_count': + report.summary.statusCounts.degraded, + 'mcp.service_health.mv_used': report.summary.materializedViewUsed, + }); + + const { data, isTrimmed } = trimToolResponse(report); + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + isTrimmed + ? { + ...data, + note: 'Trimmed for context size. Lower topN or topErrorsPerService.', + } + : data, + null, + 2, + ), + }, + ], + }; + } catch (e) { + abort.abort(); + return clickHouseErrorResult(e, { + prefix: 'Failed to compute service health', + suffix: isQueryOutOfTime(e) ? TIMEOUT_SUFFIX : undefined, + }); + } + }, + ); +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/operations.ts b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts new file mode 100644 index 0000000000..6ac525a061 --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts @@ -0,0 +1,99 @@ +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils'; +import { DisplayType } from '@hyperdx/common-utils/dist/types'; + +import { mcpQuerySettings } from '@/mcp/tools/trace/shared'; + +import type { QueryContext } from './queries'; +import { baseConfig } from './queries'; +import type { Window } from './schema'; + +const SERVICE_ALIAS = '__hdx_service'; +const OP_ALIAS = '__hdx_operation'; +const MAX_OPERATION_ROWS = 5000; +/** A new operation needs this many requests to be reported. */ +const NEW_OPERATION_MIN_REQUESTS = 10; +const MAX_NEW_OPERATIONS = 5; + +export type NewOperation = { + operation: string; + requests: number; + shareOfService: number; +}; + +async function operationCounts( + ctx: QueryContext, + window: Window, + services: string[], +): Promise>> { + const { source, client, signal } = ctx; + const list = services.map(s => `'${escapeSqlString(s)}'`).join(', '); + const result = await client.queryChartConfig({ + config: { + ...baseConfig(ctx, window, [ + { + type: 'sql', + condition: `${source.serviceNameExpression} IN (${list})`, + }, + ]), + displayType: DisplayType.Table, + select: [{ aggFn: 'count', valueExpression: '', alias: 'requests' }], + groupBy: [ + { valueExpression: source.serviceNameExpression, alias: SERVICE_ALIAS }, + { valueExpression: source.spanNameExpression, alias: OP_ALIAS }, + ], + limit: { limit: MAX_OPERATION_ROWS }, + }, + metadata: getMetadata(client), + querySettings: source.querySettings, + opts: { + clickhouse_settings: mcpQuerySettings(source.querySettings), + abort_signal: signal, + }, + }); + const out = new Map>(); + for (const row of result.data ?? []) { + const svc = String(row[SERVICE_ALIAS] ?? ''); + const ops = out.get(svc) ?? new Map(); + ops.set(String(row[OP_ALIAS] ?? ''), Number(row.requests) || 0); + out.set(svc, ops); + } + return out; +} + +/** + * Operations (span names) with traffic now and none in the comparison + * window, per service: usually a rollout. Errors on them are explained by the + * operation being new, not by a new failure mode. + */ +export async function findNewOperations( + ctx: QueryContext, + current: Window, + comparison: Window, + services: string[], +): Promise> { + if (services.length === 0) return new Map(); + const [cur, cmp] = await Promise.all([ + operationCounts(ctx, current, services), + operationCounts(ctx, comparison, services), + ]); + const out = new Map(); + for (const [svc, ops] of cur) { + const total = [...ops.values()].reduce((a, b) => a + b, 0); + const before = cmp.get(svc); + const fresh = [...ops] + .filter( + ([op, n]) => + op !== '' && n >= NEW_OPERATION_MIN_REQUESTS && !before?.get(op), + ) + .sort((a, b) => b[1] - a[1]) + .slice(0, MAX_NEW_OPERATIONS) + .map(([operation, requests]) => ({ + operation, + requests, + shareOfService: Math.round((requests / total) * 10000) / 10000, + })); + if (fresh.length) out.set(svc, fresh); + } + return out; +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts new file mode 100644 index 0000000000..160e300b0f --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts @@ -0,0 +1,281 @@ +import { tryOptimizeConfigWithMaterializedView } from '@hyperdx/common-utils/dist/core/materializedViews'; +import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { + errorStatusPredicateSql, + serverSpanKindPredicateSql, +} from '@hyperdx/common-utils/dist/core/spanPredicates'; +import { + escapeSqlString, + getFirstTimestampValueExpression, +} from '@hyperdx/common-utils/dist/core/utils'; +import { + type BuilderChartConfigWithDateRange, + DisplayType, + type Filter, + pickSampleWeightExpressionProps, + type QuerySettings, + SourceKind, + type TTraceSource, +} from '@hyperdx/common-utils/dist/types'; + +import type { ClickhouseClient } from '@/clickhouse'; +import { durationDivisor, mcpQuerySettings } from '@/mcp/tools/trace/shared'; + +import type { WindowStats } from './rank'; +import type { ServiceHealthInput, Window } from './schema'; + +const SERVICE_ALIAS = '__hdx_service'; +const ERROR_COUNT_ALIAS = '__hdx_error_count'; +const FIRST_SEEN_ALIAS = '__hdx_first_seen'; +const LAST_SEEN_ALIAS = '__hdx_last_seen'; +const errorKeyAlias = (i: number) => `__hdx_error_key_${i}`; + +/** Upper bound on error-group rows fetched, before bucketing per service. */ +const MAX_ERROR_ROWS = 1000; + +type MvSource = Parameters[4]; + +/** + * Plain copies of the trace source fields the queries read. The Mongoose + * document's subdocuments (e.g. `from`) can't go into a chart config, because + * the MV optimizer structuredClones the config. + */ +export type HealthSource = Pick< + TTraceSource, + | 'id' + | 'timestampValueExpression' + | 'implicitColumnExpression' + | 'useTextIndexForImplicitColumn' + | 'sampleRateExpression' + | 'spanNameExpression' + | 'spanKindExpression' + | 'statusMessageExpression' + | 'durationExpression' + | 'durationPrecision' + | 'metricSourceId' +> & { + connection: string; + from: { databaseName: string; tableName: string }; + serviceNameExpression: string; + statusCodeExpression: string; + querySettings: QuerySettings | undefined; + /** The original document, for the MV optimizer. Unset when it has no MVs. */ + mvSource: MvSource | undefined; +}; + +export type QueryContext = { + client: ClickhouseClient; + metadata: Metadata; + source: HealthSource; + input: ServiceHealthInput; + signal: AbortSignal; +}; + +export type ErrorGroup = { + service: string; + group: Record; + count: number; + /** Epoch ms of the first and last error in the window, if known. */ + firstSeenMs?: number; + lastSeenMs?: number; +}; + +const sqlList = (values: string[]) => + values.map(v => `'${escapeSqlString(v)}'`).join(', '); + +export function baseConfig( + { source, input }: QueryContext, + window: Window, + extraFilters: Filter[], +): Omit { + const filters: Filter[] = [...extraFilters]; + if (input.spanKind === 'server') { + filters.push({ + type: 'sql', + condition: serverSpanKindPredicateSql(source.spanKindExpression), + }); + } + if (input.services?.length) { + filters.push({ + type: 'sql', + condition: `${source.serviceNameExpression} IN (${sqlList(input.services)})`, + }); + } + return { + displayType: DisplayType.Table, + source: source.id, + connection: source.connection, + from: source.from, + timestampValueExpression: source.timestampValueExpression, + implicitColumnExpression: source.implicitColumnExpression, + useTextIndexForImplicitColumn: source.useTextIndexForImplicitColumn, + ...pickSampleWeightExpressionProps({ + kind: SourceKind.Trace, + sampleRateExpression: source.sampleRateExpression, + }), + where: input.where, + whereLanguage: input.where ? input.whereLanguage : 'sql', + filters, + dateRange: [window.start, window.end], + }; +} + +async function run( + ctx: QueryContext, + config: BuilderChartConfigWithDateRange, +): Promise<{ rows: Record[]; mvUsed: boolean }> { + const { client, metadata, source, signal } = ctx; + const optimized = source.mvSource + ? await tryOptimizeConfigWithMaterializedView( + config, + metadata, + client, + signal, + source.mvSource, + ) + : config; + const result = await client.queryChartConfig({ + config: optimized, + metadata, + querySettings: source.querySettings, + opts: { + clickhouse_settings: mcpQuerySettings(source.querySettings), + abort_signal: signal, + }, + }); + return { rows: result.data ?? [], mvUsed: optimized !== config }; +} + +const toNumberOrNull = (v: unknown) => { + const n = Number(v); + return v == null || !Number.isFinite(n) ? null : n; +}; + +/** Requests, errors, and latency quantiles per service in one window. */ +export async function fetchServiceStats( + ctx: QueryContext, + window: Window, +): Promise<{ stats: Map; mvUsed: boolean }> { + const { source } = ctx; + const quantile = (level: number, alias: string) => ({ + aggFn: 'quantile' as const, + level, + valueExpression: source.durationExpression, + aggCondition: '', + aggConditionLanguage: 'sql' as const, + alias, + }); + const { rows, mvUsed } = await run(ctx, { + ...baseConfig(ctx, window, []), + select: [ + { aggFn: 'count', valueExpression: '', alias: 'requests' }, + { + aggFn: 'count', + valueExpression: '', + aggCondition: errorStatusPredicateSql(source.statusCodeExpression), + aggConditionLanguage: 'sql', + alias: 'errors', + }, + quantile(0.5, 'p50'), + quantile(0.95, 'p95'), + quantile(0.99, 'p99'), + ], + groupBy: [ + { valueExpression: source.serviceNameExpression, alias: SERVICE_ALIAS }, + ], + }); + + const divisor = durationDivisor(source.durationPrecision); + const toMs = (v: unknown) => { + const n = toNumberOrNull(v); + return n == null ? null : Math.round((n / divisor) * 100) / 100; + }; + const stats = new Map(); + for (const row of rows) { + stats.set(String(row[SERVICE_ALIAS] ?? ''), { + requests: Number(row.requests) || 0, + errors: Number(row.errors) || 0, + p50Ms: toMs(row.p50), + p95Ms: toMs(row.p95), + p99Ms: toMs(row.p99), + }); + } + return { stats, mvUsed }; +} + +/** The expressions that define an error group, in output order. */ +export function errorGroupExpressions( + source: HealthSource, + errorGroupBy: string | undefined, + split: (s: string) => string[], +): string[] { + const exprs = errorGroupBy + ? split(errorGroupBy) + : [source.spanNameExpression, source.statusMessageExpression]; + return exprs.filter( + (e): e is string => !!e && e !== source.serviceNameExpression, + ); +} + +/** + * Error counts per (service, group) in one window. `match` restricts rows to + * exact (service, ...group) tuples, so the comparison window is looked up for + * the groups found in the current window instead of its own top-N. + */ +export async function fetchErrorGroups( + ctx: QueryContext, + window: Window, + groupExprs: string[], + scope: { services: string[] } | { match: ErrorGroup[] }, +): Promise { + const { source } = ctx; + const keyExprs = [source.serviceNameExpression, ...groupExprs]; + const epochMs = `toUnixTimestamp64Milli(toDateTime64(${getFirstTimestampValueExpression(source.timestampValueExpression)}, 3))`; + const scopeCondition = + 'services' in scope + ? `${source.serviceNameExpression} IN (${sqlList(scope.services)})` + : `tuple(${keyExprs.map(e => `toString(${e})`).join(', ')}) IN (${scope.match + .map( + g => + `tuple(${sqlList([g.service, ...groupExprs.map(e => g.group[e] ?? '')])})`, + ) + .join(', ')})`; + + const { rows } = await run(ctx, { + ...baseConfig(ctx, window, [ + { + type: 'sql', + condition: errorStatusPredicateSql(source.statusCodeExpression), + }, + { type: 'sql', condition: scopeCondition }, + ]), + select: [ + { aggFn: 'count', valueExpression: '', alias: ERROR_COUNT_ALIAS }, + // Epoch ms, because min/max cast their argument to Float64. + { aggFn: 'min', valueExpression: epochMs, alias: FIRST_SEEN_ALIAS }, + { aggFn: 'max', valueExpression: epochMs, alias: LAST_SEEN_ALIAS }, + ], + groupBy: [ + { valueExpression: source.serviceNameExpression, alias: SERVICE_ALIAS }, + ...groupExprs.map((e, i) => ({ + valueExpression: `toString(${e})`, + alias: errorKeyAlias(i), + })), + ], + orderBy: `${ERROR_COUNT_ALIAS} DESC`, + limit: { limit: MAX_ERROR_ROWS }, + }); + + return rows.map(row => ({ + service: String(row[SERVICE_ALIAS] ?? ''), + group: Object.fromEntries( + groupExprs.map((e, i) => [e, String(row[errorKeyAlias(i)] ?? '')]), + ), + count: Number(row[ERROR_COUNT_ALIAS]) || 0, + firstSeenMs: toNumberOrNull(row[FIRST_SEEN_ALIAS]) || undefined, + lastSeenMs: toNumberOrNull(row[LAST_SEEN_ALIAS]) || undefined, + })); +} + +export const errorGroupKey = (g: Pick) => + JSON.stringify([g.service, ...Object.values(g.group)]); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/rank.ts b/packages/api/src/mcp/tools/query/serviceHealth/rank.ts new file mode 100644 index 0000000000..ccf4be2724 --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/rank.ts @@ -0,0 +1,206 @@ +/** + * Thresholds that decide when a service is flagged. Each "up" rule needs a + * ratio AND an absolute delta: the ratio alone flags 0.1% → 0.3% errors or + * 4ms → 8ms latency, which is routine variance, not degraded health. + */ +export const HEALTH_THRESHOLDS = { + /** error_rate_up: current error rate at least this multiple of the comparison... */ + errorRateRatio: 2, + /** ...and at least this many percentage points higher. */ + errorRateMinDeltaPp: 1, + /** high_error_rate, and critical when combined with error_rate_up. */ + highErrorRate: 0.05, + /** latency_up: p95 at least this multiple of the comparison... */ + latencyRatio: 1.5, + /** ...and at least this many ms slower. */ + latencyMinDeltaMs: 50, + /** traffic_drop: requests at or below this fraction of the comparison. */ + trafficDropRatio: 0.5, + /** traffic_spike (informational): requests at least this multiple. */ + trafficSpikeRatio: 3, +} as const; + +export type WindowStats = { + requests: number; + errors: number; + p50Ms: number | null; + p95Ms: number | null; + p99Ms: number | null; +}; + +export type HealthFlag = + | 'error_rate_up' + | 'high_error_rate' + | 'latency_up' + | 'traffic_drop' + | 'missing_service' + // Context for the reader; these never change status. + | 'traffic_spike' + | 'new_service' + | 'low_volume' + // Set by the report: a shown error group is new or growing. + | 'new_errors'; + +const DEGRADING_FLAGS: ReadonlySet = new Set([ + 'error_rate_up', + 'high_error_rate', + 'latency_up', + 'traffic_drop', + 'missing_service', +]); +export type HealthStatus = 'critical' | 'degraded' | 'healthy'; + +export type ServiceAssessment = { + status: HealthStatus; + flags: HealthFlag[]; + score: number; + deltas: { + throughputRatio: number | null; + errorRateDeltaPp: number | null; + p95Ratio: number | null; + p95DeltaMs: number | null; + }; +}; + +export const STATUS_ORDER: Record = { + critical: 0, + degraded: 1, + healthy: 2, +}; + +const errorRate = (s: WindowStats) => + s.requests > 0 ? s.errors / s.requests : 0; + +const round = (n: number, places = 4) => { + const f = 10 ** places; + return Math.round(n * f) / f; +}; + +/** + * Assess one service. `comparison` is undefined when comparisons are off, and + * null when they are on but the service had no requests in that window. + */ +export function assessService( + current: WindowStats | null, + comparison: WindowStats | null | undefined, + minRequests: number, +): ServiceAssessment { + const cur = current ?? { + requests: 0, + errors: 0, + p50Ms: null, + p95Ms: null, + p99Ms: null, + }; + const cmp = comparison ?? null; + const comparing = comparison !== undefined; + const flags: HealthFlag[] = []; + + const curRate = errorRate(cur); + const cmpRate = cmp ? errorRate(cmp) : null; + const errorRateDeltaPp = + cmpRate != null ? round((curRate - cmpRate) * 100) : null; + const throughputRatio = + cmp && cmp.requests > 0 ? round(cur.requests / cmp.requests) : null; + const p95Ratio = + cmp?.p95Ms && cur.p95Ms != null ? round(cur.p95Ms / cmp.p95Ms) : null; + const p95DeltaMs = + cmp?.p95Ms != null && cur.p95Ms != null + ? round(cur.p95Ms - cmp.p95Ms, 2) + : null; + + const curEnough = cur.requests >= minRequests; + const cmpEnough = (cmp?.requests ?? 0) >= minRequests; + + if (!curEnough && !cmpEnough) { + flags.push('low_volume'); + } else if (comparing && !cmp) { + flags.push('new_service'); + } else if (comparing && cmp && cur.requests === 0) { + flags.push('missing_service'); + } + + if (curEnough) { + if (curRate >= HEALTH_THRESHOLDS.highErrorRate) + flags.push('high_error_rate'); + if ( + cmpEnough && + cmpRate != null && + curRate >= cmpRate * HEALTH_THRESHOLDS.errorRateRatio && + (errorRateDeltaPp ?? 0) >= HEALTH_THRESHOLDS.errorRateMinDeltaPp + ) { + flags.push('error_rate_up'); + } + if ( + cmpEnough && + p95Ratio != null && + p95Ratio >= HEALTH_THRESHOLDS.latencyRatio && + (p95DeltaMs ?? 0) >= HEALTH_THRESHOLDS.latencyMinDeltaMs + ) { + flags.push('latency_up'); + } + if ( + cmpEnough && + throughputRatio != null && + throughputRatio >= HEALTH_THRESHOLDS.trafficSpikeRatio + ) { + flags.push('traffic_spike'); + } + } + if ( + cmpEnough && + cur.requests > 0 && + throughputRatio != null && + throughputRatio <= HEALTH_THRESHOLDS.trafficDropRatio + ) { + flags.push('traffic_drop'); + } + + const status: HealthStatus = + flags.includes('error_rate_up') && + curRate >= HEALTH_THRESHOLDS.highErrorRate + ? 'critical' + : flags.some(f => DEGRADING_FLAGS.has(f)) + ? 'degraded' + : 'healthy'; + + return { + status, + flags, + score: severityScore(cur, cmp, { curRate, cmpRate, p95Ratio }), + deltas: { throughputRatio, errorRateDeltaPp, p95Ratio, p95DeltaMs }, + }; +} + +/** + * Orders services within a status. Sums how far each metric moved the wrong + * way, then scales by log volume so a busy service outranks a quiet one with + * the same shift. Error points are percentage points; latency and traffic are + * log2 ratios, so doubling p95 weighs the same as +2pp errors. + */ +function severityScore( + cur: WindowStats, + cmp: WindowStats | null, + m: { curRate: number; cmpRate: number | null; p95Ratio: number | null }, +): number { + const errorPp = Math.max(0, m.curRate - (m.cmpRate ?? 0)) * 100; + const latency = m.p95Ratio && m.p95Ratio > 1 ? 2 * Math.log2(m.p95Ratio) : 0; + const traffic = + cmp && cmp.requests > 0 && cur.requests < cmp.requests + ? Math.log2(cmp.requests / Math.max(cur.requests, 1)) + : 0; + const volume = Math.log10(Math.max(cur.requests, cmp?.requests ?? 0) + 10); + return round((errorPp + latency + traffic) * volume, 2); +} + +export function compareAssessments( + a: { assessment: ServiceAssessment; errors: number; requests: number }, + b: { assessment: ServiceAssessment; errors: number; requests: number }, +): number { + return ( + STATUS_ORDER[a.assessment.status] - STATUS_ORDER[b.assessment.status] || + b.assessment.score - a.assessment.score || + b.errors - a.errors || + b.requests - a.requests + ); +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/report.ts b/packages/api/src/mcp/tools/query/serviceHealth/report.ts new file mode 100644 index 0000000000..40d165f4eb --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/report.ts @@ -0,0 +1,285 @@ +import type { NewOperation } from './operations'; +import type { ErrorGroup } from './queries'; +import { errorGroupKey } from './queries'; +import { + assessService, + compareAssessments, + HEALTH_THRESHOLDS, + type HealthFlag, + type HealthStatus, + type ServiceAssessment, + STATUS_ORDER, + type WindowStats, +} from './rank'; +import type { Window } from './schema'; +import { errorTiming, serviceSummary } from './summary'; + +export type RankedService = { + service: string; + current: WindowStats | null; + /** undefined when comparisons are off; null when the service was absent. */ + comparison: WindowStats | null | undefined; + assessment: ServiceAssessment; +}; + +/** Assess every service seen in either window and sort worst first. */ +export function rankServices( + current: Map, + comparison: Map | null, + minRequests: number, +): RankedService[] { + const names = new Set([...current.keys(), ...(comparison?.keys() ?? [])]); + return [...names] + .map(service => { + const cur = current.get(service) ?? null; + const cmp = comparison ? (comparison.get(service) ?? null) : undefined; + return { + service, + current: cur, + comparison: cmp, + assessment: assessService(cur, cmp, minRequests), + }; + }) + .sort((a, b) => + compareAssessments( + { + assessment: a.assessment, + errors: a.current?.errors ?? 0, + requests: a.current?.requests ?? 0, + }, + { + assessment: b.assessment, + errors: b.current?.errors ?? 0, + requests: b.current?.requests ?? 0, + }, + ), + ); +} + +/** The first `perService` groups for each service; input is sorted by count. */ +export function topGroupsPerService( + groups: ErrorGroup[], + perService: number, +): Map { + const out = new Map(); + for (const g of groups) { + const list = out.get(g.service) ?? []; + if (list.length < perService) list.push(g); + out.set(g.service, list); + } + return out; +} + +/** Candidate groups fetched per service before choosing the ones to show. */ +export const ERROR_CANDIDATES_PER_SERVICE = 25; +/** A new group needs at least this many errors to be called out. */ +const NEW_ERROR_MIN = 3; +/** + * Growth must clear this z-score, treating both counts as Poisson. Small + * counts swing a lot by chance (5 -> 13 is within noise across dozens of + * groups); 2x alone flags them. + */ +const GROWTH_MIN_Z = 3; + +export type ErrorGroupStatus = 'new' | 'growing' | 'existing' | 'new_operation'; + +export function errorGroupStatus( + count: number, + comparisonCount: number, +): ErrorGroupStatus { + if (comparisonCount === 0) return 'new'; + const z = (count - comparisonCount) / Math.sqrt(count + comparisonCount); + return count >= 2 * comparisonCount && z >= GROWTH_MIN_Z + ? 'growing' + : 'existing'; +} + +const isChanged = (count: number, comparisonCount: number) => { + const status = errorGroupStatus(count, comparisonCount); + return status === 'growing' || (status === 'new' && count >= NEW_ERROR_MIN); +}; + +/** + * Pick the groups to show for one service. With a comparison, new and growing + * groups come first, largest gain first: a burst of a novel error is the + * finding, and ranking by count alone buries it under steady background + * errors. The remaining slots go to the largest groups. + */ +export function selectTopErrors( + candidates: ErrorGroup[], + comparisonCounts: Map | null, + limit: number, + explained: (g: ErrorGroup) => boolean = () => false, +): ErrorGroup[] { + if (!comparisonCounts) return candidates.slice(0, limit); + const prior = (g: ErrorGroup) => comparisonCounts.get(errorGroupKey(g)) ?? 0; + const novel = (g: ErrorGroup) => + isChanged(g.count, prior(g)) && !explained(g); + const changed = candidates + .filter(novel) + .sort((a, b) => b.count - prior(b) - (a.count - prior(a))); + const rest = candidates.filter(g => !novel(g)); + return [...changed, ...rest].slice(0, limit); +} + +function formatStats(s: WindowStats | null, windowMinutes: number) { + const requests = s?.requests ?? 0; + const errors = s?.errors ?? 0; + return { + requests, + throughputPerMin: Math.round((requests / windowMinutes) * 100) / 100, + errors, + errorRate: + requests > 0 ? Math.round((errors / requests) * 10000) / 10000 : 0, + p50Ms: s?.p50Ms ?? null, + p95Ms: s?.p95Ms ?? null, + p99Ms: s?.p99Ms ?? null, + }; +} + +const iso = (w: Window) => ({ + start: w.start.toISOString(), + end: w.end.toISOString(), +}); + +const USAGE = + 'Services are sorted by status (critical, degraded, healthy), then score. ' + + `error_rate_up: error rate >= ${HEALTH_THRESHOLDS.errorRateRatio}x the comparison AND ` + + `>= ${HEALTH_THRESHOLDS.errorRateMinDeltaPp} percentage point higher. ` + + `high_error_rate: error rate >= ${HEALTH_THRESHOLDS.highErrorRate * 100}%. ` + + 'critical = error_rate_up together with high_error_rate. ' + + `latency_up: p95 >= ${HEALTH_THRESHOLDS.latencyRatio}x AND >= ${HEALTH_THRESHOLDS.latencyMinDeltaMs}ms slower. ` + + `traffic_drop: requests <= ${HEALTH_THRESHOLDS.trafficDropRatio * 100}% of the comparison. ` + + 'missing_service: served requests in the comparison window but none now. ' + + 'Any of those makes a service degraded. traffic_spike, new_service and ' + + 'low_volume (under minRequests in both windows, never flagged) and ' + + 'new_errors (a listed error group is new or growing; such services sort ' + + 'first within their status) are context only. Healthy services are not regressions; report them as steady and do ' + + "treat moves below these thresholds as steady (no action needed); each service's " + + '`summary` states which moves crossed one. topErrors carry firstSeen / ' + + 'lastSeen / ongoing so a burst that stopped reads as transient. topErrors lists new and ' + + 'growing error groups first (largest gain first), then the largest steady ' + + 'ones. newOperations = span names with traffic now and none in the ' + + 'comparison window (usually a rollout); errors on them have status ' + + '"new_operation". status "new" = absent from the comparison window, "growing" = at ' + + 'least 2x its comparison count and beyond random variation, "existing" = ' + + 'otherwise.'; + +export function buildReport(args: { + ranked: RankedService[]; + current: Window; + comparison: Window | null; + comparisonMode: string; + comparisonNote?: string; + spanKind: string; + topN: number; + errorGroupBy: string[]; + currentErrors: Map; + comparisonErrorCounts: Map | null; + newOperations: Map; + spanNameKey: string; + metricSourceId?: string | null; + mvUsed: boolean; +}) { + const onNewOperation = (r: RankedService, g: ErrorGroup) => + (args.newOperations.get(r.service) ?? []).some( + o => o.operation === g.group[args.spanNameKey], + ); + const windowMinutes = + (args.current.end.getTime() - args.current.start.getTime()) / 60_000; + const counts: Record = { + critical: 0, + degraded: 0, + healthy: 0, + }; + for (const r of args.ranked) counts[r.assessment.status]++; + + const prior = (g: ErrorGroup) => + args.comparisonErrorCounts?.get(errorGroupKey(g)) ?? 0; + const changedErrors = (r: RankedService) => + args.comparisonErrorCounts + ? (args.currentErrors.get(r.service) ?? []).filter(g => + isChanged(g.count, prior(g)), + ) + : []; + const hasNewErrors = (r: RankedService) => changedErrors(r).length > 0; + // Within a status, a service with a new or growing error group goes first: + // its aggregate rates may be flat, but it has the most novel signal. + const shown = args.ranked + .slice(0, args.topN) + .map((r, i) => ({ r, i, novel: hasNewErrors(r) })) + .sort( + (a, b) => + STATUS_ORDER[a.r.assessment.status] - + STATUS_ORDER[b.r.assessment.status] || + Number(b.novel) - Number(a.novel) || + a.i - b.i, + ); + + const services = shown.map(({ r, novel }, i) => ({ + rank: i + 1, + service: r.service, + status: r.assessment.status, + flags: novel + ? [...r.assessment.flags, 'new_errors' satisfies HealthFlag] + : r.assessment.flags, + summary: serviceSummary({ + current: r.current, + comparison: r.comparison, + assessment: r.assessment, + changedErrors: changedErrors(r), + newOperations: args.newOperations.get(r.service) ?? [], + window: args.current, + comparisonMode: args.comparisonMode, + metricSourceId: args.metricSourceId, + }), + score: r.assessment.score, + current: formatStats(r.current, windowMinutes), + ...(r.comparison !== undefined && { + comparison: r.comparison + ? formatStats(r.comparison, windowMinutes) + : null, + deltas: r.assessment.deltas, + }), + ...(args.newOperations.has(r.service) && { + newOperations: args.newOperations.get(r.service), + }), + topErrors: (args.currentErrors.get(r.service) ?? []).map(g => { + const comparisonCount = prior(g); + return { + group: g.group, + count: g.count, + ...errorTiming(g, args.current), + ...(args.comparisonErrorCounts && { + comparisonCount, + status: onNewOperation(r, g) + ? ('new_operation' satisfies ErrorGroupStatus) + : errorGroupStatus(g.count, comparisonCount), + }), + }; + }), + })); + + return { + summary: { + currentWindow: iso(args.current), + comparisonWindow: args.comparison ? iso(args.comparison) : null, + comparison: args.comparisonMode, + spanKind: args.spanKind, + errorGroupBy: args.errorGroupBy, + servicesAssessed: args.ranked.length, + servicesReturned: services.length, + statusCounts: counts, + materializedViewUsed: args.mvUsed, + ...(args.ranked.length === 0 && { + warning: + 'No spans matched in either window. Check the time window, the ' + + 'where filter, and spanKind ("server" skips sources without server ' + + 'spans; try spanKind:"all").', + }), + ...(args.comparisonNote && { comparisonNote: args.comparisonNote }), + }, + services, + usage: USAGE, + }; +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/schema.ts b/packages/api/src/mcp/tools/query/serviceHealth/schema.ts new file mode 100644 index 0000000000..568da02af6 --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/schema.ts @@ -0,0 +1,123 @@ +import ms from 'ms'; +import { z } from 'zod'; + +import { + sourceIdSchema, + whereLanguageSchema, + whereSchema, +} from '@/mcp/tools/query/schemas'; + +const COMPARISON_MODES = [ + 'yesterday', + 'previous_period', + 'last_week', + 'none', +] as const; +export type ComparisonMode = (typeof COMPARISON_MODES)[number]; + +export const DEFAULT_WINDOW_MS = ms('1h'); + +export const serviceHealthSchema = z.object({ + sourceId: sourceIdSchema.describe( + 'The "id" of a kind="trace" source from clickstack_list_sources (24-char ' + + 'hex, e.g. "6a4c18f1d64c5f47bb275fbe"). Not the source name or table.', + ), + startTime: z + .string() + .optional() + .describe( + 'Start of the window to assess (ISO 8601). Default: 1 hour before endTime.', + ), + endTime: z + .string() + .optional() + .describe('End of the window to assess (ISO 8601). Default: now.'), + comparison: z + .enum(COMPARISON_MODES) + .optional() + .default('yesterday') + .describe( + 'Window to compare against, always the same length as the assessed window:\n' + + ' yesterday (default): shifted back 24h, so daily traffic cycles line up\n' + + ' previous_period: the window immediately before startTime\n' + + ' last_week: shifted back 7 days\n' + + ' none: no comparison; only absolute thresholds apply', + ), + where: whereSchema, + whereLanguage: whereLanguageSchema, + spanKind: z + .enum(['server', 'all']) + .optional() + .default('server') + .describe( + 'Which spans count as requests. server (default): entry-point spans only, ' + + 'matching the Services dashboard. all: every span, including internal ' + + 'and client calls (inflates throughput and dilutes error rate).', + ), + services: z + .array(z.string().min(1)) + .max(100) + .optional() + .describe('Only assess these service names. Default: every service.'), + topN: z + .number() + .int() + .min(1) + .max(100) + .optional() + .default(20) + .describe('Max services returned, worst first. Default: 20.'), + topErrorsPerService: z + .number() + .int() + .min(0) + .max(10) + .optional() + .default(3) + .describe( + 'Error groups returned per service. 0 skips the error breakdown (faster). Default: 3.', + ), + errorGroupBy: z + .string() + .optional() + .describe( + 'Comma-separated SQL expressions that define an error group. Default: the ' + + "source's span name and status message. Example: " + + '"SpanName, SpanAttributes[\'exception.type\']".', + ), + minRequests: z + .number() + .int() + .min(1) + .optional() + .default(50) + .describe( + 'Services with fewer requests than this in both windows are never flagged; ' + + 'their ratios are too noisy. Default: 50.', + ), +}); + +export type ServiceHealthInput = z.infer; + +export type Window = { start: Date; end: Date }; + +/** The comparison window for a mode, or null for `none`. */ +export function comparisonWindow( + current: Window, + mode: ComparisonMode, +): Window | null { + const lengthMs = current.end.getTime() - current.start.getTime(); + const shiftMs = + mode === 'yesterday' + ? ms('1d') + : mode === 'last_week' + ? ms('7d') + : mode === 'previous_period' + ? lengthMs + : null; + if (shiftMs == null) return null; + return { + start: new Date(current.start.getTime() - shiftMs), + end: new Date(current.end.getTime() - shiftMs), + }; +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/source.ts b/packages/api/src/mcp/tools/query/serviceHealth/source.ts new file mode 100644 index 0000000000..b409afc02a --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/source.ts @@ -0,0 +1,91 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { getSource, getSources } from '@/controllers/sources'; +import type { McpErrorResult } from '@/mcp/utils/errors'; +import { mcpUserError } from '@/mcp/utils/errors'; + +import type { HealthSource } from './queries'; + +const MAX_LISTED_SOURCES = 10; + +/** + * Agents often pass a table or source name where the ID belongs. Point them at + * the exact ID when the input names one trace source, else list the options. + */ +async function sourceNotFound( + teamId: string, + sourceId: string, + toolName: string, +): Promise { + const traces = (await getSources(teamId)).filter( + s => s.kind === SourceKind.Trace, + ); + const needle = sourceId.trim().toLowerCase(); + const describe = (s: (typeof traces)[number]) => + `${s._id.toString()} ("${s.name}", table ${s.from.databaseName}.${s.from.tableName})`; + const match = traces.find(s => + [ + s.name, + s.from.tableName, + `${s.from.databaseName}.${s.from.tableName}`, + ].some(v => v.toLowerCase() === needle), + ); + if (match) { + return mcpUserError( + `"${sourceId}" is a source name or table, not a source ID. Retry ${toolName} ` + + `with sourceId:"${match._id.toString()}".`, + ); + } + const listed = traces.slice(0, MAX_LISTED_SOURCES).map(describe); + return mcpUserError( + `Source not found: "${sourceId}". sourceId must be the "id" field from ` + + 'clickstack_list_sources, not a name or table. ' + + (listed.length + ? `Trace sources: ${listed.join('; ')}.` + : 'This team has no trace sources.'), + ); +} + +/** Load a trace source and check it has the fields the tool needs. */ +export async function loadHealthSource( + teamId: string, + sourceId: string, + toolName: string, +): Promise { + const doc = await getSource(teamId, sourceId); + if (!doc) return sourceNotFound(teamId, sourceId, toolName); + if (doc.kind !== SourceKind.Trace) { + return mcpUserError( + `Source ${sourceId} is kind="${doc.kind}". ${toolName} requires a source of kind="trace".`, + ); + } + if (!doc.serviceNameExpression || !doc.statusCodeExpression) { + return mcpUserError( + `Source ${sourceId} has no ` + + `${doc.serviceNameExpression ? 'statusCodeExpression' : 'serviceNameExpression'}. ` + + 'Set it with clickstack_save_source, then retry.', + ); + } + return { + id: doc._id.toString(), + connection: doc.connection.toString(), + from: { + databaseName: doc.from.databaseName, + tableName: doc.from.tableName, + }, + timestampValueExpression: doc.timestampValueExpression, + implicitColumnExpression: doc.implicitColumnExpression, + useTextIndexForImplicitColumn: doc.useTextIndexForImplicitColumn, + sampleRateExpression: doc.sampleRateExpression, + serviceNameExpression: doc.serviceNameExpression, + statusCodeExpression: doc.statusCodeExpression, + statusMessageExpression: doc.statusMessageExpression, + spanNameExpression: doc.spanNameExpression, + spanKindExpression: doc.spanKindExpression, + durationExpression: doc.durationExpression, + durationPrecision: doc.durationPrecision, + metricSourceId: doc.metricSourceId, + querySettings: doc.querySettings, + mvSource: doc.materializedViews?.length ? doc : undefined, + }; +} diff --git a/packages/api/src/mcp/tools/query/serviceHealth/summary.ts b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts new file mode 100644 index 0000000000..6a15f3af49 --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts @@ -0,0 +1,114 @@ +import type { NewOperation } from './operations'; +import type { ErrorGroup } from './queries'; +import { + HEALTH_THRESHOLDS, + type ServiceAssessment, + type WindowStats, +} from './rank'; +import type { Window } from './schema'; + +/** An error group whose last error is this close to the window end is ongoing. */ +const ONGOING_TAIL_MS = 2 * 60_000; + +const pct = (s: WindowStats | null) => + s && s.requests > 0 ? `${((s.errors / s.requests) * 100).toFixed(2)}%` : '0%'; +const hhmm = (ms: number) => new Date(ms).toISOString().slice(11, 16); + +export function errorTiming(g: ErrorGroup, window: Window) { + if (g.firstSeenMs == null || g.lastSeenMs == null) return {}; + const ongoing = window.end.getTime() - g.lastSeenMs <= ONGOING_TAIL_MS; + return { + firstSeen: new Date(g.firstSeenMs).toISOString(), + lastSeen: new Date(g.lastSeenMs).toISOString(), + activeMinutes: + Math.round(((g.lastSeenMs - g.firstSeenMs) / 60_000) * 10) / 10, + ongoing, + }; +} + +function describeError(g: ErrorGroup, window: Window): string { + const label = Object.values(g.group).filter(Boolean).join(' / ') || 'error'; + const t = errorTiming(g, window); + if (t.firstSeen == null || g.firstSeenMs == null || g.lastSeenMs == null) + return `"${label}" (${g.count})`; + const [from, to] = [hhmm(g.firstSeenMs), hhmm(g.lastSeenMs)]; + const when = from === to ? `at ${from} UTC` : `${from}-${to} UTC`; + return `"${label}" (${g.count}, ${when}, ${t.ongoing ? 'still occurring' : 'stopped'})`; +} + +/** + * A plain verdict per service. Agents otherwise read every nonzero delta as a + * trend ("error rate up 0.08pp"), so this states which moves crossed a + * threshold and which are ordinary variance. + */ +export function serviceSummary(args: { + current: WindowStats | null; + comparison: WindowStats | null | undefined; + assessment: ServiceAssessment; + changedErrors: ErrorGroup[]; + newOperations: NewOperation[]; + window: Window; + comparisonMode: string; + metricSourceId?: string | null; +}): string { + const { current, comparison, assessment, changedErrors, window } = args; + const f = new Set(assessment.flags); + const parts: string[] = []; + if (f.has('missing_service')) + parts.push('no requests now, but had traffic in the comparison window'); + if (f.has('error_rate_up')) + parts.push(`error rate ${pct(current)} vs ${pct(comparison ?? null)}`); + else if (f.has('high_error_rate')) + parts.push( + `error rate ${pct(current)} is at or above ${HEALTH_THRESHOLDS.highErrorRate * 100}%`, + ); + if (f.has('latency_up')) + parts.push(`p95 ${current?.p95Ms}ms vs ${comparison?.p95Ms}ms`); + if (f.has('traffic_drop')) + parts.push(`requests ${current?.requests} vs ${comparison?.requests}`); + + const head = + parts.length > 0 + ? `${assessment.status.toUpperCase()}: ${parts.join('; ')}.` + : comparison === undefined + ? 'Healthy: no absolute threshold crossed (no comparison window).' + : `Healthy: no metric crossed a threshold vs ${args.comparisonMode}; ` + + 'the deltas are normal variance; report this service as steady, no action needed.'; + + const context: string[] = []; + if (f.has('new_service')) + context.push('No traffic in the comparison window.'); + if (f.has('traffic_spike')) + context.push('Traffic is well above the comparison window.'); + if (f.has('low_volume')) context.push('Too few requests to judge.'); + if (f.has('latency_up') || f.has('error_rate_up')) + context.push( + "Before blaming dependencies, check this service's runtime metrics " + + '(memory, GC, CPU, restarts) by pod or instance' + + (args.metricSourceId + ? ` on metric source ${args.metricSourceId}` + : ' on a kind="metric" source from clickstack_list_sources') + + '; slow or failing spans are often a symptom of the service itself.', + ); + if (args.newOperations.length > 0) + context.push( + `New operations, likely a rollout: ${args.newOperations + .map( + o => + `${o.operation} (${o.requests} requests, ${(o.shareOfService * 100).toFixed(1)}% of traffic)`, + ) + .join('; ')}.`, + ); + if (changedErrors.length > 0) { + const stopped = changedErrors.some( + g => errorTiming(g, window).ongoing === false, + ); + context.push( + `New or growing errors: ${changedErrors.map(g => describeError(g, window)).join('; ')}.` + + (stopped + ? ' A burst that stopped is transient, not an ongoing problem.' + : ''), + ); + } + return [head, ...context].join(' '); +} diff --git a/packages/app/src/llm/lib/expressions.ts b/packages/app/src/llm/lib/expressions.ts index ae5743b164..27140cde54 100644 --- a/packages/app/src/llm/lib/expressions.ts +++ b/packages/app/src/llm/lib/expressions.ts @@ -1,7 +1,6 @@ +import { errorStatusPredicateSql } from '@hyperdx/common-utils/dist/core/spanPredicates'; import { TLogSource, TTraceSource } from '@hyperdx/common-utils/dist/types'; -import { errorStatusPredicateSql } from '@/source'; - import { generateCostSqlExpression } from './cost'; import { buildAnyKeyExistsSql, buildLLMSpanSqlPredicate } from './detect'; diff --git a/packages/app/src/serviceDashboard.ts b/packages/app/src/serviceDashboard.ts index 51aa0d6fc4..8208a8e544 100644 --- a/packages/app/src/serviceDashboard.ts +++ b/packages/app/src/serviceDashboard.ts @@ -1,11 +1,14 @@ import { useMemo } from 'react'; import { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { + errorStatusPredicateSql, + serverSpanKindPredicateSql, +} from '@hyperdx/common-utils/dist/core/spanPredicates'; import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils'; import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types'; import { useColumns, useJsonColumns } from './hooks/useMetadata'; -import { errorStatusPredicateSql } from './source'; const COALESCE_FIELDS_LIMIT = 100; @@ -227,7 +230,7 @@ export function getExpressions( const filterExpressions = { isEndpointNonEmpty: `NOT empty(${auxExpressions.endpoint})`, isError: errorStatusPredicateSql(fieldExpressions.severityText), - isSpanKindServer: `${fieldExpressions.spanKind} IN ('Server', 'SPAN_KIND_SERVER')`, + isSpanKindServer: serverSpanKindPredicateSql(fieldExpressions.spanKind), isDbSpan: `${fieldExpressions.dbStatement} <> ''`, }; diff --git a/packages/app/src/source.ts b/packages/app/src/source.ts index b98ee52b2b..f235e01786 100644 --- a/packages/app/src/source.ts +++ b/packages/app/src/source.ts @@ -540,20 +540,6 @@ export function getDurationSecondsExpression(source: TTraceSource) { return `(${source.durationExpression})/1e${source.durationPrecision ?? 9}`; } -/** - * The single definition of an errored span, as a SQL predicate over a span - * status-code expression. Matches both the short OTel status ('Error'/'ERROR') - * and the legacy collector form ('STATUS_CODE_ERROR'), which some collectors - * write instead. - * - * Span status only. Log severity text is a different vocabulary (its error - * values are 'fatal', 'crit' and friends, never 'status_code_error'), so log - * callers keep their own predicate rather than reusing this one. - */ -export function errorStatusPredicateSql(statusCodeExpression: string): string { - return `lower(${statusCodeExpression}) IN ('error', 'status_code_error')`; -} - // Aggregate functions whose output preserves the unit of the input value. // count and count_distinct produce dimensionless counts and should not // inherit the duration format. diff --git a/packages/common-utils/src/__tests__/spanPredicates.test.ts b/packages/common-utils/src/__tests__/spanPredicates.test.ts new file mode 100644 index 0000000000..e5ee7c0b96 --- /dev/null +++ b/packages/common-utils/src/__tests__/spanPredicates.test.ts @@ -0,0 +1,20 @@ +import { + errorStatusPredicateSql, + serverSpanKindPredicateSql, +} from '@/core/spanPredicates'; + +describe('errorStatusPredicateSql', () => { + it('matches both short and legacy error statuses case-insensitively', () => { + expect(errorStatusPredicateSql('StatusCode')).toBe( + "lower(StatusCode) IN ('error', 'status_code_error')", + ); + }); +}); + +describe('serverSpanKindPredicateSql', () => { + it('matches both short and legacy server span kinds', () => { + expect(serverSpanKindPredicateSql('SpanKind')).toBe( + "SpanKind IN ('Server', 'SPAN_KIND_SERVER')", + ); + }); +}); diff --git a/packages/common-utils/src/core/spanPredicates.ts b/packages/common-utils/src/core/spanPredicates.ts new file mode 100644 index 0000000000..e2d37549e6 --- /dev/null +++ b/packages/common-utils/src/core/spanPredicates.ts @@ -0,0 +1,18 @@ +/** + * The single definition of an errored span, as a SQL predicate over a span + * status-code expression. Matches both the short OTel status ('Error'/'ERROR') + * and the legacy collector form ('STATUS_CODE_ERROR'), which some collectors + * write instead. + * + * Span status only. Log severity text is a different vocabulary (its error + * values are 'fatal', 'crit' and friends, never 'status_code_error'), so log + * callers keep their own predicate rather than reusing this one. + */ +export function errorStatusPredicateSql(statusCodeExpression: string): string { + return `lower(${statusCodeExpression}) IN ('error', 'status_code_error')`; +} + +/** Server (request entry-point) spans, in both short and legacy enum forms. */ +export function serverSpanKindPredicateSql(spanKindExpression: string): string { + return `${spanKindExpression} IN ('Server', 'SPAN_KIND_SERVER')`; +} From 51c40037b514ee4dc1bffe6a1f97262587bb6a59 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 19 Oct 2026 13:24:20 -0600 Subject: [PATCH 02/15] fix(mcp): fit service health description in the 2048-byte limit and declare all behavior hints --- .../mcp/tools/query/serviceHealth/index.ts | 57 +++++++------------ .../mcp/tools/query/serviceHealth/report.ts | 1 + .../mcp/tools/query/serviceHealth/schema.ts | 17 +++--- .../mcp/tools/query/serviceHealth/summary.ts | 6 ++ 4 files changed, 37 insertions(+), 44 deletions(-) diff --git a/packages/api/src/mcp/tools/query/serviceHealth/index.ts b/packages/api/src/mcp/tools/query/serviceHealth/index.ts index c7376844c6..f8783668e1 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/index.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/index.ts @@ -2,6 +2,7 @@ import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { trace } from '@opentelemetry/api'; +import { READ_ONLY_TOOL } from '@/mcp/tools/annotations'; import { PREFER_BUILDER_OVER_SQL_NUDGE } from '@/mcp/tools/query/builderCatalog'; import { clickHouseErrorResult, @@ -37,41 +38,25 @@ const TIMEOUT_SUFFIX = const DESCRIPTION = 'One-call health report for every service on a trace source: throughput, ' + - 'error rate, p50/p95/p99 latency and top error groups, each compared with ' + - 'the same window yesterday (configurable), ranked worst first with a ' + - 'status (critical / degraded / healthy) and the flags that explain it. ' + - 'Error groups that are new or growing versus the comparison are listed ' + - 'first.\n\n' + - 'USE THIS FIRST for status checks, "how are my services doing", "is X ' + - 'healthy", post-deploy checks, and as the entry point for triage. It ' + - 'replaces several clickstack_table / clickstack_timeseries calls.\n\n' + - 'sourceId is the "id" of a kind="trace" source from ' + - 'clickstack_list_sources (a 24-character hex string), not its name or ' + - 'table.\n\n' + - 'This report is per service and per window, so it cannot show WHEN ' + - 'something happened inside the window, or per-endpoint shifts. For a ' + - 'status report, follow up with:\n' + - ' - clickstack_timeseries (error count and p99 by minute) to place a ' + - 'new error group or latency change in time: a one-minute burst that ' + - 'recovered is not an ongoing problem\n' + - ' - clickstack_table grouped by span name to spot new or shifting ' + - 'endpoints\n' + - ' - clickstack_emerging_signals on the related log source for new log ' + - 'patterns, which span data does not show\n' + - 'For triage of a flagged service, check its own runtime metrics first ' + - '(memory, GC, CPU, restarts: clickstack_list_metrics then ' + - 'clickstack_timeseries on the metric source, grouped by pod or instance). ' + - 'Slow or failing spans are often a symptom of the service itself, not ' + - 'its dependencies. Then use clickstack_event_deltas to find what ' + - 'separates its failing or slow spans, and ' + - 'clickstack_trace_top_time_consuming_operations for slow downstream ' + - 'calls.\n\n' + - 'Requires serviceName and statusCode expressions on the source (see ' + - 'clickstack_describe_source). Counts server spans by default.\n\n' + - 'CALIBRATION: flags only fire past fixed thresholds (see `usage` in the ' + - 'response). A healthy service with small swings is steady, not a ' + - 'regression and needs no action. An all-healthy report is a ' + - 'valid answer.\n\n' + + 'error rate, p50/p95/p99 latency and top error groups, compared with the ' + + 'same window yesterday and ranked worst first. Each service gets a status ' + + '(critical / degraded / healthy), flags, and a one-line summary of which ' + + 'changes crossed a threshold. New or growing error groups come first, ' + + 'with first/last-seen times; operations with no prior traffic are listed ' + + 'as likely rollouts.\n\n' + + 'USE THIS FIRST for status checks, "is X healthy", post-deploy checks, and ' + + 'as the entry point for triage.\n\n' + + 'It is per service and per window. To follow up:\n' + + ' - clickstack_timeseries by minute to see when a change happened\n' + + ' - clickstack_table grouped by span name for per-endpoint shifts\n' + + ' - clickstack_emerging_signals on the log source for new log patterns\n' + + 'For a flagged service, check its own runtime metrics (memory, GC, CPU, ' + + 'restarts by pod) on the metric source before blaming dependencies, then ' + + 'use clickstack_event_deltas and ' + + 'clickstack_trace_top_time_consuming_operations.\n\n' + + 'CALIBRATION: flags fire only past fixed thresholds (see `usage` in the ' + + 'response). Small swings on a healthy service are normal variance. An ' + + 'all-healthy report is a valid answer.\n\n' + PREFER_BUILDER_OVER_SQL_NUDGE; export function registerServiceHealth({ @@ -84,7 +69,7 @@ export function registerServiceHealth({ TOOL_NAME, { title: 'Service Health', - annotations: { readOnlyHint: true }, + annotations: READ_ONLY_TOOL, description: DESCRIPTION, inputSchema: serviceHealthSchema, }, diff --git a/packages/api/src/mcp/tools/query/serviceHealth/report.ts b/packages/api/src/mcp/tools/query/serviceHealth/report.ts index 40d165f4eb..0f1f7101c1 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/report.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/report.ts @@ -232,6 +232,7 @@ export function buildReport(args: { window: args.current, comparisonMode: args.comparisonMode, metricSourceId: args.metricSourceId, + spanKind: args.spanKind, }), score: r.assessment.score, current: formatStats(r.current, windowMinutes), diff --git a/packages/api/src/mcp/tools/query/serviceHealth/schema.ts b/packages/api/src/mcp/tools/query/serviceHealth/schema.ts index 568da02af6..def9e23751 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/schema.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/schema.ts @@ -31,7 +31,7 @@ export const serviceHealthSchema = z.object({ endTime: z .string() .optional() - .describe('End of the window to assess (ISO 8601). Default: now.'), + .describe('End of the window to assess (ISO 8601). Defaults to now.'), comparison: z .enum(COMPARISON_MODES) .optional() @@ -41,7 +41,8 @@ export const serviceHealthSchema = z.object({ ' yesterday (default): shifted back 24h, so daily traffic cycles line up\n' + ' previous_period: the window immediately before startTime\n' + ' last_week: shifted back 7 days\n' + - ' none: no comparison; only absolute thresholds apply', + ' none: no comparison; only absolute thresholds apply\n' + + 'If the chosen window has no data, the previous period is used instead.', ), where: whereSchema, whereLanguage: whereLanguageSchema, @@ -52,13 +53,13 @@ export const serviceHealthSchema = z.object({ .describe( 'Which spans count as requests. server (default): entry-point spans only, ' + 'matching the Services dashboard. all: every span, including internal ' + - 'and client calls (inflates throughput and dilutes error rate).', + 'and client calls, which inflates throughput and dilutes error rate and latency changes.', ), services: z .array(z.string().min(1)) .max(100) .optional() - .describe('Only assess these service names. Default: every service.'), + .describe('Only assess these service names. Omit to assess every service.'), topN: z .number() .int() @@ -66,7 +67,7 @@ export const serviceHealthSchema = z.object({ .max(100) .optional() .default(20) - .describe('Max services returned, worst first. Default: 20.'), + .describe('Max services returned, worst first.'), topErrorsPerService: z .number() .int() @@ -75,13 +76,13 @@ export const serviceHealthSchema = z.object({ .optional() .default(3) .describe( - 'Error groups returned per service. 0 skips the error breakdown (faster). Default: 3.', + 'Error groups returned per service. 0 skips the error breakdown, which is faster.', ), errorGroupBy: z .string() .optional() .describe( - 'Comma-separated SQL expressions that define an error group. Default: the ' + + 'Comma-separated SQL expressions that define an error group. Defaults to the ' + "source's span name and status message. Example: " + '"SpanName, SpanAttributes[\'exception.type\']".', ), @@ -93,7 +94,7 @@ export const serviceHealthSchema = z.object({ .default(50) .describe( 'Services with fewer requests than this in both windows are never flagged; ' + - 'their ratios are too noisy. Default: 50.', + 'their ratios are too noisy.', ), }); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/summary.ts b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts index 6a15f3af49..ea9e7012ed 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/summary.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts @@ -50,6 +50,7 @@ export function serviceSummary(args: { window: Window; comparisonMode: string; metricSourceId?: string | null; + spanKind?: string; }): string { const { current, comparison, assessment, changedErrors, window } = args; const f = new Set(assessment.flags); @@ -110,5 +111,10 @@ export function serviceSummary(args: { : ''), ); } + if (args.spanKind === 'all' && assessment.status === 'healthy') + context.push( + 'Internal and client spans are included, which dilutes server-side ' + + 'changes; check spanKind:"server" before concluding nothing changed.', + ); return [head, ...context].join(' '); } From b0c5527e7bb3fcb45d84faeb8401cf1a7fc68069 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 19 Oct 2026 14:24:09 -0600 Subject: [PATCH 03/15] fix(mcp): find novel errors beyond the top N and stop double-counting window boundaries - Assess error novelty for up to 100 services before cutting to topN. A service ranked below topN by aggregate score was never checked for new errors, so a new failure on a quiet service could not surface. - Make every window half-open. With an inclusive end, a span at exactly the boundary counted in both windows, so an empty comparison could look like a one-request baseline. - Treat fewer than minRequests prior requests as no baseline (new_service), and do not call that service's errors or operations new. - Exclude errors on a brand-new operation from new_errors and the summary. This was intended earlier but the change had not applied. - Drop "no action needed" from a healthy summary that lists new errors. --- .../query/__tests__/serviceHealthRank.test.ts | 99 +++++++++++++++++++ .../mcp/tools/query/serviceHealth/collect.ts | 14 ++- .../mcp/tools/query/serviceHealth/index.ts | 6 +- .../tools/query/serviceHealth/operations.ts | 5 + .../mcp/tools/query/serviceHealth/queries.ts | 5 +- .../src/mcp/tools/query/serviceHealth/rank.ts | 3 +- .../mcp/tools/query/serviceHealth/report.ts | 14 ++- .../mcp/tools/query/serviceHealth/summary.ts | 5 +- 8 files changed, 139 insertions(+), 12 deletions(-) diff --git a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts index 4b526291bf..2abbf106aa 100644 --- a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts +++ b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts @@ -1,3 +1,4 @@ +import type { NewOperation } from '@/mcp/tools/query/serviceHealth/operations'; import { errorGroupKey } from '@/mcp/tools/query/serviceHealth/queries'; import { assessService, @@ -5,6 +6,7 @@ import { type WindowStats, } from '@/mcp/tools/query/serviceHealth/rank'; import { + buildReport, errorGroupStatus, rankServices, selectTopErrors, @@ -90,6 +92,12 @@ describe('assessService', () => { expect(gone.status).toBe('degraded'); }); + it('treats a handful of prior requests as no baseline', () => { + const a = assessService(stats(), stats({ requests: 1, errors: 1 }), 50); + expect(a.flags).toEqual(['new_service']); + expect(a.status).toBe('healthy'); + }); + it('never flags services under minRequests in both windows', () => { const a = assessService( stats({ requests: 10, errors: 9 }), @@ -230,3 +238,94 @@ describe('error group selection', () => { expect(picked.map(x => x.group.StatusMessage)).toEqual(['a', 'b']); }); }); + +describe('buildReport', () => { + it('lifts a service with a new error group into the top N', () => { + const ranked = rankServices( + new Map([ + ['busy', stats({ requests: 100_000, errors: 600 })], + ['quiet', stats({ requests: 1000, errors: 5 })], + ]), + new Map([ + ['busy', stats({ requests: 100_000, errors: 500 })], + ['quiet', stats({ requests: 1000, errors: 5 })], + ]), + 50, + ); + expect(ranked.map(r => r.service)).toEqual(['busy', 'quiet']); + const novel = { + service: 'quiet', + group: { SpanName: 'POST /pay', StatusMessage: 'payment unavailable' }, + count: 8, + }; + const report = buildReport({ + ranked, + current: { start: new Date(0), end: new Date(600_000) }, + comparison: { start: new Date(-600_000), end: new Date(0) }, + comparisonMode: 'previous_period', + spanKind: 'server', + topN: 1, + errorGroupBy: ['SpanName', 'StatusMessage'], + currentErrors: new Map([['quiet', [novel]]]), + comparisonErrorCounts: new Map(), + newOperations: new Map(), + spanNameKey: 'SpanName', + mvUsed: false, + }); + expect(report.services.map(s => s.service)).toEqual(['quiet']); + expect(report.services[0].flags).toContain('new_errors'); + }); + + const reportFor = ( + cur: WindowStats, + cmp: WindowStats, + newOperations: Map = new Map(), + ) => + buildReport({ + ranked: rankServices( + new Map([['svc', cur]]), + new Map([['svc', cmp]]), + 50, + ), + current: { start: new Date(0), end: new Date(600_000) }, + comparison: { start: new Date(-600_000), end: new Date(0) }, + comparisonMode: 'previous_period', + spanKind: 'server', + topN: 5, + errorGroupBy: ['SpanName', 'StatusMessage'], + currentErrors: new Map([ + [ + 'svc', + [ + { + service: 'svc', + group: { SpanName: 'GET /v2', StatusMessage: 'x' }, + count: 20, + }, + ], + ], + ]), + comparisonErrorCounts: new Map(), + newOperations, + spanNameKey: 'SpanName', + mvUsed: false, + }).services[0]; + + it('does not call errors on a brand-new operation new errors', () => { + const svc = reportFor( + stats(), + stats(), + new Map([ + ['svc', [{ operation: 'GET /v2', requests: 100, shareOfService: 0.1 }]], + ]), + ); + expect(svc.flags).not.toContain('new_errors'); + expect(svc.topErrors[0].status).toBe('new_operation'); + }); + + it('does not call errors new when the service has no baseline', () => { + const svc = reportFor(stats(), stats({ requests: 1, errors: 0 })); + expect(svc.flags).toEqual(['new_service']); + expect(svc.summary).not.toMatch(/New or growing errors/); + }); +}); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/collect.ts b/packages/api/src/mcp/tools/query/serviceHealth/collect.ts index aecd3ce568..3f0d2e55a1 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/collect.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/collect.ts @@ -7,6 +7,7 @@ import { } from './queries'; import { ERROR_CANDIDATES_PER_SERVICE, + hasBaseline, type RankedService, selectTopErrors, topGroupsPerService, @@ -32,9 +33,11 @@ export async function collectErrors( shown: RankedService[]; perService: number; groupExprs: string[]; + minRequests: number; }, ): Promise { - const { current, comparison, shown, perService, groupExprs } = args; + const { current, comparison, shown, perService, groupExprs, minRequests } = + args; const withErrors = shown .filter(r => (r.current?.errors ?? 0) > 0) .map(r => r.service); @@ -50,6 +53,7 @@ export async function collectErrors( current, comparison, shown.filter(r => r.current).map(r => r.service), + minRequests, ) : Promise.resolve(new Map()), ]); @@ -70,14 +74,18 @@ export async function collectErrors( const spanKey = ctx.source.spanNameExpression; const currentErrors = new Map(); + const baseline = new Map(shown.map(r => [r.service, hasBaseline(r)])); for (const [service, groups] of candidates) { const newOps = new Set( (newOperations.get(service) ?? []).map(o => o.operation), ); currentErrors.set( service, - selectTopErrors(groups, comparisonErrorCounts, perService, g => - newOps.has(g.group[spanKey] ?? ''), + selectTopErrors( + groups, + comparisonErrorCounts, + perService, + g => !baseline.get(service) || newOps.has(g.group[spanKey] ?? ''), ), ); } diff --git a/packages/api/src/mcp/tools/query/serviceHealth/index.ts b/packages/api/src/mcp/tools/query/serviceHealth/index.ts index f8783668e1..e0e3a99153 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/index.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/index.ts @@ -30,6 +30,7 @@ import { import { loadHealthSource } from './source'; const TOOL_NAME = 'clickstack_service_health'; +const MAX_ERROR_SERVICES = 100; const TIMEOUT_SUFFIX = 'Service health aggregates every span in both windows. Retry with a ' + @@ -136,9 +137,12 @@ export function registerServiceHealth({ await collectErrors(ctx, { current, comparison, - shown: ranked.slice(0, input.topN), + // Novelty can lift a service into the top N, so assess errors + // beyond it rather than only for services already ranked there. + shown: ranked.slice(0, MAX_ERROR_SERVICES), perService: input.topErrorsPerService, groupExprs, + minRequests: input.minRequests, }); const report = buildReport({ diff --git a/packages/api/src/mcp/tools/query/serviceHealth/operations.ts b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts index 6ac525a061..5fbc132fac 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/operations.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts @@ -71,6 +71,7 @@ export async function findNewOperations( current: Window, comparison: Window, services: string[], + minPriorRequests: number, ): Promise> { if (services.length === 0) return new Map(); const [cur, cmp] = await Promise.all([ @@ -81,6 +82,10 @@ export async function findNewOperations( for (const [svc, ops] of cur) { const total = [...ops.values()].reduce((a, b) => a + b, 0); const before = cmp.get(svc); + // A service with no real prior traffic is new as a whole (new_service); + // calling each of its operations a rollout would be noise. + const priorTotal = [...(before?.values() ?? [])].reduce((a, b) => a + b, 0); + if (priorTotal < minPriorRequests) continue; const fresh = [...ops] .filter( ([op, n]) => diff --git a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts index 160e300b0f..3248d767bd 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts @@ -31,7 +31,7 @@ const LAST_SEEN_ALIAS = '__hdx_last_seen'; const errorKeyAlias = (i: number) => `__hdx_error_key_${i}`; /** Upper bound on error-group rows fetched, before bucketing per service. */ -const MAX_ERROR_ROWS = 1000; +const MAX_ERROR_ROWS = 2500; type MvSource = Parameters[4]; @@ -117,6 +117,9 @@ export function baseConfig( whereLanguage: input.where ? input.whereLanguage : 'sql', filters, dateRange: [window.start, window.end], + // Half-open windows: with an inclusive end, a span at exactly the + // boundary lands in both the comparison and the current window. + dateRangeEndInclusive: false, }; } diff --git a/packages/api/src/mcp/tools/query/serviceHealth/rank.ts b/packages/api/src/mcp/tools/query/serviceHealth/rank.ts index ccf4be2724..d998a3b190 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/rank.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/rank.ts @@ -114,7 +114,8 @@ export function assessService( if (!curEnough && !cmpEnough) { flags.push('low_volume'); - } else if (comparing && !cmp) { + } else if (comparing && !cmpEnough && curEnough) { + // A handful of prior requests is not a baseline; treat it like none. flags.push('new_service'); } else if (comparing && cmp && cur.requests === 0) { flags.push('missing_service'); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/report.ts b/packages/api/src/mcp/tools/query/serviceHealth/report.ts index 0f1f7101c1..2cac60a7c6 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/report.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/report.ts @@ -165,6 +165,10 @@ const USAGE = 'least 2x its comparison count and beyond random variation, "existing" = ' + 'otherwise.'; +/** Without real prior traffic every error group looks new; skip novelty. */ +export const hasBaseline = (r: RankedService) => + !r.assessment.flags.some(f => f === 'new_service' || f === 'low_volume'); + export function buildReport(args: { ranked: RankedService[]; current: Window; @@ -197,16 +201,15 @@ export function buildReport(args: { const prior = (g: ErrorGroup) => args.comparisonErrorCounts?.get(errorGroupKey(g)) ?? 0; const changedErrors = (r: RankedService) => - args.comparisonErrorCounts - ? (args.currentErrors.get(r.service) ?? []).filter(g => - isChanged(g.count, prior(g)), + args.comparisonErrorCounts && hasBaseline(r) + ? (args.currentErrors.get(r.service) ?? []).filter( + g => isChanged(g.count, prior(g)) && !onNewOperation(r, g), ) : []; const hasNewErrors = (r: RankedService) => changedErrors(r).length > 0; // Within a status, a service with a new or growing error group goes first: // its aggregate rates may be flat, but it has the most novel signal. const shown = args.ranked - .slice(0, args.topN) .map((r, i) => ({ r, i, novel: hasNewErrors(r) })) .sort( (a, b) => @@ -214,7 +217,8 @@ export function buildReport(args: { STATUS_ORDER[b.r.assessment.status] || Number(b.novel) - Number(a.novel) || a.i - b.i, - ); + ) + .slice(0, args.topN); const services = shown.map(({ r, novel }, i) => ({ rank: i + 1, diff --git a/packages/api/src/mcp/tools/query/serviceHealth/summary.ts b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts index ea9e7012ed..83d941cbe8 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/summary.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts @@ -74,7 +74,10 @@ export function serviceSummary(args: { : comparison === undefined ? 'Healthy: no absolute threshold crossed (no comparison window).' : `Healthy: no metric crossed a threshold vs ${args.comparisonMode}; ` + - 'the deltas are normal variance; report this service as steady, no action needed.'; + 'the deltas are normal variance' + + (changedErrors.length > 0 + ? ', but see the new errors below.' + : '; report this service as steady, no action needed.'); const context: string[] = []; if (f.has('new_service')) From 304c0fa8c0bd1a0b26a46409d8d5d43a38dfc3e1 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 19 Oct 2026 19:29:31 -0600 Subject: [PATCH 04/15] chore: update service health changeset --- .changeset/mcp-service-health-tool.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/mcp-service-health-tool.md b/.changeset/mcp-service-health-tool.md index bd5b4aa9ac..9df837c658 100644 --- a/.changeset/mcp-service-health-tool.md +++ b/.changeset/mcp-service-health-tool.md @@ -4,4 +4,4 @@ '@hyperdx/app': patch --- -feat(mcp): add `clickstack_service_health`, a one-call per-service health report (throughput, error rate, p50/p95, top errors) compared with yesterday and ranked worst first +feat(mcp): add `clickstack_service_health`, a one-call per-service health report for a trace source (throughput, error rate, p50/p95/p99, top error groups), compared with yesterday and ranked worst first From 40cb7cb6ae79b2d1c1511a02cf739cec2e8e61b4 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 19 Oct 2026 19:44:08 -0600 Subject: [PATCH 05/15] fix(mcp): flag incomplete novelty checks and keep service health lint-clean - Report a coverageNote when a limit (services checked, error rows, groups per service) kept errors out of the novelty check (greptile P1). - Skip new-operation detection when either operations query is truncated, so an existing operation cannot look new (greptile P1). - Classify rollout errors against every new operation; cap only the returned list (greptile P2). - Omit new/growing labels on errors of services without a baseline (greptile P2). - Read result rows through a Map and type the test's parsed body, keeping the api lint warning count at its cap (lint check). --- .../__tests__/serviceHealthTool.int.test.ts | 5 ++- .../query/__tests__/serviceHealthRank.test.ts | 1 + .../mcp/tools/query/serviceHealth/collect.ts | 42 +++++++++++++++---- .../mcp/tools/query/serviceHealth/index.ts | 28 ++++++++----- .../tools/query/serviceHealth/operations.ts | 35 +++++++++------- .../mcp/tools/query/serviceHealth/queries.ts | 24 +++++++---- .../mcp/tools/query/serviceHealth/report.ts | 30 ++++++++----- 7 files changed, 111 insertions(+), 54 deletions(-) diff --git a/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts b/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts index 1578f4a85b..a3ab5df719 100644 --- a/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts +++ b/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts @@ -148,10 +148,11 @@ describe('MCP Service Health Tool', () => { const parse = (result: Awaited>) => { expect(result.isError).toBeFalsy(); - return JSON.parse(getFirstText(result)) as { + const body: { summary: Record; services: ServiceReport[]; - }; + } = JSON.parse(getFirstText(result)); + return body; }; it('is listed with its input schema', async () => { diff --git a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts index 2abbf106aa..39c6f21346 100644 --- a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts +++ b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts @@ -327,5 +327,6 @@ describe('buildReport', () => { const svc = reportFor(stats(), stats({ requests: 1, errors: 0 })); expect(svc.flags).toEqual(['new_service']); expect(svc.summary).not.toMatch(/New or growing errors/); + expect(svc.topErrors[0]).not.toHaveProperty('status'); }); }); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/collect.ts b/packages/api/src/mcp/tools/query/serviceHealth/collect.ts index 3f0d2e55a1..9d175f5616 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/collect.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/collect.ts @@ -3,6 +3,8 @@ import { type ErrorGroup, errorGroupKey, fetchErrorGroups, + groupValue, + MAX_ERROR_ROWS, type QueryContext, } from './queries'; import { @@ -18,6 +20,8 @@ export type CollectedErrors = { currentErrors: Map; comparisonErrorCounts: Map | null; newOperations: Map; + /** Set when a limit kept some services or error groups out of the check. */ + coverageNote: string | undefined; }; /** @@ -34,6 +38,8 @@ export async function collectErrors( perService: number; groupExprs: string[]; minRequests: number; + /** Services assessed in total; `shown` may be a prefix of them. */ + totalServices: number; }, ): Promise { const { current, comparison, shown, perService, groupExprs, minRequests } = @@ -43,7 +49,7 @@ export async function collectErrors( .map(r => r.service); const fetchErrors = perService > 0 && withErrors.length > 0; - const [rows, newOperations] = await Promise.all([ + const [rows, ops] = await Promise.all([ fetchErrors ? fetchErrorGroups(ctx, current, groupExprs, { services: withErrors }) : Promise.resolve([]), @@ -55,13 +61,15 @@ export async function collectErrors( shown.filter(r => r.current).map(r => r.service), minRequests, ) - : Promise.resolve(new Map()), + : Promise.resolve({ + newOperations: new Map(), + complete: true, + }), ]); + const { newOperations } = ops; - const candidates = topGroupsPerService( - rows, - Math.max(ERROR_CANDIDATES_PER_SERVICE, perService), - ); + const perServiceCap = Math.max(ERROR_CANDIDATES_PER_SERVICE, perService); + const candidates = topGroupsPerService(rows, perServiceCap); const all = [...candidates.values()].flat(); const comparisonErrorCounts = comparison && all.length @@ -85,9 +93,27 @@ export async function collectErrors( groups, comparisonErrorCounts, perService, - g => !baseline.get(service) || newOps.has(g.group[spanKey] ?? ''), + g => !baseline.get(service) || newOps.has(groupValue(g, spanKey) ?? ''), ), ); } - return { currentErrors, comparisonErrorCounts, newOperations }; + const groupsPerService = new Map(); + for (const g of rows) { + groupsPerService.set(g.service, (groupsPerService.get(g.service) ?? 0) + 1); + } + const limits = [ + args.totalServices > shown.length && + `only the first ${shown.length} of ${args.totalServices} services were checked for new errors`, + rows.length >= MAX_ERROR_ROWS && + `the error query hit its ${MAX_ERROR_ROWS}-row limit`, + [...groupsPerService.values()].some(n => n > perServiceCap) && + `only the ${perServiceCap} largest error groups per service were compared`, + !ops.complete && + 'too many operations to detect new ones reliably, so none are reported', + ].filter((x): x is string => typeof x === 'string'); + const coverageNote = limits.length + ? `Incomplete: ${limits.join('; ')}. Narrow with services or a where filter for a full check.` + : undefined; + + return { currentErrors, comparisonErrorCounts, newOperations, coverageNote }; } diff --git a/packages/api/src/mcp/tools/query/serviceHealth/index.ts b/packages/api/src/mcp/tools/query/serviceHealth/index.ts index e0e3a99153..493125f3b3 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/index.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/index.ts @@ -133,17 +133,22 @@ export function registerServiceHealth({ input.minRequests, ); - const { currentErrors, comparisonErrorCounts, newOperations } = - await collectErrors(ctx, { - current, - comparison, - // Novelty can lift a service into the top N, so assess errors - // beyond it rather than only for services already ranked there. - shown: ranked.slice(0, MAX_ERROR_SERVICES), - perService: input.topErrorsPerService, - groupExprs, - minRequests: input.minRequests, - }); + const { + currentErrors, + comparisonErrorCounts, + newOperations, + coverageNote, + } = await collectErrors(ctx, { + current, + comparison, + // Novelty can lift a service into the top N, so assess errors + // beyond it rather than only for services already ranked there. + shown: ranked.slice(0, MAX_ERROR_SERVICES), + perService: input.topErrorsPerService, + groupExprs, + minRequests: input.minRequests, + totalServices: ranked.length, + }); const report = buildReport({ ranked, @@ -151,6 +156,7 @@ export function registerServiceHealth({ comparison, comparisonMode, comparisonNote, + coverageNote, spanKind: input.spanKind, topN: input.topN, errorGroupBy: groupExprs, diff --git a/packages/api/src/mcp/tools/query/serviceHealth/operations.ts b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts index 5fbc132fac..ccda0c76f2 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/operations.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts @@ -5,7 +5,7 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types'; import { mcpQuerySettings } from '@/mcp/tools/trace/shared'; import type { QueryContext } from './queries'; -import { baseConfig } from './queries'; +import { baseConfig, col } from './queries'; import type { Window } from './schema'; const SERVICE_ALIAS = '__hdx_service'; @@ -13,7 +13,8 @@ const OP_ALIAS = '__hdx_operation'; const MAX_OPERATION_ROWS = 5000; /** A new operation needs this many requests to be reported. */ const NEW_OPERATION_MIN_REQUESTS = 10; -const MAX_NEW_OPERATIONS = 5; +/** New operations shown per service; all of them still classify errors. */ +export const MAX_NEW_OPERATIONS = 5; export type NewOperation = { operation: string; @@ -25,7 +26,7 @@ async function operationCounts( ctx: QueryContext, window: Window, services: string[], -): Promise>> { +): Promise<{ counts: Map>; truncated: boolean }> { const { source, client, signal } = ctx; const list = services.map(s => `'${escapeSqlString(s)}'`).join(', '); const result = await client.queryChartConfig({ @@ -51,14 +52,15 @@ async function operationCounts( abort_signal: signal, }, }); + const rows = result.data ?? []; const out = new Map>(); - for (const row of result.data ?? []) { - const svc = String(row[SERVICE_ALIAS] ?? ''); + for (const row of rows) { + const svc = String(col(row, SERVICE_ALIAS) ?? ''); const ops = out.get(svc) ?? new Map(); - ops.set(String(row[OP_ALIAS] ?? ''), Number(row.requests) || 0); + ops.set(String(col(row, OP_ALIAS) ?? ''), Number(row.requests) || 0); out.set(svc, ops); } - return out; + return { counts: out, truncated: rows.length >= MAX_OPERATION_ROWS }; } /** @@ -72,14 +74,20 @@ export async function findNewOperations( comparison: Window, services: string[], minPriorRequests: number, -): Promise> { - if (services.length === 0) return new Map(); - const [cur, cmp] = await Promise.all([ +): Promise<{ newOperations: Map; complete: boolean }> { + const out = new Map(); + if (services.length === 0) return { newOperations: out, complete: true }; + const [curRes, cmpRes] = await Promise.all([ operationCounts(ctx, current, services), operationCounts(ctx, comparison, services), ]); - const out = new Map(); - for (const [svc, ops] of cur) { + // A truncated comparison can omit operations that did exist, which would + // make them look new. Report nothing rather than guess. + if (curRes.truncated || cmpRes.truncated) { + return { newOperations: out, complete: false }; + } + const cmp = cmpRes.counts; + for (const [svc, ops] of curRes.counts) { const total = [...ops.values()].reduce((a, b) => a + b, 0); const before = cmp.get(svc); // A service with no real prior traffic is new as a whole (new_service); @@ -92,7 +100,6 @@ export async function findNewOperations( op !== '' && n >= NEW_OPERATION_MIN_REQUESTS && !before?.get(op), ) .sort((a, b) => b[1] - a[1]) - .slice(0, MAX_NEW_OPERATIONS) .map(([operation, requests]) => ({ operation, requests, @@ -100,5 +107,5 @@ export async function findNewOperations( })); if (fresh.length) out.set(svc, fresh); } - return out; + return { newOperations: out, complete: true }; } diff --git a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts index 3248d767bd..d10ee79445 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts @@ -31,7 +31,7 @@ const LAST_SEEN_ALIAS = '__hdx_last_seen'; const errorKeyAlias = (i: number) => `__hdx_error_key_${i}`; /** Upper bound on error-group rows fetched, before bucketing per service. */ -const MAX_ERROR_ROWS = 2500; +export const MAX_ERROR_ROWS = 2500; type MvSource = Parameters[4]; @@ -80,6 +80,14 @@ export type ErrorGroup = { lastSeenMs?: number; }; +/** Read a column from a result row (rows are keyed by alias). */ +export const col = (row: Record, key: string) => + new Map(Object.entries(row)).get(key); + +/** The value of one grouping expression in an error group. */ +export const groupValue = (g: Pick, key: string) => + new Map(Object.entries(g.group)).get(key); + const sqlList = (values: string[]) => values.map(v => `'${escapeSqlString(v)}'`).join(', '); @@ -195,7 +203,7 @@ export async function fetchServiceStats( }; const stats = new Map(); for (const row of rows) { - stats.set(String(row[SERVICE_ALIAS] ?? ''), { + stats.set(String(col(row, SERVICE_ALIAS) ?? ''), { requests: Number(row.requests) || 0, errors: Number(row.errors) || 0, p50Ms: toMs(row.p50), @@ -240,7 +248,7 @@ export async function fetchErrorGroups( : `tuple(${keyExprs.map(e => `toString(${e})`).join(', ')}) IN (${scope.match .map( g => - `tuple(${sqlList([g.service, ...groupExprs.map(e => g.group[e] ?? '')])})`, + `tuple(${sqlList([g.service, ...groupExprs.map(e => groupValue(g, e) ?? '')])})`, ) .join(', ')})`; @@ -270,13 +278,13 @@ export async function fetchErrorGroups( }); return rows.map(row => ({ - service: String(row[SERVICE_ALIAS] ?? ''), + service: String(col(row, SERVICE_ALIAS) ?? ''), group: Object.fromEntries( - groupExprs.map((e, i) => [e, String(row[errorKeyAlias(i)] ?? '')]), + groupExprs.map((e, i) => [e, String(col(row, errorKeyAlias(i)) ?? '')]), ), - count: Number(row[ERROR_COUNT_ALIAS]) || 0, - firstSeenMs: toNumberOrNull(row[FIRST_SEEN_ALIAS]) || undefined, - lastSeenMs: toNumberOrNull(row[LAST_SEEN_ALIAS]) || undefined, + count: Number(col(row, ERROR_COUNT_ALIAS)) || 0, + firstSeenMs: toNumberOrNull(col(row, FIRST_SEEN_ALIAS)) || undefined, + lastSeenMs: toNumberOrNull(col(row, LAST_SEEN_ALIAS)) || undefined, })); } diff --git a/packages/api/src/mcp/tools/query/serviceHealth/report.ts b/packages/api/src/mcp/tools/query/serviceHealth/report.ts index 2cac60a7c6..9a89b1e733 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/report.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/report.ts @@ -1,6 +1,6 @@ -import type { NewOperation } from './operations'; +import { MAX_NEW_OPERATIONS, type NewOperation } from './operations'; import type { ErrorGroup } from './queries'; -import { errorGroupKey } from './queries'; +import { errorGroupKey, groupValue } from './queries'; import { assessService, compareAssessments, @@ -175,6 +175,7 @@ export function buildReport(args: { comparison: Window | null; comparisonMode: string; comparisonNote?: string; + coverageNote?: string; spanKind: string; topN: number; errorGroupBy: string[]; @@ -187,7 +188,7 @@ export function buildReport(args: { }) { const onNewOperation = (r: RankedService, g: ErrorGroup) => (args.newOperations.get(r.service) ?? []).some( - o => o.operation === g.group[args.spanNameKey], + o => o.operation === groupValue(g, args.spanNameKey), ); const windowMinutes = (args.current.end.getTime() - args.current.start.getTime()) / 60_000; @@ -232,7 +233,10 @@ export function buildReport(args: { comparison: r.comparison, assessment: r.assessment, changedErrors: changedErrors(r), - newOperations: args.newOperations.get(r.service) ?? [], + newOperations: (args.newOperations.get(r.service) ?? []).slice( + 0, + MAX_NEW_OPERATIONS, + ), window: args.current, comparisonMode: args.comparisonMode, metricSourceId: args.metricSourceId, @@ -247,7 +251,9 @@ export function buildReport(args: { deltas: r.assessment.deltas, }), ...(args.newOperations.has(r.service) && { - newOperations: args.newOperations.get(r.service), + newOperations: args.newOperations + .get(r.service) + ?.slice(0, MAX_NEW_OPERATIONS), }), topErrors: (args.currentErrors.get(r.service) ?? []).map(g => { const comparisonCount = prior(g); @@ -255,12 +261,13 @@ export function buildReport(args: { group: g.group, count: g.count, ...errorTiming(g, args.current), - ...(args.comparisonErrorCounts && { - comparisonCount, - status: onNewOperation(r, g) - ? ('new_operation' satisfies ErrorGroupStatus) - : errorGroupStatus(g.count, comparisonCount), - }), + ...(args.comparisonErrorCounts && + hasBaseline(r) && { + comparisonCount, + status: onNewOperation(r, g) + ? ('new_operation' satisfies ErrorGroupStatus) + : errorGroupStatus(g.count, comparisonCount), + }), }; }), })); @@ -283,6 +290,7 @@ export function buildReport(args: { 'spans; try spanKind:"all").', }), ...(args.comparisonNote && { comparisonNote: args.comparisonNote }), + ...(args.coverageNote && { coverageNote: args.coverageNote }), }, services, usage: USAGE, From 80b488ad6c932f55d39ef759dbe07bac4469ac30 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 19 Oct 2026 20:35:14 -0600 Subject: [PATCH 06/15] perf(mcp): build each result row's column map once (deep-review P3) --- .../tools/query/serviceHealth/operations.ts | 7 +++-- .../mcp/tools/query/serviceHealth/queries.ts | 29 ++++++++++--------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/packages/api/src/mcp/tools/query/serviceHealth/operations.ts b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts index ccda0c76f2..15b235371f 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/operations.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts @@ -5,7 +5,7 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types'; import { mcpQuerySettings } from '@/mcp/tools/trace/shared'; import type { QueryContext } from './queries'; -import { baseConfig, col } from './queries'; +import { baseConfig, columns } from './queries'; import type { Window } from './schema'; const SERVICE_ALIAS = '__hdx_service'; @@ -55,9 +55,10 @@ async function operationCounts( const rows = result.data ?? []; const out = new Map>(); for (const row of rows) { - const svc = String(col(row, SERVICE_ALIAS) ?? ''); + const c = columns(row); + const svc = String(c.get(SERVICE_ALIAS) ?? ''); const ops = out.get(svc) ?? new Map(); - ops.set(String(col(row, OP_ALIAS) ?? ''), Number(row.requests) || 0); + ops.set(String(c.get(OP_ALIAS) ?? ''), Number(row.requests) || 0); out.set(svc, ops); } return { counts: out, truncated: rows.length >= MAX_OPERATION_ROWS }; diff --git a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts index d10ee79445..a516aa4cf1 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts @@ -80,9 +80,9 @@ export type ErrorGroup = { lastSeenMs?: number; }; -/** Read a column from a result row (rows are keyed by alias). */ -export const col = (row: Record, key: string) => - new Map(Object.entries(row)).get(key); +/** A result row's columns by alias. Build once per row, then read with get. */ +export const columns = (row: Record) => + new Map(Object.entries(row)); /** The value of one grouping expression in an error group. */ export const groupValue = (g: Pick, key: string) => @@ -203,7 +203,7 @@ export async function fetchServiceStats( }; const stats = new Map(); for (const row of rows) { - stats.set(String(col(row, SERVICE_ALIAS) ?? ''), { + stats.set(String(columns(row).get(SERVICE_ALIAS) ?? ''), { requests: Number(row.requests) || 0, errors: Number(row.errors) || 0, p50Ms: toMs(row.p50), @@ -277,15 +277,18 @@ export async function fetchErrorGroups( limit: { limit: MAX_ERROR_ROWS }, }); - return rows.map(row => ({ - service: String(col(row, SERVICE_ALIAS) ?? ''), - group: Object.fromEntries( - groupExprs.map((e, i) => [e, String(col(row, errorKeyAlias(i)) ?? '')]), - ), - count: Number(col(row, ERROR_COUNT_ALIAS)) || 0, - firstSeenMs: toNumberOrNull(col(row, FIRST_SEEN_ALIAS)) || undefined, - lastSeenMs: toNumberOrNull(col(row, LAST_SEEN_ALIAS)) || undefined, - })); + return rows.map(row => { + const c = columns(row); + return { + service: String(c.get(SERVICE_ALIAS) ?? ''), + group: Object.fromEntries( + groupExprs.map((e, i) => [e, String(c.get(errorKeyAlias(i)) ?? '')]), + ), + count: Number(c.get(ERROR_COUNT_ALIAS)) || 0, + firstSeenMs: toNumberOrNull(c.get(FIRST_SEEN_ALIAS)) || undefined, + lastSeenMs: toNumberOrNull(c.get(LAST_SEEN_ALIAS)) || undefined, + }; + }); } export const errorGroupKey = (g: Pick) => From 5001c464cf306b1e2c600c56ae92b55c8ce9d65e Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 19 Oct 2026 20:42:47 -0600 Subject: [PATCH 07/15] test(mcp): cover service health comparison fallback and new-operation detection --- .../__tests__/serviceHealthComparison.test.ts | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts diff --git a/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts b/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts new file mode 100644 index 0000000000..f097999a84 --- /dev/null +++ b/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts @@ -0,0 +1,147 @@ +import { resolveComparison } from '@/mcp/tools/query/serviceHealth/comparison'; +import { findNewOperations } from '@/mcp/tools/query/serviceHealth/operations'; +import type { QueryContext } from '@/mcp/tools/query/serviceHealth/queries'; +import { fetchServiceStats } from '@/mcp/tools/query/serviceHealth/queries'; +import { comparisonWindow } from '@/mcp/tools/query/serviceHealth/schema'; + +jest.mock('@/mcp/tools/query/serviceHealth/queries', () => ({ + ...jest.requireActual('@/mcp/tools/query/serviceHealth/queries'), + fetchServiceStats: jest.fn(), +})); + +const fetchStats = jest.mocked(fetchServiceStats); + +const current = { + start: new Date('2026-10-16T10:00:00Z'), + end: new Date('2026-10-16T11:00:00Z'), +}; +const yesterday = comparisonWindow(current, 'yesterday'); +const stats = (services: string[]) => ({ + stats: new Map( + services.map(s => [ + s, + { requests: 100, errors: 0, p50Ms: 1, p95Ms: 2, p99Ms: 3 }, + ]), + ), + mvUsed: false, +}); + +const source = { + id: 'src', + connection: 'conn', + from: { databaseName: 'db', tableName: 't' }, + timestampValueExpression: 'Timestamp', + serviceNameExpression: 'ServiceName', + statusCodeExpression: 'StatusCode', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + durationExpression: 'Duration', + durationPrecision: 9, + querySettings: undefined, + mvSource: undefined, +}; + +const ctxWith = (queryChartConfig: jest.Mock): QueryContext => + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test double with only the members the code under test reads + ({ + client: { queryChartConfig }, + metadata: {}, + source, + input: { spanKind: 'server', where: '', whereLanguage: 'lucene' }, + signal: new AbortController().signal, + }) as unknown as QueryContext; + +describe('resolveComparison', () => { + beforeEach(() => fetchStats.mockReset()); + + it('keeps the requested window when it has data', async () => { + const r = await resolveComparison(ctxWith(jest.fn()), { + current, + requested: yesterday, + requestedMode: 'yesterday', + curHasData: true, + requestedCmp: stats(['a']), + }); + expect(r.comparisonMode).toBe('yesterday'); + expect(r.comparisonNote).toBeUndefined(); + expect(fetchStats).not.toHaveBeenCalled(); + }); + + it('falls back to the previous period when the requested window is empty', async () => { + fetchStats.mockResolvedValue(stats(['a'])); + const r = await resolveComparison(ctxWith(jest.fn()), { + current, + requested: yesterday, + requestedMode: 'yesterday', + curHasData: true, + requestedCmp: stats([]), + }); + expect(r.comparisonMode).toBe('previous_period'); + expect(r.comparison?.end.toISOString()).toBe(current.start.toISOString()); + expect(r.comparisonNote).toMatch(/previous period/); + }); + + it('drops the comparison when the previous period is empty too', async () => { + fetchStats.mockResolvedValue(stats([])); + const r = await resolveComparison(ctxWith(jest.fn()), { + current, + requested: yesterday, + requestedMode: 'yesterday', + curHasData: true, + requestedCmp: stats([]), + }); + expect(r).toMatchObject({ + comparison: null, + cmp: null, + comparisonMode: 'none', + }); + expect(r.comparisonNote).toMatch(/nothing is labelled new/); + }); +}); + +describe('findNewOperations', () => { + const row = (service: string, op: string, requests: number) => ({ + __hdx_service: service, + __hdx_operation: op, + requests, + }); + + it('reports operations with no prior traffic', async () => { + const query = jest + .fn() + .mockResolvedValueOnce({ + data: [row('svc', 'GET /old', 900), row('svc', 'GET /new', 100)], + }) + .mockResolvedValueOnce({ data: [row('svc', 'GET /old', 900)] }); + const r = await findNewOperations( + ctxWith(query), + current, + yesterday!, + ['svc'], + 50, + ); + expect(r.complete).toBe(true); + expect(r.newOperations.get('svc')).toEqual([ + { operation: 'GET /new', requests: 100, shareOfService: 0.1 }, + ]); + }); + + it('reports nothing when a result is truncated', async () => { + const many = Array.from({ length: 5000 }, (_, i) => + row('svc', `op${i}`, 20), + ); + const query = jest + .fn() + .mockResolvedValueOnce({ data: many }) + .mockResolvedValueOnce({ data: [row('svc', 'op0', 900)] }); + const r = await findNewOperations( + ctxWith(query), + current, + yesterday!, + ['svc'], + 50, + ); + expect(r.complete).toBe(false); + expect(r.newOperations.size).toBe(0); + }); +}); From e73937dee8b9aa97e998c82f2fa742c788c8fd68 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 19 Oct 2026 20:50:00 -0600 Subject: [PATCH 08/15] test(mcp): build a typed context instead of an eslint-disable cast (ratchet) --- .../__tests__/serviceHealthComparison.test.ts | 70 +++++++++---------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts b/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts index f097999a84..823d36adc7 100644 --- a/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts +++ b/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts @@ -1,8 +1,17 @@ +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; + +import { ClickhouseClient } from '@/clickhouse'; import { resolveComparison } from '@/mcp/tools/query/serviceHealth/comparison'; import { findNewOperations } from '@/mcp/tools/query/serviceHealth/operations'; -import type { QueryContext } from '@/mcp/tools/query/serviceHealth/queries'; +import type { + HealthSource, + QueryContext, +} from '@/mcp/tools/query/serviceHealth/queries'; import { fetchServiceStats } from '@/mcp/tools/query/serviceHealth/queries'; -import { comparisonWindow } from '@/mcp/tools/query/serviceHealth/schema'; +import { + comparisonWindow, + serviceHealthSchema, +} from '@/mcp/tools/query/serviceHealth/schema'; jest.mock('@/mcp/tools/query/serviceHealth/queries', () => ({ ...jest.requireActual('@/mcp/tools/query/serviceHealth/queries'), @@ -26,7 +35,7 @@ const stats = (services: string[]) => ({ mvUsed: false, }); -const source = { +const source: HealthSource = { id: 'src', connection: 'conn', from: { databaseName: 'db', tableName: 't' }, @@ -41,21 +50,25 @@ const source = { mvSource: undefined, }; -const ctxWith = (queryChartConfig: jest.Mock): QueryContext => - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test double with only the members the code under test reads - ({ - client: { queryChartConfig }, - metadata: {}, +const ctxWith = (results: { data: Record[] }[]) => { + const client = new ClickhouseClient({ host: 'http://localhost:1' }); + const query = jest.spyOn(client, 'queryChartConfig'); + for (const r of results) query.mockResolvedValueOnce({ data: r.data }); + const ctx: QueryContext = { + client, + metadata: getMetadata(client), source, - input: { spanKind: 'server', where: '', whereLanguage: 'lucene' }, + input: serviceHealthSchema.parse({ sourceId: 'src' }), signal: new AbortController().signal, - }) as unknown as QueryContext; + }; + return ctx; +}; describe('resolveComparison', () => { beforeEach(() => fetchStats.mockReset()); it('keeps the requested window when it has data', async () => { - const r = await resolveComparison(ctxWith(jest.fn()), { + const r = await resolveComparison(ctxWith([]), { current, requested: yesterday, requestedMode: 'yesterday', @@ -69,7 +82,7 @@ describe('resolveComparison', () => { it('falls back to the previous period when the requested window is empty', async () => { fetchStats.mockResolvedValue(stats(['a'])); - const r = await resolveComparison(ctxWith(jest.fn()), { + const r = await resolveComparison(ctxWith([]), { current, requested: yesterday, requestedMode: 'yesterday', @@ -83,7 +96,7 @@ describe('resolveComparison', () => { it('drops the comparison when the previous period is empty too', async () => { fetchStats.mockResolvedValue(stats([])); - const r = await resolveComparison(ctxWith(jest.fn()), { + const r = await resolveComparison(ctxWith([]), { current, requested: yesterday, requestedMode: 'yesterday', @@ -107,19 +120,11 @@ describe('findNewOperations', () => { }); it('reports operations with no prior traffic', async () => { - const query = jest - .fn() - .mockResolvedValueOnce({ - data: [row('svc', 'GET /old', 900), row('svc', 'GET /new', 100)], - }) - .mockResolvedValueOnce({ data: [row('svc', 'GET /old', 900)] }); - const r = await findNewOperations( - ctxWith(query), - current, - yesterday!, - ['svc'], - 50, - ); + const ctx = ctxWith([ + { data: [row('svc', 'GET /old', 900), row('svc', 'GET /new', 100)] }, + { data: [row('svc', 'GET /old', 900)] }, + ]); + const r = await findNewOperations(ctx, current, yesterday!, ['svc'], 50); expect(r.complete).toBe(true); expect(r.newOperations.get('svc')).toEqual([ { operation: 'GET /new', requests: 100, shareOfService: 0.1 }, @@ -130,17 +135,8 @@ describe('findNewOperations', () => { const many = Array.from({ length: 5000 }, (_, i) => row('svc', `op${i}`, 20), ); - const query = jest - .fn() - .mockResolvedValueOnce({ data: many }) - .mockResolvedValueOnce({ data: [row('svc', 'op0', 900)] }); - const r = await findNewOperations( - ctxWith(query), - current, - yesterday!, - ['svc'], - 50, - ); + const ctx = ctxWith([{ data: many }, { data: [row('svc', 'op0', 900)] }]); + const r = await findNewOperations(ctx, current, yesterday!, ['svc'], 50); expect(r.complete).toBe(false); expect(r.newOperations.size).toBe(0); }); From a2e368ad94fcbae6c1ccf019742857f5f9f16a6a Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 20 Oct 2026 10:50:36 -0600 Subject: [PATCH 09/15] fix(mcp): match error groups across windows by hash and reuse the shared query path (review major) --- .../tools/query/serviceHealth/operations.ts | 47 ++++++------------- .../mcp/tools/query/serviceHealth/queries.ts | 25 ++++++---- .../mcp/tools/query/serviceHealth/report.ts | 5 +- 3 files changed, 35 insertions(+), 42 deletions(-) diff --git a/packages/api/src/mcp/tools/query/serviceHealth/operations.ts b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts index 15b235371f..d215ae18a9 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/operations.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts @@ -1,11 +1,5 @@ -import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; -import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils'; -import { DisplayType } from '@hyperdx/common-utils/dist/types'; - -import { mcpQuerySettings } from '@/mcp/tools/trace/shared'; - import type { QueryContext } from './queries'; -import { baseConfig, columns } from './queries'; +import { baseConfig, columns, run, sqlList } from './queries'; import type { Window } from './schema'; const SERVICE_ALIAS = '__hdx_service'; @@ -27,32 +21,21 @@ async function operationCounts( window: Window, services: string[], ): Promise<{ counts: Map>; truncated: boolean }> { - const { source, client, signal } = ctx; - const list = services.map(s => `'${escapeSqlString(s)}'`).join(', '); - const result = await client.queryChartConfig({ - config: { - ...baseConfig(ctx, window, [ - { - type: 'sql', - condition: `${source.serviceNameExpression} IN (${list})`, - }, - ]), - displayType: DisplayType.Table, - select: [{ aggFn: 'count', valueExpression: '', alias: 'requests' }], - groupBy: [ - { valueExpression: source.serviceNameExpression, alias: SERVICE_ALIAS }, - { valueExpression: source.spanNameExpression, alias: OP_ALIAS }, - ], - limit: { limit: MAX_OPERATION_ROWS }, - }, - metadata: getMetadata(client), - querySettings: source.querySettings, - opts: { - clickhouse_settings: mcpQuerySettings(source.querySettings), - abort_signal: signal, - }, + const { source } = ctx; + const { rows } = await run(ctx, { + ...baseConfig(ctx, window, [ + { + type: 'sql', + condition: `${source.serviceNameExpression} IN (${sqlList(services)})`, + }, + ]), + select: [{ aggFn: 'count', valueExpression: '', alias: 'requests' }], + groupBy: [ + { valueExpression: source.serviceNameExpression, alias: SERVICE_ALIAS }, + { valueExpression: source.spanNameExpression, alias: OP_ALIAS }, + ], + limit: { limit: MAX_OPERATION_ROWS }, }); - const rows = result.data ?? []; const out = new Map>(); for (const row of rows) { const c = columns(row); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts index a516aa4cf1..7da992c462 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts @@ -27,6 +27,7 @@ import type { ServiceHealthInput, Window } from './schema'; const SERVICE_ALIAS = '__hdx_service'; const ERROR_COUNT_ALIAS = '__hdx_error_count'; const FIRST_SEEN_ALIAS = '__hdx_first_seen'; +const HASH_ALIAS = '__hdx_group_hash'; const LAST_SEEN_ALIAS = '__hdx_last_seen'; const errorKeyAlias = (i: number) => `__hdx_error_key_${i}`; @@ -75,6 +76,8 @@ export type ErrorGroup = { service: string; group: Record; count: number; + /** cityHash64 of the group's key, used to look it up in another window. */ + hash?: string; /** Epoch ms of the first and last error in the window, if known. */ firstSeenMs?: number; lastSeenMs?: number; @@ -88,7 +91,7 @@ export const columns = (row: Record) => export const groupValue = (g: Pick, key: string) => new Map(Object.entries(g.group)).get(key); -const sqlList = (values: string[]) => +export const sqlList = (values: string[]) => values.map(v => `'${escapeSqlString(v)}'`).join(', '); export function baseConfig( @@ -131,7 +134,7 @@ export function baseConfig( }; } -async function run( +export async function run( ctx: QueryContext, config: BuilderChartConfigWithDateRange, ): Promise<{ rows: Record[]; mvUsed: boolean }> { @@ -241,16 +244,19 @@ export async function fetchErrorGroups( ): Promise { const { source } = ctx; const keyExprs = [source.serviceNameExpression, ...groupExprs]; + // Match groups across windows by a fixed-size hash. Inlining the raw key + // text (status messages can be stack traces) can exceed max_query_size. + const hashExpr = `cityHash64(${keyExprs.map(e => `toString(${e})`).join(', ')})`; const epochMs = `toUnixTimestamp64Milli(toDateTime64(${getFirstTimestampValueExpression(source.timestampValueExpression)}, 3))`; const scopeCondition = 'services' in scope ? `${source.serviceNameExpression} IN (${sqlList(scope.services)})` - : `tuple(${keyExprs.map(e => `toString(${e})`).join(', ')}) IN (${scope.match - .map( - g => - `tuple(${sqlList([g.service, ...groupExprs.map(e => groupValue(g, e) ?? '')])})`, - ) - .join(', ')})`; + : `${hashExpr} IN (${ + scope.match + .map(g => g.hash) + .filter((h): h is string => !!h && /^\d+$/.test(h)) + .join(', ') || 'NULL' + })`; const { rows } = await run(ctx, { ...baseConfig(ctx, window, [ @@ -265,6 +271,8 @@ export async function fetchErrorGroups( // Epoch ms, because min/max cast their argument to Float64. { aggFn: 'min', valueExpression: epochMs, alias: FIRST_SEEN_ALIAS }, { aggFn: 'max', valueExpression: epochMs, alias: LAST_SEEN_ALIAS }, + // 'none' passes the expression through on the sample-weighted path too. + { aggFn: 'none', valueExpression: `any(${hashExpr})`, alias: HASH_ALIAS }, ], groupBy: [ { valueExpression: source.serviceNameExpression, alias: SERVICE_ALIAS }, @@ -285,6 +293,7 @@ export async function fetchErrorGroups( groupExprs.map((e, i) => [e, String(c.get(errorKeyAlias(i)) ?? '')]), ), count: Number(c.get(ERROR_COUNT_ALIAS)) || 0, + hash: String(c.get(HASH_ALIAS) ?? '') || undefined, firstSeenMs: toNumberOrNull(c.get(FIRST_SEEN_ALIAS)) || undefined, lastSeenMs: toNumberOrNull(c.get(LAST_SEEN_ALIAS)) || undefined, }; diff --git a/packages/api/src/mcp/tools/query/serviceHealth/report.ts b/packages/api/src/mcp/tools/query/serviceHealth/report.ts index 9a89b1e733..491d450b19 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/report.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/report.ts @@ -154,8 +154,9 @@ const USAGE = 'Any of those makes a service degraded. traffic_spike, new_service and ' + 'low_volume (under minRequests in both windows, never flagged) and ' + 'new_errors (a listed error group is new or growing; such services sort ' + - 'first within their status) are context only. Healthy services are not regressions; report them as steady and do ' + - "treat moves below these thresholds as steady (no action needed); each service's " + + 'first within their status) are context only. Healthy services are not ' + + 'regressions: report them as steady, and treat moves below these ' + + "thresholds as normal variance. Each service's " + '`summary` states which moves crossed one. topErrors carry firstSeen / ' + 'lastSeen / ongoing so a burst that stopped reads as transient. topErrors lists new and ' + 'growing error groups first (largest gain first), then the largest steady ' + From 65514af8a2f9cfc24d5f4ba0e3d08833a5ba7f3d Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 20 Oct 2026 11:00:46 -0600 Subject: [PATCH 10/15] refactor(mcp): move the error-group query out of queries.ts to stay under 300 lines (greptile P2) --- .../query/__tests__/serviceHealthRank.test.ts | 2 +- .../mcp/tools/query/serviceHealth/collect.ts | 6 +- .../tools/query/serviceHealth/errorGroups.ts | 124 ++++++++++++++++++ .../mcp/tools/query/serviceHealth/index.ts | 7 +- .../mcp/tools/query/serviceHealth/queries.ts | 118 +---------------- .../mcp/tools/query/serviceHealth/report.ts | 3 +- .../mcp/tools/query/serviceHealth/summary.ts | 2 +- 7 files changed, 135 insertions(+), 127 deletions(-) create mode 100644 packages/api/src/mcp/tools/query/serviceHealth/errorGroups.ts diff --git a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts index 39c6f21346..27f358da21 100644 --- a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts +++ b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts @@ -1,5 +1,5 @@ +import { errorGroupKey } from '@/mcp/tools/query/serviceHealth/errorGroups'; import type { NewOperation } from '@/mcp/tools/query/serviceHealth/operations'; -import { errorGroupKey } from '@/mcp/tools/query/serviceHealth/queries'; import { assessService, HEALTH_THRESHOLDS, diff --git a/packages/api/src/mcp/tools/query/serviceHealth/collect.ts b/packages/api/src/mcp/tools/query/serviceHealth/collect.ts index 9d175f5616..13216b603b 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/collect.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/collect.ts @@ -1,12 +1,12 @@ -import { findNewOperations, type NewOperation } from './operations'; import { type ErrorGroup, errorGroupKey, fetchErrorGroups, groupValue, MAX_ERROR_ROWS, - type QueryContext, -} from './queries'; +} from './errorGroups'; +import { findNewOperations, type NewOperation } from './operations'; +import type { QueryContext } from './queries'; import { ERROR_CANDIDATES_PER_SERVICE, hasBaseline, diff --git a/packages/api/src/mcp/tools/query/serviceHealth/errorGroups.ts b/packages/api/src/mcp/tools/query/serviceHealth/errorGroups.ts new file mode 100644 index 0000000000..c7220bb2fc --- /dev/null +++ b/packages/api/src/mcp/tools/query/serviceHealth/errorGroups.ts @@ -0,0 +1,124 @@ +import { errorStatusPredicateSql } from '@hyperdx/common-utils/dist/core/spanPredicates'; +import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils'; + +import { + baseConfig, + columns, + type HealthSource, + type QueryContext, + run, + SERVICE_ALIAS, + sqlList, + toNumberOrNull, +} from './queries'; +import type { Window } from './schema'; + +const ERROR_COUNT_ALIAS = '__hdx_error_count'; +const FIRST_SEEN_ALIAS = '__hdx_first_seen'; +const HASH_ALIAS = '__hdx_group_hash'; +const LAST_SEEN_ALIAS = '__hdx_last_seen'; +const errorKeyAlias = (i: number) => `__hdx_error_key_${i}`; + +/** Upper bound on error-group rows fetched, before bucketing per service. */ +export const MAX_ERROR_ROWS = 2500; + +export type ErrorGroup = { + service: string; + group: Record; + count: number; + /** cityHash64 of the group's key, used to look it up in another window. */ + hash?: string; + /** Epoch ms of the first and last error in the window, if known. */ + firstSeenMs?: number; + lastSeenMs?: number; +}; + +/** The value of one grouping expression in an error group. */ +export const groupValue = (g: Pick, key: string) => + new Map(Object.entries(g.group)).get(key); + +/** The expressions that define an error group, in output order. */ +export function errorGroupExpressions( + source: HealthSource, + errorGroupBy: string | undefined, + split: (s: string) => string[], +): string[] { + const exprs = errorGroupBy + ? split(errorGroupBy) + : [source.spanNameExpression, source.statusMessageExpression]; + return exprs.filter( + (e): e is string => !!e && e !== source.serviceNameExpression, + ); +} + +/** + * Error counts per (service, group) in one window. `match` restricts rows to + * exact (service, ...group) tuples, so the comparison window is looked up for + * the groups found in the current window instead of its own top-N. + */ +export async function fetchErrorGroups( + ctx: QueryContext, + window: Window, + groupExprs: string[], + scope: { services: string[] } | { match: ErrorGroup[] }, +): Promise { + const { source } = ctx; + const keyExprs = [source.serviceNameExpression, ...groupExprs]; + // Match groups across windows by a fixed-size hash. Inlining the raw key + // text (status messages can be stack traces) can exceed max_query_size. + const hashExpr = `cityHash64(${keyExprs.map(e => `toString(${e})`).join(', ')})`; + const epochMs = `toUnixTimestamp64Milli(toDateTime64(${getFirstTimestampValueExpression(source.timestampValueExpression)}, 3))`; + const scopeCondition = + 'services' in scope + ? `${source.serviceNameExpression} IN (${sqlList(scope.services)})` + : `${hashExpr} IN (${ + scope.match + .map(g => g.hash) + .filter((h): h is string => !!h && /^\d+$/.test(h)) + .join(', ') || 'NULL' + })`; + + const { rows } = await run(ctx, { + ...baseConfig(ctx, window, [ + { + type: 'sql', + condition: errorStatusPredicateSql(source.statusCodeExpression), + }, + { type: 'sql', condition: scopeCondition }, + ]), + select: [ + { aggFn: 'count', valueExpression: '', alias: ERROR_COUNT_ALIAS }, + // Epoch ms, because min/max cast their argument to Float64. + { aggFn: 'min', valueExpression: epochMs, alias: FIRST_SEEN_ALIAS }, + { aggFn: 'max', valueExpression: epochMs, alias: LAST_SEEN_ALIAS }, + // 'none' passes the expression through on the sample-weighted path too. + { aggFn: 'none', valueExpression: `any(${hashExpr})`, alias: HASH_ALIAS }, + ], + groupBy: [ + { valueExpression: source.serviceNameExpression, alias: SERVICE_ALIAS }, + ...groupExprs.map((e, i) => ({ + valueExpression: `toString(${e})`, + alias: errorKeyAlias(i), + })), + ], + orderBy: `${ERROR_COUNT_ALIAS} DESC`, + limit: { limit: MAX_ERROR_ROWS }, + }); + + return rows.map(row => { + const c = columns(row); + return { + service: String(c.get(SERVICE_ALIAS) ?? ''), + group: Object.fromEntries( + groupExprs.map((e, i) => [e, String(c.get(errorKeyAlias(i)) ?? '')]), + ), + count: Number(c.get(ERROR_COUNT_ALIAS)) || 0, + hash: String(c.get(HASH_ALIAS) ?? '') || undefined, + firstSeenMs: toNumberOrNull(c.get(FIRST_SEEN_ALIAS)) || undefined, + lastSeenMs: toNumberOrNull(c.get(LAST_SEEN_ALIAS)) || undefined, + }; + }); +} + +export const errorGroupKey = (g: Pick) => + JSON.stringify([g.service, ...Object.values(g.group)]); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/index.ts b/packages/api/src/mcp/tools/query/serviceHealth/index.ts index 493125f3b3..6ff3f6e2dd 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/index.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/index.ts @@ -16,11 +16,8 @@ import { trimToolResponse } from '@/utils/trimToolResponse'; import { collectErrors } from './collect'; import { resolveComparison } from './comparison'; -import { - errorGroupExpressions, - fetchServiceStats, - type QueryContext, -} from './queries'; +import { errorGroupExpressions } from './errorGroups'; +import { fetchServiceStats, type QueryContext } from './queries'; import { buildReport, rankServices } from './report'; import { comparisonWindow, diff --git a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts index 7da992c462..a9b7f33978 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts @@ -4,10 +4,7 @@ import { errorStatusPredicateSql, serverSpanKindPredicateSql, } from '@hyperdx/common-utils/dist/core/spanPredicates'; -import { - escapeSqlString, - getFirstTimestampValueExpression, -} from '@hyperdx/common-utils/dist/core/utils'; +import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils'; import { type BuilderChartConfigWithDateRange, DisplayType, @@ -24,15 +21,7 @@ import { durationDivisor, mcpQuerySettings } from '@/mcp/tools/trace/shared'; import type { WindowStats } from './rank'; import type { ServiceHealthInput, Window } from './schema'; -const SERVICE_ALIAS = '__hdx_service'; -const ERROR_COUNT_ALIAS = '__hdx_error_count'; -const FIRST_SEEN_ALIAS = '__hdx_first_seen'; -const HASH_ALIAS = '__hdx_group_hash'; -const LAST_SEEN_ALIAS = '__hdx_last_seen'; -const errorKeyAlias = (i: number) => `__hdx_error_key_${i}`; - -/** Upper bound on error-group rows fetched, before bucketing per service. */ -export const MAX_ERROR_ROWS = 2500; +export const SERVICE_ALIAS = '__hdx_service'; type MvSource = Parameters[4]; @@ -72,25 +61,10 @@ export type QueryContext = { signal: AbortSignal; }; -export type ErrorGroup = { - service: string; - group: Record; - count: number; - /** cityHash64 of the group's key, used to look it up in another window. */ - hash?: string; - /** Epoch ms of the first and last error in the window, if known. */ - firstSeenMs?: number; - lastSeenMs?: number; -}; - /** A result row's columns by alias. Build once per row, then read with get. */ export const columns = (row: Record) => new Map(Object.entries(row)); -/** The value of one grouping expression in an error group. */ -export const groupValue = (g: Pick, key: string) => - new Map(Object.entries(g.group)).get(key); - export const sqlList = (values: string[]) => values.map(v => `'${escapeSqlString(v)}'`).join(', '); @@ -160,7 +134,7 @@ export async function run( return { rows: result.data ?? [], mvUsed: optimized !== config }; } -const toNumberOrNull = (v: unknown) => { +export const toNumberOrNull = (v: unknown) => { const n = Number(v); return v == null || !Number.isFinite(n) ? null : n; }; @@ -216,89 +190,3 @@ export async function fetchServiceStats( } return { stats, mvUsed }; } - -/** The expressions that define an error group, in output order. */ -export function errorGroupExpressions( - source: HealthSource, - errorGroupBy: string | undefined, - split: (s: string) => string[], -): string[] { - const exprs = errorGroupBy - ? split(errorGroupBy) - : [source.spanNameExpression, source.statusMessageExpression]; - return exprs.filter( - (e): e is string => !!e && e !== source.serviceNameExpression, - ); -} - -/** - * Error counts per (service, group) in one window. `match` restricts rows to - * exact (service, ...group) tuples, so the comparison window is looked up for - * the groups found in the current window instead of its own top-N. - */ -export async function fetchErrorGroups( - ctx: QueryContext, - window: Window, - groupExprs: string[], - scope: { services: string[] } | { match: ErrorGroup[] }, -): Promise { - const { source } = ctx; - const keyExprs = [source.serviceNameExpression, ...groupExprs]; - // Match groups across windows by a fixed-size hash. Inlining the raw key - // text (status messages can be stack traces) can exceed max_query_size. - const hashExpr = `cityHash64(${keyExprs.map(e => `toString(${e})`).join(', ')})`; - const epochMs = `toUnixTimestamp64Milli(toDateTime64(${getFirstTimestampValueExpression(source.timestampValueExpression)}, 3))`; - const scopeCondition = - 'services' in scope - ? `${source.serviceNameExpression} IN (${sqlList(scope.services)})` - : `${hashExpr} IN (${ - scope.match - .map(g => g.hash) - .filter((h): h is string => !!h && /^\d+$/.test(h)) - .join(', ') || 'NULL' - })`; - - const { rows } = await run(ctx, { - ...baseConfig(ctx, window, [ - { - type: 'sql', - condition: errorStatusPredicateSql(source.statusCodeExpression), - }, - { type: 'sql', condition: scopeCondition }, - ]), - select: [ - { aggFn: 'count', valueExpression: '', alias: ERROR_COUNT_ALIAS }, - // Epoch ms, because min/max cast their argument to Float64. - { aggFn: 'min', valueExpression: epochMs, alias: FIRST_SEEN_ALIAS }, - { aggFn: 'max', valueExpression: epochMs, alias: LAST_SEEN_ALIAS }, - // 'none' passes the expression through on the sample-weighted path too. - { aggFn: 'none', valueExpression: `any(${hashExpr})`, alias: HASH_ALIAS }, - ], - groupBy: [ - { valueExpression: source.serviceNameExpression, alias: SERVICE_ALIAS }, - ...groupExprs.map((e, i) => ({ - valueExpression: `toString(${e})`, - alias: errorKeyAlias(i), - })), - ], - orderBy: `${ERROR_COUNT_ALIAS} DESC`, - limit: { limit: MAX_ERROR_ROWS }, - }); - - return rows.map(row => { - const c = columns(row); - return { - service: String(c.get(SERVICE_ALIAS) ?? ''), - group: Object.fromEntries( - groupExprs.map((e, i) => [e, String(c.get(errorKeyAlias(i)) ?? '')]), - ), - count: Number(c.get(ERROR_COUNT_ALIAS)) || 0, - hash: String(c.get(HASH_ALIAS) ?? '') || undefined, - firstSeenMs: toNumberOrNull(c.get(FIRST_SEEN_ALIAS)) || undefined, - lastSeenMs: toNumberOrNull(c.get(LAST_SEEN_ALIAS)) || undefined, - }; - }); -} - -export const errorGroupKey = (g: Pick) => - JSON.stringify([g.service, ...Object.values(g.group)]); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/report.ts b/packages/api/src/mcp/tools/query/serviceHealth/report.ts index 491d450b19..74f3789a04 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/report.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/report.ts @@ -1,6 +1,5 @@ +import { type ErrorGroup, errorGroupKey, groupValue } from './errorGroups'; import { MAX_NEW_OPERATIONS, type NewOperation } from './operations'; -import type { ErrorGroup } from './queries'; -import { errorGroupKey, groupValue } from './queries'; import { assessService, compareAssessments, diff --git a/packages/api/src/mcp/tools/query/serviceHealth/summary.ts b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts index 83d941cbe8..d939c950b8 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/summary.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts @@ -1,5 +1,5 @@ +import type { ErrorGroup } from './errorGroups'; import type { NewOperation } from './operations'; -import type { ErrorGroup } from './queries'; import { HEALTH_THRESHOLDS, type ServiceAssessment, From 9767585b42738a3be2933e357bbb3ccf084ea71a Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 20 Oct 2026 11:10:58 -0600 Subject: [PATCH 11/15] test(mcp): cover the table-name sourceId hint; reuse SERVICE_ALIAS (review minors) --- .../api/src/mcp/__tests__/serviceHealthTool.int.test.ts | 8 ++++++++ .../api/src/mcp/tools/query/serviceHealth/operations.ts | 3 +-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts b/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts index a3ab5df719..be604d69eb 100644 --- a/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts +++ b/packages/api/src/mcp/__tests__/serviceHealthTool.int.test.ts @@ -175,6 +175,14 @@ describe('MCP Service Health Tool', () => { expect(getFirstText(result)).toContain('kind="trace"'); }); + it('points a table name passed as sourceId at the right source ID', async () => { + const result = await call({ sourceId: DEFAULT_TRACES_TABLE }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain( + `sourceId:"${traceSource._id.toString()}"`, + ); + }); + it('rejects a window longer than the comparison shift', async () => { const result = await call({ startTime: new Date(now - 30 * HOUR).toISOString(), diff --git a/packages/api/src/mcp/tools/query/serviceHealth/operations.ts b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts index d215ae18a9..c3665267a7 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/operations.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/operations.ts @@ -1,8 +1,7 @@ import type { QueryContext } from './queries'; -import { baseConfig, columns, run, sqlList } from './queries'; +import { baseConfig, columns, run, SERVICE_ALIAS, sqlList } from './queries'; import type { Window } from './schema'; -const SERVICE_ALIAS = '__hdx_service'; const OP_ALIAS = '__hdx_operation'; const MAX_OPERATION_ROWS = 5000; /** A new operation needs this many requests to be reported. */ From 94c88d9388e487ba08f707263749f4d0cb6cd812 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 20 Oct 2026 11:19:31 -0600 Subject: [PATCH 12/15] fix(mcp): zero the score of low-volume services; only name a source when one matches (review minors) --- .../src/mcp/tools/query/__tests__/serviceHealthRank.test.ts | 1 + packages/api/src/mcp/tools/query/serviceHealth/rank.ts | 5 ++++- packages/api/src/mcp/tools/query/serviceHealth/source.ts | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts index 27f358da21..fea250e4d8 100644 --- a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts +++ b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts @@ -105,6 +105,7 @@ describe('assessService', () => { 50, ); expect(a.flags).toEqual(['low_volume']); + expect(a.score).toBe(0); expect(a.status).toBe('healthy'); }); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/rank.ts b/packages/api/src/mcp/tools/query/serviceHealth/rank.ts index d998a3b190..3c4ebeb1a8 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/rank.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/rank.ts @@ -168,7 +168,10 @@ export function assessService( return { status, flags, - score: severityScore(cur, cmp, { curRate, cmpRate, p95Ratio }), + // Too few requests to judge, so it should not outrank real services. + score: flags.includes('low_volume') + ? 0 + : severityScore(cur, cmp, { curRate, cmpRate, p95Ratio }), deltas: { throughputRatio, errorRateDeltaPp, p95Ratio, p95DeltaMs }, }; } diff --git a/packages/api/src/mcp/tools/query/serviceHealth/source.ts b/packages/api/src/mcp/tools/query/serviceHealth/source.ts index b409afc02a..6bde0000c7 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/source.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/source.ts @@ -23,13 +23,15 @@ async function sourceNotFound( const needle = sourceId.trim().toLowerCase(); const describe = (s: (typeof traces)[number]) => `${s._id.toString()} ("${s.name}", table ${s.from.databaseName}.${s.from.tableName})`; - const match = traces.find(s => + const matches = traces.filter(s => [ s.name, s.from.tableName, `${s.from.databaseName}.${s.from.tableName}`, ].some(v => v.toLowerCase() === needle), ); + // With several matches, naming one would be a guess; list them instead. + const match = matches.length === 1 ? matches[0] : undefined; if (match) { return mcpUserError( `"${sourceId}" is a source name or table, not a source ID. Retry ${toolName} ` + From 9114430db914be498b53d5efac50bfae43f68093 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 20 Oct 2026 12:12:32 -0600 Subject: [PATCH 13/15] fix(mcp): call a long error trickle intermittent instead of a transient burst --- .../query/__tests__/serviceHealthRank.test.ts | 54 +++++++++++++++++++ .../mcp/tools/query/serviceHealth/report.ts | 3 +- .../mcp/tools/query/serviceHealth/summary.ts | 37 ++++++++++--- 3 files changed, 87 insertions(+), 7 deletions(-) diff --git a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts index fea250e4d8..c17778bf86 100644 --- a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts +++ b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts @@ -13,6 +13,10 @@ import { topGroupsPerService, } from '@/mcp/tools/query/serviceHealth/report'; import { comparisonWindow } from '@/mcp/tools/query/serviceHealth/schema'; +import { + errorTiming, + serviceSummary, +} from '@/mcp/tools/query/serviceHealth/summary'; const stats = (s: Partial = {}): WindowStats => ({ requests: 1000, @@ -331,3 +335,53 @@ describe('buildReport', () => { expect(svc.topErrors[0]).not.toHaveProperty('status'); }); }); + +describe('errorTiming', () => { + const window = { + start: new Date('2026-10-20T17:05:00Z'), + end: new Date('2026-10-20T18:05:00Z'), + }; + const at = (hhmm: string) => Date.parse(`2026-10-20T${hhmm}:00Z`); + const group = (first: string, last: string) => ({ + service: 'svc', + group: { SpanName: 'POST /x' }, + count: 30, + firstSeenMs: at(first), + lastSeenMs: at(last), + }); + + it('calls a group still firing at the window end ongoing', () => { + expect(errorTiming(group('17:30', '18:04'), window).pattern).toBe( + 'ongoing', + ); + }); + + it('calls a long trickle that paused intermittent, not a burst', () => { + // 52 minutes of errors, last one 7 minutes before the window end. + expect(errorTiming(group('17:06', '17:58'), window).pattern).toBe( + 'intermittent', + ); + }); + + it('calls a short spike that ended a burst', () => { + expect(errorTiming(group('17:20', '17:21'), window).pattern).toBe('burst'); + }); + + it('only calls bursts transient in the summary', () => { + const summary = (g: ReturnType) => + serviceSummary({ + current: stats(), + comparison: stats(), + assessment: assessService(stats(), stats(), 50), + changedErrors: [g], + newOperations: [], + window, + comparisonMode: 'yesterday', + }); + const trickle = summary(group('17:06', '17:58')); + expect(trickle).toMatch(/on and off from 17:06 to 17:58 UTC/); + expect(trickle).not.toMatch(/transient/); + expect(trickle).toMatch(/still a live problem/); + expect(summary(group('17:20', '17:21'))).toMatch(/transient/); + }); +}); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/report.ts b/packages/api/src/mcp/tools/query/serviceHealth/report.ts index 74f3789a04..34e1f5fcd3 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/report.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/report.ts @@ -157,7 +157,8 @@ const USAGE = 'regressions: report them as steady, and treat moves below these ' + "thresholds as normal variance. Each service's " + '`summary` states which moves crossed one. topErrors carry firstSeen / ' + - 'lastSeen / ongoing so a burst that stopped reads as transient. topErrors lists new and ' + + 'lastSeen and a pattern: ongoing (still firing), intermittent (on and ' + + 'off across most of the window) or burst (short and over, so transient). topErrors lists new and ' + 'growing error groups first (largest gain first), then the largest steady ' + 'ones. newOperations = span names with traffic now and none in the ' + 'comparison window (usually a rollout); errors on them have status ' + diff --git a/packages/api/src/mcp/tools/query/serviceHealth/summary.ts b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts index d939c950b8..88c92e1196 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/summary.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts @@ -9,6 +9,14 @@ import type { Window } from './schema'; /** An error group whose last error is this close to the window end is ongoing. */ const ONGOING_TAIL_MS = 2 * 60_000; +/** + * A group that is not ongoing but spans this share of the window is a slow + * trickle that paused, not a burst that ended. Calling it transient would + * tell the agent to dismiss a recurring failure. + */ +const INTERMITTENT_SHARE = 0.5; + +type ErrorPattern = 'ongoing' | 'intermittent' | 'burst'; const pct = (s: WindowStats | null) => s && s.requests > 0 ? `${((s.errors / s.requests) * 100).toFixed(2)}%` : '0%'; @@ -17,23 +25,37 @@ const hhmm = (ms: number) => new Date(ms).toISOString().slice(11, 16); export function errorTiming(g: ErrorGroup, window: Window) { if (g.firstSeenMs == null || g.lastSeenMs == null) return {}; const ongoing = window.end.getTime() - g.lastSeenMs <= ONGOING_TAIL_MS; + const windowMs = window.end.getTime() - window.start.getTime(); + const spread = windowMs > 0 ? (g.lastSeenMs - g.firstSeenMs) / windowMs : 0; + const pattern: ErrorPattern = ongoing + ? 'ongoing' + : spread >= INTERMITTENT_SHARE + ? 'intermittent' + : 'burst'; return { firstSeen: new Date(g.firstSeenMs).toISOString(), lastSeen: new Date(g.lastSeenMs).toISOString(), activeMinutes: Math.round(((g.lastSeenMs - g.firstSeenMs) / 60_000) * 10) / 10, ongoing, + pattern, }; } function describeError(g: ErrorGroup, window: Window): string { const label = Object.values(g.group).filter(Boolean).join(' / ') || 'error'; const t = errorTiming(g, window); - if (t.firstSeen == null || g.firstSeenMs == null || g.lastSeenMs == null) + if (t.pattern == null || g.firstSeenMs == null || g.lastSeenMs == null) return `"${label}" (${g.count})`; const [from, to] = [hhmm(g.firstSeenMs), hhmm(g.lastSeenMs)]; const when = from === to ? `at ${from} UTC` : `${from}-${to} UTC`; - return `"${label}" (${g.count}, ${when}, ${t.ongoing ? 'still occurring' : 'stopped'})`; + const how = + t.pattern === 'ongoing' + ? `${when}, still occurring` + : t.pattern === 'intermittent' + ? `on and off from ${from} to ${to} UTC, last at ${to}` + : `${when}, stopped`; + return `"${label}" (${g.count}, ${how})`; } /** @@ -104,13 +126,16 @@ export function serviceSummary(args: { .join('; ')}.`, ); if (changedErrors.length > 0) { - const stopped = changedErrors.some( - g => errorTiming(g, window).ongoing === false, + const patterns = new Set( + changedErrors.map(g => errorTiming(g, window).pattern), ); context.push( `New or growing errors: ${changedErrors.map(g => describeError(g, window)).join('; ')}.` + - (stopped - ? ' A burst that stopped is transient, not an ongoing problem.' + (patterns.has('burst') + ? ' A short burst that stopped is transient, not an ongoing problem.' + : '') + + (patterns.has('intermittent') + ? ' An error that recurs on and off across the window is still a live problem even if it is quiet right now.' : ''), ); } From 18c6c82fae6f7e14f975083dbf6906b62ae73459 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 20 Oct 2026 12:54:45 -0600 Subject: [PATCH 14/15] fix(mcp): describe spread-out errors by what was observed (greptile P2) --- .../query/__tests__/serviceHealthRank.test.ts | 10 ++++----- .../mcp/tools/query/serviceHealth/report.ts | 4 ++-- .../mcp/tools/query/serviceHealth/summary.ts | 22 +++++++++---------- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts index c17778bf86..da6a7cb330 100644 --- a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts +++ b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts @@ -356,11 +356,9 @@ describe('errorTiming', () => { ); }); - it('calls a long trickle that paused intermittent, not a burst', () => { + it('calls errors spread across the window spread, not a burst', () => { // 52 minutes of errors, last one 7 minutes before the window end. - expect(errorTiming(group('17:06', '17:58'), window).pattern).toBe( - 'intermittent', - ); + expect(errorTiming(group('17:06', '17:58'), window).pattern).toBe('spread'); }); it('calls a short spike that ended a burst', () => { @@ -379,9 +377,9 @@ describe('errorTiming', () => { comparisonMode: 'yesterday', }); const trickle = summary(group('17:06', '17:58')); - expect(trickle).toMatch(/on and off from 17:06 to 17:58 UTC/); + expect(trickle).toMatch(/spread across 17:06-17:58 UTC, none since 17:58/); expect(trickle).not.toMatch(/transient/); - expect(trickle).toMatch(/still a live problem/); + expect(trickle).toMatch(/check clickstack_timeseries/); expect(summary(group('17:20', '17:21'))).toMatch(/transient/); }); }); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/report.ts b/packages/api/src/mcp/tools/query/serviceHealth/report.ts index 34e1f5fcd3..442a65dabe 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/report.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/report.ts @@ -157,8 +157,8 @@ const USAGE = 'regressions: report them as steady, and treat moves below these ' + "thresholds as normal variance. Each service's " + '`summary` states which moves crossed one. topErrors carry firstSeen / ' + - 'lastSeen and a pattern: ongoing (still firing), intermittent (on and ' + - 'off across most of the window) or burst (short and over, so transient). topErrors lists new and ' + + 'lastSeen and a pattern: ongoing (still firing), spread (most of the ' + + 'window, quiet now; not proven resolved) or burst (short, so transient). topErrors lists new and ' + 'growing error groups first (largest gain first), then the largest steady ' + 'ones. newOperations = span names with traffic now and none in the ' + 'comparison window (usually a rollout); errors on them have status ' + diff --git a/packages/api/src/mcp/tools/query/serviceHealth/summary.ts b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts index 88c92e1196..a7737af20a 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/summary.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/summary.ts @@ -10,13 +10,13 @@ import type { Window } from './schema'; /** An error group whose last error is this close to the window end is ongoing. */ const ONGOING_TAIL_MS = 2 * 60_000; /** - * A group that is not ongoing but spans this share of the window is a slow - * trickle that paused, not a burst that ended. Calling it transient would - * tell the agent to dismiss a recurring failure. + * A group that is not ongoing but spans this share of the window is not a + * short burst. First/last times can't tell a trickle from one long incident, + * so it is only called spread, never transient. */ -const INTERMITTENT_SHARE = 0.5; +const SPREAD_SHARE = 0.5; -type ErrorPattern = 'ongoing' | 'intermittent' | 'burst'; +type ErrorPattern = 'ongoing' | 'spread' | 'burst'; const pct = (s: WindowStats | null) => s && s.requests > 0 ? `${((s.errors / s.requests) * 100).toFixed(2)}%` : '0%'; @@ -29,8 +29,8 @@ export function errorTiming(g: ErrorGroup, window: Window) { const spread = windowMs > 0 ? (g.lastSeenMs - g.firstSeenMs) / windowMs : 0; const pattern: ErrorPattern = ongoing ? 'ongoing' - : spread >= INTERMITTENT_SHARE - ? 'intermittent' + : spread >= SPREAD_SHARE + ? 'spread' : 'burst'; return { firstSeen: new Date(g.firstSeenMs).toISOString(), @@ -52,8 +52,8 @@ function describeError(g: ErrorGroup, window: Window): string { const how = t.pattern === 'ongoing' ? `${when}, still occurring` - : t.pattern === 'intermittent' - ? `on and off from ${from} to ${to} UTC, last at ${to}` + : t.pattern === 'spread' + ? `spread across ${from}-${to} UTC, none since ${to}` : `${when}, stopped`; return `"${label}" (${g.count}, ${how})`; } @@ -134,8 +134,8 @@ export function serviceSummary(args: { (patterns.has('burst') ? ' A short burst that stopped is transient, not an ongoing problem.' : '') + - (patterns.has('intermittent') - ? ' An error that recurs on and off across the window is still a live problem even if it is quiet right now.' + (patterns.has('spread') + ? ' An error spread across much of the window may still recur; check clickstack_timeseries by minute before calling it resolved.' : ''), ); } From f0b1f965a6f11b9506d915757422b7520e49bfe8 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 20 Oct 2026 13:06:57 -0600 Subject: [PATCH 15/15] refactor(mcp): drop materialized-view support from service health No trace source we checked has MVs, the error-group queries cannot use one, and the MV rewrite rounds windows to the view's granularity, which broke the half-open comparison windows (review minors). --- .../__tests__/serviceHealthComparison.test.ts | 2 -- .../query/__tests__/serviceHealthRank.test.ts | 2 -- .../mcp/tools/query/serviceHealth/index.ts | 2 -- .../mcp/tools/query/serviceHealth/queries.ts | 33 +++++++------------ .../mcp/tools/query/serviceHealth/report.ts | 2 -- .../mcp/tools/query/serviceHealth/source.ts | 1 - 6 files changed, 11 insertions(+), 31 deletions(-) diff --git a/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts b/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts index 823d36adc7..4a75cdec5b 100644 --- a/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts +++ b/packages/api/src/mcp/tools/query/__tests__/serviceHealthComparison.test.ts @@ -32,7 +32,6 @@ const stats = (services: string[]) => ({ { requests: 100, errors: 0, p50Ms: 1, p95Ms: 2, p99Ms: 3 }, ]), ), - mvUsed: false, }); const source: HealthSource = { @@ -47,7 +46,6 @@ const source: HealthSource = { durationExpression: 'Duration', durationPrecision: 9, querySettings: undefined, - mvSource: undefined, }; const ctxWith = (results: { data: Record[] }[]) => { diff --git a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts index da6a7cb330..a0a2140c54 100644 --- a/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts +++ b/packages/api/src/mcp/tools/query/__tests__/serviceHealthRank.test.ts @@ -275,7 +275,6 @@ describe('buildReport', () => { comparisonErrorCounts: new Map(), newOperations: new Map(), spanNameKey: 'SpanName', - mvUsed: false, }); expect(report.services.map(s => s.service)).toEqual(['quiet']); expect(report.services[0].flags).toContain('new_errors'); @@ -313,7 +312,6 @@ describe('buildReport', () => { comparisonErrorCounts: new Map(), newOperations, spanNameKey: 'SpanName', - mvUsed: false, }).services[0]; it('does not call errors on a brand-new operation new errors', () => { diff --git a/packages/api/src/mcp/tools/query/serviceHealth/index.ts b/packages/api/src/mcp/tools/query/serviceHealth/index.ts index 6ff3f6e2dd..d0d9e72128 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/index.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/index.ts @@ -162,7 +162,6 @@ export function registerServiceHealth({ newOperations, spanNameKey: source.spanNameExpression, metricSourceId: source.metricSourceId, - mvUsed: cur.mvUsed || !!cmp?.mvUsed, }); trace.getActiveSpan()?.setAttributes({ @@ -177,7 +176,6 @@ export function registerServiceHealth({ report.summary.statusCounts.critical, 'mcp.service_health.degraded_count': report.summary.statusCounts.degraded, - 'mcp.service_health.mv_used': report.summary.materializedViewUsed, }); const { data, isTrimmed } = trimToolResponse(report); diff --git a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts index a9b7f33978..f71a893670 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/queries.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/queries.ts @@ -1,4 +1,3 @@ -import { tryOptimizeConfigWithMaterializedView } from '@hyperdx/common-utils/dist/core/materializedViews'; import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; import { errorStatusPredicateSql, @@ -23,12 +22,10 @@ import type { ServiceHealthInput, Window } from './schema'; export const SERVICE_ALIAS = '__hdx_service'; -type MvSource = Parameters[4]; - /** * Plain copies of the trace source fields the queries read. The Mongoose - * document's subdocuments (e.g. `from`) can't go into a chart config, because - * the MV optimizer structuredClones the config. + * document's subdocuments (e.g. `from`) don't belong in a chart config, which + * the renderer may clone. */ export type HealthSource = Pick< TTraceSource, @@ -49,8 +46,6 @@ export type HealthSource = Pick< serviceNameExpression: string; statusCodeExpression: string; querySettings: QuerySettings | undefined; - /** The original document, for the MV optimizer. Unset when it has no MVs. */ - mvSource: MvSource | undefined; }; export type QueryContext = { @@ -111,19 +106,13 @@ export function baseConfig( export async function run( ctx: QueryContext, config: BuilderChartConfigWithDateRange, -): Promise<{ rows: Record[]; mvUsed: boolean }> { +): Promise<{ rows: Record[] }> { const { client, metadata, source, signal } = ctx; - const optimized = source.mvSource - ? await tryOptimizeConfigWithMaterializedView( - config, - metadata, - client, - signal, - source.mvSource, - ) - : config; + // No materialized-view rewrite: it rounds the window to the view's + // granularity, which would break the half-open windows the comparison + // relies on, and the error-group queries can't use a view anyway. const result = await client.queryChartConfig({ - config: optimized, + config, metadata, querySettings: source.querySettings, opts: { @@ -131,7 +120,7 @@ export async function run( abort_signal: signal, }, }); - return { rows: result.data ?? [], mvUsed: optimized !== config }; + return { rows: result.data ?? [] }; } export const toNumberOrNull = (v: unknown) => { @@ -143,7 +132,7 @@ export const toNumberOrNull = (v: unknown) => { export async function fetchServiceStats( ctx: QueryContext, window: Window, -): Promise<{ stats: Map; mvUsed: boolean }> { +): Promise<{ stats: Map }> { const { source } = ctx; const quantile = (level: number, alias: string) => ({ aggFn: 'quantile' as const, @@ -153,7 +142,7 @@ export async function fetchServiceStats( aggConditionLanguage: 'sql' as const, alias, }); - const { rows, mvUsed } = await run(ctx, { + const { rows } = await run(ctx, { ...baseConfig(ctx, window, []), select: [ { aggFn: 'count', valueExpression: '', alias: 'requests' }, @@ -188,5 +177,5 @@ export async function fetchServiceStats( p99Ms: toMs(row.p99), }); } - return { stats, mvUsed }; + return { stats }; } diff --git a/packages/api/src/mcp/tools/query/serviceHealth/report.ts b/packages/api/src/mcp/tools/query/serviceHealth/report.ts index 442a65dabe..c5f7fc2b2c 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/report.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/report.ts @@ -185,7 +185,6 @@ export function buildReport(args: { newOperations: Map; spanNameKey: string; metricSourceId?: string | null; - mvUsed: boolean; }) { const onNewOperation = (r: RankedService, g: ErrorGroup) => (args.newOperations.get(r.service) ?? []).some( @@ -283,7 +282,6 @@ export function buildReport(args: { servicesAssessed: args.ranked.length, servicesReturned: services.length, statusCounts: counts, - materializedViewUsed: args.mvUsed, ...(args.ranked.length === 0 && { warning: 'No spans matched in either window. Check the time window, the ' + diff --git a/packages/api/src/mcp/tools/query/serviceHealth/source.ts b/packages/api/src/mcp/tools/query/serviceHealth/source.ts index 6bde0000c7..4e7553693f 100644 --- a/packages/api/src/mcp/tools/query/serviceHealth/source.ts +++ b/packages/api/src/mcp/tools/query/serviceHealth/source.ts @@ -88,6 +88,5 @@ export async function loadHealthSource( durationPrecision: doc.durationPrecision, metricSourceId: doc.metricSourceId, querySettings: doc.querySettings, - mvSource: doc.materializedViews?.length ? doc : undefined, }; }