From 5800f86f6a9fb1e8d979ef72af9f0931a3a56818 Mon Sep 17 00:00:00 2001 From: Mohd Quamar Tyagi <104281681+Tyagiquamar@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:06:25 +0530 Subject: [PATCH] fix: count legacy STATUS_CODE_ERROR spans as errors Service map counts and the trace waterfall only matched the short status Error, so collectors that still write STATUS_CODE_ERROR showed no errors. --- .changeset/legacy-span-error-status.md | 7 +++++ .../src/components/DBTraceWaterfallChart.tsx | 6 ++-- .../ServiceMap/ServiceMapTooltip.tsx | 16 +++++----- .../ServiceMap/__tests__/utils.test.ts | 16 ++++++++++ .../app/src/components/ServiceMap/utils.ts | 24 ++++++++++++++ .../src/hooks/__tests__/useServiceMap.test.ts | 9 ++++++ packages/app/src/hooks/useServiceMap.tsx | 7 ++++- .../src/__tests__/spanPredicates.test.ts | 28 +++++++++++++++++ .../common-utils/src/core/spanPredicates.ts | 31 ++++++++++++++----- 9 files changed, 122 insertions(+), 22 deletions(-) create mode 100644 .changeset/legacy-span-error-status.md diff --git a/.changeset/legacy-span-error-status.md b/.changeset/legacy-span-error-status.md new file mode 100644 index 0000000000..783bf8f374 --- /dev/null +++ b/.changeset/legacy-span-error-status.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +--- + +fix: count legacy STATUS_CODE_ERROR spans as errors on the service map and trace +waterfall diff --git a/packages/app/src/components/DBTraceWaterfallChart.tsx b/packages/app/src/components/DBTraceWaterfallChart.tsx index eaadd947ee..7811018e2f 100644 --- a/packages/app/src/components/DBTraceWaterfallChart.tsx +++ b/packages/app/src/components/DBTraceWaterfallChart.tsx @@ -4,6 +4,7 @@ import { useForm } from 'react-hook-form'; import SqlString from 'sqlstring'; import TimestampNano from 'timestamp-nano'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { isErrorStatusCode } from '@hyperdx/common-utils/dist/core/spanPredicates'; import { ChartConfig, ChartConfigWithDateRange, @@ -154,7 +155,7 @@ function getRowSeverity( severityText: row.SeverityText, }); } - return row.StatusCode === 'Error' ? 'error' : undefined; + return isErrorStatusCode(row.StatusCode) ? 'error' : undefined; } // Per-service span palette: the full categorical palette minus the two hues we @@ -1208,9 +1209,6 @@ export function DBTraceWaterfallChartContainer({ }) : []; - // Extract status logic - // TODO: Legacy schemas will have STATUS_CODE_ERROR - // See: https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/34799/files#diff-1ec84547ed93f2c8bfb21c371ca0b5304f01371e748d4b02bf397313a4b1dfa4L197 const severity = getRowSeverity(result); const isError = severity === 'error'; const status = result.StatusCode || result.SeverityText; diff --git a/packages/app/src/components/ServiceMap/ServiceMapTooltip.tsx b/packages/app/src/components/ServiceMap/ServiceMapTooltip.tsx index aeb2d0c058..8dafdf4bd1 100644 --- a/packages/app/src/components/ServiceMap/ServiceMapTooltip.tsx +++ b/packages/app/src/components/ServiceMap/ServiceMapTooltip.tsx @@ -13,6 +13,7 @@ import { import { formatDurationMs } from '@/utils'; import { + buildServiceMapErrorWhere, ERROR_RATE_HIGH, formatApproximateNumber, formatRate, @@ -77,15 +78,12 @@ export default function ServiceMapTooltip({ navigateToTraceSearch({ dateRange, source, - where: SqlString.format( - "? = ? AND ? IN ('Server', 'Consumer') AND ? = 'Error'", - [ - SqlString.raw(source.serviceNameExpression ?? 'ServiceName'), - serviceName, - SqlString.raw(source.spanKindExpression ?? 'SpanKind'), - SqlString.raw(source.statusCodeExpression ?? 'StatusCode'), - ], - ), + where: buildServiceMapErrorWhere({ + serviceNameExpression: source.serviceNameExpression ?? 'ServiceName', + serviceName, + spanKindExpression: source.spanKindExpression ?? 'SpanKind', + statusCodeExpression: source.statusCodeExpression ?? 'StatusCode', + }), }); }, [dateRange, source, serviceName]); diff --git a/packages/app/src/components/ServiceMap/__tests__/utils.test.ts b/packages/app/src/components/ServiceMap/__tests__/utils.test.ts index 134f59dde7..77faff2319 100644 --- a/packages/app/src/components/ServiceMap/__tests__/utils.test.ts +++ b/packages/app/src/components/ServiceMap/__tests__/utils.test.ts @@ -2,6 +2,7 @@ import router from 'next/router'; import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types'; import { + buildServiceMapErrorWhere, deriveDisplayMetrics, ERROR_RATE_HIGH, formatApproximateNumber, @@ -25,6 +26,21 @@ jest.mock('next/router', () => ({ }, })); +describe('buildServiceMapErrorWhere', () => { + it('includes legacy STATUS_CODE_ERROR spans in the error drill-down', () => { + expect( + buildServiceMapErrorWhere({ + serviceNameExpression: 'ServiceName', + serviceName: 'api', + spanKindExpression: 'SpanKind', + statusCodeExpression: 'StatusCode', + }), + ).toBe( + "ServiceName = 'api' AND SpanKind IN ('Server', 'Consumer') AND lower(StatusCode) IN ('error', 'status_code_error')", + ); + }); +}); + describe('navigateToTraceSearch', () => { const mockSource: TTraceSource = { id: 'test-source-id', diff --git a/packages/app/src/components/ServiceMap/utils.ts b/packages/app/src/components/ServiceMap/utils.ts index bf898574d3..dca2201441 100644 --- a/packages/app/src/components/ServiceMap/utils.ts +++ b/packages/app/src/components/ServiceMap/utils.ts @@ -1,8 +1,32 @@ import router from 'next/router'; +import SqlString from 'sqlstring'; +import { errorStatusPredicateSql } from '@hyperdx/common-utils/dist/core/spanPredicates'; import { TTraceSource } from '@hyperdx/common-utils/dist/types'; import type { ServiceAggregation } from '@/hooks/useServiceMap'; +export function buildServiceMapErrorWhere({ + serviceNameExpression, + serviceName, + spanKindExpression, + statusCodeExpression, +}: { + serviceNameExpression: string; + serviceName: string; + spanKindExpression: string; + statusCodeExpression: string; +}): string { + return SqlString.format( + `? = ? AND ? IN ('Server', 'Consumer') AND ${errorStatusPredicateSql('?')}`, + [ + SqlString.raw(serviceNameExpression), + serviceName, + SqlString.raw(spanKindExpression), + SqlString.raw(statusCodeExpression), + ], + ); +} + export function navigateToTraceSearch({ dateRange, source, diff --git a/packages/app/src/hooks/__tests__/useServiceMap.test.ts b/packages/app/src/hooks/__tests__/useServiceMap.test.ts index 9c66e1d038..d5e9167773 100644 --- a/packages/app/src/hooks/__tests__/useServiceMap.test.ts +++ b/packages/app/src/hooks/__tests__/useServiceMap.test.ts @@ -1,8 +1,17 @@ import { aggregateServiceMapData, + serviceMapErrorCountSql, SpanAggregationRow, } from '@/hooks/useServiceMap'; +describe('serviceMapErrorCountSql', () => { + it('counts short and legacy error statuses, not only Error', () => { + expect(serviceMapErrorCountSql()).toBe( + "countIf(lower(ServerSpans.statusCode) IN ('error', 'status_code_error'))", + ); + }); +}); + // Convenience builders for the GROUPING SETS row shapes. function nodeRow( serverServiceName: string, diff --git a/packages/app/src/hooks/useServiceMap.tsx b/packages/app/src/hooks/useServiceMap.tsx index a220e340f4..64002f9a3e 100644 --- a/packages/app/src/hooks/useServiceMap.tsx +++ b/packages/app/src/hooks/useServiceMap.tsx @@ -2,6 +2,7 @@ import SqlString from 'sqlstring'; import { chSql } from '@hyperdx/common-utils/dist/clickhouse'; import { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { errorStatusPredicateSql } from '@hyperdx/common-utils/dist/core/spanPredicates'; import { TTraceSource } from '@hyperdx/common-utils/dist/types'; import { useQuery } from '@tanstack/react-query'; @@ -25,6 +26,10 @@ export type SpanAggregationRow = { p99?: number; }; +export function serviceMapErrorCountSql(): string { + return `countIf(${errorStatusPredicateSql('ServerSpans.statusCode')})`; +} + async function getServiceMapQuery({ source, dateRange, @@ -175,7 +180,7 @@ async function getServiceMapQuery({ ClientSpans.serviceName AS clientServiceName, GROUPING(ClientSpans.serviceName) AS isNodeLevel, count(*) * ${{ Int64: effectiveSamplingLevel }} as requestCount, - countIf(ServerSpans.statusCode = 'Error') * ${{ Int64: effectiveSamplingLevel }} as errorCount${latencySelect} + ${serviceMapErrorCountSql()} * ${{ Int64: effectiveSamplingLevel }} as errorCount${latencySelect} FROM ServerSpans LEFT JOIN ClientSpans ON ServerSpans.traceId = ClientSpans.traceId diff --git a/packages/common-utils/src/__tests__/spanPredicates.test.ts b/packages/common-utils/src/__tests__/spanPredicates.test.ts index e5ee7c0b96..dccfe11ba0 100644 --- a/packages/common-utils/src/__tests__/spanPredicates.test.ts +++ b/packages/common-utils/src/__tests__/spanPredicates.test.ts @@ -1,5 +1,6 @@ import { errorStatusPredicateSql, + isErrorStatusCode, serverSpanKindPredicateSql, } from '@/core/spanPredicates'; @@ -11,6 +12,33 @@ describe('errorStatusPredicateSql', () => { }); }); +describe('isErrorStatusCode', () => { + it.each([ + 'Error', + 'ERROR', + 'error', + 'STATUS_CODE_ERROR', + 'status_code_error', + 'Status_Code_Error', + ])('treats %s as an error', status => { + expect(isErrorStatusCode(status)).toBe(true); + }); + + it.each([ + 'Ok', + 'OK', + 'STATUS_CODE_OK', + 'Unset', + 'STATUS_CODE_UNSET', + 'erroring', + '', + undefined, + null, + ])('does not treat %s as an error', status => { + expect(isErrorStatusCode(status)).toBe(false); + }); +}); + describe('serverSpanKindPredicateSql', () => { it('matches both short and legacy server span kinds', () => { expect(serverSpanKindPredicateSql('SpanKind')).toBe( diff --git a/packages/common-utils/src/core/spanPredicates.ts b/packages/common-utils/src/core/spanPredicates.ts index e2d37549e6..2889e0a218 100644 --- a/packages/common-utils/src/core/spanPredicates.ts +++ b/packages/common-utils/src/core/spanPredicates.ts @@ -1,15 +1,30 @@ +/** + * Span status values that mean the span errored, lowercased. + * Short OTel status ('Error'/'ERROR') and the legacy collector form + * ('STATUS_CODE_ERROR'). Log severity text is a different vocabulary + * ('fatal', 'crit', and friends, never 'status_code_error'), so log callers + * keep their own predicate rather than reusing this list. + */ +const ERROR_STATUS_CODES = ['error', 'status_code_error'] as const; + /** * The single definition of an errored span, as a SQL predicate over a span - * status-code expression. Matches both the short OTel status ('Error'/'ERROR') - * and the legacy collector form ('STATUS_CODE_ERROR'), which some collectors - * write instead. - * - * Span status only. Log severity text is a different vocabulary (its error - * values are 'fatal', 'crit' and friends, never 'status_code_error'), so log - * callers keep their own predicate rather than reusing this one. + * status-code expression. */ export function errorStatusPredicateSql(statusCodeExpression: string): string { - return `lower(${statusCodeExpression}) IN ('error', 'status_code_error')`; + const quoted = ERROR_STATUS_CODES.map(code => `'${code}'`).join(', '); + return `lower(${statusCodeExpression}) IN (${quoted})`; +} + +/** Same match as {@link errorStatusPredicateSql}, for rows already fetched. */ +export function isErrorStatusCode( + statusCode: string | null | undefined, +): boolean { + if (!statusCode) { + return false; + } + const normalized = statusCode.toLowerCase(); + return ERROR_STATUS_CODES.some(code => code === normalized); } /** Server (request entry-point) spans, in both short and legacy enum forms. */