Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/legacy-span-error-status.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
'@hyperdx/common-utils': patch
'@hyperdx/app': patch
---

fix: count legacy STATUS_CODE_ERROR spans as errors on the service map and trace
waterfall
6 changes: 2 additions & 4 deletions packages/app/src/components/DBTraceWaterfallChart.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { useForm } from 'react-hook-form';
import SqlString from 'sqlstring';
import TimestampNano from 'timestamp-nano';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import { isErrorStatusCode } from '@hyperdx/common-utils/dist/core/spanPredicates';
import {
ChartConfig,
ChartConfigWithDateRange,
Expand Down Expand Up @@ -154,7 +155,7 @@ function getRowSeverity(
severityText: row.SeverityText,
});
}
return row.StatusCode === 'Error' ? 'error' : undefined;
return isErrorStatusCode(row.StatusCode) ? 'error' : undefined;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔵 minor — Waterfall legacy-status fix has no component test

The waterfall's error count and row highlight now depend on isErrorStatusCode, but DBTraceWaterfallChart.test.tsx has no fixture with StatusCode: 'STATUS_CODE_ERROR'. A regression back to === 'Error' in getRowSeverity would still pass. Add a waterfall test with a legacy-status span and assert that the error count includes it and the row is marked as an error.

Advisory. Fix if it is a small defect in code this PR changes; otherwise reply in the thread. Do not widen the PR or touch files it did not already change. How to respond

}

// Per-service span palette: the full categorical palette minus the two hues we
Expand Down Expand Up @@ -1208,9 +1209,6 @@ export function DBTraceWaterfallChartContainer({
})
: [];

// Extract status logic
// TODO: Legacy schemas will have STATUS_CODE_ERROR
// See: https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/34799/files#diff-1ec84547ed93f2c8bfb21c371ca0b5304f01371e748d4b02bf397313a4b1dfa4L197
const severity = getRowSeverity(result);
const isError = severity === 'error';
const status = result.StatusCode || result.SeverityText;
Expand Down
16 changes: 7 additions & 9 deletions packages/app/src/components/ServiceMap/ServiceMapTooltip.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
import { formatDurationMs } from '@/utils';

import {
buildServiceMapErrorWhere,
ERROR_RATE_HIGH,
formatApproximateNumber,
formatRate,
Expand Down Expand Up @@ -77,15 +78,12 @@ export default function ServiceMapTooltip({
navigateToTraceSearch({
dateRange,
source,
where: SqlString.format(
"? = ? AND ? IN ('Server', 'Consumer') AND ? = 'Error'",
[
SqlString.raw(source.serviceNameExpression ?? 'ServiceName'),
serviceName,
SqlString.raw(source.spanKindExpression ?? 'SpanKind'),
SqlString.raw(source.statusCodeExpression ?? 'StatusCode'),
],
),
where: buildServiceMapErrorWhere({
serviceNameExpression: source.serviceNameExpression ?? 'ServiceName',
serviceName,
spanKindExpression: source.spanKindExpression ?? 'SpanKind',
statusCodeExpression: source.statusCodeExpression ?? 'StatusCode',
}),
});
}, [dateRange, source, serviceName]);

Expand Down
16 changes: 16 additions & 0 deletions packages/app/src/components/ServiceMap/__tests__/utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import router from 'next/router';
import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types';

import {
buildServiceMapErrorWhere,
deriveDisplayMetrics,
ERROR_RATE_HIGH,
formatApproximateNumber,
Expand All @@ -25,6 +26,21 @@ jest.mock('next/router', () => ({
},
}));

describe('buildServiceMapErrorWhere', () => {
it('includes legacy STATUS_CODE_ERROR spans in the error drill-down', () => {
expect(
buildServiceMapErrorWhere({
serviceNameExpression: 'ServiceName',
serviceName: 'api',
spanKindExpression: 'SpanKind',
statusCodeExpression: 'StatusCode',
}),
).toBe(
"ServiceName = 'api' AND SpanKind IN ('Server', 'Consumer') AND lower(StatusCode) IN ('error', 'status_code_error')",
);
});
});

describe('navigateToTraceSearch', () => {
const mockSource: TTraceSource = {
id: 'test-source-id',
Expand Down
24 changes: 24 additions & 0 deletions packages/app/src/components/ServiceMap/utils.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,32 @@
import router from 'next/router';
import SqlString from 'sqlstring';
import { errorStatusPredicateSql } from '@hyperdx/common-utils/dist/core/spanPredicates';
import { TTraceSource } from '@hyperdx/common-utils/dist/types';

import type { ServiceAggregation } from '@/hooks/useServiceMap';

export function buildServiceMapErrorWhere({
serviceNameExpression,
serviceName,
spanKindExpression,
statusCodeExpression,
}: {
serviceNameExpression: string;
serviceName: string;
spanKindExpression: string;
statusCodeExpression: string;
}): string {
return SqlString.format(
`? = ? AND ? IN ('Server', 'Consumer') AND ${errorStatusPredicateSql('?')}`,
[
SqlString.raw(serviceNameExpression),
serviceName,
SqlString.raw(spanKindExpression),
SqlString.raw(statusCodeExpression),
],
);
}

export function navigateToTraceSearch({
dateRange,
source,
Expand Down
9 changes: 9 additions & 0 deletions packages/app/src/hooks/__tests__/useServiceMap.test.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
import {
aggregateServiceMapData,
serviceMapErrorCountSql,
SpanAggregationRow,
} from '@/hooks/useServiceMap';

describe('serviceMapErrorCountSql', () => {
it('counts short and legacy error statuses, not only Error', () => {
expect(serviceMapErrorCountSql()).toBe(
"countIf(lower(ServerSpans.statusCode) IN ('error', 'status_code_error'))",
);
});
});

// Convenience builders for the GROUPING SETS row shapes.
function nodeRow(
serverServiceName: string,
Expand Down
7 changes: 6 additions & 1 deletion packages/app/src/hooks/useServiceMap.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import SqlString from 'sqlstring';
import { chSql } from '@hyperdx/common-utils/dist/clickhouse';
import { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig';
import { errorStatusPredicateSql } from '@hyperdx/common-utils/dist/core/spanPredicates';
import { TTraceSource } from '@hyperdx/common-utils/dist/types';
import { useQuery } from '@tanstack/react-query';

Expand All @@ -25,6 +26,10 @@ export type SpanAggregationRow = {
p99?: number;
};

export function serviceMapErrorCountSql(): string {
return `countIf(${errorStatusPredicateSql('ServerSpans.statusCode')})`;
}

async function getServiceMapQuery({
source,
dateRange,
Expand Down Expand Up @@ -175,7 +180,7 @@ async function getServiceMapQuery({
ClientSpans.serviceName AS clientServiceName,
GROUPING(ClientSpans.serviceName) AS isNodeLevel,
count(*) * ${{ Int64: effectiveSamplingLevel }} as requestCount,
countIf(ServerSpans.statusCode = 'Error') * ${{ Int64: effectiveSamplingLevel }} as errorCount${latencySelect}
${serviceMapErrorCountSql()} * ${{ Int64: effectiveSamplingLevel }} as errorCount${latencySelect}
FROM ServerSpans
LEFT JOIN ClientSpans
ON ServerSpans.traceId = ClientSpans.traceId
Expand Down
28 changes: 28 additions & 0 deletions packages/common-utils/src/__tests__/spanPredicates.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import {
errorStatusPredicateSql,
isErrorStatusCode,
serverSpanKindPredicateSql,
} from '@/core/spanPredicates';

Expand All @@ -11,6 +12,33 @@ describe('errorStatusPredicateSql', () => {
});
});

describe('isErrorStatusCode', () => {
it.each([
'Error',
'ERROR',
'error',
'STATUS_CODE_ERROR',
'status_code_error',
'Status_Code_Error',
])('treats %s as an error', status => {
expect(isErrorStatusCode(status)).toBe(true);
});

it.each([
'Ok',
'OK',
'STATUS_CODE_OK',
'Unset',
'STATUS_CODE_UNSET',
'erroring',
'',
undefined,
null,
])('does not treat %s as an error', status => {
expect(isErrorStatusCode(status)).toBe(false);
});
});

describe('serverSpanKindPredicateSql', () => {
it('matches both short and legacy server span kinds', () => {
expect(serverSpanKindPredicateSql('SpanKind')).toBe(
Expand Down
31 changes: 23 additions & 8 deletions packages/common-utils/src/core/spanPredicates.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,30 @@
/**
* Span status values that mean the span errored, lowercased.
* Short OTel status ('Error'/'ERROR') and the legacy collector form
* ('STATUS_CODE_ERROR'). Log severity text is a different vocabulary
* ('fatal', 'crit', and friends, never 'status_code_error'), so log callers
* keep their own predicate rather than reusing this list.
*/
const ERROR_STATUS_CODES = ['error', 'status_code_error'] as const;

/**
* The single definition of an errored span, as a SQL predicate over a span
* status-code expression. Matches both the short OTel status ('Error'/'ERROR')
* and the legacy collector form ('STATUS_CODE_ERROR'), which some collectors
* write instead.
*
* Span status only. Log severity text is a different vocabulary (its error
* values are 'fatal', 'crit' and friends, never 'status_code_error'), so log
* callers keep their own predicate rather than reusing this one.
* status-code expression.
*/
export function errorStatusPredicateSql(statusCodeExpression: string): string {
return `lower(${statusCodeExpression}) IN ('error', 'status_code_error')`;
const quoted = ERROR_STATUS_CODES.map(code => `'${code}'`).join(', ');
return `lower(${statusCodeExpression}) IN (${quoted})`;
}

/** Same match as {@link errorStatusPredicateSql}, for rows already fetched. */
export function isErrorStatusCode(
statusCode: string | null | undefined,
): boolean {
if (!statusCode) {
return false;
}
const normalized = statusCode.toLowerCase();
return ERROR_STATUS_CODES.some(code => code === normalized);
}

/** Server (request entry-point) spans, in both short and legacy enum forms. */
Expand Down
Loading