diff --git a/.changeset/error-span-predicate.md b/.changeset/error-span-predicate.md new file mode 100644 index 0000000000..6856153a68 --- /dev/null +++ b/.changeset/error-span-predicate.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +Count spans recorded with the legacy `STATUS_CODE_ERROR` status as errors on the Services and LLM dashboards. The errored-span definition now lives in one place and matches both that value and the short `Error` form, so those dashboards no longer undercount errors from collectors that emit the long form. diff --git a/packages/app/src/__tests__/serviceDashboard.test.ts b/packages/app/src/__tests__/serviceDashboard.test.ts index a6c3866f61..ae862f6b63 100644 --- a/packages/app/src/__tests__/serviceDashboard.test.ts +++ b/packages/app/src/__tests__/serviceDashboard.test.ts @@ -460,7 +460,7 @@ describe('Service Dashboard', () => { expect(result.current.isLoading).toBe(false); expect(result.current.expressions?.isError).toBe( - "lower(StatusCode) = 'error'", + "lower(StatusCode) IN ('error', 'status_code_error')", ); expect(result.current.expressions?.isSpanKindServer).toContain( 'SpanKind IN', diff --git a/packages/app/src/llm/__tests__/expressions.test.ts b/packages/app/src/llm/__tests__/expressions.test.ts index 5852854ccc..6643b13965 100644 --- a/packages/app/src/llm/__tests__/expressions.test.ts +++ b/packages/app/src/llm/__tests__/expressions.test.ts @@ -32,7 +32,9 @@ describe('getLLMExpressions', () => { expect(expressions.isLLMSpan).toContain( "mapContains(SpanAttributes, 'gen_ai.operation.name')", ); - expect(expressions.isError).toBe("lower(StatusCode) = 'error'"); + expect(expressions.isError).toBe( + "lower(StatusCode) IN ('error', 'status_code_error')", + ); expect(expressions.durationInMillis).toBe('Duration/1e6'); // Provided cost takes precedence in the cost expression. expect(expressions.costUsd.startsWith('if((greatest(')).toBe(true); @@ -274,7 +276,9 @@ describe('getLLMExpressions', () => { [], ); expect(expressions.model).toContain("Attrs['gen_ai.response.model']"); - expect(expressions.isError).toBe("lower(Status) = 'error'"); + expect(expressions.isError).toBe( + "lower(Status) IN ('error', 'status_code_error')", + ); expect(expressions.durationInMillis).toBe('DurationNs/1e6'); }); }); diff --git a/packages/app/src/llm/lib/expressions.ts b/packages/app/src/llm/lib/expressions.ts index 72f901a18a..ae5743b164 100644 --- a/packages/app/src/llm/lib/expressions.ts +++ b/packages/app/src/llm/lib/expressions.ts @@ -1,5 +1,7 @@ import { TLogSource, TTraceSource } from '@hyperdx/common-utils/dist/types'; +import { errorStatusPredicateSql } from '@/source'; + import { generateCostSqlExpression } from './cost'; import { buildAnyKeyExistsSql, buildLLMSpanSqlPredicate } from './detect'; @@ -452,7 +454,7 @@ export function getLLMExpressions(source: TTraceSource, jsonColumns: string[]) { ...attributeExpressions, ...fieldExpressions, ...auxExpressions, - isError: `lower(${fieldExpressions.severityText}) = 'error'`, + isError: errorStatusPredicateSql(fieldExpressions.severityText), hasTokens: `${attributeExpressions.totalTokens} > 0`, }; } @@ -481,7 +483,13 @@ export function getLLMLogExpressions( ...attributeExpressions, service: source.serviceNameExpression || 'ServiceName', severityText, - /** Error log events (mirrors the serviceDashboard convention). */ + /** + * Error log events. Severity text has its own vocabulary, so this keeps a + * predicate of its own rather than reusing the span-status one: the extra + * status value that predicate matches cannot appear in a severity-text + * column, and the values severity text does use for errors are a + * different set. + */ isError: `lower(${severityText}) = 'error'`, /** * Log events that belong to LLM activity: explicit LLM markers or any diff --git a/packages/app/src/serviceDashboard.ts b/packages/app/src/serviceDashboard.ts index d48221e43c..51aa0d6fc4 100644 --- a/packages/app/src/serviceDashboard.ts +++ b/packages/app/src/serviceDashboard.ts @@ -5,6 +5,7 @@ import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils'; import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types'; import { useColumns, useJsonColumns } from './hooks/useMetadata'; +import { errorStatusPredicateSql } from './source'; const COALESCE_FIELDS_LIMIT = 100; @@ -225,7 +226,7 @@ export function getExpressions( const filterExpressions = { isEndpointNonEmpty: `NOT empty(${auxExpressions.endpoint})`, - isError: `lower(${fieldExpressions.severityText}) = 'error'`, + isError: errorStatusPredicateSql(fieldExpressions.severityText), isSpanKindServer: `${fieldExpressions.spanKind} IN ('Server', 'SPAN_KIND_SERVER')`, isDbSpan: `${fieldExpressions.dbStatement} <> ''`, }; diff --git a/packages/app/src/source.ts b/packages/app/src/source.ts index 2479582e63..6b956a27b2 100644 --- a/packages/app/src/source.ts +++ b/packages/app/src/source.ts @@ -529,6 +529,20 @@ export function getDurationSecondsExpression(source: TTraceSource) { return `(${source.durationExpression})/1e${source.durationPrecision ?? 9}`; } +/** + * The single definition of an errored span, as a SQL predicate over a span + * status-code expression. Matches both the short OTel status ('Error'/'ERROR') + * and the legacy collector form ('STATUS_CODE_ERROR'), which some collectors + * write instead. + * + * Span status only. Log severity text is a different vocabulary (its error + * values are 'fatal', 'crit' and friends, never 'status_code_error'), so log + * callers keep their own predicate rather than reusing this one. + */ +export function errorStatusPredicateSql(statusCodeExpression: string): string { + return `lower(${statusCodeExpression}) IN ('error', 'status_code_error')`; +} + // Aggregate functions whose output preserves the unit of the input value. // count and count_distinct produce dimensionless counts and should not // inherit the duration format.