From e5e089e1a8f17a446139791ae06dec37fd446285 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 28 Aug 2026 14:46:32 -0400 Subject: [PATCH 01/24] feat(node): distributed tracing spans Adds startSpan, withSpan and getActiveSpan to posthog-node behind a new traces client option, encoding spans as OTLP JSON without an OpenTelemetry dependency. --- .changeset/node-distributed-tracing.md | 9 + packages/core/package.json | 2 +- .../core/src/__tests__/posthog.flush.spec.ts | 3 +- .../src/__tests__/posthog.otlp-auth.spec.ts | 49 + packages/core/src/index.ts | 15 + packages/core/src/posthog-core-stateless.ts | 41 +- packages/core/src/traces/context.ts | 28 + packages/core/src/traces/ids.spec.ts | 115 ++ packages/core/src/traces/ids.ts | 75 + packages/core/src/traces/index.spec.ts | 1220 +++++++++++++++++ packages/core/src/traces/index.ts | 559 ++++++++ packages/core/src/traces/live-spans.spec.ts | 61 + packages/core/src/traces/otlp.spec.ts | 315 +++++ packages/core/src/traces/otlp.ts | 171 +++ packages/core/src/traces/sanitize.ts | 140 ++ packages/core/src/traces/span.spec.ts | 378 +++++ packages/core/src/traces/span.ts | 250 ++++ packages/core/src/traces/traceparent.spec.ts | 96 ++ packages/core/src/traces/traceparent.ts | 82 ++ packages/core/src/traces/types.ts | 101 ++ packages/node/package.json | 1 + .../posthog-node-references-latest.json | 111 ++ .../src/__tests__/traces-defaults.spec.ts | 112 ++ .../node/src/__tests__/traces-edge.spec.ts | 81 ++ packages/node/src/__tests__/traces.spec.ts | 478 +++++++ packages/node/src/client.ts | 161 ++- packages/node/src/entrypoints/index.node.ts | 6 + packages/node/src/exports.ts | 13 + .../extensions/context/span-context.node.ts | 22 + packages/node/src/traces-defaults.ts | 64 + packages/node/src/types.ts | 46 + ...osthog-react-native-references-latest.json | 7 + packages/types/src/index.ts | 17 + packages/types/src/traces.ts | 286 ++++ pnpm-lock.yaml | 3 + 35 files changed, 5102 insertions(+), 16 deletions(-) create mode 100644 .changeset/node-distributed-tracing.md create mode 100644 packages/core/src/__tests__/posthog.otlp-auth.spec.ts create mode 100644 packages/core/src/traces/context.ts create mode 100644 packages/core/src/traces/ids.spec.ts create mode 100644 packages/core/src/traces/ids.ts create mode 100644 packages/core/src/traces/index.spec.ts create mode 100644 packages/core/src/traces/index.ts create mode 100644 packages/core/src/traces/live-spans.spec.ts create mode 100644 packages/core/src/traces/otlp.spec.ts create mode 100644 packages/core/src/traces/otlp.ts create mode 100644 packages/core/src/traces/sanitize.ts create mode 100644 packages/core/src/traces/span.spec.ts create mode 100644 packages/core/src/traces/span.ts create mode 100644 packages/core/src/traces/traceparent.spec.ts create mode 100644 packages/core/src/traces/traceparent.ts create mode 100644 packages/core/src/traces/types.ts create mode 100644 packages/node/src/__tests__/traces-defaults.spec.ts create mode 100644 packages/node/src/__tests__/traces-edge.spec.ts create mode 100644 packages/node/src/__tests__/traces.spec.ts create mode 100644 packages/node/src/extensions/context/span-context.node.ts create mode 100644 packages/node/src/traces-defaults.ts create mode 100644 packages/types/src/traces.ts diff --git a/.changeset/node-distributed-tracing.md b/.changeset/node-distributed-tracing.md new file mode 100644 index 0000000000..52aef9f86d --- /dev/null +++ b/.changeset/node-distributed-tracing.md @@ -0,0 +1,9 @@ +--- +'posthog-node': minor +'@posthog/core': minor +'@posthog/types': minor +--- + +Add distributed tracing to `posthog-node` — experimental. `withSpan`, `startSpan` and `getActiveSpan` record spans against a new `traces` client option; spans started inside a request context carry the distinct ID and session ID, and `parent` / `span.traceparent()` continue a W3C trace across services. + +`IPostHog` gains these three members, so anything implementing that interface (hand-written test doubles, DI wrappers) needs them added, or can extend `PostHogBackendClient` instead. diff --git a/packages/core/package.json b/packages/core/package.json index 3564b3b351..c004ae8117 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -43,7 +43,7 @@ "lint:fix": "eslint src --fix", "build": "rslib build", "dev": "rslib build -w", - "test:unit": "jest", + "test:unit": "NODE_OPTIONS=--expose-gc jest", "package": "pnpm pack --out $PACKAGE_DEST/%s.tgz" }, "exports": { diff --git a/packages/core/src/__tests__/posthog.flush.spec.ts b/packages/core/src/__tests__/posthog.flush.spec.ts index 1372d6ff87..d3a2f1bded 100644 --- a/packages/core/src/__tests__/posthog.flush.spec.ts +++ b/packages/core/src/__tests__/posthog.flush.spec.ts @@ -762,11 +762,12 @@ describe('PostHog Core', () => { }) describe('OTLP batch senders', () => { - // Both share one `_sendOtlpBatch`; the table pins them to the same + // All three share one `_sendOtlpBatch`; the table pins them to the same // classification so a wrapper can't reintroduce a per-signal retry policy. const senders = { logs: (client: PostHogCoreTestClient) => client._sendLogsBatch({ resourceLogs: [] }), metrics: (client: PostHogCoreTestClient) => client._sendMetricsBatch({ resourceMetrics: [] }), + traces: (client: PostHogCoreTestClient) => client._sendTracesBatch({ resourceSpans: [] }), } const cases: [number, string][] = [ diff --git a/packages/core/src/__tests__/posthog.otlp-auth.spec.ts b/packages/core/src/__tests__/posthog.otlp-auth.spec.ts new file mode 100644 index 0000000000..78ea3ba8df --- /dev/null +++ b/packages/core/src/__tests__/posthog.otlp-auth.spec.ts @@ -0,0 +1,49 @@ +import { createTestClient, PostHogCoreTestClient, PostHogCoreTestClientMocks } from '@/testing' + +// One `_sendOtlpBatch` serves logs, metrics and traces, and the endpoint each one +// authenticates against is a single argument away from the wrong scheme. These pin +// all three so a flipped argument fails here rather than shipping the project key +// in a header the logs endpoint does not expect. +describe('OTLP batch auth', () => { + let posthog: PostHogCoreTestClient + let mocks: PostHogCoreTestClientMocks + + beforeEach(() => { + ;[posthog, mocks] = createTestClient('TEST_API_KEY', { + host: 'http://example.com', + preloadFeatureFlags: false, + disableCompression: true, + }) + mocks.fetch.mockResolvedValue({ + status: 200, + text: () => Promise.resolve('ok'), + json: () => Promise.resolve({ status: 'ok' }), + }) + }) + + const lastCall = (): [string, any] => mocks.fetch.mock.calls[mocks.fetch.mock.calls.length - 1] as [string, any] + + it('sends logs to the query-token endpoint with no Authorization header', async () => { + await posthog._sendLogsBatch({ resourceLogs: [] } as any) + + const [url, options] = lastCall() + expect(url).toBe('http://example.com/i/v1/logs?token=TEST_API_KEY') + expect(options.headers).not.toHaveProperty('Authorization') + }) + + it('sends metrics to the query-token endpoint with no Authorization header', async () => { + await posthog._sendMetricsBatch({ resourceMetrics: [] } as any) + + const [url, options] = lastCall() + expect(url).toBe('http://example.com/i/v1/metrics?token=TEST_API_KEY') + expect(options.headers).not.toHaveProperty('Authorization') + }) + + it('sends traces with bearer auth and no token in the query string', async () => { + await posthog._sendTracesBatch({ resourceSpans: [] } as any) + + const [url, options] = lastCall() + expect(url).toBe('http://example.com/i/v1/traces') + expect(options.headers.Authorization).toBe('Bearer TEST_API_KEY') + }) +}) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 8219875000..5e2af15b69 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -86,6 +86,21 @@ export type { Metrics, MetricsConfig, } from './metrics/types' +export { PostHogTraces } from './traces' +export { SyncSpanContextManager } from './traces/context' +export { NOOP_SPAN } from './traces/span' +export type { ResolvedTracesConfig, SpanContextManager, TraceSdkContext } from './traces/types' +// Same barrel convention as logs and metrics for the user-facing tracing types. +export type { + Span, + SpanAttributes, + SpanAttributeValue, + SpanKind, + SpanStatusCode, + SpanTimeInput, + StartSpanOptions, + TracesConfig, +} from './traces/types' export { uuidv7 } from './vendor/uuidv7' export * from './cookie' export * from './posthog-core' diff --git a/packages/core/src/posthog-core-stateless.ts b/packages/core/src/posthog-core-stateless.ts index 1001cd90f1..73565120ec 100644 --- a/packages/core/src/posthog-core-stateless.ts +++ b/packages/core/src/posthog-core-stateless.ts @@ -1,5 +1,6 @@ -import type { OtlpLogsPayload, OtlpMetricsPayload } from '@posthog/types' +import type { OtlpLogsPayload, OtlpMetricsPayload, OtlpTracesPayload } from '@posthog/types' import type { SendMetricsBatchOutcome } from './metrics/types' +import type { SendTracesBatchOutcome } from './traces/types' import { SimpleEventEmitter } from './eventemitter' import { getFeatureFlagValue, minimizeFlagCalledEventProperties, normalizeFlagsResponse } from './featureFlagUtils' import { gzipCompress, isGzipSupported } from './gzip' @@ -238,8 +239,8 @@ export type SendLogsBatchOutcome = /** * Each signal keeps its own exported outcome type because each belongs to a - * separate host contract. The wrappers return this value directly, so one - * drifting out of shape fails to compile. + * separate host contract. The wrappers return this value directly, so any of + * the three drifting out of shape fails to compile. */ type SendOtlpBatchOutcome = | { kind: 'ok' } @@ -1642,9 +1643,9 @@ export abstract class PostHogCoreStateless { } /** - * Shared implementation behind the OTLP senders, which differ only in path. - * Returns a tagged outcome instead of throwing so the queue owners don't - * have to know the core's error class hierarchy. + * Shared implementation behind the three OTLP senders, which differ only in + * path and auth style. Returns a tagged outcome instead of throwing so the + * queue owners don't have to know the core's error class hierarchy. * * Exhausted 408/429/5xx stay `retry-later`, unlike the events `_flush()` * which drops anything that isn't a network error: every OTLP queue is @@ -1653,17 +1654,22 @@ export abstract class PostHogCoreStateless { */ private async _sendOtlpBatch({ path, + auth, payload, }: { - path: 'logs' | 'metrics' - payload: OtlpLogsPayload | OtlpMetricsPayload + path: 'logs' | 'metrics' | 'traces' + auth: 'query-token' | 'bearer' + payload: OtlpLogsPayload | OtlpMetricsPayload | OtlpTracesPayload }): Promise { if (this.disabled) { return { kind: 'fatal', error: new Error('The client is disabled') } } const serialized = JSON.stringify(payload) - const url = `${this.host}/i/v1/${path}?token=${encodeURIComponent(this.apiKey)}` + const url = + auth === 'bearer' + ? `${this.host}/i/v1/${path}` + : `${this.host}/i/v1/${path}?token=${encodeURIComponent(this.apiKey)}` const gzippedPayload = !this.disableCompression ? await this.compressPayload(serialized) : null const fetchOptions: PostHogFetchOptions = { @@ -1671,6 +1677,7 @@ export abstract class PostHogCoreStateless { headers: { ...this.getCustomHeaders(), 'Content-Type': 'application/json', + ...(auth === 'bearer' && { Authorization: `Bearer ${this.apiKey}` }), ...(gzippedPayload !== null && { 'Content-Encoding': 'gzip' }), }, body: gzippedPayload || serialized, @@ -1703,11 +1710,23 @@ export abstract class PostHogCoreStateless { } async _sendLogsBatch(payload: OtlpLogsPayload): Promise { - return this._sendOtlpBatch({ path: 'logs', payload }) + return this._sendOtlpBatch({ path: 'logs', auth: 'query-token', payload }) } async _sendMetricsBatch(payload: OtlpMetricsPayload): Promise { - return this._sendOtlpBatch({ path: 'metrics', payload }) + return this._sendOtlpBatch({ path: 'metrics', auth: 'query-token', payload }) + } + + /** + * The `TracesHost._sendTracesBatch` implementation, so `PostHogTraces` can + * use any core-based SDK as its host. + * + * Authenticates with `Authorization: Bearer` rather than the `?token=` query + * parameter the logs and metrics senders use: it's the service's primary auth + * path, and server runtimes have no CORS preflight to avoid. + */ + async _sendTracesBatch(payload: OtlpTracesPayload): Promise { + return this._sendOtlpBatch({ path: 'traces', auth: 'bearer', payload }) } private fetchWithRetry( diff --git a/packages/core/src/traces/context.ts b/packages/core/src/traces/context.ts new file mode 100644 index 0000000000..8744a8c335 --- /dev/null +++ b/packages/core/src/traces/context.ts @@ -0,0 +1,28 @@ +import type { Span } from '@posthog/types' +import type { SpanContextManager } from './types' + +/** + * Synchronous active-span tracking: restores the previous active span when the + * callback returns, which for an async callback means when it returns its + * promise — so spans started after an `await` won't see it as active. + * + * The fallback for runtimes with no ambient async context; Node injects an + * `AsyncLocalStorage`-backed manager instead. `parent` is the escape hatch. + */ +export class SyncSpanContextManager implements SpanContextManager { + private _active: Span | undefined + + active(): Span | undefined { + return this._active + } + + with(span: Span, fn: () => T): T { + const previous = this._active + this._active = span + try { + return fn() + } finally { + this._active = previous + } + } +} diff --git a/packages/core/src/traces/ids.spec.ts b/packages/core/src/traces/ids.spec.ts new file mode 100644 index 0000000000..bbc3833b59 --- /dev/null +++ b/packages/core/src/traces/ids.spec.ts @@ -0,0 +1,115 @@ +import { getRandomBytes, isValidSpanId, isValidTraceId, newSpanId, newTraceId } from './ids' + +describe('trace and span ids', () => { + describe('newTraceId', () => { + it('is 32 lowercase hex characters', () => { + for (let i = 0; i < 50; i++) { + expect(newTraceId()).toMatch(/^[0-9a-f]{32}$/) + } + }) + + it('is never all zeros', () => { + for (let i = 0; i < 50; i++) { + expect(newTraceId()).not.toBe('0'.repeat(32)) + } + }) + + it('does not repeat', () => { + const ids = new Set(Array.from({ length: 200 }, newTraceId)) + expect(ids.size).toBe(200) + }) + }) + + describe('newSpanId', () => { + it('is 16 lowercase hex characters', () => { + for (let i = 0; i < 50; i++) { + expect(newSpanId()).toMatch(/^[0-9a-f]{16}$/) + } + }) + + it('does not repeat', () => { + const ids = new Set(Array.from({ length: 200 }, newSpanId)) + expect(ids.size).toBe(200) + }) + }) + + describe('getRandomBytes', () => { + it('returns the requested length', () => { + expect(getRandomBytes(8)).toHaveLength(8) + expect(getRandomBytes(16)).toHaveLength(16) + }) + + it('falls back to Math.random when crypto is unavailable', () => { + const original = Object.getOwnPropertyDescriptor(globalThis, 'crypto') + // React Native has no global crypto without a polyfill — the fallback path + // is what keeps span ids working there. + Object.defineProperty(globalThis, 'crypto', { value: undefined, configurable: true }) + try { + expect(newTraceId()).toMatch(/^[0-9a-f]{32}$/) + expect(newSpanId()).toMatch(/^[0-9a-f]{16}$/) + } finally { + if (original) { + Object.defineProperty(globalThis, 'crypto', original) + } + } + }) + + it('falls back when getRandomValues throws', () => { + const original = Object.getOwnPropertyDescriptor(globalThis, 'crypto') + Object.defineProperty(globalThis, 'crypto', { + value: { + getRandomValues: () => { + throw new Error('not allowed') + }, + }, + configurable: true, + }) + try { + expect(newTraceId()).toMatch(/^[0-9a-f]{32}$/) + } finally { + if (original) { + Object.defineProperty(globalThis, 'crypto', original) + } + } + }) + + it('never emits an all-zero id even when the random source is broken', () => { + const original = Object.getOwnPropertyDescriptor(globalThis, 'crypto') + Object.defineProperty(globalThis, 'crypto', { + value: { getRandomValues: (array: Uint8Array) => array.fill(0) }, + configurable: true, + }) + try { + // The server zeroes ids it can't use, so an all-zero id would be stored + // and silently orphaned rather than rejected. + expect(newTraceId()).not.toBe('0'.repeat(32)) + expect(newSpanId()).not.toBe('0'.repeat(16)) + } finally { + if (original) { + Object.defineProperty(globalThis, 'crypto', original) + } + } + }) + }) + + describe('validation', () => { + it.each([ + ['a valid trace id', '4bf92f3577b34da6a3ce929d0e0e4736', true], + ['an all-zero trace id', '0'.repeat(32), false], + ['a short trace id', 'abc', false], + ['uppercase hex', '4BF92F3577B34DA6A3CE929D0E0E4736', false], + ['a non-hex string', 'zzf92f3577b34da6a3ce929d0e0e4736', false], + ['a non-string', 12345, false], + ])('isValidTraceId rejects/accepts %s', (_name, value, expected) => { + expect(isValidTraceId(value)).toBe(expected) + }) + + it.each([ + ['a valid span id', '00f067aa0ba902b7', true], + ['an all-zero span id', '0'.repeat(16), false], + ['a trace-length id', '4bf92f3577b34da6a3ce929d0e0e4736', false], + ])('isValidSpanId rejects/accepts %s', (_name, value, expected) => { + expect(isValidSpanId(value)).toBe(expected) + }) + }) +}) diff --git a/packages/core/src/traces/ids.ts b/packages/core/src/traces/ids.ts new file mode 100644 index 0000000000..52a445bf5a --- /dev/null +++ b/packages/core/src/traces/ids.ts @@ -0,0 +1,75 @@ +// W3C Trace Context identifier generation. Trace ids are 16 bytes, span ids 8, +// both lowercase hex on the JSON wire. The ingestion service *zeroes* ids that +// aren't exactly the right length rather than rejecting them, silently orphaning +// the span — so length is load-bearing and every id is validated before it ships. + +const TRACE_ID_BYTES = 16 +const SPAN_ID_BYTES = 8 + +const TRACE_ID_HEX = TRACE_ID_BYTES * 2 +const SPAN_ID_HEX = SPAN_ID_BYTES * 2 + +const INVALID_TRACE_ID = '0'.repeat(TRACE_ID_HEX) +const INVALID_SPAN_ID = '0'.repeat(SPAN_ID_HEX) + +const HEX_RE = /^[0-9a-f]+$/ + +type CryptoLike = { getRandomValues?: (array: Uint8Array) => Uint8Array } + +/** + * Random bytes from the platform's CSPRNG, falling back to `Math.random`. + * + * The fallback exists for React Native, which has no global `crypto` without a + * polyfill. Trace ids need collision resistance, not unpredictability. + */ +export function getRandomBytes(byteLength: number): Uint8Array { + const bytes = new Uint8Array(byteLength) + const cryptoLike = (globalThis as { crypto?: CryptoLike }).crypto + if (cryptoLike && typeof cryptoLike.getRandomValues === 'function') { + try { + cryptoLike.getRandomValues(bytes) + return bytes + } catch { + // A locked-down `crypto` throws; fall through to the `Math.random` path. + } + } + for (let i = 0; i < byteLength; i++) { + bytes[i] = Math.floor(Math.random() * 256) + } + return bytes +} + +function bytesToHex(bytes: Uint8Array): string { + let hex = '' + for (let i = 0; i < bytes.length; i++) { + hex += bytes[i].toString(16).padStart(2, '0') + } + return hex +} + +function randomHexId(byteLength: number): string { + const hex = bytesToHex(getRandomBytes(byteLength)) + // An all-zero id is invalid per W3C and the server treats one as absent, so the + // span would be stored and silently orphaned. Only reachable from a broken source. + return /[^0]/.test(hex) ? hex : hex.slice(0, -1) + '1' +} + +export function newTraceId(): string { + return randomHexId(TRACE_ID_BYTES) +} + +export function newSpanId(): string { + return randomHexId(SPAN_ID_BYTES) +} + +function isValidHexId(value: unknown, length: number, invalid: string): value is string { + return typeof value === 'string' && value.length === length && value !== invalid && HEX_RE.test(value) +} + +export function isValidTraceId(value: unknown): value is string { + return isValidHexId(value, TRACE_ID_HEX, INVALID_TRACE_ID) +} + +export function isValidSpanId(value: unknown): value is string { + return isValidHexId(value, SPAN_ID_HEX, INVALID_SPAN_ID) +} diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts new file mode 100644 index 0000000000..cc5d2f21fe --- /dev/null +++ b/packages/core/src/traces/index.spec.ts @@ -0,0 +1,1220 @@ +import { PostHogTraces } from './index' +import { SyncSpanContextManager } from './context' +import { NOOP_SPAN } from './span' +import type { + OtlpSpan, + OtlpTracesPayload, + ResolvedTracesConfig, + SendTracesBatchOutcome, + TraceSdkContext, +} from './types' +import type { Logger } from '../types' +import { createMockLogger } from '@/testing' + +const TRACE_ID = '4bf92f3577b34da6a3ce929d0e0e4736' +const REMOTE_SPAN_ID = '00f067aa0ba902b7' + +const resolveForTest = (partial?: Partial): ResolvedTracesConfig => ({ + flushIntervalMs: 5000, + maxExportBatchSize: 512, + maxQueueSize: 2048, + ...partial, +}) + +const createMockInstance = (overrides: Record = {}): any => ({ + isDisabled: false, + optedOut: false, + getLibraryId: jest.fn(() => 'posthog-core-tests'), + getLibraryVersion: jest.fn(() => '0.0.0-test'), + _sendTracesBatch: jest.fn((): Promise => Promise.resolve({ kind: 'ok' })), + ...overrides, +}) + +describe('PostHogTraces', () => { + let mockInstance: any + let logger: Logger + let context: TraceSdkContext + + const createTraces = (config?: Partial, instance?: any): PostHogTraces => + new PostHogTraces( + instance ?? mockInstance, + resolveForTest(config), + logger, + () => context, + new SyncSpanContextManager() + ) + + const flushMicrotasks = async (): Promise => { + for (let i = 0; i < 5; i++) { + await Promise.resolve() + } + } + + const sentPayloads = (instance?: any): OtlpTracesPayload[] => + (instance ?? mockInstance)._sendTracesBatch.mock.calls.map((c: any[]) => c[0]) + + const sentSpans = (instance?: any): OtlpSpan[] => + sentPayloads(instance).flatMap((p) => p.resourceSpans[0].scopeSpans[0].spans) + + beforeEach(() => { + mockInstance = createMockInstance() + logger = createMockLogger() + context = {} + }) + + describe('startSpan', () => { + it('enqueues exactly one record per span', async () => { + const traces = createTraces() + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans()).toHaveLength(1) + expect(sentSpans()[0].name).toBe('checkout') + }) + + it('gives a root span a fresh trace id and no parent', async () => { + const traces = createTraces() + traces.startSpan('root').end() + await traces.flush() + + const [span] = sentSpans() + expect(span.traceId).toMatch(/^[0-9a-f]{32}$/) + expect(span.spanId).toMatch(/^[0-9a-f]{16}$/) + expect(span.parentSpanId).toBeUndefined() + }) + + it('does not activate the span it returns', () => { + const traces = createTraces() + const manual = traces.startSpan('manual') + expect(traces.getActiveSpan()).toBeNull() + manual.end() + }) + + it('parents a child to an explicit span handle', async () => { + const traces = createTraces() + const parent = traces.startSpan('parent') + const child = traces.startSpan('child', { parent }) + child.end() + parent.end() + await traces.flush() + + const [childSpan, parentSpan] = sentSpans() + expect(childSpan.traceId).toBe(parentSpan.traceId) + expect(childSpan.parentSpanId).toBe(parentSpan.spanId) + }) + + it('defaults kind to internal and honours an explicit kind', async () => { + const traces = createTraces() + traces.startSpan('a').end() + traces.startSpan('b', { kind: 'server' }).end() + await traces.flush() + + expect(sentSpans().map((s) => s.kind)).toEqual([1, 2]) + }) + + it('returns an inert handle when the SDK is disabled', async () => { + const traces = createTraces({}, createMockInstance({ isDisabled: true })) + const span = traces.startSpan('checkout') + span.end() + + expect(span).toBe(NOOP_SPAN) + expect(span.traceparent()).toBeNull() + }) + + it('returns an inert handle when the user has opted out', () => { + const traces = createTraces({}, createMockInstance({ optedOut: true })) + expect(traces.startSpan('checkout')).toBe(NOOP_SPAN) + }) + + it('makes a child of a no-op handle a no-op rather than an orphan', () => { + const traces = createTraces() + expect(traces.startSpan('child', { parent: NOOP_SPAN })).toBe(NOOP_SPAN) + }) + }) + + describe('startTime', () => { + it('backdates the span to a supplied start', async () => { + const traces = createTraces() + const start = Date.now() - 60_000 + traces.startSpan('backdated', { startTime: start }).end() + await traces.flush() + + expect(sentSpans()[0].startTimeUnixNano).toBe(`${start}000000`) + }) + + it('accepts a Date', async () => { + const traces = createTraces() + const start = new Date(Date.now() - 5_000) + traces.startSpan('backdated', { startTime: start }).end() + await traces.flush() + + expect(sentSpans()[0].startTimeUnixNano).toBe(`${start.getTime()}000000`) + }) + + it('falls back to now for an unusable start, keeping the record well formed', async () => { + const traces = createTraces() + traces.startSpan('bad', { startTime: Number.NaN }).end() + await traces.flush() + + const [span] = sentSpans() + expect(span.startTimeUnixNano).toMatch(/^\d+$/) + expect(Number(span.endTimeUnixNano)).toBeGreaterThanOrEqual(Number(span.startTimeUnixNano)) + }) + + it('warns when a start is old enough for the server to clamp it', async () => { + const traces = createTraces() + traces.startSpan('stale', { startTime: Date.now() - 48 * 60 * 60 * 1000 }).end() + await traces.flush() + + expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('24 hours')) + expect(sentSpans()).toHaveLength(1) + }) + }) + + describe('trace continuation', () => { + it('continues a remote trace from a traceparent string', async () => { + const traces = createTraces() + traces.startSpan('handler', { parent: `00-${TRACE_ID}-${REMOTE_SPAN_ID}-01` }).end() + await traces.flush() + + const [span] = sentSpans() + expect(span.traceId).toBe(TRACE_ID) + expect(span.parentSpanId).toBe(REMOTE_SPAN_ID) + }) + + it('continues a trace the caller sampled out', async () => { + const traces = createTraces() + traces.startSpan('handler', { parent: `00-${TRACE_ID}-${REMOTE_SPAN_ID}-00` }).end() + await traces.flush() + + expect(sentSpans()[0].traceId).toBe(TRACE_ID) + }) + + it('preserves tracestate opaquely and passes it to children', async () => { + const traces = createTraces() + const parent = traces.startSpan('handler', { + parent: `00-${TRACE_ID}-${REMOTE_SPAN_ID}-01`, + tracestate: 'vendor=abc', + }) + const child = traces.startSpan('inner', { parent }) + expect(parent.tracestate()).toBe('vendor=abc') + + child.end() + parent.end() + await traces.flush() + + expect(sentSpans().map((s) => s.traceState)).toEqual(['vendor=abc', 'vendor=abc']) + }) + + it('starts a fresh root on a malformed traceparent without throwing', async () => { + const traces = createTraces() + expect(() => traces.startSpan('handler', { parent: 'garbage' }).end()).not.toThrow() + await traces.flush() + + const [span] = sentSpans() + expect(span.traceId).not.toBe(TRACE_ID) + expect(span.parentSpanId).toBeUndefined() + }) + + it('starts a fresh root when the parent is not a span, as a duplicated header is', async () => { + const traces = createTraces() + traces.startSpan('handler', { parent: [`00-${TRACE_ID}-${REMOTE_SPAN_ID}-01`] as unknown as string }).end() + await traces.flush() + + const [span] = sentSpans() + expect(span.traceId).not.toBe(TRACE_ID) + expect(span.parentSpanId).toBeUndefined() + }) + + it('parents to the active span when the parent is not a span', async () => { + const traces = createTraces() + traces.withSpan('handler', () => { + traces.startSpan('child', { parent: [`00-${TRACE_ID}-${REMOTE_SPAN_ID}-01`] as unknown as string }).end() + }) + await traces.flush() + + const child = sentSpans().find((s) => s.name === 'child')! + const handler = sentSpans().find((s) => s.name === 'handler')! + expect(child.traceId).toBe(handler.traceId) + expect(child.parentSpanId).toBe(handler.spanId) + }) + }) + + describe('withSpan', () => { + it('ends the span and returns the callback result', async () => { + const traces = createTraces() + const result = traces.withSpan('job', () => 'value') + await traces.flush() + + expect(result).toBe('value') + expect(sentSpans()).toHaveLength(1) + expect(sentSpans()[0].status).toBeUndefined() + }) + + it('accepts options before the callback', async () => { + const traces = createTraces() + traces.withSpan('job', { kind: 'server', attributes: { plan: 'pro' } }, () => undefined) + await traces.flush() + + const [span] = sentSpans() + expect(span.kind).toBe(2) + expect(span.attributes).toContainEqual({ key: 'plan', value: { stringValue: 'pro' } }) + }) + + it('runs the callback when an attribute getter throws', async () => { + const traces = createTraces() + const attributes: any = { ok: 1 } + Object.defineProperty(attributes, 'boom', { + enumerable: true, + get() { + throw new Error('getter exploded') + }, + }) + + expect(traces.withSpan('job', { attributes }, () => 'value')).toBe('value') + await traces.flush() + + expect(sentSpans().map((s) => s.name)).toEqual(['job']) + }) + + it('makes the span active for the callback', () => { + const traces = createTraces() + traces.withSpan('outer', (span) => { + expect(traces.getActiveSpan()).toBe(span) + }) + expect(traces.getActiveSpan()).toBeNull() + }) + + it('nests spans started inside the callback', async () => { + const traces = createTraces() + traces.withSpan('outer', () => { + traces.withSpan('inner', () => undefined) + }) + await traces.flush() + + const inner = sentSpans().find((s) => s.name === 'inner')! + const outer = sentSpans().find((s) => s.name === 'outer')! + expect(inner.traceId).toBe(outer.traceId) + expect(inner.parentSpanId).toBe(outer.spanId) + }) + + it('lets an explicit parent override the active span', async () => { + const traces = createTraces() + const detached = traces.startSpan('detached') + traces.withSpan('outer', () => { + traces.withSpan('inner', { parent: detached }, () => undefined) + }) + detached.end() + await traces.flush() + + const inner = sentSpans().find((s) => s.name === 'inner')! + const detachedSpan = sentSpans().find((s) => s.name === 'detached')! + expect(inner.parentSpanId).toBe(detachedSpan.spanId) + }) + + it('records a thrown error and rethrows it unmodified', async () => { + const traces = createTraces() + const thrown = new TypeError('boom') + + expect(() => + traces.withSpan('job', () => { + throw thrown + }) + ).toThrow(thrown) + + await traces.flush() + const [span] = sentSpans() + expect(span.status).toEqual({ code: 2, message: 'boom' }) + expect(span.events?.[0]).toMatchObject({ + name: 'exception', + attributes: [ + { key: 'exception.type', value: { stringValue: 'TypeError' } }, + { key: 'exception.message', value: { stringValue: 'boom' } }, + ], + }) + }) + + it('ends an async callback at settle, not when it returns its promise', async () => { + const traces = createTraces({ maxExportBatchSize: 1 }) + let finishWork!: () => void + const work = new Promise((resolve) => { + finishWork = resolve + }) + + const pending = traces.withSpan('job', () => work) + + await Promise.resolve() + expect(mockInstance._sendTracesBatch).not.toHaveBeenCalled() + + finishWork() + await pending + await traces.flush() + + expect(sentSpans()).toHaveLength(1) + }) + + it('covers the awaited duration', async () => { + const traces = createTraces() + const pending = traces.withSpan('job', async () => { + await new Promise((resolve) => setTimeout(resolve, 80)) + }) + await jest.advanceTimersByTimeAsync(80) + await pending + await traces.flush() + + const [span] = sentSpans() + expect(Number(span.endTimeUnixNano)).toBeGreaterThan(Number(span.startTimeUnixNano)) + }) + + it('records a rejection and rethrows it unmodified', async () => { + const traces = createTraces() + const thrown = new Error('async boom') + + await expect(traces.withSpan('job', async () => Promise.reject(thrown))).rejects.toBe(thrown) + + await traces.flush() + expect(sentSpans()[0].status).toEqual({ code: 2, message: 'async boom' }) + }) + + it('treats an explicit ok status as final when the callback throws', async () => { + const traces = createTraces() + expect(() => + traces.withSpan('job', (span) => { + span.setStatus('ok') + throw new Error('boom') + }) + ).toThrow('boom') + + await traces.flush() + const [span] = sentSpans() + expect(span.status).toEqual({ code: 1 }) + // The exception event is still attached — only the status is protected. + expect(span.events?.[0].name).toBe('exception') + }) + + it('runs the callback once with an inert handle when tracing cannot run', async () => { + const traces = createTraces({}, createMockInstance({ optedOut: true })) + const fn = jest.fn(() => 'value') + + expect(traces.withSpan('job', fn)).toBe('value') + expect(fn).toHaveBeenCalledTimes(1) + expect(fn).toHaveBeenCalledWith(NOOP_SPAN) + expect(traces.getActiveSpan()).toBeNull() + await traces.flush() + expect(sentSpans()).toHaveLength(0) + }) + }) + + describe('auto-context', () => { + it('attaches the distinct id and session id as the product join keys', async () => { + context = { distinctId: 'user-123', sessionId: 'session-123' } + const traces = createTraces() + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans()[0].attributes).toEqual( + expect.arrayContaining([ + { key: 'posthogDistinctId', value: { stringValue: 'user-123' } }, + { key: 'sessionId', value: { stringValue: 'session-123' } }, + ]) + ) + }) + + it('omits keys with no value', async () => { + context = { distinctId: 'user-123' } + const traces = createTraces() + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans()[0].attributes?.map((a) => a.key)).toEqual(['posthogDistinctId']) + }) + + it('freezes the snapshot at span start', async () => { + context = { distinctId: 'a' } + const traces = createTraces() + const span = traces.startSpan('checkout') + context = { distinctId: 'b' } + span.end() + await traces.flush() + + expect(sentSpans()[0].attributes).toContainEqual({ + key: 'posthogDistinctId', + value: { stringValue: 'a' }, + }) + }) + + it('lets user attributes win on collision', async () => { + context = { distinctId: 'a' } + const traces = createTraces() + traces.startSpan('checkout', { attributes: { posthogDistinctId: 'override' } }).end() + await traces.flush() + + expect(sentSpans()[0].attributes).toContainEqual({ + key: 'posthogDistinctId', + value: { stringValue: 'override' }, + }) + }) + + it('maps the client-platform navigation keys', async () => { + context = { currentUrl: 'https://example.com/cart', screenName: 'Cart', appState: 'foreground' } + const traces = createTraces() + traces.startSpan('checkout').end() + await traces.flush() + + const attributes = sentSpans()[0].attributes ?? [] + expect(attributes).toEqual( + expect.arrayContaining([ + { key: 'url.full', value: { stringValue: 'https://example.com/cart' } }, + { key: 'screen.name', value: { stringValue: 'Cart' } }, + { key: 'app.state', value: { stringValue: 'foreground' } }, + ]) + ) + }) + + it('still records the span when reading context throws', async () => { + const traces = new PostHogTraces( + mockInstance, + resolveForTest(), + logger, + () => { + throw new Error('no context') + }, + new SyncSpanContextManager() + ) + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans()).toHaveLength(1) + }) + }) + + describe('gating', () => { + it('drops a span whose user opted out mid-trace, without throwing', async () => { + const instance = createMockInstance() + const traces = createTraces({}, instance) + const span = traces.startSpan('checkout') + + instance.optedOut = true + expect(() => span.end()).not.toThrow() + + await traces.flush() + expect(instance._sendTracesBatch).not.toHaveBeenCalled() + }) + }) + + describe('export', () => { + it('flushes when the queue reaches the batch size', async () => { + const traces = createTraces({ maxExportBatchSize: 2 }) + traces.startSpan('a').end() + expect(mockInstance._sendTracesBatch).not.toHaveBeenCalled() + + traces.startSpan('b').end() + await flushMicrotasks() + + expect(sentSpans()).toHaveLength(2) + }) + + it('does not re-post on every span end while a flush is failing', async () => { + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('down') }) + const traces = createTraces({ maxExportBatchSize: 2, maxQueueSize: 10 }) + for (let i = 0; i < 10; i++) { + traces.startSpan(`span-${i}`).end() + await flushMicrotasks() + } + + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) + }) + + it('flushes on the interval timer', async () => { + const traces = createTraces({ flushIntervalMs: 1000 }) + traces.startSpan('a').end() + expect(mockInstance._sendTracesBatch).not.toHaveBeenCalled() + + await jest.advanceTimersByTimeAsync(1000) + expect(sentSpans()).toHaveLength(1) + }) + + it('sends one resource and one scope per batch', async () => { + const traces = createTraces({ serviceName: 'checkout-api' }) + traces.startSpan('a').end() + traces.startSpan('b').end() + await traces.flush() + + const [payload] = sentPayloads() + expect(payload.resourceSpans).toHaveLength(1) + expect(payload.resourceSpans[0].scopeSpans).toHaveLength(1) + expect(payload.resourceSpans[0].scopeSpans[0].spans).toHaveLength(2) + expect(payload.resourceSpans[0].resource.attributes).toContainEqual({ + key: 'service.name', + value: { stringValue: 'checkout-api' }, + }) + }) + + it('splits a backlog across batches', async () => { + const traces = createTraces({ maxExportBatchSize: 2 }) + for (let i = 0; i < 5; i++) { + traces.startSpan(`span-${i}`).end() + } + await traces.flush() + + expect(sentPayloads().length).toBeGreaterThanOrEqual(3) + expect(sentSpans()).toHaveLength(5) + }) + + it('joins an in-flight flush rather than double-sending', async () => { + const traces = createTraces() + traces.startSpan('a').end() + + const [first, second] = [traces.flush(), traces.flush()] + await Promise.all([first, second]) + + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) + expect(sentSpans()).toHaveLength(1) + }) + + it('drops the incoming span when the queue is full, keeping queued parents', async () => { + // Queued spans are completed parents whose children may already have been + // exported; evicting them would break assembled traces retroactively. + const traces = createTraces({ maxQueueSize: 2, maxExportBatchSize: 100 }) + traces.startSpan('first').end() + traces.startSpan('second').end() + traces.startSpan('third').end() + await traces.flush() + + expect(sentSpans().map((s) => s.name)).toEqual(['first', 'second']) + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('the queue is full')) + }) + }) + + describe('export failures', () => { + it('halves the batch and resends the same spans on 413', async () => { + const outcomes: SendTracesBatchOutcome[] = [{ kind: 'too-large' }, { kind: 'ok' }, { kind: 'ok' }] + const instance = createMockInstance({ + _sendTracesBatch: jest.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })), + }) + const traces = createTraces({ maxExportBatchSize: 4 }, instance) + for (let i = 0; i < 4; i++) { + traces.startSpan(`span-${i}`).end() + } + await traces.flush() + + const batchSizes = sentPayloads(instance).map((p) => p.resourceSpans[0].scopeSpans[0].spans.length) + expect(batchSizes).toEqual([4, 2, 2]) + expect(sentSpans(instance)).toHaveLength(8) + }) + + it('shrinks below the queue depth on 413 rather than resending the same body', async () => { + // The batch the server rejected is what has to get smaller. Halving the + // configured maximum leaves `size` unchanged whenever the queue is + // shallower than it — the ordinary timer-flush case. + const instance = createMockInstance({ + _sendTracesBatch: jest.fn().mockResolvedValueOnce({ kind: 'too-large' }).mockResolvedValue({ kind: 'ok' }), + }) + const traces = createTraces({ maxExportBatchSize: 512 }, instance) + for (let i = 0; i < 3; i++) { + traces.startSpan(`span-${i}`).end() + } + await traces.flush() + + const batchSizes = sentPayloads(instance).map((p) => p.resourceSpans[0].scopeSpans[0].spans.length) + expect(batchSizes).toEqual([3, 1, 2]) + }) + + it('ramps the batch size back up after a 413 shrink', async () => { + // A one-off oversized payload shouldn't permanently halve throughput. + const instance = createMockInstance({ + _sendTracesBatch: jest.fn().mockResolvedValueOnce({ kind: 'too-large' }).mockResolvedValue({ kind: 'ok' }), + }) + const traces = createTraces({ maxExportBatchSize: 4 }, instance) + for (let i = 0; i < 4; i++) { + traces.startSpan(`span-${i}`).end() + } + await traces.flush() + + // Shrunk to 2, then +1 per healthy send across the two batches that drained it. + instance._sendTracesBatch.mockClear() + for (let i = 0; i < 4; i++) { + traces.startSpan(`later-${i}`).end() + } + await traces.flush() + + expect(sentPayloads(instance)[0].resourceSpans[0].scopeSpans[0].spans.length).toBeGreaterThan(2) + }) + + it('drops a single span the server rejects as too large', async () => { + const instance = createMockInstance({ + _sendTracesBatch: jest.fn(() => Promise.resolve({ kind: 'too-large' as const })), + }) + const traces = createTraces({ maxExportBatchSize: 1 }, instance) + traces.startSpan('huge').end() + await traces.flush() + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('too large')) + + // The span must actually leave the queue, or it is re-POSTed on every + // flush for the life of the process. + instance._sendTracesBatch.mockResolvedValue({ kind: 'ok' }) + traces.startSpan('later').end() + await traces.flush() + expect(sentSpans(instance).map((s) => s.name)).toEqual(['huge', 'later']) + }) + + it('names the reason for each kind of drop', async () => { + const instance = createMockInstance({ + _sendTracesBatch: jest.fn(() => Promise.resolve({ kind: 'fatal' as const, error: new Error('400') })), + }) + const traces = createTraces({ maxExportBatchSize: 1 }, instance) + traces.startSpan('poison').end() + await traces.flush() + + // A poison batch is not a full queue; telling an operator to reduce span + // volume would send them after the wrong problem. + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('rejected the batch')) + expect(logger.warn).not.toHaveBeenCalledWith(expect.stringContaining('queue is full')) + }) + + it('warns again about drops on a later flush', async () => { + // Warning once per process would leave the SDK silent about every + // subsequent drop for the life of the app. + const instance = createMockInstance({ + _sendTracesBatch: jest.fn(() => Promise.resolve({ kind: 'fatal' as const, error: new Error('400') })), + }) + const traces = createTraces({ maxExportBatchSize: 1 }, instance) + + traces.startSpan('a').end() + await traces.flush() + traces.startSpan('b').end() + await traces.flush() + + expect((logger.warn as jest.Mock).mock.calls.length).toBeGreaterThan(1) + }) + + it('keeps spans queued on a retriable failure', async () => { + const instance = createMockInstance({ + _sendTracesBatch: jest + .fn() + .mockResolvedValueOnce({ kind: 'retry-later', error: new Error('network') }) + .mockResolvedValue({ kind: 'ok' }), + }) + const traces = createTraces({}, instance) + traces.startSpan('a').end() + + await traces.flush() + expect(sentSpans(instance)).toHaveLength(1) + + await traces.flush() + expect(sentSpans(instance)).toHaveLength(2) + expect(sentSpans(instance)[1].name).toBe('a') + }) + + it('backs off exponentially while sends keep failing', async () => { + const instance = createMockInstance({ + _sendTracesBatch: jest.fn(() => Promise.resolve({ kind: 'retry-later' as const, error: new Error('network') })), + }) + const traces = createTraces({ flushIntervalMs: 5000 }, instance) + traces.startSpan('a').end() + + const sendsPerWindow: number[] = [] + for (let i = 0; i < 8; i++) { + await jest.advanceTimersByTimeAsync(5000) + sendsPerWindow.push(instance._sendTracesBatch.mock.calls.length) + } + + expect(sendsPerWindow).toEqual([1, 2, 2, 3, 3, 3, 3, 4]) + }) + + it('returns to the base interval after a send succeeds', async () => { + const instance = createMockInstance({ + _sendTracesBatch: jest + .fn() + .mockResolvedValueOnce({ kind: 'retry-later', error: new Error('network') }) + .mockResolvedValueOnce({ kind: 'retry-later', error: new Error('network') }) + .mockResolvedValue({ kind: 'ok' }), + }) + const traces = createTraces({ flushIntervalMs: 5000 }, instance) + traces.startSpan('a').end() + + await jest.advanceTimersByTimeAsync(5000) + await jest.advanceTimersByTimeAsync(5000) + await jest.advanceTimersByTimeAsync(10000) + expect(instance._sendTracesBatch).toHaveBeenCalledTimes(3) + + traces.startSpan('b').end() + await jest.advanceTimersByTimeAsync(5000) + expect(instance._sendTracesBatch).toHaveBeenCalledTimes(4) + }) + + it('drops a poison batch rather than wedging the queue', async () => { + const instance = createMockInstance({ + _sendTracesBatch: jest + .fn() + .mockResolvedValueOnce({ kind: 'fatal', error: new Error('400') }) + .mockResolvedValue({ kind: 'ok' }), + }) + const traces = createTraces({ maxExportBatchSize: 1 }, instance) + traces.startSpan('poison').end() + traces.startSpan('good').end() + await traces.flush() + + expect(sentSpans(instance).map((s) => s.name)).toEqual(['poison', 'good']) + + instance._sendTracesBatch.mockClear() + await traces.flush() + expect(instance._sendTracesBatch).not.toHaveBeenCalled() + }) + + it('does not surface a transport failure through span.end()', async () => { + // Ending a span is application control flow — it must never throw because + // the exporter is broken. + const instance = createMockInstance({ + _sendTracesBatch: jest.fn(() => Promise.reject(new Error('transport exploded'))), + }) + const traces = createTraces({ maxExportBatchSize: 1 }, instance) + + expect(() => traces.startSpan('a').end()).not.toThrow() + // Let the background flush settle; the rejection is swallowed there. + await jest.advanceTimersByTimeAsync(0) + }) + + it('surfaces a transport failure through an explicit flush()', async () => { + // flush() is the caller asking to be told, so it propagates — matching + // how the logs and metrics pipelines behave. + const instance = createMockInstance({ + _sendTracesBatch: jest.fn(() => Promise.reject(new Error('transport exploded'))), + }) + const traces = createTraces({ maxExportBatchSize: 100 }, instance) + traces.startSpan('a').end() + + await expect(traces.flush()).rejects.toThrow('transport exploded') + }) + }) + + describe('poison attributes', () => { + it('encodes a circular attribute instead of blowing the stack', async () => { + const traces = createTraces() + const cyclic: any = { name: 'order' } + cyclic.self = cyclic + + traces.startSpan('checkout', { attributes: { payload: cyclic } }).end() + await traces.flush() + + expect(sentSpans()).toHaveLength(1) + expect(JSON.stringify(sentPayloads()[0])).toContain('[Circular]') + }) + + it('treats a repeated sibling reference as duplication, not a cycle', async () => { + const traces = createTraces() + const shared = { id: 1 } + + traces.startSpan('checkout', { attributes: { a: shared, b: shared } as any }).end() + await traces.flush() + + expect(JSON.stringify(sentPayloads()[0])).not.toContain('[Circular]') + }) + + it('keeps a span whose attribute getter throws, marking only that key', async () => { + // The shared encoder contains a throwing getter at the key it belongs to, + // so the span keeps its name, timing and every other attribute instead of + // being dropped whole. + const traces = createTraces({ maxExportBatchSize: 1 }) + const exploding = { + ok: 1, + get boom() { + throw new Error('getter exploded') + }, + } + + traces.startSpan('poison', { attributes: { payload: exploding as any } }).end() + traces.startSpan('healthy').end() + await traces.flush() + + expect(sentSpans().map((s) => s.name)).toEqual(['poison', 'healthy']) + expect(JSON.stringify(sentPayloads()[0])).toContain('[Unserializable]') + expect(JSON.stringify(sentPayloads()[0])).toContain('"intValue":"1"') + }) + + it('keeps a span whose top-level attribute getter throws, marking only that key', async () => { + const traces = createTraces({ maxExportBatchSize: 1 }) + const attributes: any = { ok: 1 } + Object.defineProperty(attributes, 'boom', { + enumerable: true, + get() { + throw new Error('getter exploded') + }, + }) + + expect(() => traces.startSpan('poison', { attributes }).end()).not.toThrow() + await traces.flush() + + expect(sentSpans().map((s) => s.name)).toEqual(['poison']) + expect(JSON.stringify(sentPayloads()[0])).toContain('[Unserializable]') + expect(JSON.stringify(sentPayloads()[0])).toContain('"intValue":"1"') + }) + }) + + describe('consent withdrawn after a span is queued', () => { + it('does not export spans queued before optOut()', async () => { + const instance = createMockInstance() + const traces = createTraces({}, instance) + context = { distinctId: 'alice', sessionId: 'session-1' } + traces.startSpan('checkout').end() + + instance.optedOut = true + await traces.flush() + + expect(instance._sendTracesBatch).not.toHaveBeenCalled() + }) + + it('does not export spans queued before the client is disabled', async () => { + const instance = createMockInstance() + const traces = createTraces({}, instance) + traces.startSpan('checkout').end() + + instance.isDisabled = true + await traces.flush() + + expect(instance._sendTracesBatch).not.toHaveBeenCalled() + }) + }) + + describe('drop accounting', () => { + it('still warns about queue-full drops while the endpoint is failing', async () => { + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('down') }) + const traces = createTraces({ maxExportBatchSize: 2, maxQueueSize: 2 }) + for (let i = 0; i < 10; i++) { + traces.startSpan(`span-${i}`).end() + await flushMicrotasks() + } + + const warnings = logger.warn.mock.calls.map((call: any[]) => call[0]) + expect(warnings.join(' ')).toContain('queue is full') + }) + + it('rate-limits the warning instead of one per dropped span', async () => { + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('down') }) + const traces = createTraces({ maxExportBatchSize: 2, maxQueueSize: 2, flushIntervalMs: 10_000 }) + for (let i = 0; i < 30; i++) { + traces.startSpan(`span-${i}`).end() + await flushMicrotasks() + } + + expect(logger.warn.mock.calls.length).toBeLessThanOrEqual(2) + }) + + it('surfaces queue-full drops even when every flush pass exits early', async () => { + // The retriable branch returns before the drain loop ends, so only the + // pass-level `finally` can emit this warning. + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('down') }) + const traces = createTraces({ maxExportBatchSize: 2, maxQueueSize: 2, flushIntervalMs: 1000 }) + traces.startSpan('queued-a').end() + traces.startSpan('queued-b').end() + await flushMicrotasks() + + // The first drop opens the rate-limit window itself. + traces.startSpan('dropped-first').end() + await flushMicrotasks() + logger.warn.mockClear() + + // The second lands inside that window, so `_recordDrop` stays quiet and + // only the flush pass's own `finally` can report it. + traces.startSpan('dropped-second').end() + await flushMicrotasks() + expect(logger.warn).not.toHaveBeenCalled() + + await jest.advanceTimersByTimeAsync(1000) + + expect(logger.warn.mock.calls.map((call: any[]) => call[0]).join(' ')).toContain('queue is full') + }) + + it('warns again once the flush interval has passed', async () => { + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('down') }) + const traces = createTraces({ maxExportBatchSize: 2, maxQueueSize: 2, flushIntervalMs: 1000 }) + for (let i = 0; i < 4; i++) { + traces.startSpan(`first-${i}`).end() + await flushMicrotasks() + } + const afterFirstWindow = logger.warn.mock.calls.length + + await jest.advanceTimersByTimeAsync(1000) + traces.startSpan('later').end() + await flushMicrotasks() + + expect(afterFirstWindow).toBe(1) + expect(logger.warn.mock.calls.length).toBeGreaterThan(afterFirstWindow) + }) + + it('warns once per flush with the total, not the first drop', async () => { + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'fatal', error: new Error('bad key') }) + const traces = createTraces({ maxExportBatchSize: 5, maxQueueSize: 5 }) + traces.startSpan('a').end() + traces.startSpan('b').end() + traces.startSpan('c').end() + await traces.flush() + + const warnings = logger.warn.mock.calls.map((call: any[]) => call[0]) + expect(warnings).toHaveLength(1) + expect(warnings[0]).toContain('3 span(s)') + expect(warnings[0]).toContain('rejected the batch') + }) + }) + + describe('hostile input', () => { + it('still exports when a resourceAttributes accessor throws', async () => { + const hostile: Record = {} + Object.defineProperty(hostile, 'host.name', { + enumerable: true, + get() { + throw new Error('accessor exploded') + }, + }) + // This runs before the flush pass's own error handling, so an unguarded + // read would rethrow on every flush and export nothing, ever. + const traces = createTraces({ resourceAttributes: hostile as never }) + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans().map((span) => span.name)).toEqual(['checkout']) + }) + + it('does not throw on a Date-like object with no Date slot', () => { + const traces = createTraces() + const fakeDate = Object.create(Date.prototype) + + expect(() => traces.startSpan('checkout', { startTime: fakeDate }).end(fakeDate)).not.toThrow() + }) + + it('does not throw when the parent has a throwing accessor', () => { + const traces = createTraces() + const hostile = { + get traceparent() { + throw new Error('accessor exploded') + }, + } + + expect(() => traces.startSpan('checkout', { parent: hostile as any })).not.toThrow() + }) + }) + + describe('background flush triggers', () => { + it('runs one background drain at a time while the queue stays saturated', async () => { + const traces = createTraces({ maxExportBatchSize: 8, maxQueueSize: 64 }) + let live = 0 + let peak = 0 + const drain = traces.flush.bind(traces) + jest.spyOn(traces, 'flush').mockImplementation(() => { + live++ + peak = Math.max(peak, live) + return drain().finally(() => { + live-- + }) + }) + + for (let i = 0; i < 500; i++) { + traces.startSpan(`span-${i}`).end() + if (i % 50 === 0) { + await flushMicrotasks() + } + } + + // A drain per span end would stack a loop per span, each retaining frames. + expect(peak).toBeLessThanOrEqual(2) + }) + }) + + describe('background flush re-arming', () => { + it('leaves a timer behind for a span that ends as a drain finishes', async () => { + const traces = createTraces({ maxExportBatchSize: 1, flushIntervalMs: 5000 }) + traces.startSpan('a').end() + // Four microtasks in: the drain has returned but its finally has not run, + // so the dedupe guard is still set and the queue was empty when it armed. + let chain: Promise = Promise.resolve() + for (let hop = 0; hop < 4; hop++) { + chain = chain.then(() => undefined) + } + await chain.then(() => { + traces.startSpan('b').end() + }) + await flushMicrotasks() + await jest.advanceTimersByTimeAsync(5000) + + expect(sentSpans().map((span) => span.name)).toEqual(['a', 'b']) + }) + }) + + describe('retry backoff', () => { + it('caps the retry delay while the endpoint keeps failing', async () => { + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('down') }) + const traces = createTraces({ flushIntervalMs: 1000, maxExportBatchSize: 1 }) + traces.startSpan('a').end() + + const delays: number[] = [] + // Seven retries: the eighth spends the batch's retry budget and drops it. + for (let attempt = 0; attempt < 7; attempt++) { + const before = mockInstance._sendTracesBatch.mock.calls.length + let waited = 0 + while (mockInstance._sendTracesBatch.mock.calls.length === before && waited < 120_000) { + await jest.advanceTimersByTimeAsync(1000) + waited += 1000 + } + delays.push(waited) + } + + expect(Math.max(...delays)).toBeLessThanOrEqual(30_000) + expect(delays.slice(-2)).toEqual([30_000, 30_000]) + }) + }) + + describe('retry budget', () => { + it('drops a batch the endpoint keeps refusing and moves to the next one', async () => { + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('down') }) + const traces = createTraces({ flushIntervalMs: 1000, maxExportBatchSize: 1 }) + traces.startSpan('stuck').end() + traces.startSpan('fresher').end() + + // Eight retriable failures spend the head batch's budget. + for (let attempt = 0; attempt < 12; attempt++) { + await jest.advanceTimersByTimeAsync(30_000) + } + + const attempted = mockInstance._sendTracesBatch.mock.calls.flatMap((call: any[]) => + call[0].resourceSpans[0].scopeSpans[0].spans.map((span: OtlpSpan) => span.name) + ) + // The stuck span is given up on, and the one behind it gets its turn. + expect(attempted).toContain('fresher') + expect(logger.warn.mock.calls.map((call: any[]) => call[0]).join(' ')).toContain('8 times in a row') + }) + + it('does not charge fresh spans to a budget they never spent', async () => { + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('down') }) + const traces = createTraces({ flushIntervalMs: 1000, maxExportBatchSize: 512, maxQueueSize: 2048 }) + traces.startSpan('old').end() + while (mockInstance._sendTracesBatch.mock.calls.length < 7) { + await jest.advanceTimersByTimeAsync(30_000) + } + for (let i = 0; i < 20; i++) { + traces.startSpan(`fresh-${i}`).end() + } + await jest.advanceTimersByTimeAsync(30_000) + + const eighth = mockInstance._sendTracesBatch.mock.calls[7][0] as OtlpTracesPayload + // The head cannot grow to sweep in spans that have never been retried. + expect(eighth.resourceSpans[0].scopeSpans[0].spans.map((span) => span.name)).toEqual(['old']) + }) + + it('gives the halved batch its own budget after a 413', async () => { + let attempt = 0 + const attempted: string[][] = [] + mockInstance._sendTracesBatch.mockImplementation(async (payload: OtlpTracesPayload) => { + attempted.push(payload.resourceSpans[0].scopeSpans[0].spans.map((span) => span.name)) + attempt++ + if (attempt <= 7) { + return { kind: 'retry-later', error: new Error('down') } + } + // The 413 replaces the head batch, so its failure count must not carry over. + return attempt === 8 ? { kind: 'too-large' } : { kind: 'retry-later', error: new Error('down') } + }) + const traces = createTraces({ flushIntervalMs: 1000, maxExportBatchSize: 4, maxQueueSize: 32 }) + for (let i = 0; i < 8; i++) { + traces.startSpan(`s${i}`).end() + } + for (let tick = 0; tick < 6; tick++) { + await jest.advanceTimersByTimeAsync(30_000) + } + + const halved = attempted.filter((names) => names.join() === 's0,s1') + expect(halved.length).toBeGreaterThan(1) + expect(logger.warn).not.toHaveBeenCalled() + }) + + it('gives a later batch its full budget after a success', async () => { + let attempt = 0 + // Four failures, a success, then four more: seven consecutive failures + // would spend the budget, but the success in between must reset it. + const succeedsOn = [5, 10] + mockInstance._sendTracesBatch.mockImplementation(async () => { + attempt++ + return succeedsOn.includes(attempt) ? { kind: 'ok' } : { kind: 'retry-later', error: new Error('blip') } + }) + const traces = createTraces({ flushIntervalMs: 1000, maxExportBatchSize: 1 }) + traces.startSpan('a').end() + traces.startSpan('b').end() + for (let tick = 0; tick < 12; tick++) { + await jest.advanceTimersByTimeAsync(30_000) + } + + expect([...new Set(sentSpans().map((span) => span.name))]).toEqual(['a', 'b']) + expect(logger.warn).not.toHaveBeenCalled() + }) + }) + + describe('drain progress', () => { + it('drains a span that arrives while a send is in flight', async () => { + // Queue length can't measure progress: one span out and one in leaves it + // unchanged, which would read as "no progress" and strand the new span — + // and shutdown() then discards it. + let onSend = (): void => {} + const instance = createMockInstance({ + _sendTracesBatch: jest.fn(() => { + onSend() + onSend = (): void => {} + return Promise.resolve({ kind: 'ok' as const }) + }), + }) + const traces = createTraces({ maxExportBatchSize: 10 }, instance) + onSend = (): void => traces.startSpan('arrived-mid-flight').end() + + traces.startSpan('first').end() + await traces.flush() + + expect(sentSpans(instance).map((s) => s.name)).toEqual(['first', 'arrived-mid-flight']) + }) + + it('terminates rather than spinning when a batch size of zero slips through', async () => { + // Core must not depend on every host clamping its config. + const traces = createTraces({ maxExportBatchSize: 0 }) + traces.startSpan('a').end() + + await traces.flush() + + expect(sentSpans()).toHaveLength(1) + }) + }) + + describe('reset', () => { + it('abandons an in-flight pass instead of splicing spans it never sent', async () => { + let release!: (outcome: SendTracesBatchOutcome) => void + const instance = createMockInstance({ + _sendTracesBatch: jest.fn( + () => + new Promise((resolve) => { + release = resolve + }) + ), + }) + const traces = createTraces({ maxExportBatchSize: 10 }, instance) + + traces.startSpan('sent-a').end() + traces.startSpan('sent-b').end() + const inFlight = traces.flush() + await Promise.resolve() + + // shutdown() lost the race and tore the pipeline down. + traces.reset() + traces.startSpan('after-reset').end() + + release({ kind: 'ok' }) + await inFlight + + expect((traces as any)._queue.map((r: any) => r.name)).toEqual(['after-reset']) + }) + + it('clears the queue', async () => { + const traces = createTraces() + traces.startSpan('a').end() + traces.reset() + await traces.flush() + + expect(mockInstance._sendTracesBatch).not.toHaveBeenCalled() + }) + }) +}) diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts new file mode 100644 index 0000000000..0c837a5d75 --- /dev/null +++ b/packages/core/src/traces/index.ts @@ -0,0 +1,559 @@ +import type { Span, SpanAttributes, StartSpanOptions } from '@posthog/types' +import type { Logger } from '../types' +import type { + OtlpSpan, + ResolvedTracesConfig, + SpanContextManager, + SpanRecord, + TraceSdkContext, + TracesHost, +} from './types' +import { NOOP_SPAN, PostHogSpan, describeError } from './span' +import { newSpanId, newTraceId } from './ids' +import { parseTraceparent, sanitizeTracestate } from './traceparent' +import { assignUserAttributes, resolveStartTime, sanitizeName } from './sanitize' +import { buildOtlpSpan, buildOtlpTracesPayload, buildTracesResourceAttributes } from './otlp' +import { isPromise, safeSetTimeout } from '../utils' + +// Retriable failures on the same head batch before it is dropped, so a stuck +// batch cannot pin the queue while fresher spans are refused at the cap. The +// budget counts attempts, not elapsed time: on the timer path the backoff +// spreads them over minutes, while a host that calls `flush()` per request +// spends them as fast as the requests arrive. +const MAX_RETRIES_PER_BATCH = 8 + +const MAX_FLUSH_BACKOFF_EXPONENT = 6 +const MAX_FLUSH_BACKOFF_MS = 30_000 + +type SpanCallback = (span: Span) => T + +/** `instanceof` and property access both throw on a hostile proxy; `startSpan` must not. */ +function isOwnSpan(value: unknown): value is PostHogSpan { + try { + return value instanceof PostHogSpan + } catch { + return false + } +} + +function looksLikeSpan(value: unknown): boolean { + try { + return typeof (value as Span).traceparent === 'function' + } catch { + return false + } +} + +interface ParentContext { + traceId: string + parentSpanId?: string + traceState?: string +} + +/** + * The traces pipeline: span creation, active-span parenting, and OTLP export. + * Separate from the analytics-events pipeline — own queue, endpoint and flush + * cycle — mirroring logs and metrics. + */ +export class PostHogTraces { + private _queue: SpanRecord[] = [] + private _flushTimer?: ReturnType + // Serializes flushes: a second caller joins the first instead of double-sending the head. + private _flushPromise: Promise | null = null + // A trigger no-ops while a background drain is already pending. + private _backgroundFlush?: Promise + private _maxExportBatchSize: number + // Reset when the warning is emitted, so each warning reports its own window. + private _droppedSinceWarning = 0 + private _lastDropWarningAt = 0 + private _dropReasons = new Set() + private _consecutiveFlushFailures = 0 + // Separate from the backoff counter: this one belongs to whatever batch is at + // the head, and resets whenever that batch is removed or shrunk. + private _headBatchFailures = 0 + // Read only while a budget is in flight, so the head cannot grow to sweep in + // fresh spans and drop them on a budget they never spent. + private _headBatchSize = 0 + // Bumped by reset(); a pass whose generation is stale abandons the queue. + private _generation = 0 + + constructor( + private readonly _instance: TracesHost, + private readonly _config: ResolvedTracesConfig, + private readonly _logger: Logger, + private readonly _getContext: () => TraceSdkContext, + private readonly _contextManager: SpanContextManager, + /** Told when a span joins the queue, so a serverless host can keep the invocation alive. */ + private readonly _onSpanQueued?: () => void + ) { + this._maxExportBatchSize = _config.maxExportBatchSize + } + + /** + * Starts a span without making it active. Always returns a handle — an inert + * one when tracing cannot run — so calling code never branches. + */ + startSpan(name: string, options?: StartSpanOptions): Span { + if (this._instance.isDisabled || this._instance.optedOut) { + return NOOP_SPAN + } + + const explicitParent = options?.parent + if (explicitParent && typeof explicitParent !== 'string' && !isOwnSpan(explicitParent)) { + if (looksLikeSpan(explicitParent)) { + // A child of a no-op is itself a no-op, never an orphan with invented ids. + this._logger.debug('Span parent is not a span from this SDK; returning an inert span') + return NOOP_SPAN + } + // Not a span at all — `req.headers.traceparent` is `string[]` when the header + // arrives twice. Ignored: falls back to the active span, or to a new trace. + this._logger.debug('Ignoring an unusable span parent') + } + + const parent = this._resolveParent(options) + + const now = Date.now() + const startTime = resolveStartTime(options?.startTime, now, this._logger) + + return new PostHogSpan( + { + traceId: parent?.traceId ?? newTraceId(), + spanId: newSpanId(), + parentSpanId: parent?.parentSpanId, + traceState: parent?.traceState, + name: sanitizeName(name, 'Span name', this._logger), + kind: options?.kind ?? 'internal', + // Auto-context first so user-supplied attributes win on collision. + attributes: assignUserAttributes(this._autoContextAttributes(), options?.attributes), + startTime, + backdated: startTime !== now, + }, + (record) => this._onSpanEnd(record), + this._logger + ) + } + + /** + * Runs a callback with a span active for its duration and guarantees the span + * ends — at return for a sync callback, at settle for an async one. + * + * A throw or rejection is recorded on the span and rethrown unmodified: the + * SDK never swallows application control flow. + */ + withSpan(name: string, fn: SpanCallback): T + withSpan(name: string, options: StartSpanOptions, fn: SpanCallback): T + withSpan(name: string, optionsOrFn: StartSpanOptions | SpanCallback, maybeFn?: SpanCallback): T { + const options = typeof optionsOrFn === 'function' ? undefined : optionsOrFn + const fn = (typeof optionsOrFn === 'function' ? optionsOrFn : maybeFn) as SpanCallback + + const span = this.startSpan(name, options) + + try { + // A no-op span is never activated, so `getActiveSpan()` inside the + // callback reads null — callbacks should use the handle they're given. + const result = span === NOOP_SPAN ? fn(span) : this._contextManager.with(span, () => fn(span)) + + if (isPromise(result)) { + return result.then( + (value: unknown) => { + span.end() + return value + }, + (error: unknown) => { + this._recordCallbackError(span, error) + span.end() + throw error + } + ) as T + } + + span.end() + return result + } catch (error) { + this._recordCallbackError(span, error) + span.end() + throw error + } + } + + /** The active span, or `null` outside any `withSpan` callback. */ + getActiveSpan(): Span | null { + return this._contextManager.active() ?? null + } + + /** + * Drains the span queue in repeated passes: joining a single in-flight pass + * would leave spans enqueued after its watermark behind. + * + * A pass reports spans removed — queue length can't stand in, since a send + * concurrent with an arrival leaves it unchanged. + */ + async flush(): Promise { + for (;;) { + if (!this._queue.length) { + return + } + + const inFlight = this._flushPromise + const removed = await (inFlight ?? this._startFlush()) + + // No progress means a retriable failure, an abandoned pass, or spans + // arriving as fast as we send them. Either way, stop rather than spin. + if (!removed) { + return + } + } + } + + private _startFlush(): Promise { + this._clearFlushTimer() + const promise = this._flushInner().finally(() => { + // Only clear the slot this call installed: a `reset()` mid-flight may + // already have installed a newer one. + if (this._flushPromise === promise) { + this._flushPromise = null + } + this._armFlushTimerIfQueued() + }) + this._flushPromise = promise + return promise + } + + /** Clears the queue and timer. Used on shutdown and between tests. */ + reset(): void { + this._clearFlushTimer() + this._queue = [] + this._flushPromise = null + // Abandons any in-flight pass, which would otherwise splice out spans it never sent. + this._generation++ + this._maxExportBatchSize = this._config.maxExportBatchSize + this._droppedSinceWarning = 0 + this._dropReasons.clear() + this._lastDropWarningAt = 0 + this._consecutiveFlushFailures = 0 + this._headBatchFailures = 0 + } + + /** + * Resolves a span's parent: an explicit `parent`, then the active span, then a + * fresh root. A no-op explicit parent is rejected earlier, in `startSpan`. + */ + private _resolveParent(options?: StartSpanOptions): ParentContext | undefined { + const explicit = options?.parent + + if (typeof explicit === 'string') { + const remote = parseTraceparent(explicit) + if (!remote) { + this._logger.debug('Ignoring malformed traceparent; starting a new trace') + return undefined + } + return { + traceId: remote.traceId, + parentSpanId: remote.spanId, + traceState: sanitizeTracestate(options?.tracestate), + } + } + + if (isOwnSpan(explicit)) { + // `tracestate` is ignored for handle parents — the child inherits the + // parent span's tracestate instead. + return explicit.childContext() + } + + const active = this._contextManager.active() + return isOwnSpan(active) ? active.childContext() : undefined + } + + /** + * PostHog context snapshotted at span start. These are the product's join + * keys — they're what makes a span reachable from a person or a session. + */ + private _autoContextAttributes(): SpanAttributes { + let context: TraceSdkContext + try { + context = this._getContext() + } catch (error) { + this._logger.debug('Failed to read tracing context; span will carry no PostHog attributes', error) + return {} + } + + const attributes: SpanAttributes = {} + if (context.distinctId) { + attributes.posthogDistinctId = context.distinctId + } + if (context.sessionId) { + attributes.sessionId = context.sessionId + } + if (context.currentUrl) { + attributes['url.full'] = context.currentUrl + } + if (context.screenName) { + attributes['screen.name'] = context.screenName + } + if (context.appState) { + attributes['app.state'] = context.appState + } + return attributes + } + + /** + * Records a callback failure on the span: an `exception` event always, plus + * status `error` unless the callback explicitly marked the span `ok`. + */ + private _recordCallbackError(span: Span, error: unknown): void { + if (!(span instanceof PostHogSpan)) { + return + } + const { type, message } = describeError(error) + span.addEvent('exception', { 'exception.type': type, 'exception.message': message }) + if (!span.statusIsExplicitlyOk) { + span.setStatus('error', message) + } + } + + private _onSpanEnd(record: SpanRecord): void { + // Re-checked at end: opting out mid-trace must stop the span exporting. + if (this._instance.isDisabled || this._instance.optedOut) { + return + } + + if (this._queue.length >= this._config.maxQueueSize) { + // Drop the incoming span, not queued ones: those are completed parents whose + // children may already have shipped. + this._recordDrop( + 1, + `the queue is full (${this._config.maxQueueSize}) — raise the flush frequency or reduce span volume` + ) + return + } + + this._queue.push(record) + try { + this._onSpanQueued?.() + } catch (error) { + this._logger.debug('Span queue notification failed', error) + } + + // Not while a flush is failing: the queue stays above the batch size for the + // whole outage, so every further span end would re-POST immediately and the + // retry backoff would never apply. + if (this._queue.length >= this._maxExportBatchSize && !this._consecutiveFlushFailures) { + this._flushInBackground() + } else { + this._armFlushTimerIfQueued() + } + } + + private _recordDrop(count: number, reason: string): void { + this._droppedSinceWarning += count + this._dropReasons.add(reason) + // Drops also happen with no flush in sight — a full queue during an outage — + // so the warning is paced by the clock rather than by the flush loop. + if (Date.now() - this._lastDropWarningAt >= this._config.flushIntervalMs) { + this._warnAboutDrops() + } + } + + /** At most one warning per flush pass, naming the total and every reason behind it. */ + private _warnAboutDrops(): void { + if (!this._droppedSinceWarning) { + return + } + this._lastDropWarningAt = Date.now() + this._logger.warn(`Dropping ${this._droppedSinceWarning} span(s): ${[...this._dropReasons].join('; ')}`) + this._droppedSinceWarning = 0 + this._dropReasons.clear() + } + + /** + * Encodes a batch, dropping any span whose attributes can't be encoded. + * An unguarded throw here would leave the queue unspliced, so every later + * flush would die on the same span. + */ + private _encodeBatch(batch: SpanRecord[]): OtlpSpan[] { + const encoded: OtlpSpan[] = [] + for (const record of batch) { + try { + encoded.push(buildOtlpSpan(record, this._logger)) + } catch (error) { + this._logger.debug('Failed to encode a span; dropping it', error) + this._recordDrop(1, 'its attributes could not be encoded') + } + } + return encoded + } + + /** Returns how many spans it removed from the queue, sent or dropped. */ + private async _flushInner(): Promise { + if (!this._queue.length) { + return 0 + } + + // Consent can flip between a span being queued and this pass running. Spans + // carry `posthogDistinctId` and `sessionId`, so anything still queued when + // the user opts out must be discarded rather than exported. + if (this._instance.isDisabled || this._instance.optedOut) { + const discarded = this._queue.length + this._queue = [] + return discarded + } + + const resourceAttributes = buildTracesResourceAttributes( + this._config, + this._instance.getLibraryId(), + this._instance.getLibraryVersion() + ) + const scopeName = this._instance.getLibraryId() + const scopeVersion = this._instance.getLibraryVersion() + + // Bounded by queue depth at flush start, so mid-drain arrivals ride the next flush. + let remaining = this._queue.length + let removed = 0 + const generation = this._generation + + try { + while (remaining > 0 && this._queue.length > 0) { + // Floor at one, or a non-positive batch size loops forever on an empty batch. + const cap = + this._headBatchFailures > 0 + ? Math.min(this._maxExportBatchSize, this._headBatchSize) + : this._maxExportBatchSize + const size = Math.max(1, Math.min(cap, remaining, this._queue.length)) + const batch = this._queue.slice(0, size) + const spans = this._encodeBatch(batch) + + if (!spans.length) { + // Nothing survived encoding; drop the batch rather than re-encoding it forever. + this._queue.splice(0, size) + remaining -= size + removed += size + this._headBatchFailures = 0 + continue + } + + const outcome = await this._instance._sendTracesBatch( + buildOtlpTracesPayload(spans, resourceAttributes, scopeName, scopeVersion, this._logger) + ) + + if (generation !== this._generation) { + // reset() ran mid-send: this pass no longer owns the queue. + return removed + } + + if (outcome.kind === 'ok') { + this._consecutiveFlushFailures = 0 + this._headBatchFailures = 0 + this._queue.splice(0, size) + remaining -= size + removed += size + // Ramp back toward the configured max after a 413 shrink. + if (this._maxExportBatchSize < this._config.maxExportBatchSize) { + this._maxExportBatchSize++ + } + continue + } + + if (outcome.kind === 'too-large') { + if (size === 1) { + // A single span the server won't accept at any size; drop it or it wedges the queue. + this._queue.splice(0, 1) + remaining -= 1 + removed += 1 + this._recordDrop(1, 'the ingestion endpoint rejected it as too large') + this._headBatchFailures = 0 + continue + } + // Halve the batch the server rejected, not the configured maximum: when the + // queue is shallower than the maximum, shrinking it resends an identical body. + this._maxExportBatchSize = Math.max(1, Math.floor(size / 2)) + // A different batch from here on, so its budget starts fresh. + this._headBatchFailures = 0 + this._logger.debug(`Batch too large; retrying the same spans in batches of ${this._maxExportBatchSize}`) + continue + } + + if (outcome.kind === 'retry-later') { + this._consecutiveFlushFailures++ + this._headBatchFailures++ + this._headBatchSize = size + if (this._headBatchFailures < MAX_RETRIES_PER_BATCH) { + // Keep the spans queued; the flush timer picks them up again. + this._logger.debug('Span export failed; retrying on the next flush', outcome.error) + return removed + } + // Out of retries. Drop this batch and start clean on the next one, so a + // permanently failing head cannot hold the queue against fresher spans. + this._queue.splice(0, size) + remaining -= size + removed += size + this._consecutiveFlushFailures = 0 + this._headBatchFailures = 0 + this._recordDrop(size, `the ingestion endpoint failed ${MAX_RETRIES_PER_BATCH} times in a row`) + continue + } + + // Non-retriable (poison batch or bad key); drop it so it can't wedge the queue. + this._logger.debug('Dropping a span batch the ingestion endpoint rejected', outcome.error) + this._queue.splice(0, size) + remaining -= size + removed += size + this._headBatchFailures = 0 + this._recordDrop(size, 'the ingestion endpoint rejected the batch') + } + + return removed + } finally { + // Every exit path, so a queue-full drop during an outage still surfaces — + // the retriable branch returns early. + this._warnAboutDrops() + } + } + + /** + * One background drain at a time. `flush()` is a multi-pass loop that keeps + * going while the queue stays above the batch size, so a trigger per span end + * would stack a loop per span on a busy service — each retaining its frames. + */ + private _flushInBackground(): void { + if (this._backgroundFlush) { + return + } + this._backgroundFlush = this.flush() + .catch((error) => { + // Background flushes have no caller to surface to; an explicit flush() + // still rejects. + this._logger.debug('Background span flush failed', error) + }) + .finally(() => { + this._backgroundFlush = undefined + // A trigger that arrived while this drain was finishing found the guard + // set and the queue empty, so neither path armed a timer. + this._armFlushTimerIfQueued() + }) + } + + private _armFlushTimerIfQueued(): void { + if (this._flushTimer || !this._queue.length) { + return + } + this._flushTimer = safeSetTimeout(() => { + this._flushTimer = undefined + this._flushInBackground() + }, this._nextFlushDelay()) + } + + // Retry delay: base interval, doubling, capped at 30s — never below an interval + // a host configured above the cap. + private _nextFlushDelay(): number { + const exponent = Math.min(Math.max(0, this._consecutiveFlushFailures - 1), MAX_FLUSH_BACKOFF_EXPONENT) + const delay = this._config.flushIntervalMs * 2 ** exponent + return Math.min(delay, Math.max(MAX_FLUSH_BACKOFF_MS, this._config.flushIntervalMs)) + } + + private _clearFlushTimer(): void { + if (this._flushTimer) { + clearTimeout(this._flushTimer) + this._flushTimer = undefined + } + } +} diff --git a/packages/core/src/traces/live-spans.spec.ts b/packages/core/src/traces/live-spans.spec.ts new file mode 100644 index 0000000000..64e6efa7dc --- /dev/null +++ b/packages/core/src/traces/live-spans.spec.ts @@ -0,0 +1,61 @@ +import { PostHogTraces } from './index' +import { SyncSpanContextManager } from './context' +import type { ResolvedTracesConfig } from './types' +import type { Logger } from '../types' + +// The pipeline holds no reference to a span until that span ends, so a handle the +// caller drops is collectable like any other object. +const gc = (globalThis as { gc?: () => void }).gc + +// `--expose-gc` is set by the `test:unit` script. A runner that invokes jest +// directly has no `gc`, and a probe that cannot force a collection proves nothing. +const itWithGc = gc ? it : process.env.CI ? it : it.skip + +describe('live spans', () => { + const config: ResolvedTracesConfig = { + serviceName: 'svc', + flushIntervalMs: 5000, + maxExportBatchSize: 512, + maxQueueSize: 2048, + } + + const createTraces = (): PostHogTraces => + new PostHogTraces( + { + isDisabled: false, + optedOut: false, + getLibraryId: () => 'posthog-core', + getLibraryVersion: () => '0.0.0', + _sendTracesBatch: async () => ({ kind: 'ok' }), + }, + config, + { debug: jest.fn(), warn: jest.fn() } as unknown as Logger, + () => ({}), + new SyncSpanContextManager() + ) + + itWithGc('does not retain spans that never end', async () => { + if (!gc) { + throw new Error('Run this suite with NODE_OPTIONS=--expose-gc; see packages/core test:unit') + } + jest.useRealTimers() + try { + const traces = createTraces() + // Started in their own frame so the handles are unreachable once it returns. + const refs = ((): WeakRef[] => + Array.from({ length: 1000 }, (_unused, i) => new WeakRef(traces.startSpan(`leaked-${i}`) as object)))() + + await new Promise((resolve) => setTimeout(resolve, 50)) + gc() + await new Promise((resolve) => setTimeout(resolve, 50)) + gc() + + // A threshold, not zero: collection timing is not guaranteed, but a registry + // holding every span would keep all 1000 alive. + const alive = refs.filter((ref) => ref.deref() !== undefined).length + expect(alive).toBeLessThan(100) + } finally { + jest.useFakeTimers() + } + }) +}) diff --git a/packages/core/src/traces/otlp.spec.ts b/packages/core/src/traces/otlp.spec.ts new file mode 100644 index 0000000000..a728cd12ac --- /dev/null +++ b/packages/core/src/traces/otlp.spec.ts @@ -0,0 +1,315 @@ +import { + buildOtlpSpan, + buildOtlpTracesPayload, + buildTracesResourceAttributes, + msToUnixNanoString, + spanKindToOtlp, +} from './otlp' +import type { OtlpSpan, ResolvedTracesConfig, SpanRecord } from './types' + +const record = (overrides: Partial = {}): SpanRecord => ({ + traceId: '4bf92f3577b34da6a3ce929d0e0e4736', + spanId: '00f067aa0ba902b7', + name: 'checkout', + kind: 'internal', + attributes: {}, + events: [], + startTime: 1_700_000_000_000, + endTime: 1_700_000_000_080, + ...overrides, +}) + +describe('OTLP span encoding', () => { + describe('msToUnixNanoString', () => { + it('encodes milliseconds as a nanosecond string', () => { + expect(msToUnixNanoString(1_700_000_000_000)).toBe('1700000000000000000') + }) + + it('keeps sub-millisecond precision', () => { + expect(msToUnixNanoString(1_700_000_000_000.5)).toBe('1700000000000500000') + }) + + it('stays exact beyond Number.MAX_SAFE_INTEGER', () => { + // The whole point of string concatenation over `ms * 1e6`, which would + // silently lose precision at this magnitude. + const encoded = msToUnixNanoString(1_700_000_000_123) + expect(encoded).toBe('1700000000123000000') + expect(Number(encoded)).toBeGreaterThan(Number.MAX_SAFE_INTEGER) + }) + + it.each([0.9999999, 1.9999999, 999.9999999])( + 'carries a rounded-up fraction into the next millisecond for %p', + (ms) => { + // Without the carry the padded fraction gains a seventh digit, producing + // a malformed timestamp that 400s the whole request. Unreachable from a + // real clock — float64 quantization at epoch-ms magnitude keeps the + // fraction well below the carry — but reachable via a caller-supplied + // `startTime`, which the validity check accepts anywhere in [0, MAX]. + const encoded = msToUnixNanoString(ms) + expect(encoded).toHaveLength(String(Math.round(ms)).length + 6) + expect(encoded).toMatch(/^\d+$/) + } + ) + }) + + describe('spanKindToOtlp', () => { + it.each([ + ['internal', 1], + ['server', 2], + ['client', 3], + ['producer', 4], + ['consumer', 5], + ] as const)('maps %s to %i', (kind, expected) => { + expect(spanKindToOtlp(kind)).toBe(expected) + }) + + it('defaults to internal', () => { + expect(spanKindToOtlp(undefined)).toBe(1) + }) + }) + + describe('buildOtlpSpan', () => { + it('builds the minimal shape', () => { + expect(buildOtlpSpan(record())).toEqual({ + traceId: '4bf92f3577b34da6a3ce929d0e0e4736', + spanId: '00f067aa0ba902b7', + name: 'checkout', + kind: 1, + startTimeUnixNano: '1700000000000000000', + endTimeUnixNano: '1700000000080000000', + flags: 1, + }) + }) + + it('omits status when it was never set', () => { + expect(buildOtlpSpan(record())).not.toHaveProperty('status') + }) + + it('encodes ok and error status codes', () => { + expect(buildOtlpSpan(record({ status: { code: 'ok' } })).status).toEqual({ code: 1 }) + expect(buildOtlpSpan(record({ status: { code: 'error', message: 'boom' } })).status).toEqual({ + code: 2, + message: 'boom', + }) + }) + + it('includes parent, tracestate, attributes and events when present', () => { + const span = buildOtlpSpan( + record({ + parentSpanId: 'b7ad6b7169203331', + traceState: 'vendor=abc', + attributes: { plan: 'pro' }, + events: [{ name: 'cache miss', timestamp: 1_700_000_000_040 }], + }) + ) + expect(span.parentSpanId).toBe('b7ad6b7169203331') + expect(span.traceState).toBe('vendor=abc') + expect(span.attributes).toEqual([{ key: 'plan', value: { stringValue: 'pro' } }]) + expect(span.events).toEqual([{ name: 'cache miss', timeUnixNano: '1700000000040000000' }]) + }) + + it('always sets the sampled trace flag', () => { + expect(buildOtlpSpan(record()).flags).toBe(1) + }) + }) + + describe('buildTracesResourceAttributes', () => { + const config = (partial: Partial = {}): ResolvedTracesConfig => ({ + flushIntervalMs: 5000, + maxExportBatchSize: 512, + maxQueueSize: 2048, + ...partial, + }) + + it('always emits service.name', () => { + // The server reads service_name only from this attribute and stores an + // empty string when it's missing, leaving spans unattributable. + expect(buildTracesResourceAttributes(config(), 'posthog-node', '1.0.0')['service.name']).toBe('unknown_service') + }) + + it('uses the configured service name', () => { + expect(buildTracesResourceAttributes(config({ serviceName: 'checkout' }), 'posthog-node', '1.0.0')).toMatchObject( + { + 'service.name': 'checkout', + } + ) + }) + + it('includes environment and version only when set', () => { + const attributes = buildTracesResourceAttributes( + config({ environment: 'production', serviceVersion: '2.1.0' }), + 'posthog-node', + '1.0.0' + ) + expect(attributes['deployment.environment']).toBe('production') + expect(attributes['service.version']).toBe('2.1.0') + expect(buildTracesResourceAttributes(config(), 'posthog-node', '1.0.0')).not.toHaveProperty( + 'deployment.environment' + ) + }) + + it('protects SDK identity keys from user resource attributes', () => { + const attributes = buildTracesResourceAttributes( + config({ resourceAttributes: { 'telemetry.sdk.name': 'custom', 'host.name': 'web-01' } }), + 'posthog-node', + '1.0.0' + ) + expect(attributes['telemetry.sdk.name']).toBe('posthog-node') + expect(attributes['host.name']).toBe('web-01') + }) + }) + + describe('buildOtlpTracesPayload', () => { + it('produces one resource, one scope, N spans', () => { + const spans = [buildOtlpSpan(record()), buildOtlpSpan(record({ name: 'other' }))] + const payload = buildOtlpTracesPayload(spans, { 'service.name': 'checkout' }, 'posthog-node', '1.0.0') + + expect(payload.resourceSpans).toHaveLength(1) + expect(payload.resourceSpans[0].scopeSpans).toHaveLength(1) + expect(payload.resourceSpans[0].scopeSpans[0].spans).toHaveLength(2) + expect(payload.resourceSpans[0].scopeSpans[0].scope).toEqual({ name: 'posthog-node', version: '1.0.0' }) + expect(payload.resourceSpans[0].resource.attributes).toEqual([ + { key: 'service.name', value: { stringValue: 'checkout' } }, + ]) + }) + }) + + describe('golden wire fixture', () => { + it('matches the shape the ingestion service accepts', () => { + // Pinned against the OTLP/JSON encoding the capture-logs service's own + // trace fixtures use: hex ids, string nanosecond timestamps, integer kind + // and status enums, and stringified int64 attribute values. + const payload = buildOtlpTracesPayload( + [ + buildOtlpSpan( + record({ + parentSpanId: 'b7ad6b7169203331', + name: 'GET /users/:id', + kind: 'server', + status: { code: 'error', message: 'boom' }, + attributes: { + posthogDistinctId: 'user-123', + sessionId: 'session-123', + 'http.status_code': 500, + 'http.duration_ratio': 0.25, + cached: false, + }, + events: [ + { + name: 'exception', + timestamp: 1_700_000_000_040, + attributes: { 'exception.type': 'TypeError', 'exception.message': 'boom' }, + }, + ], + }) + ), + ], + { 'service.name': 'checkout-api', 'telemetry.sdk.name': 'posthog-node' }, + 'posthog-node', + '1.0.0' + ) + + expect(payload).toEqual({ + resourceSpans: [ + { + resource: { + attributes: [ + { key: 'service.name', value: { stringValue: 'checkout-api' } }, + { key: 'telemetry.sdk.name', value: { stringValue: 'posthog-node' } }, + ], + }, + scopeSpans: [ + { + scope: { name: 'posthog-node', version: '1.0.0' }, + spans: [ + { + traceId: '4bf92f3577b34da6a3ce929d0e0e4736', + spanId: '00f067aa0ba902b7', + parentSpanId: 'b7ad6b7169203331', + name: 'GET /users/:id', + kind: 2, + startTimeUnixNano: '1700000000000000000', + endTimeUnixNano: '1700000000080000000', + flags: 1, + attributes: [ + { key: 'posthogDistinctId', value: { stringValue: 'user-123' } }, + { key: 'sessionId', value: { stringValue: 'session-123' } }, + { key: 'http.status_code', value: { intValue: '500' } }, + { key: 'http.duration_ratio', value: { doubleValue: 0.25 } }, + { key: 'cached', value: { boolValue: false } }, + ], + events: [ + { + name: 'exception', + timeUnixNano: '1700000000040000000', + attributes: [ + { key: 'exception.type', value: { stringValue: 'TypeError' } }, + { key: 'exception.message', value: { stringValue: 'boom' } }, + ], + }, + ], + status: { code: 2, message: 'boom' }, + }, + ], + }, + ], + }, + ], + }) + }) + }) +}) + +describe('wire string safety', () => { + const lone = 'value \u{1F680}'.slice(0, 7) + + const encode = (record: Partial = {}): OtlpSpan => + buildOtlpSpan({ + traceId: 'a'.repeat(32), + spanId: 'b'.repeat(16), + name: 'checkout', + kind: 'internal', + attributes: {}, + events: [], + startTime: 1, + endTime: 2, + ...record, + } as SpanRecord) + + it('replaces a lone surrogate in every free-text field', () => { + const span = encode({ + name: lone, + traceState: `vendor=${lone}`, + status: { code: 'error', message: lone }, + events: [{ name: lone, timestamp: 1 }], + }) + + expect(span.name).not.toContain('\ud83d') + expect(span.events?.[0].name).not.toContain('\ud83d') + expect(span.status?.message).not.toContain('\ud83d') + expect(span.traceState).not.toContain('\ud83d') + }) + + it('keeps the span when a status message cannot be stringified', () => { + const hostile = { + toString() { + throw new Error('toString exploded') + }, + } + const span = encode({ status: { code: 'error', message: hostile as unknown as string } }) + + expect(span.name).toBe('checkout') + expect(span.status?.message).toBe('[Unserializable]') + }) + + it('coerces a non-string status message', () => { + const span = encode({ status: { code: 'error', message: 500 as unknown as string } }) + + expect(span.status?.message).toBe('500') + }) + + it('falls back to internal for a prototype key passed as a kind', () => { + expect(spanKindToOtlp('__proto__' as never)).toBe(1) + expect(spanKindToOtlp('toString' as never)).toBe(1) + }) +}) diff --git a/packages/core/src/traces/otlp.ts b/packages/core/src/traces/otlp.ts new file mode 100644 index 0000000000..a7fabf8368 --- /dev/null +++ b/packages/core/src/traces/otlp.ts @@ -0,0 +1,171 @@ +import type { + OtlpSpan, + OtlpSpanEvent, + OtlpSpanKeyValue, + OtlpTracesPayload, + SpanAttributes, + SpanKind, + SpanStatusCode, +} from '@posthog/types' +import type { Logger } from '../types' +import type { ResolvedTracesConfig, SpanRecord } from './types' +import { toOtlpKeyValueList } from '../utils/otlp-any-value' +import { UNSERIALIZABLE_VALUE, sanitizeString } from '../utils/json-utils' +import { assignUserAttributes } from './sanitize' + +const SPAN_KIND_TO_OTLP: Record = { + internal: 1, + server: 2, + client: 3, + producer: 4, + consumer: 5, +} + +const SPAN_STATUS_TO_OTLP: Record = { + ok: 1, + error: 2, +} + +/** W3C trace flags: the sampled bit, always set because every captured span is recorded. */ +const TRACE_FLAGS_SAMPLED = 1 + +/** + * Every free-text string this encoder puts on the wire. A lone surrogate survives + * `JSON.stringify` as a `\uD800` escape that strict parsers refuse, and the + * ingestion service refuses the whole request rather than the one row — so the + * safe path has to be the default, not the exception. + */ +function wireString(value: unknown): string { + if (typeof value === 'string') { + return sanitizeString(value) + } + try { + return sanitizeString(String(value)) + } catch { + // A hostile `toString` costs its own field, not the whole span. + return UNSERIALIZABLE_VALUE + } +} + +export function spanKindToOtlp(kind: SpanKind | undefined): number { + // `hasOwnProperty`, not a plain lookup: `kind: '__proto__'` from an untyped + // caller otherwise resolves to `Object.prototype` and ships `"kind":{}`. + if (kind && Object.prototype.hasOwnProperty.call(SPAN_KIND_TO_OTLP, kind)) { + return SPAN_KIND_TO_OTLP[kind] + } + return SPAN_KIND_TO_OTLP.internal +} + +/** + * Converts a millisecond epoch to the unix-nanosecond string OTLP expects. + * Concatenation rather than multiplication: `Date.now() * 1e6` exceeds + * `Number.MAX_SAFE_INTEGER`. + */ +export function msToUnixNanoString(ms: number): string { + let whole = Math.floor(ms) + let fractionalNanos = Math.round((ms - whole) * 1e6) + // Rounding can carry into the next millisecond. Without this the padded + // fraction gains a seventh digit and the concatenated timestamp is malformed, + // which 400s the entire request — the exact failure client-side validity + // exists to prevent. + if (fractionalNanos >= 1e6) { + whole += 1 + fractionalNanos = 0 + } + return String(whole) + String(fractionalNanos).padStart(6, '0') +} + +function toOtlpEvent(event: SpanRecord['events'][number], logger?: Logger): OtlpSpanEvent { + const encoded: OtlpSpanEvent = { + name: wireString(event.name), + timeUnixNano: msToUnixNanoString(event.timestamp), + } + if (event.attributes) { + const attributes = toOtlpKeyValueList(event.attributes, logger) + if (attributes.length) { + encoded.attributes = attributes + } + } + return encoded +} + +export function buildOtlpSpan(record: SpanRecord, logger?: Logger): OtlpSpan { + const span: OtlpSpan = { + traceId: record.traceId, + spanId: record.spanId, + name: wireString(record.name), + kind: spanKindToOtlp(record.kind), + startTimeUnixNano: msToUnixNanoString(record.startTime), + endTimeUnixNano: msToUnixNanoString(record.endTime), + flags: TRACE_FLAGS_SAMPLED, + } + if (record.parentSpanId) { + span.parentSpanId = record.parentSpanId + } + if (record.traceState) { + span.traceState = wireString(record.traceState) + } + const attributes = toOtlpKeyValueList(record.attributes, logger) + if (attributes.length) { + span.attributes = attributes + } + if (record.events.length) { + span.events = record.events.map((event) => toOtlpEvent(event, logger)) + } + if (record.status) { + span.status = { + code: SPAN_STATUS_TO_OTLP[record.status.code], + ...(record.status.message && { message: wireString(record.status.message) }), + } + } + return span +} + +/** + * OTLP resource attributes for every batch. User `resourceAttributes` are spread + * first, then SDK-controlled identity keys on top so a stray key can't clobber + * them. `service.name` is always emitted: the server reads `service_name` only + * from that attribute, and spans are unattributable without it. + */ +export function buildTracesResourceAttributes( + config: ResolvedTracesConfig, + sdkName: string, + sdkVersion: string +): SpanAttributes { + return { + // Read through the shared guard: a throwing accessor here runs on every + // flush, before the pass's own error handling. + ...assignUserAttributes({}, config.resourceAttributes), + 'service.name': config.serviceName || 'unknown_service', + ...(config.environment && { 'deployment.environment': config.environment }), + ...(config.serviceVersion && { 'service.version': config.serviceVersion }), + 'telemetry.sdk.name': sdkName, + 'telemetry.sdk.version': sdkVersion, + } +} + +/** + * Wraps spans in the OTLP `resourceSpans` envelope: one resource, one scope, N + * spans per batch. The server flattens the scope to `{name}@{version}`. + */ +export function buildOtlpTracesPayload( + spans: OtlpSpan[], + resourceAttributes: SpanAttributes, + scopeName: string, + scopeVersion: string, + logger?: Logger +): OtlpTracesPayload { + return { + resourceSpans: [ + { + resource: { attributes: toOtlpKeyValueList(resourceAttributes, logger) }, + scopeSpans: [ + { + scope: { name: scopeName, version: scopeVersion }, + spans, + }, + ], + }, + ], + } +} diff --git a/packages/core/src/traces/sanitize.ts b/packages/core/src/traces/sanitize.ts new file mode 100644 index 0000000000..1ba5b97bbd --- /dev/null +++ b/packages/core/src/traces/sanitize.ts @@ -0,0 +1,140 @@ +// Client-side validity. +// +// The ingestion service 400s the *entire request* when one span fails row +// conversion — a timestamp outside signed 64-bit nanoseconds, say — and a 400 is +// non-retriable, so one bad span destroys every other span in the batch. + +import type { Logger } from '../types' +import type { SpanAttributes, SpanTimeInput } from '@posthog/types' +import { UNSERIALIZABLE_VALUE } from '../utils/json-utils' + +const FALLBACK_SPAN_NAME = 'unknown' + +// OTLP declares the timestamp fields `fixed64`, but the service parses them as +// signed 64-bit, so a negative (pre-epoch) value is as invalid as an overflow. +const MAX_TIMESTAMP_MS = 9223372036854 // floor(i64::MAX nanoseconds / 1e6) +const MIN_TIMESTAMP_MS = 0 + +// The server clamps timestamps more than 24h from receive time to now, keeping +// the original in `$originalTimestamp`. +const DEEP_BACKDATE_WARNING_MS = 24 * 60 * 60 * 1000 + +/** + * Span and event names must be non-empty. An empty or non-string name is + * replaced rather than dropped, so a mis-instrumented call site loses its name, + * not its span. `label` names what is being sanitized in the warning. + */ +export function sanitizeName(name: unknown, label: string, logger?: Logger): string { + if (typeof name === 'string' && name.trim()) { + return name + } + logger?.debug(`${label} must be a non-empty string; using "${FALLBACK_SPAN_NAME}"`) + return FALLBACK_SPAN_NAME +} + +/** + * Normalizes a caller-supplied time to a millisecond epoch. + * + * Returns `undefined` for anything unusable — the wrong type, `NaN`, or outside + * the representable range — leaving the caller to fall back to a derived time. + */ +export function toEpochMs(value: SpanTimeInput | undefined): number | undefined { + if (value === undefined || value === null) { + return undefined + } + let ms: unknown = value + if (value instanceof Date) { + try { + ms = value.getTime() + } catch { + // `Object.create(Date.prototype)` passes `instanceof` without a Date slot. + return undefined + } + } + if (typeof ms !== 'number' || !Number.isFinite(ms)) { + return undefined + } + if (ms < MIN_TIMESTAMP_MS || ms > MAX_TIMESTAMP_MS) { + return undefined + } + return ms +} + +/** + * Resolves a caller-supplied start time, warning when it is deep enough in the + * past that the server will clamp it. + */ +export function resolveStartTime(value: SpanTimeInput | undefined, now: number, logger?: Logger): number { + const supplied = toEpochMs(value) + if (supplied === undefined) { + if (value !== undefined) { + logger?.debug('Span startTime is out of range or not a valid time; using the current time') + } + return now + } + if (now - supplied > DEEP_BACKDATE_WARNING_MS) { + logger?.debug( + 'Span startTime is more than 24 hours in the past; the server will clamp it to receive time and keep the original in $originalTimestamp' + ) + } + return supplied +} + +/** + * Corrects an end time that precedes its start, producing a zero-duration span + * rather than a negative one the server would reject. + */ +export function clampEndTime(endTime: number, startTime: number): number { + return endTime < startTime ? startTime : endTime +} + +/** + * Keeps a caller-supplied end or event time inside the representable range, + * falling back to the span's own clock basis when it is unusable. `label` names + * what is being sanitized in the warning. + */ +export function resolveSuppliedTime( + value: SpanTimeInput | undefined, + derived: number, + label: string, + logger?: Logger +): number { + const supplied = toEpochMs(value) + if (supplied === undefined) { + if (value !== undefined) { + logger?.debug(`Span ${label} is out of range or not a valid time; using the derived time`) + } + return derived + } + return supplied +} + +/** + * Copies caller-supplied attributes onto `target`, own enumerable keys only. + * + * Read key by key rather than spread: a getter over a disposed resource or a + * revoked proxy throws on the read itself, before the encoder's guards see it. + */ +export function assignUserAttributes>(target: T, source: SpanAttributes | undefined): T { + if (!source) { + return target + } + let keys: string[] = [] + try { + keys = Object.keys(source) + } catch { + keys = [] + } + for (const key of keys) { + let value: unknown + try { + value = source[key] + } catch { + value = UNSERIALIZABLE_VALUE + } + // defineProperty, not assignment: `attributes['__proto__'] = v` hits the + // prototype setter and the attribute vanishes. + Object.defineProperty(target, key, { value, enumerable: true, writable: true, configurable: true }) + } + return target +} diff --git a/packages/core/src/traces/span.spec.ts b/packages/core/src/traces/span.spec.ts new file mode 100644 index 0000000000..c980c5f423 --- /dev/null +++ b/packages/core/src/traces/span.spec.ts @@ -0,0 +1,378 @@ +import { NOOP_SPAN, PostHogSpan, describeError } from './span' +import type { SpanInit } from './span' +import type { SpanRecord } from './types' +import type { Logger } from '../types' +import { createMockLogger } from '@/testing' + +const TRACE_ID = '4bf92f3577b34da6a3ce929d0e0e4736' +const SPAN_ID = '00f067aa0ba902b7' + +describe('PostHogSpan', () => { + let ended: SpanRecord[] + let logger: Logger + + const createSpan = (init: Partial = {}): PostHogSpan => + new PostHogSpan( + { + traceId: TRACE_ID, + spanId: SPAN_ID, + name: 'checkout', + kind: 'internal', + attributes: {}, + startTime: Date.now(), + backdated: false, + ...init, + }, + (record) => ended.push(record), + logger + ) + + beforeEach(() => { + ended = [] + logger = createMockLogger() + }) + + describe('monotonic clock', () => { + const withMonotonic = (readings: number[], run: () => void): void => { + const original = (globalThis as any).performance + let index = 0 + ;(globalThis as any).performance = { now: () => readings[Math.min(index++, readings.length - 1)] } + try { + run() + } finally { + ;(globalThis as any).performance = original + } + } + + it('measures duration against the monotonic reading, not the wall clock', () => { + const start = Date.now() + withMonotonic([1000, 1025], () => { + const span = createSpan({ startTime: start }) + jest.spyOn(Date, 'now').mockReturnValue(start - 60_000) + span.end() + }) + jest.spyOn(Date, 'now').mockRestore() + + expect(ended[0].endTime - ended[0].startTime).toBe(25) + }) + + it('never reports a negative duration when the monotonic source goes backwards', () => { + const start = Date.now() + withMonotonic([1000, 900], () => { + createSpan({ startTime: start }).end() + }) + + expect(ended[0].endTime).toBe(ended[0].startTime) + }) + + it('places an event inside the span window', () => { + const start = Date.now() + withMonotonic([1000, 1010, 1040], () => { + const span = createSpan({ startTime: start }) + span.addEvent('cache-miss') + span.end() + }) + + const [record] = ended + expect(record.events[0].timestamp).toBeGreaterThanOrEqual(record.startTime) + expect(record.events[0].timestamp).toBeLessThanOrEqual(record.endTime) + }) + + it('uses the wall clock for a backdated span', () => { + withMonotonic([1000, 9999], () => { + createSpan({ startTime: Date.now() - 5000, backdated: true }).end() + }) + + expect(ended[0].endTime - ended[0].startTime).toBeGreaterThanOrEqual(5000) + }) + }) + + it('produces exactly one record on end', () => { + createSpan().end() + expect(ended).toHaveLength(1) + expect(ended[0].name).toBe('checkout') + }) + + it('is idempotent on end', () => { + const span = createSpan() + span.end() + span.end() + expect(ended).toHaveLength(1) + }) + + it('ignores operations after end', () => { + const span = createSpan() + span.end() + span.setAttribute('k', 'v') + span.updateName('renamed') + span.addEvent('late') + + expect(ended[0].attributes).not.toHaveProperty('k') + expect(ended[0].name).toBe('checkout') + expect(ended[0].events).toHaveLength(0) + }) + + it('chains mutators', () => { + const span = createSpan() + span.setAttribute('a', 1).setAttributes({ b: 2 }).setStatus('ok').updateName('renamed') + span.end() + + expect(ended[0].attributes).toEqual({ a: 1, b: 2 }) + expect(ended[0].name).toBe('renamed') + expect(ended[0].status).toEqual({ code: 'ok' }) + }) + + it('replaces the name up until end', () => { + // A route template is often only knowable after routing resolves, and the + // product aggregates by (service, name) — so renaming has to be possible. + const span = createSpan({ name: 'HTTP request' }) + span.updateName('GET /users/:id') + span.end() + expect(ended[0].name).toBe('GET /users/:id') + }) + + it('replaces an empty name rather than dropping the span', () => { + const span = createSpan() + span.updateName(' ') + span.end() + expect(ended[0].name).toBe('unknown') + }) + + it('applies last-write-wins to status', () => { + const span = createSpan() + span.setStatus('ok') + span.setStatus('error', 'boom') + span.end() + expect(ended[0].status).toEqual({ code: 'error', message: 'boom' }) + }) + + it('omits status when never set', () => { + createSpan().end() + expect(ended[0].status).toBeUndefined() + }) + + it('ignores an unrecognized status along with its message', () => { + const rejected = createSpan() + expect((rejected as any).setStatus('OK', 'all good')).toBe(rejected) + rejected.end() + + const corrected = createSpan() + ;(corrected as any).setStatus('OK', 'all good') + corrected.setStatus('error', 'boom') + corrected.end() + + expect(ended[0].status).toBeUndefined() + expect(ended[1].status).toEqual({ code: 'error', message: 'boom' }) + expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('OK')) + }) + + describe('poison attributes', () => { + const withThrowingGetter = (): any => { + const attributes: any = { ok: 1 } + Object.defineProperty(attributes, 'boom', { + enumerable: true, + get() { + throw new Error('getter exploded') + }, + }) + return attributes + } + + it('marks only the throwing key on setAttributes', () => { + const span = createSpan() + expect(() => span.setAttributes(withThrowingGetter())).not.toThrow() + span.end() + + expect(ended[0].attributes).toEqual({ ok: 1, boom: '[Unserializable]' }) + }) + + it('marks only the throwing key on addEvent', () => { + const span = createSpan() + expect(() => span.addEvent('checkout.step', withThrowingGetter())).not.toThrow() + span.end() + + expect(ended[0].events[0].attributes).toEqual({ ok: 1, boom: '[Unserializable]' }) + }) + + it('keeps a __proto__ key on setAttribute', () => { + const span = createSpan() + span.setAttribute('__proto__', 'polluted') + span.end() + + expect(Object.keys(ended[0].attributes)).toContain('__proto__') + expect(Object.getPrototypeOf(ended[0].attributes)).toBe(Object.prototype) + }) + + it('copies only own enumerable keys on setAttributes', () => { + const span = createSpan() + span.setAttributes(Object.create({ inherited: 'proto' }, { own: { value: 'yes', enumerable: true } })) + span.end() + + expect(ended[0].attributes).toEqual({ own: 'yes' }) + }) + }) + + describe('recordException', () => { + it('sets error status and attaches an exception event without ending', () => { + const span = createSpan() + span.recordException(new TypeError('boom')) + + expect(ended).toHaveLength(0) + + span.end() + expect(ended[0].status).toEqual({ code: 'error', message: 'boom' }) + expect(ended[0].events).toEqual([ + expect.objectContaining({ + name: 'exception', + attributes: { 'exception.type': 'TypeError', 'exception.message': 'boom' }, + }), + ]) + }) + }) + + describe('timestamps', () => { + it('records an end at or after the start', () => { + const span = createSpan() + span.end() + expect(ended[0].endTime).toBeGreaterThanOrEqual(ended[0].startTime) + }) + + it('honours an explicit end time', () => { + const start = 1_700_000_000_000 + const span = createSpan({ startTime: start, backdated: true }) + span.end(start + 5000) + expect(ended[0].endTime).toBe(start + 5000) + }) + + it('accepts a Date as an end time', () => { + const start = 1_700_000_000_000 + const span = createSpan({ startTime: start, backdated: true }) + span.end(new Date(start + 1000)) + expect(ended[0].endTime).toBe(start + 1000) + }) + + it('corrects an end before the start to a zero duration', () => { + const start = 1_700_000_000_000 + const span = createSpan({ startTime: start, backdated: true }) + span.end(start - 5000) + expect(ended[0].endTime).toBe(start) + }) + + it('falls back to the derived end for an out-of-range end time', () => { + const start = 1_700_000_000_000 + const span = createSpan({ startTime: start, backdated: true }) + span.end(Number.MAX_SAFE_INTEGER) + expect(ended[0].endTime).toBeGreaterThanOrEqual(start) + expect(ended[0].endTime).toBeLessThan(9_223_372_036_854) + }) + + it('keeps event timestamps inside the span window', () => { + const span = createSpan() + span.addEvent('cache miss') + span.end() + + const [event] = ended[0].events + expect(event.timestamp).toBeGreaterThanOrEqual(ended[0].startTime) + expect(event.timestamp).toBeLessThanOrEqual(ended[0].endTime) + }) + + it('honours an explicit event timestamp', () => { + const start = 1_700_000_000_000 + const span = createSpan({ startTime: start, backdated: true }) + span.addEvent('cache miss', undefined, start + 40) + span.end(start + 80) + expect(ended[0].events[0].timestamp).toBe(start + 40) + }) + + it('snapshots event attributes so a reused object cannot mutate them', () => { + const span = createSpan() + const reused = { attempt: 1 } + span.addEvent('retry', reused) + reused.attempt = 2 + span.addEvent('retry', reused) + span.end() + + expect(ended[0].events.map((event) => event.attributes)).toEqual([{ attempt: 1 }, { attempt: 2 }]) + }) + }) + + describe('context propagation', () => { + it('produces a sampled traceparent', () => { + expect(createSpan().traceparent()).toBe(`00-${TRACE_ID}-${SPAN_ID}-01`) + }) + + it('returns null tracestate when it has none', () => { + expect(createSpan().tracestate()).toBeNull() + }) + + it('returns the tracestate it was created with', () => { + expect(createSpan({ traceState: 'vendor=abc' }).tracestate()).toBe('vendor=abc') + }) + + it('exposes a child context carrying its own span id as the parent', () => { + expect(createSpan({ traceState: 'vendor=abc' }).childContext()).toEqual({ + traceId: TRACE_ID, + parentSpanId: SPAN_ID, + traceState: 'vendor=abc', + }) + }) + }) +}) + +describe('NoopSpan', () => { + it('supports the full surface without throwing', () => { + expect(() => { + NOOP_SPAN.setAttribute('a', 1) + .setAttributes({ b: 2 }) + .addEvent('x') + .setStatus('error', 'boom') + .recordException(new Error('boom')) + .updateName('renamed') + .end() + }).not.toThrow() + }) + + it('never produces a well-formed traceparent', () => { + // An id that was never recorded must not propagate to another service. + expect(NOOP_SPAN.traceparent()).toBeNull() + expect(NOOP_SPAN.tracestate()).toBeNull() + }) +}) + +describe('describeError', () => { + it.each([ + ['an Error', new Error('boom'), { type: 'Error', message: 'boom' }], + ['a TypeError', new TypeError('bad type'), { type: 'TypeError', message: 'bad type' }], + ['a string', 'just a string', { type: 'string', message: 'just a string' }], + ['an object with a message', { name: 'CustomError', message: 'oops' }, { type: 'CustomError', message: 'oops' }], + ['an object without a name', { message: 'oops' }, { type: 'Object', message: 'oops' }], + ])('describes %s', (_name, error, expected) => { + expect(describeError(error)).toEqual(expected) + }) + + it('describes a thrown primitive', () => { + // Anything can be thrown in JS, so a non-Error must still produce a usable + // exception event rather than being dropped. + expect(describeError(42)).toEqual({ type: 'number', message: '42' }) + }) + + it('survives a value whose toString throws', () => { + const hostile = { + message: 123, + toString() { + throw new Error('boom from toString') + }, + } + expect(() => describeError(hostile)).not.toThrow() + expect(describeError(hostile)).toEqual({ type: 'object', message: '' }) + }) + + it('survives a value whose message getter throws', () => { + const hostile = { + get message(): string { + throw new Error('boom from getter') + }, + } + expect(() => describeError(hostile)).not.toThrow() + }) +}) diff --git a/packages/core/src/traces/span.ts b/packages/core/src/traces/span.ts new file mode 100644 index 0000000000..dc2c336d61 --- /dev/null +++ b/packages/core/src/traces/span.ts @@ -0,0 +1,250 @@ +import type { Span, SpanAttributes, SpanAttributeValue, SpanKind, SpanStatusCode, SpanTimeInput } from '@posthog/types' +import type { Logger } from '../types' +import type { SpanEventRecord, SpanRecord } from './types' +import { formatTraceparent } from './traceparent' +import { assignUserAttributes, clampEndTime, resolveSuppliedTime, sanitizeName } from './sanitize' +import { isError } from '../utils' + +/** + * A monotonic millisecond reading where the platform has one, so an NTP + * correction mid-span can't produce a negative duration. + */ +function monotonicNow(): number | undefined { + const perf = (globalThis as { performance?: { now?: () => number } }).performance + return typeof perf?.now === 'function' ? perf.now() : undefined +} + +export interface SpanInit { + traceId: string + spanId: string + parentSpanId?: string + traceState?: string + name: string + kind: SpanKind + attributes: SpanAttributes + /** ms epoch. */ + startTime: number + /** True when the caller supplied an explicit `startTime`. */ + backdated: boolean +} + +export class PostHogSpan implements Span { + private readonly _traceId: string + private readonly _spanId: string + private readonly _parentSpanId?: string + private readonly _traceState?: string + private readonly _startTime: number + // Absent on backdated spans and on platforms with no monotonic source. + private readonly _startMono?: number + + private _name: string + private _kind: SpanKind + private _attributes: SpanAttributes + private _events: SpanEventRecord[] = [] + private _status?: { code: SpanStatusCode; message?: string } + private _ended = false + + constructor( + init: SpanInit, + private readonly _onEnd: (record: SpanRecord) => void, + private readonly _logger?: Logger + ) { + this._traceId = init.traceId + this._spanId = init.spanId + this._parentSpanId = init.parentSpanId + this._traceState = init.traceState + this._name = init.name + this._kind = init.kind + this._attributes = init.attributes + this._startTime = init.startTime + this._startMono = init.backdated ? undefined : monotonicNow() + } + + /** + * "Now" on this span's clock basis: start plus monotonic elapsed where we + * have it, wall clock otherwise. + */ + private _now(): number { + if (this._startMono !== undefined) { + const mono = monotonicNow() + if (mono !== undefined) { + return this._startTime + Math.max(0, mono - this._startMono) + } + } + return Date.now() + } + + /** Guards every mutator: operations after `end()` no-op with a debug warning. */ + private _mutable(operation: string): boolean { + if (this._ended) { + this._logger?.debug(`Ignoring ${operation} on a span that has already ended`) + return false + } + return true + } + + setAttribute(key: string, value: SpanAttributeValue): this { + if (this._mutable('setAttribute')) { + Object.defineProperty(this._attributes, key, { value, enumerable: true, writable: true, configurable: true }) + } + return this + } + + setAttributes(attributes: SpanAttributes): this { + if (this._mutable('setAttributes')) { + assignUserAttributes(this._attributes, attributes) + } + return this + } + + addEvent(name: string, attributes?: SpanAttributes, timestamp?: SpanTimeInput): this { + if (this._mutable('addEvent')) { + this._events.push({ + name: sanitizeName(name, 'Span event name', this._logger), + timestamp: resolveSuppliedTime(timestamp, this._now(), 'event timestamp', this._logger), + // Copied so a caller reusing one object across events can't mutate a recorded one. + ...(attributes && { attributes: assignUserAttributes({}, attributes) }), + }) + } + return this + } + + setStatus(status: SpanStatusCode, message?: string): this { + if (this._mutable('setStatus')) { + if (status !== 'ok' && status !== 'error') { + this._logger?.debug(`Ignoring unknown span status "${String(status)}"; expected "ok" or "error"`) + return this + } + this._status = { code: status, ...(message && { message }) } + } + return this + } + + /** True when the caller explicitly marked the span `ok`; `withSpan` treats that as final. */ + get statusIsExplicitlyOk(): boolean { + return this._status?.code === 'ok' + } + + recordException(error: unknown): this { + if (!this._mutable('recordException')) { + return this + } + const { type, message } = describeError(error) + this.addEvent('exception', { + 'exception.type': type, + 'exception.message': message, + }) + // recordException is itself an explicit call, so it follows last-write-wins + // rather than deferring to an earlier `ok`. + return this.setStatus('error', message) + } + + updateName(name: string): this { + if (this._mutable('updateName')) { + this._name = sanitizeName(name, 'Span name', this._logger) + } + return this + } + + traceparent(): string | null { + return formatTraceparent(this._traceId, this._spanId) + } + + tracestate(): string | null { + return this._traceState ?? null + } + + /** Context a child span inherits when this handle is its parent. */ + childContext(): { traceId: string; parentSpanId: string; traceState?: string } { + return { traceId: this._traceId, parentSpanId: this._spanId, traceState: this._traceState } + } + + end(endTime?: SpanTimeInput): void { + if (this._ended) { + this._logger?.debug('Ignoring end() on a span that has already ended') + return + } + this._ended = true + + const derived = this._now() + const resolved = resolveSuppliedTime(endTime, derived, 'end time', this._logger) + + this._onEnd({ + traceId: this._traceId, + spanId: this._spanId, + ...(this._parentSpanId && { parentSpanId: this._parentSpanId }), + ...(this._traceState && { traceState: this._traceState }), + name: this._name, + kind: this._kind, + ...(this._status && { status: this._status }), + attributes: this._attributes, + events: this._events, + startTime: this._startTime, + endTime: clampEndTime(resolved, this._startTime), + }) + } +} + +/** + * An inert handle returned whenever tracing cannot run — traces unconfigured, + * SDK disabled, user opted out. Supports the full surface so caller code never + * branches, is never activated, and returns `null` from `traceparent()` so an id + * that was never recorded cannot propagate. + */ +export class NoopSpan implements Span { + setAttribute(): this { + return this + } + setAttributes(): this { + return this + } + addEvent(): this { + return this + } + setStatus(): this { + return this + } + recordException(): this { + return this + } + updateName(): this { + return this + } + traceparent(): string | null { + return null + } + tracestate(): string | null { + return null + } + end(): void {} +} + +// Typed as `Span`, not `NoopSpan`: the class's methods take no parameters, so the +// concrete type would reject calls the interface allows. +export const NOOP_SPAN: Span = /* @__PURE__ */ new NoopSpan() + +/** + * Extracts the OTel `exception.type` / `exception.message` pair from whatever was + * thrown. Anything can be thrown in JS, so non-Errors are described by type. + */ +export function describeError(error: unknown): { type: string; message: string } { + try { + if (isError(error)) { + return { type: error.name || 'Error', message: error.message || '' } + } + if (typeof error === 'string') { + return { type: 'string', message: error } + } + if (error && typeof error === 'object') { + const maybe = error as { name?: unknown; message?: unknown } + if (typeof maybe.message === 'string') { + return { type: typeof maybe.name === 'string' ? maybe.name : 'Object', message: maybe.message } + } + } + return { type: typeof error, message: String(error) } + } catch { + // A hostile `toString` or accessor must not throw a second error: in `withSpan` + // that would replace the application's error and skip the span's `end()`. + return { type: typeof error, message: '' } + } +} diff --git a/packages/core/src/traces/traceparent.spec.ts b/packages/core/src/traces/traceparent.spec.ts new file mode 100644 index 0000000000..589ce273de --- /dev/null +++ b/packages/core/src/traces/traceparent.spec.ts @@ -0,0 +1,96 @@ +import { formatTraceparent, parseTraceparent, sanitizeTracestate } from './traceparent' + +const TRACE_ID = '4bf92f3577b34da6a3ce929d0e0e4736' +const SPAN_ID = '00f067aa0ba902b7' + +describe('traceparent', () => { + describe('parseTraceparent', () => { + it('parses a sampled header', () => { + expect(parseTraceparent(`00-${TRACE_ID}-${SPAN_ID}-01`)).toEqual({ traceId: TRACE_ID, spanId: SPAN_ID }) + }) + + it('continues the trace even when the caller sampled it out', () => { + // Every captured span is recorded, so honouring an inbound `00` would + // orphan our own spans rather than save anything. + expect(parseTraceparent(`00-${TRACE_ID}-${SPAN_ID}-00`)).toEqual({ traceId: TRACE_ID, spanId: SPAN_ID }) + }) + + it('accepts a future version with extra fields', () => { + expect(parseTraceparent(`01-${TRACE_ID}-${SPAN_ID}-01-something`)).toEqual({ + traceId: TRACE_ID, + spanId: SPAN_ID, + }) + }) + + it('normalizes case and surrounding whitespace', () => { + expect(parseTraceparent(` 00-${TRACE_ID.toUpperCase()}-${SPAN_ID.toUpperCase()}-01 `)).toEqual({ + traceId: TRACE_ID, + spanId: SPAN_ID, + }) + }) + + it.each([ + ['garbage', 'garbage'], + ['an empty string', ''], + ['version ff', `ff-${TRACE_ID}-${SPAN_ID}-01`], + ['an all-zero trace id', `00-${'0'.repeat(32)}-${SPAN_ID}-01`], + ['an all-zero span id', `00-${TRACE_ID}-${'0'.repeat(16)}-01`], + ['a short trace id', `00-abc-${SPAN_ID}-01`], + ['a missing field', `00-${TRACE_ID}-${SPAN_ID}`], + ['a non-string', 42], + ['undefined', undefined], + ])('returns undefined for %s', (_name, value) => { + expect(parseTraceparent(value)).toBeUndefined() + }) + }) + + describe('formatTraceparent', () => { + it('always sets the sampled flag', () => { + expect(formatTraceparent(TRACE_ID, SPAN_ID)).toBe(`00-${TRACE_ID}-${SPAN_ID}-01`) + }) + + it('round-trips through the parser', () => { + expect(parseTraceparent(formatTraceparent(TRACE_ID, SPAN_ID))).toEqual({ traceId: TRACE_ID, spanId: SPAN_ID }) + }) + }) + + describe('sanitizeTracestate', () => { + it('preserves a valid vendor list unchanged', () => { + expect(sanitizeTracestate('vendor=abc,other=def')).toBe('vendor=abc,other=def') + }) + + it('trims surrounding whitespace', () => { + expect(sanitizeTracestate(' vendor=abc ')).toBe('vendor=abc') + }) + + it.each([ + ['an empty string', ''], + ['a member without a value', 'vendor'], + ['a non-string', 42], + ['undefined', undefined], + ['more than 32 members', Array.from({ length: 33 }, (_v, i) => `k${i}=v`).join(',')], + ['an overlong value', `vendor=${'a'.repeat(600)}`], + ])('discards %s', (_name, value) => { + expect(sanitizeTracestate(value)).toBeUndefined() + }) + }) +}) + +describe('tracestate character safety', () => { + it('discards a value carrying CRLF or a lone surrogate', () => { + expect(sanitizeTracestate('vendor=abc\r\nx-injected: 1')).toBeUndefined() + expect(sanitizeTracestate('vendor=\ud800')).toBeUndefined() + }) + + it('keeps a tab-separated vendor list, which W3C allows', () => { + expect(sanitizeTracestate('rojo=00f067aa0ba902b7,\tcongo=t61rcWkgMzE')).toBe( + 'rojo=00f067aa0ba902b7,\tcongo=t61rcWkgMzE' + ) + }) + + it('keeps an ordinary vendor list', () => { + expect(sanitizeTracestate('rojo=00f067aa0ba902b7,congo=t61rcWkgMzE')).toBe( + 'rojo=00f067aa0ba902b7,congo=t61rcWkgMzE' + ) + }) +}) diff --git a/packages/core/src/traces/traceparent.ts b/packages/core/src/traces/traceparent.ts new file mode 100644 index 0000000000..22466e74ed --- /dev/null +++ b/packages/core/src/traces/traceparent.ts @@ -0,0 +1,82 @@ +import { isValidSpanId, isValidTraceId } from './ids' + +export interface RemoteSpanContext { + traceId: string + spanId: string +} + +// `00-<32 hex>-<16 hex>-<2 hex>`. Version `ff` is invalid per the spec; other +// unknown versions are forwards-compatible, so we parse the first four fields only. +const TRACEPARENT_RE = /^([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})(-.*)?$/ + +/** + * Parses an incoming `traceparent` header value, returning `undefined` for + * anything malformed so a bad header starts a fresh root rather than throwing. + * + * Incoming trace flags are deliberately ignored: we continue the trace even when + * the caller sampled it out (`00`), because PostHog records every captured span + * and dropping the parentage would orphan our own spans. + */ +export function parseTraceparent(value: unknown): RemoteSpanContext | undefined { + if (typeof value !== 'string') { + return undefined + } + const match = TRACEPARENT_RE.exec(value.trim().toLowerCase()) + if (!match) { + return undefined + } + const [, version, traceId, spanId] = match + if (version === 'ff') { + return undefined + } + if (!isValidTraceId(traceId) || !isValidSpanId(spanId)) { + return undefined + } + return { traceId, spanId } +} + +/** + * Builds the `traceparent` header value for a span. The sampled flag is always + * set, because a span we exported is by definition recorded. + */ +export function formatTraceparent(traceId: string, spanId: string): string { + return `00-${traceId}-${spanId}-01` +} + +// tracestate is a comma-separated list of at most 32 `key=value` members, and +// is carried opaquely — we never interpret the vendor entries. +const TRACESTATE_MAX_MEMBERS = 32 +const TRACESTATE_MAX_LENGTH = 512 + +/** + * Validates an incoming `tracestate` far enough to know it is safe to echo back. + * An invalid one is discarded without invalidating its traceparent, so a + * malformed vendor entry never costs us the trace continuation. + */ +export function sanitizeTracestate(value: unknown): string | undefined { + if (typeof value !== 'string') { + return undefined + } + const trimmed = value.trim() + if (!trimmed || trimmed.length > TRACESTATE_MAX_LENGTH) { + return undefined + } + // W3C restricts tracestate to printable ASCII plus HTAB as optional whitespace. + // A CRLF would make the caller's own propagation throw, and a lone surrogate + // refuses the whole OTLP request. + if (/[^\x20-\x7e\t]/.test(trimmed)) { + return undefined + } + const members = trimmed.split(',') + if (members.length > TRACESTATE_MAX_MEMBERS) { + return undefined + } + for (const member of members) { + // An empty member is tolerated by the spec (list optional-white-space), but + // a member without a `=` is not a key/value pair at all. + if (member.trim() && !member.includes('=')) { + return undefined + } + } + return trimmed +} diff --git a/packages/core/src/traces/types.ts b/packages/core/src/traces/types.ts new file mode 100644 index 0000000000..fb330fa760 --- /dev/null +++ b/packages/core/src/traces/types.ts @@ -0,0 +1,101 @@ +export type { + Span, + SpanAttributes, + SpanAttributeValue, + SpanKind, + SpanStatusCode, + SpanTimeInput, + StartSpanOptions, + TracesConfig, + OtlpSpan, + OtlpSpanEvent, + OtlpSpanKeyValue, + OtlpSpanStatus, + OtlpTracesPayload, +} from '@posthog/types' + +import type { OtlpTracesPayload, Span, SpanAttributes, SpanKind, SpanStatusCode, TracesConfig } from '@posthog/types' + +/** Same tagged outcome shape as `SendLogsBatchOutcome` — one policy for all three signals. */ +export type SendTracesBatchOutcome = + | { kind: 'ok' } + | { kind: 'retry-later'; error: unknown } + | { kind: 'too-large' } + | { kind: 'fatal'; error: unknown } + +/** The minimal host surface `PostHogTraces` depends on; `PostHogCoreStateless` satisfies it structurally. */ +export interface TracesHost { + readonly isDisabled: boolean + readonly optedOut: boolean + _sendTracesBatch(payload: OtlpTracesPayload): Promise + getLibraryId(): string + getLibraryVersion(): string +} + +/** + * PostHog context snapshotted onto every span at start, so traces join back to + * persons and sessions. Each SDK fills the fields that apply to it; absent + * fields add no attribute. Internal to `@posthog/core`. + */ +export interface TraceSdkContext { + distinctId?: string + sessionId?: string + /** Web-only — current page URL. */ + currentUrl?: string + /** Mobile-only — current screen / view name. */ + screenName?: string + /** Mobile-only — app foreground/background state. */ + appState?: 'foreground' | 'background' +} + +export interface SpanEventRecord { + name: string + /** ms epoch. */ + timestamp: number + attributes?: SpanAttributes +} + +/** + * A completed span in plain, pre-encoding form: strings for kind and status, a + * plain attribute map, ms-epoch timestamps. + */ +export interface SpanRecord { + traceId: string + spanId: string + parentSpanId?: string + traceState?: string + name: string + kind: SpanKind + status?: { code: SpanStatusCode; message?: string } + attributes: SpanAttributes + events: SpanEventRecord[] + /** ms epoch. */ + startTime: number + endTime: number +} + +/** + * Tracks which span is active, so spans nest without manual parent plumbing. The + * mechanism is platform-specific and stays out of core: node injects an + * `AsyncLocalStorage` implementation over the synchronous default. + */ +export interface SpanContextManager { + /** The active span, or `undefined` when none is active. */ + active(): Span | undefined + /** Run `fn` with `span` active for its (synchronous and async) duration. */ + with(span: Span, fn: () => T): T +} + +/** + * Fields `PostHogTraces` needs resolved at runtime. The host SDK applies its own + * defaults and hands the resolved config to the constructor. + */ +export interface ResolvedTracesConfig extends TracesConfig { + flushIntervalMs: number + maxExportBatchSize: number + /** + * Bound on the in-memory export queue. On overflow the *incoming* span is + * dropped rather than queued ones, whose children may already have shipped. + */ + maxQueueSize: number +} diff --git a/packages/node/package.json b/packages/node/package.json index 8b633ce62e..ebc0ca9dda 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -46,6 +46,7 @@ "devDependencies": { "@edge-runtime/jest-environment": "^4.0.0", "@posthog-tooling/tsconfig-base": "workspace:*", + "@posthog/types": "workspace:^", "@rslib/core": "catalog:", "@types/express": "^5.0.6", "@types/jest": "catalog:", diff --git a/packages/node/references/posthog-node-references-latest.json b/packages/node/references/posthog-node-references-latest.json index 9eb1756884..8b8c6ecf76 100644 --- a/packages/node/references/posthog-node-references-latest.json +++ b/packages/node/references/posthog-node-references-latest.json @@ -547,6 +547,28 @@ }, "path": "src/entrypoints/index.node.ts" }, + { + "category": "Traces", + "description": "The span currently active on this async execution path, or `null` outside any `withSpan` callback.\nOn the edge build this returns `null` after an `await`, because the active span is tracked synchronously there.\n\nSubject to change in a minor release.", + "details": null, + "id": "getActiveSpan", + "showDocs": true, + "title": "getActiveSpan", + "examples": [ + { + "id": "propagate_the_trace_to_another_service", + "name": "Propagate the trace to another service", + "code": "\n\n// Propagate the trace to another service\nconst traceparent = posthog.getActiveSpan()?.traceparent()\nawait fetch(url, { headers: traceparent ? { traceparent } : {} })\n\n\n\n" + } + ], + "releaseTag": "public", + "params": [], + "returnType": { + "id": "Span | null", + "name": "Span | null" + }, + "path": "src/entrypoints/index.node.ts" + }, { "category": "Feature flags", "description": "Get all feature flag values for a specific user.", @@ -1279,6 +1301,41 @@ }, "path": "src/entrypoints/index.node.ts" }, + { + "category": "Traces", + "description": "Starts a span without making it active — for work that can't wrap a callback. Prefer `withSpan`, which ends the span for you.\nAlways returns a handle, so calling code never has to branch: when the `traces` option is absent, the SDK is disabled, or the user has opted out, the handle is inert and nothing is exported.\n\nSubject to change in a minor release.", + "details": null, + "id": "startSpan", + "showDocs": true, + "title": "startSpan", + "examples": [ + { + "id": "", + "name": "", + "code": "\n\nconst span = posthog.startSpan('checkout', { attributes: { plan: 'pro' } })\nspan.setAttribute('cart.items', 3)\nspan.end()\n\n\n\n" + } + ], + "releaseTag": "public", + "params": [ + { + "description": "", + "isOptional": false, + "type": "string", + "name": "name" + }, + { + "description": "", + "isOptional": true, + "type": "StartSpanOptions", + "name": "options" + } + ], + "returnType": { + "id": "Span", + "name": "Span" + }, + "path": "src/entrypoints/index.node.ts" + }, { "category": "Identification", "description": "Remove properties from a person profile.", @@ -1389,6 +1446,41 @@ }, "path": "src/entrypoints/index.node.ts" }, + { + "category": "Traces", + "description": "Runs a callback with a span active for its duration and ends the span for you — at return for a sync callback, at settle for an async one.\nSpans started inside the callback nest under it automatically. If the callback throws or rejects, the span records the exception and the original error is rethrown unchanged.\nSpans nest across `await` only on the Node runtime, which tracks the active span with `AsyncLocalStorage`. The edge build restores the active span when the callback returns its promise, so spans started after an `await` there begin a new trace.\n\nSubject to change in a minor release.", + "details": null, + "id": "withSpan", + "showDocs": true, + "title": "withSpan", + "examples": [ + { + "id": "", + "name": "", + "code": "\n\nawait posthog.withSpan('POST /checkout', { parent: req.get('traceparent') }, async (span) => {\n span.setAttribute('plan', user.plan)\n return processOrder()\n})\n\n\n\n" + } + ], + "releaseTag": "public", + "params": [ + { + "description": "", + "isOptional": false, + "type": "string", + "name": "name" + }, + { + "description": "", + "isOptional": false, + "type": "(span: Span) => T", + "name": "fn" + } + ], + "returnType": { + "id": "T", + "name": "T" + }, + "path": "src/entrypoints/index.node.ts" + }, { "category": "", "details": null, @@ -3502,6 +3594,11 @@ "type": "MetricsConfig", "name": "metrics" }, + { + "description": "Configuration for distributed tracing (`startSpan` / `withSpan`). Tracing is\noff until this is set; supplying it is all that's needed to turn it on.\n\nSet `serviceName` so spans can be attributed and grouped per service — the\nproduct aggregates operations by service and span name.\n\n`shutdown()` drains spans that have already ended, within the shutdown\ntimeout; spans still open at that point are discarded.", + "type": "TracesConfig", + "name": "traces" + }, { "description": "Credential that enables local feature flag evaluation and remote config.\n\nAccepts either a Personal API Key (`phx_...`) or a Project Secret API Key (`phs_...`).\nWhen provided, the client can evaluate feature flags locally and decrypt remote\nconfig payloads via `getRemoteConfigPayload`. Prefer this over the deprecated\n`personalApiKey` option; when both are set, `secretKey` takes precedence.", "type": "string", @@ -4003,6 +4100,13 @@ "path": "../core/src/metrics/types.ts", "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, + { + "id": "SendTracesBatchOutcome", + "name": "SendTracesBatchOutcome", + "properties": [], + "path": "../core/src/traces/types.ts", + "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" + }, { "id": "SentryIntegrationOptions", "name": "SentryIntegrationOptions", @@ -4056,6 +4160,12 @@ "path": "../core/src/error-tracking/types.ts", "example": "(typeof severityLevels)[number]" }, + { + "id": "SpanContextManager", + "name": "SpanContextManager", + "properties": [], + "path": "../core/src/traces/types.ts" + }, { "id": "SpecificQuestionBranching", "name": "SpecificQuestionBranching", @@ -4654,6 +4764,7 @@ "Error tracking", "Privacy", "Feature flags", + "Traces", "Context" ] } \ No newline at end of file diff --git a/packages/node/src/__tests__/traces-defaults.spec.ts b/packages/node/src/__tests__/traces-defaults.spec.ts new file mode 100644 index 0000000000..c8b7b5a225 --- /dev/null +++ b/packages/node/src/__tests__/traces-defaults.spec.ts @@ -0,0 +1,112 @@ +import { resolveTracesConfig } from '../traces-defaults' + +describe('resolveTracesConfig', () => { + it('applies the documented defaults', () => { + expect(resolveTracesConfig(undefined)).toMatchObject({ + flushIntervalMs: 5000, + maxExportBatchSize: 512, + maxQueueSize: 2048, + }) + }) + + it('leaves serviceName unset so core supplies unknown_service', () => { + expect(resolveTracesConfig({}).serviceName).toBeUndefined() + }) + + it('honours explicit values', () => { + expect( + resolveTracesConfig({ serviceName: 'checkout', flushIntervalMs: 1000, maxExportBatchSize: 50 }) + ).toMatchObject({ + serviceName: 'checkout', + flushIntervalMs: 1000, + maxExportBatchSize: 50, + }) + }) + + it('lets OTLP resource attributes override the named fields', () => { + const resolved = resolveTracesConfig({ + serviceName: 'named', + serviceVersion: '1.0.0', + environment: 'staging', + resourceAttributes: { + 'service.name': 'from-attributes', + 'service.version': '2.0.0', + 'deployment.environment': 'production', + }, + }) + + expect(resolved.serviceName).toBe('from-attributes') + expect(resolved.serviceVersion).toBe('2.0.0') + expect(resolved.environment).toBe('production') + }) + + it.each([0, -1, 0.5, Number.NaN, Number.POSITIVE_INFINITY])( + 'falls back to the default for an unusable maxExportBatchSize (%p)', + (value) => { + // A non-positive batch size reaches an export loop that cannot make + // progress with it, so it spins forever posting empty batches. + expect(resolveTracesConfig({ maxExportBatchSize: value }).maxExportBatchSize).toBe(512) + } + ) + + it('floors a fractional batch size to an integer', () => { + expect(resolveTracesConfig({ maxExportBatchSize: 10.9 }).maxExportBatchSize).toBe(10) + }) + + it.each([0, -1, Number.NaN])('falls back to the default for an unusable flushIntervalMs (%p)', (value) => { + expect(resolveTracesConfig({ flushIntervalMs: value }).flushIntervalMs).toBe(5000) + }) + + it('keeps the queue at least as large as the export batch', () => { + // A queue smaller than the flush trigger would stop the depth-based flush + // from ever firing. + expect(resolveTracesConfig({ maxExportBatchSize: 5000 }).maxQueueSize).toBe(5000) + expect(resolveTracesConfig({ maxExportBatchSize: 10 }).maxQueueSize).toBe(2048) + }) + + it('honours an explicit maxQueueSize', () => { + expect(resolveTracesConfig({ maxQueueSize: 100_000 }).maxQueueSize).toBe(100_000) + }) + + it('floors an explicit maxQueueSize at the export batch size', () => { + expect(resolveTracesConfig({ maxExportBatchSize: 512, maxQueueSize: 10 }).maxQueueSize).toBe(512) + }) +}) + +describe('resourceAttributes guarding', () => { + it('ignores a non-object value', () => { + const { resourceAttributes } = resolveTracesConfig({ resourceAttributes: 'oops' as never }) + + expect(Object.keys(resourceAttributes ?? {})).toEqual([]) + }) + + it('ignores an array, which would otherwise spread as numeric keys', () => { + const { resourceAttributes } = resolveTracesConfig({ + resourceAttributes: [{ key: 'service.name' }] as never, + }) + + expect(Object.keys(resourceAttributes ?? {})).not.toContain('0') + }) + + it('drops an identity key that is not a string', () => { + const resolved = resolveTracesConfig({ + serviceName: 'checkout-api', + resourceAttributes: { 'service.name': 12345 as never, region: 'us' }, + }) + + expect(resolved.serviceName).toBe('checkout-api') + expect(resolved.resourceAttributes).toEqual({ region: 'us' }) + }) + + it('does not throw when an identity accessor throws', () => { + const hostile = {} + Object.defineProperty(hostile, 'service.name', { + enumerable: true, + get() { + throw new Error('config getter exploded') + }, + }) + + expect(() => resolveTracesConfig({ resourceAttributes: hostile as never })).not.toThrow() + }) +}) diff --git a/packages/node/src/__tests__/traces-edge.spec.ts b/packages/node/src/__tests__/traces-edge.spec.ts new file mode 100644 index 0000000000..91c9edbe92 --- /dev/null +++ b/packages/node/src/__tests__/traces-edge.spec.ts @@ -0,0 +1,81 @@ +import { PostHog } from '@/entrypoints/index.edge' +import type { OtlpSpan, OtlpTracesPayload } from '@posthog/types' + +jest.mock('../version', () => ({ version: '1.2.3' })) + +const mockedFetch = jest.spyOn(globalThis, 'fetch').mockImplementation() + +describe('PostHog traces on the edge build', () => { + const createClient = (): PostHog => + new PostHog('phc_test_key', { + host: 'http://example.com', + flushAt: 1, + fetchRetryCount: 0, + disableCompression: true, + traces: { serviceName: 'edge-api' }, + }) + + const sentSpans = (): OtlpSpan[] => + mockedFetch.mock.calls + .filter((call) => (call[0] as string).includes('/i/v1/traces')) + .map(([, init]) => JSON.parse((init as any).body as string) as OtlpTracesPayload) + .flatMap((payload) => payload.resourceSpans[0].scopeSpans[0].spans) + + beforeEach(() => { + mockedFetch.mockReset() + mockedFetch.mockResolvedValue({ status: 200, text: async () => '{}', json: async () => ({}) } as any) + }) + + it('exports spans', async () => { + const client = createClient() + client.startSpan('edge-work').end() + await client.shutdown() + + expect(sentSpans().map((span) => span.name)).toEqual(['edge-work']) + }) + + it('nests spans created synchronously inside withSpan', async () => { + const client = createClient() + client.withSpan('parent', () => { + client.startSpan('child', { parent: client.getActiveSpan()! }).end() + }) + await client.shutdown() + + const spans = sentSpans() + const parent = spans.find((span) => span.name === 'parent')! + const child = spans.find((span) => span.name === 'child')! + expect(child.traceId).toBe(parent.traceId) + expect(child.parentSpanId).toBe(parent.spanId) + }) + + it('starts a new trace for a span created after an await, as documented', async () => { + const client = createClient() + let parentTraceId = '' + await client.withSpan('parent', async (span) => { + parentTraceId = span.traceparent()!.split('-')[1] + await Promise.resolve() + expect(client.getActiveSpan()).toBeNull() + client.startSpan('after-await').end() + }) + await client.shutdown() + + const orphan = sentSpans().find((span) => span.name === 'after-await')! + expect(orphan.traceId).not.toBe(parentTraceId) + expect(orphan.parentSpanId).toBeUndefined() + }) + + it('still nests across an await when the parent is passed explicitly', async () => { + const client = createClient() + await client.withSpan('parent', async (span) => { + await Promise.resolve() + client.startSpan('after-await', { parent: span }).end() + }) + await client.shutdown() + + const spans = sentSpans() + const parent = spans.find((span) => span.name === 'parent')! + const child = spans.find((span) => span.name === 'after-await')! + expect(child.traceId).toBe(parent.traceId) + expect(child.parentSpanId).toBe(parent.spanId) + }) +}) diff --git a/packages/node/src/__tests__/traces.spec.ts b/packages/node/src/__tests__/traces.spec.ts new file mode 100644 index 0000000000..9f4723c6b2 --- /dev/null +++ b/packages/node/src/__tests__/traces.spec.ts @@ -0,0 +1,478 @@ +import { PostHog } from '@/entrypoints/index.node' +import type { OtlpSpan, OtlpTracesPayload } from '@posthog/types' +import { waitForPromises } from './utils' +import { isGzipSupported } from '@posthog/core' + +jest.mock('../version', () => ({ version: '1.2.3' })) + +const mockedFetch = jest.spyOn(globalThis, 'fetch').mockImplementation() + +describe('PostHog traces', () => { + let posthog: PostHog + + const createClient = (options: Record = {}): PostHog => + new PostHog('phc_test_key', { + host: 'http://example.com', + flushAt: 1, + fetchRetryCount: 0, + disableCompression: true, + traces: { serviceName: 'checkout-api' }, + ...options, + }) + + const traceRequests = (): [string, any][] => + mockedFetch.mock.calls.filter((call) => (call[0] as string).includes('/i/v1/traces')) as [string, any][] + + const sentPayloads = (): OtlpTracesPayload[] => + traceRequests().map(([, init]) => JSON.parse(init.body as string) as OtlpTracesPayload) + + const sentSpans = (): OtlpSpan[] => sentPayloads().flatMap((p) => p.resourceSpans[0].scopeSpans[0].spans) + + const attributeOf = (span: OtlpSpan, key: string): any => span.attributes?.find((a) => a.key === key)?.value + + // Traces run their own flush cycle; this advances it without calling flush(). + const DEFAULT_TRACES_FLUSH_INTERVAL_MS = 5000 + const flushTraces = async (): Promise => { + await jest.advanceTimersByTimeAsync(DEFAULT_TRACES_FLUSH_INTERVAL_MS) + await waitForPromises() + } + + beforeEach(() => { + jest.clearAllMocks() + mockedFetch.mockResolvedValue({ + status: 200, + text: () => Promise.resolve('{}'), + json: () => Promise.resolve({}), + } as any) + posthog = createClient() + }) + + afterEach(async () => { + await posthog.shutdown() + }) + + describe('configuration', () => { + it('is off until the traces option is supplied', async () => { + const untraced = createClient({ traces: undefined }) + const span = untraced.startSpan('checkout') + span.end() + await untraced.shutdown() + + expect(span.traceparent()).toBeNull() + expect(traceRequests()).toHaveLength(0) + }) + + it('still runs a withSpan callback when tracing is off', async () => { + const untraced = createClient({ traces: undefined }) + const fn = jest.fn(() => 'value') + + expect(untraced.withSpan('job', fn)).toBe('value') + expect(fn).toHaveBeenCalledTimes(1) + expect(untraced.getActiveSpan()).toBeNull() + await untraced.shutdown() + }) + }) + + describe('transport', () => { + it('posts to /i/v1/traces with bearer auth', async () => { + posthog.startSpan('checkout').end() + await flushTraces() + + const [url, init] = traceRequests()[0] + expect(url).toBe('http://example.com/i/v1/traces') + expect(init.method).toBe('POST') + expect(init.headers.Authorization).toBe('Bearer phc_test_key') + expect(init.headers['Content-Type']).toBe('application/json') + }) + + it('does not put the project key in the query string', async () => { + posthog.startSpan('checkout').end() + await flushTraces() + + expect(traceRequests()[0][0]).not.toContain('token=') + }) + + it('sends the service name as a resource attribute', async () => { + posthog.startSpan('checkout').end() + await flushTraces() + + // The server reads service_name only from this attribute; without it the + // spans are stored with an empty service and are unattributable. + expect(sentPayloads()[0].resourceSpans[0].resource.attributes).toContainEqual({ + key: 'service.name', + value: { stringValue: 'checkout-api' }, + }) + }) + + it('identifies the SDK in the scope and resource', async () => { + posthog.startSpan('checkout').end() + await flushTraces() + + const [resourceSpan] = sentPayloads()[0].resourceSpans + expect(resourceSpan.scopeSpans[0].scope).toEqual({ name: 'posthog-node', version: '1.2.3' }) + expect(resourceSpan.resource.attributes).toContainEqual({ + key: 'telemetry.sdk.name', + value: { stringValue: 'posthog-node' }, + }) + }) + }) + + describe('span shape', () => { + it('exports well-formed W3C identifiers', async () => { + posthog.startSpan('checkout').end() + await flushTraces() + + const [span] = sentSpans() + expect(span.traceId).toMatch(/^[0-9a-f]{32}$/) + expect(span.spanId).toMatch(/^[0-9a-f]{16}$/) + }) + + it('encodes timestamps as nanosecond strings', async () => { + posthog.startSpan('checkout').end() + await flushTraces() + + const [span] = sentSpans() + expect(typeof span.startTimeUnixNano).toBe('string') + expect(Number(span.endTimeUnixNano)).toBeGreaterThanOrEqual(Number(span.startTimeUnixNano)) + }) + + it('encodes integer attributes as stringified int64', async () => { + posthog.startSpan('checkout', { attributes: { 'http.status_code': 200 } }).end() + await flushTraces() + + expect(attributeOf(sentSpans()[0], 'http.status_code')).toEqual({ intValue: '200' }) + }) + + it('replaces an empty span name rather than poisoning the batch', async () => { + // A malformed span 400s the entire request, and 400 is non-retriable — + // one bad name would silently destroy every other span in the batch. + posthog.startSpan('').end() + await flushTraces() + + expect(sentSpans()[0].name).toBe('unknown') + }) + }) + + describe('active span context', () => { + it('nests spans started inside a withSpan callback', async () => { + posthog.withSpan('outer', () => { + posthog.withSpan('inner', () => undefined) + }) + await flushTraces() + + const inner = sentSpans().find((s) => s.name === 'inner')! + const outer = sentSpans().find((s) => s.name === 'outer')! + expect(inner.traceId).toBe(outer.traceId) + expect(inner.parentSpanId).toBe(outer.spanId) + }) + + it('keeps the span active across an await', async () => { + // This is what AsyncLocalStorage buys over the synchronous fallback: a + // span started after an await still nests correctly. + await posthog.withSpan('outer', async () => { + await Promise.resolve() + posthog.withSpan('inner', () => undefined) + }) + await flushTraces() + + const inner = sentSpans().find((s) => s.name === 'inner')! + const outer = sentSpans().find((s) => s.name === 'outer')! + expect(inner.parentSpanId).toBe(outer.spanId) + }) + + it('isolates concurrent requests from each other', async () => { + await Promise.all([ + posthog.withSpan('request-a', async () => { + await Promise.resolve() + posthog.withSpan('child-a', () => undefined) + }), + posthog.withSpan('request-b', async () => { + await Promise.resolve() + posthog.withSpan('child-b', () => undefined) + }), + ]) + await flushTraces() + + const byName = (name: string): OtlpSpan => sentSpans().find((s) => s.name === name)! + expect(byName('child-a').parentSpanId).toBe(byName('request-a').spanId) + expect(byName('child-b').parentSpanId).toBe(byName('request-b').spanId) + expect(byName('request-a').traceId).not.toBe(byName('request-b').traceId) + }) + + it('reads null outside any callback', () => { + expect(posthog.getActiveSpan()).toBeNull() + }) + }) + + describe('auto-context from the request context', () => { + it('attaches the request distinct id and session id', async () => { + // Fed by the Express/NestJS middleware from the X-POSTHOG-DISTINCT-ID and + // X-POSTHOG-SESSION-ID tracing headers. + posthog.withContext({ distinctId: 'user-123', sessionId: 'session-123' }, () => { + posthog.startSpan('checkout').end() + }) + await flushTraces() + + const [span] = sentSpans() + expect(attributeOf(span, 'posthogDistinctId')).toEqual({ stringValue: 'user-123' }) + expect(attributeOf(span, 'sessionId')).toEqual({ stringValue: 'session-123' }) + }) + + it('omits the keys outside a request context', async () => { + posthog.startSpan('background-job').end() + await flushTraces() + + expect(attributeOf(sentSpans()[0], 'posthogDistinctId')).toBeUndefined() + expect(attributeOf(sentSpans()[0], 'sessionId')).toBeUndefined() + }) + }) + + describe('distributed tracing', () => { + it('continues a trace from an inbound traceparent header', async () => { + const traceId = '4bf92f3577b34da6a3ce929d0e0e4736' + const spanId = '00f067aa0ba902b7' + + posthog.withSpan('POST /checkout', { parent: `00-${traceId}-${spanId}-01` }, () => undefined) + await flushTraces() + + const [span] = sentSpans() + expect(span.traceId).toBe(traceId) + expect(span.parentSpanId).toBe(spanId) + }) + + it('produces a traceparent for the next service', async () => { + let traceparent: string | null = null + posthog.withSpan('POST /checkout', () => { + traceparent = posthog.getActiveSpan()!.traceparent() + }) + await flushTraces() + + const [span] = sentSpans() + expect(traceparent).toBe(`00-${span.traceId}-${span.spanId}-01`) + }) + }) + + describe('errors', () => { + it('records a thrown error and rethrows it unchanged', async () => { + const thrown = new TypeError('boom') + expect(() => + posthog.withSpan('job', () => { + throw thrown + }) + ).toThrow(thrown) + + await flushTraces() + + const [span] = sentSpans() + expect(span.status).toEqual({ code: 2, message: 'boom' }) + expect(span.events?.[0].name).toBe('exception') + }) + }) + + describe('compression', () => { + it('advertises the encoding exactly when it compresses', async () => { + const client = createClient({ disableCompression: false }) + client.startSpan('checkout').end() + await client.shutdown() + + const [, init] = traceRequests()[0] + // Runtimes without gzip send plain text; a header that disagrees with the + // body is what 400s the batch, so the two must always match. + expect(init.headers['Content-Encoding'] === 'gzip').toBe(typeof init.body !== 'string') + expect(typeof init.body !== 'string').toBe(isGzipSupported()) + }) + + it('sends an uncompressed body when compression is disabled', async () => { + const client = createClient() + client.startSpan('checkout').end() + await client.shutdown() + + const [, init] = traceRequests()[0] + expect(init.headers['Content-Encoding']).toBeUndefined() + expect(typeof init.body).toBe('string') + }) + }) + + describe('identity resource attributes', () => { + it('ignores a non-string service.name and keeps the configured one', async () => { + const client = createClient({ + traces: { serviceName: 'checkout-api', resourceAttributes: { 'service.name': 12345 } }, + }) + client.startSpan('checkout').end() + await client.shutdown() + + const resource = sentPayloads()[0].resourceSpans[0].resource.attributes + expect(resource.find((a) => a.key === 'service.name')?.value).toEqual({ stringValue: 'checkout-api' }) + }) + + it('ignores a resourceAttributes value that is not an object', async () => { + const client = createClient({ + traces: { serviceName: 'checkout-api', resourceAttributes: 'oops' as any }, + }) + expect(() => client.startSpan('checkout').end()).not.toThrow() + await client.shutdown() + + const keys = sentPayloads()[0].resourceSpans[0].resource.attributes.map((a) => a.key) + // A spread primitive would arrive as attributes keyed "0", "1", "2", "3". + expect(keys).not.toContain('0') + }) + + it('does not throw when a resourceAttributes accessor throws', () => { + const hostile = {} + Object.defineProperty(hostile, 'service.name', { + enumerable: true, + get() { + throw new Error('config getter exploded') + }, + }) + const client = createClient({ traces: { serviceName: 'checkout-api', resourceAttributes: hostile as any } }) + + expect(() => client.startSpan('checkout')).not.toThrow() + }) + + it('falls back to unknown_service when the only service.name is not a string', async () => { + // Nothing re-emits a valid name here, so this is what actually covers the + // resolver's type check rather than the encoder's key ordering. + const client = createClient({ traces: { resourceAttributes: { 'service.name': 12345 } } }) + client.startSpan('checkout').end() + await client.shutdown() + + const resource = sentPayloads()[0].resourceSpans[0].resource.attributes + expect(resource.find((a) => a.key === 'service.name')?.value).toEqual({ stringValue: 'unknown_service' }) + }) + + it('drops a non-string deployment.environment instead of shipping it as an int', async () => { + const client = createClient({ + traces: { serviceName: 'checkout-api', resourceAttributes: { 'deployment.environment': 42 } }, + }) + client.startSpan('checkout').end() + await client.shutdown() + + const resource = sentPayloads()[0].resourceSpans[0].resource.attributes + expect(resource.find((a) => a.key === 'deployment.environment')).toBeUndefined() + }) + + it('lets a string service.name in resourceAttributes win', async () => { + const client = createClient({ + traces: { serviceName: 'checkout-api', resourceAttributes: { 'service.name': 'from-attributes' } }, + }) + client.startSpan('checkout').end() + await client.shutdown() + + const resource = sentPayloads()[0].resourceSpans[0].resource.attributes + expect(resource.find((a) => a.key === 'service.name')?.value).toEqual({ stringValue: 'from-attributes' }) + }) + }) + + describe('413 handling', () => { + it('halves the batch on a real 413 rather than retrying it forever', async () => { + // `_sendTracesBatch` classifies the response itself; the core halving + // logic is dead unless that mapping is right, and every core test mocks + // the outcome rather than the status. + let requests = 0 + mockedFetch.mockImplementation(((url: string) => { + if (!url.includes('/i/v1/traces')) { + return Promise.resolve({ status: 200, text: () => Promise.resolve('ok') } as any) + } + requests++ + return Promise.resolve({ + status: requests === 1 ? 413 : 200, + text: () => Promise.resolve(requests === 1 ? 'too large' : '{}'), + } as any) + }) as any) + + const client = createClient({ traces: { serviceName: 'checkout-api', maxExportBatchSize: 2 } }) + client.startSpan('a').end() + client.startSpan('b').end() + await client.shutdown() + + const batchSizes = sentPayloads().map((p) => p.resourceSpans[0].scopeSpans[0].spans.length) + expect(batchSizes).toEqual([2, 1, 1]) + }) + }) + + describe('flush cycle', () => { + it('drains queued spans', async () => { + posthog.startSpan('checkout').end() + await posthog.flush() + + expect(sentSpans()).toHaveLength(1) + }) + + it('resolves when the span export fails', async () => { + mockedFetch.mockRejectedValue(new Error('network down')) + posthog.startSpan('checkout').end() + + await expect(posthog.flush()).resolves.toBeUndefined() + }) + + it('still flushes on its own interval', async () => { + posthog.startSpan('checkout').end() + await flushTraces() + + expect(sentSpans()).toHaveLength(1) + }) + }) + + describe('serverless waitUntil', () => { + it('drains spans on the debounced waitUntil flush, not only on shutdown', async () => { + const waitUntil = jest.fn() + // High flushAt and a long event interval so the only thing that can flush + // within the window is the debounced waitUntil cycle. + const client = createClient({ + waitUntil, + waitUntilDebounceMs: 50, + flushAt: 100, + flushInterval: 60_000, + traces: { serviceName: 'checkout-api', flushIntervalMs: 60_000 }, + }) + // No capture(): a handler that only traces must still hold the invocation open. + client.startSpan('handler').end() + + await jest.advanceTimersByTimeAsync(100) + await waitForPromises() + + expect(waitUntil).toHaveBeenCalled() + expect(sentSpans().map((span) => span.name)).toEqual(['handler']) + await client.shutdown() + }) + }) + + describe('shutdown', () => { + it('is bounded by the shutdown timeout when the transport hangs', async () => { + const client = createClient() + client.startSpan('checkout').end() + mockedFetch.mockImplementation(() => new Promise(() => {}) as any) + + const shutdown = client.shutdown(500) + await jest.advanceTimersByTimeAsync(600) + + await expect(shutdown).resolves.toBeUndefined() + }) + + it('discards spans whose raced-out flush settles after teardown', async () => { + let rejectFetch!: (error: Error) => void + mockedFetch.mockImplementation(() => new Promise((_resolve, reject) => (rejectFetch = reject)) as any) + + const client = createClient() + client.startSpan('checkout').end() + const shutdown = client.shutdown(100) + await jest.advanceTimersByTimeAsync(150) + await shutdown + const afterShutdown = traceRequests().length + + rejectFetch(new Error('connection refused')) + await jest.advanceTimersByTimeAsync(60_000) + + expect(traceRequests()).toHaveLength(afterShutdown) + }) + + it('drains queued spans', async () => { + posthog.startSpan('a').end() + posthog.startSpan('b').end() + await posthog.shutdown() + + expect(sentSpans()).toHaveLength(2) + }) + }) +}) diff --git a/packages/node/src/client.ts b/packages/node/src/client.ts index e02e9f079b..50a2950b72 100644 --- a/packages/node/src/client.ts +++ b/packages/node/src/client.ts @@ -17,14 +17,18 @@ import { PostHogFlagsResponse, PostHogMetrics, PostHogPersistedProperty, + PostHogTraces, + NOOP_SPAN, Properties, resolveMetricsConfig, RetriableOptions, raceWithTimeout, safeSetTimeout, + SyncSpanContextManager, uuidv7, } from '@posthog/core' -import type { Metrics } from '@posthog/core' +import type { Metrics, Span, SpanContextManager, StartSpanOptions, TraceSdkContext } from '@posthog/core' +import { resolveTracesConfig } from './traces-defaults' import { AllFlagsOptions, EventMessage, @@ -144,6 +148,7 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen public readonly options: PostHogOptions protected readonly context?: IPostHogContext private _metrics?: PostHogMetrics + private _traces?: PostHogTraces private readonly captureMode: CaptureMode private _v1Sender?: V1CaptureSender @@ -281,8 +286,21 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen this.scheduleDebouncedFlush() } + /** + * Concurrent so a serverless handler waits for one round trip, not two. A + * failed span export leaves the spans queued rather than rejecting, since + * callers already treat `flush()` as safe to leave unwrapped. + */ + private _flushEventsAndSpans(): Promise { + const events = this.flushWithPendingPromises() + if (!this._traces) { + return events + } + return Promise.all([events, this._traces.flush().catch(() => {})]).then(() => undefined) + } + override async flush(): Promise { - const flushPromise = this.flushWithPendingPromises() + const flushPromise = this._flushEventsAndSpans() const waitUntil = this.options.waitUntil // Only register when no debounce promise is already keeping runtime alive if (waitUntil && !this._waitUntilCycle) { @@ -355,7 +373,7 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen private async resolveWaitUntilFlush(): Promise { const resolve = this._consumeWaitUntilCycle() try { - await this.flushWithPendingPromises() + await this._flushEventsAndSpans() } catch { // Flush errors are already logged by flush() internals } finally { @@ -601,6 +619,134 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen return this._metrics } + /** + * Active-span tracking. Overridden by the Node entrypoint with an + * `AsyncLocalStorage`-backed manager; the edge build keeps this synchronous + * fallback, matching how `initializeContext` already differs between them. + */ + protected initializeSpanContextManager(): SpanContextManager { + return new SyncSpanContextManager() + } + + /** + * The traces pipeline, built on first use. Returns `undefined` when the + * `traces` client option is absent — tracing is off until configured. + */ + private get _tracesPipeline(): PostHogTraces | undefined { + if (!this.options.traces) { + return undefined + } + if (!this._traces) { + this._traces = new PostHogTraces( + this, + resolveTracesConfig(this.options.traces), + this._logger, + () => this._tracingContext(), + this.initializeSpanContextManager(), + // A handler that only traces still has to hold the invocation open. + () => this.scheduleDebouncedFlush() + ) + } + return this._traces + } + + /** + * PostHog context attached to every span, so traces join back to persons and + * sessions. A server process has no ambient identity, so these come from the + * current request context — the Express/NestJS middleware or `withContext`. + */ + private _tracingContext(): TraceSdkContext { + const context = this.context?.get() + return { distinctId: context?.distinctId, sessionId: context?.sessionId } + } + + /** + * Starts a span without making it active — for work that can't wrap a + * callback. Prefer `withSpan`, which ends the span for you. + * + * Always returns a handle, so calling code never has to branch: when the + * `traces` option is absent, the SDK is disabled, or the user has opted out, + * the handle is inert and nothing is exported. + * + * {@label Traces} + * + * @experimental Subject to change in a minor release. + * + * @example + * ```ts + * const span = posthog.startSpan('checkout', { attributes: { plan: 'pro' } }) + * span.setAttribute('cart.items', 3) + * span.end() + * ``` + */ + startSpan(name: string, options?: StartSpanOptions): Span { + return this._tracesPipeline?.startSpan(name, options) ?? NOOP_SPAN + } + + /** + * Runs a callback with a span active for its duration and ends the span for + * you — at return for a sync callback, at settle for an async one. Takes an + * optional `StartSpanOptions` between the name and the callback: + * `withSpan(name, fn)` or `withSpan(name, options, fn)`. + * + * Spans started inside the callback nest under it automatically. If the + * callback throws or rejects, the span records the exception and the original + * error is rethrown unchanged. A callback that ends the span itself gets the + * rethrow but not the recording, since the span is already exported by then. + * + * Spans nest across `await` only on the Node runtime, which tracks the active + * span with `AsyncLocalStorage`. The edge build restores the active span when + * the callback returns its promise, so spans started after an `await` there + * begin a new trace. + * + * {@label Traces} + * + * @experimental Subject to change in a minor release. + * + * @example + * ```ts + * await posthog.withSpan('POST /checkout', { parent: req.get('traceparent') }, async (span) => { + * span.setAttribute('plan', user.plan) + * return processOrder() + * }) + * ``` + */ + withSpan(name: string, fn: (span: Span) => T): T + withSpan(name: string, options: StartSpanOptions, fn: (span: Span) => T): T + withSpan(name: string, optionsOrFn: StartSpanOptions | ((span: Span) => T), maybeFn?: (span: Span) => T): T { + const options = typeof optionsOrFn === 'function' ? undefined : optionsOrFn + const fn = (typeof optionsOrFn === 'function' ? optionsOrFn : maybeFn) as (span: Span) => T + + const pipeline = this._tracesPipeline + if (!pipeline) { + // Tracing off: still run the callback exactly once, with an inert handle. + return fn(NOOP_SPAN) + } + return options ? pipeline.withSpan(name, options, fn) : pipeline.withSpan(name, fn) + } + + /** + * The span currently active on this async execution path, or `null` outside + * any `withSpan` callback. + * + * On the edge build this returns `null` after an `await`, because the active + * span is tracked synchronously there. + * + * {@label Traces} + * + * @experimental Subject to change in a minor release. + * + * @example + * ```ts + * // Propagate the trace to another service + * const traceparent = posthog.getActiveSpan()?.traceparent() + * await fetch(url, { headers: traceparent ? { traceparent } : {} }) + * ``` + */ + getActiveSpan(): Span | null { + return this._tracesPipeline?.getActiveSpan() ?? null + } + /** * Get the custom user agent string for this client. * @@ -2617,6 +2763,15 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen // merged back onto a re-armed timer after teardown. this._metrics.reset() } + if (this._traces) { + // Same treatment as metrics: send what's queued, raced against the shared + // shutdown budget, then reset so a losing flush can't re-arm a timer. + await raceWithTimeout( + this._traces.flush().catch(() => {}), + Math.max(0, shutdownDeadlineMs - Date.now()) + ) + this._traces.reset() + } try { return await super._shutdown(Math.max(0, shutdownDeadlineMs - Date.now())) } finally { diff --git a/packages/node/src/entrypoints/index.node.ts b/packages/node/src/entrypoints/index.node.ts index 1c872b4ed2..96a9fed515 100644 --- a/packages/node/src/entrypoints/index.node.ts +++ b/packages/node/src/entrypoints/index.node.ts @@ -7,7 +7,9 @@ import { createRelativePathModifier } from '../extensions/error-tracking/modifie import type { PostHogFetchBodyBytes } from '@posthog/core' import { PostHogBackendClient } from '../client' import { ErrorTracking as CoreErrorTracking } from '@posthog/core' +import type { SpanContextManager } from '@posthog/core' import { PostHogContext } from '../extensions/context/context' +import { AsyncLocalStorageSpanContextManager } from '../extensions/context/span-context.node' import { gzipCompress } from '../gzip.node' export class PostHog extends PostHogBackendClient { @@ -23,6 +25,10 @@ export class PostHog extends PostHogBackendClient { return new PostHogContext() } + protected override initializeSpanContextManager(): SpanContextManager { + return new AsyncLocalStorageSpanContextManager() + } + protected override createErrorPropertiesBuilder(): CoreErrorTracking.ErrorPropertiesBuilder { return new CoreErrorTracking.ErrorPropertiesBuilder( [ diff --git a/packages/node/src/exports.ts b/packages/node/src/exports.ts index a47ed77e8f..23e4897a93 100644 --- a/packages/node/src/exports.ts +++ b/packages/node/src/exports.ts @@ -14,6 +14,19 @@ export type { FeatureFlagErrorType } from '@posthog/core' // and API surface without a direct @posthog/core dependency. export type { CaptureMetricOptions, Metrics, MetricsConfig } from '@posthog/core' +// Tracing types re-exported so consumers can name the `traces` client option and +// the span API without a direct @posthog/core dependency. +export type { + Span, + SpanAttributes, + SpanAttributeValue, + SpanKind, + SpanStatusCode, + SpanTimeInput, + StartSpanOptions, + TracesConfig, +} from '@posthog/core' + // Identity helpers re-exported from core for posthog-node consumers managing // distinct_id outside the browser SDK (e.g. Lambda functions handing out // `download-app` redirects). Closes #2143. diff --git a/packages/node/src/extensions/context/span-context.node.ts b/packages/node/src/extensions/context/span-context.node.ts new file mode 100644 index 0000000000..bf4e1bacd8 --- /dev/null +++ b/packages/node/src/extensions/context/span-context.node.ts @@ -0,0 +1,22 @@ +import { AsyncLocalStorage } from 'node:async_hooks' +import type { Span, SpanContextManager } from '@posthog/core' + +/** + * Active-span tracking backed by `AsyncLocalStorage`, so a span stays active + * across `await` boundaries and through any async work its callback starts. + * + * Lives here rather than in core because core ships to browsers, edge runtimes + * and React Native and must not import `node:async_hooks`; the edge entrypoint + * falls back to core's synchronous manager. + */ +export class AsyncLocalStorageSpanContextManager implements SpanContextManager { + private readonly _storage = new AsyncLocalStorage() + + active(): Span | undefined { + return this._storage.getStore() + } + + with(span: Span, fn: () => T): T { + return this._storage.run(span, fn) + } +} diff --git a/packages/node/src/traces-defaults.ts b/packages/node/src/traces-defaults.ts new file mode 100644 index 0000000000..2e1be86c67 --- /dev/null +++ b/packages/node/src/traces-defaults.ts @@ -0,0 +1,64 @@ +import type { ResolvedTracesConfig, TracesConfig } from '@posthog/core' + +// OpenTelemetry's BatchSpanProcessor defaults, which sit comfortably under the +// server's 2 MB body cap. +const DEFAULT_FLUSH_INTERVAL_MS = 5000 +const DEFAULT_MAX_EXPORT_BATCH_SIZE = 512 +const DEFAULT_MAX_QUEUE_SIZE = 2048 + +/** + * Coerces a caller-supplied positive-integer option. `0`, a negative, or `NaN` + * reaching the export loop would stall it. + */ +function positiveInteger(value: number | undefined, fallback: number): number { + return typeof value === 'number' && Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback +} + +const IDENTITY_KEYS = ['service.name', 'service.version', 'deployment.environment'] as const + +/** + * Drops a non-string identity key rather than letting it through: the resolver + * would ignore it, and it would still reach the wire as an int and leave the + * spans unattributable. + */ +function withUsableIdentityKeys(attributes: TracesConfig['resourceAttributes']): TracesConfig['resourceAttributes'] { + // A primitive or array would otherwise be spread into attributes keyed "0", "1", "2". + if (!attributes || typeof attributes !== 'object' || Array.isArray(attributes)) { + return undefined + } + try { + if (IDENTITY_KEYS.every((key) => !(key in attributes) || typeof attributes[key] === 'string')) { + return attributes + } + const usable = { ...attributes } + for (const key of IDENTITY_KEYS) { + if (key in usable && typeof usable[key] !== 'string') { + delete usable[key] + } + } + return usable + } catch { + // A throwing accessor on the config object must not escape `startSpan`. + return undefined + } +} + +/** + * Resolves the public `traces` config into the shape core `PostHogTraces` consumes. + * OTLP resource attributes take precedence over the named fields, matching the + * logs config — a user who sets `service.name` directly means it. + */ +export function resolveTracesConfig(config: TracesConfig | undefined): ResolvedTracesConfig { + const resourceAttributes = withUsableIdentityKeys(config?.resourceAttributes) + const maxExportBatchSize = positiveInteger(config?.maxExportBatchSize, DEFAULT_MAX_EXPORT_BATCH_SIZE) + return { + serviceName: (resourceAttributes?.['service.name'] as string | undefined) ?? config?.serviceName, + serviceVersion: (resourceAttributes?.['service.version'] as string | undefined) ?? config?.serviceVersion, + environment: (resourceAttributes?.['deployment.environment'] as string | undefined) ?? config?.environment, + resourceAttributes, + flushIntervalMs: positiveInteger(config?.flushIntervalMs, DEFAULT_FLUSH_INTERVAL_MS), + maxExportBatchSize, + // Never below the flush trigger, or the depth-based flush could never fire. + maxQueueSize: Math.max(positiveInteger(config?.maxQueueSize, DEFAULT_MAX_QUEUE_SIZE), maxExportBatchSize), + } +} diff --git a/packages/node/src/types.ts b/packages/node/src/types.ts index edb13a64ef..0580e404ef 100644 --- a/packages/node/src/types.ts +++ b/packages/node/src/types.ts @@ -9,6 +9,9 @@ import type { PostHogFetchResponse, PostHogFlagsAndPayloadsResponse, Properties, + Span, + StartSpanOptions, + TracesConfig, } from '@posthog/core' import { ContextData, ContextOptions } from './extensions/context/types' @@ -181,6 +184,25 @@ export type PostHogOptions = Omit stripe.charge(order)) + * ``` + * + * @experimental Subject to change in a minor release. + */ + traces?: TracesConfig /** * Credential that enables local feature flag evaluation and remote config. * @@ -809,6 +831,30 @@ export interface IPostHog { */ readonly metrics: Metrics + /** + * @description Starts a span without making it active, for work that can't wrap a callback. + * Prefer `withSpan`. Always returns a handle — an inert one when tracing is off — so calling + * code never has to branch. + * @experimental Subject to change in a minor release. + */ + startSpan(name: string, options?: StartSpanOptions): Span + + /** + * @description Runs a callback with a span active for its duration and ends the span at return + * (sync) or settle (async). Spans started inside nest automatically; a throw or rejection is + * recorded on the span and rethrown unchanged. + * @experimental Subject to change in a minor release. + */ + withSpan(name: string, fn: (span: Span) => T): T + withSpan(name: string, options: StartSpanOptions, fn: (span: Span) => T): T + + /** + * @description The span currently active on this async execution path, or null outside any + * `withSpan` callback. + * @experimental Subject to change in a minor release. + */ + getActiveSpan(): Span | null + /** * @description Flushes the events still in the queue and clears the feature flags poller to allow for * a clean shutdown. diff --git a/packages/react-native/references/posthog-react-native-references-latest.json b/packages/react-native/references/posthog-react-native-references-latest.json index a1bbae980a..326f31634e 100644 --- a/packages/react-native/references/posthog-react-native-references-latest.json +++ b/packages/react-native/references/posthog-react-native-references-latest.json @@ -4453,6 +4453,13 @@ "path": "../core/src/metrics/types.ts", "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, + { + "id": "SendTracesBatchOutcome", + "name": "SendTracesBatchOutcome", + "properties": [], + "path": "../core/src/traces/types.ts", + "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" + }, { "id": "SeverityLevel", "name": "SeverityLevel", diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index d1a399c486..04e2974cce 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -128,3 +128,20 @@ export type { OtlpMetricsPayload, } from './capture-metric' export { OTLP_AGGREGATION_TEMPORALITY_DELTA } from './capture-metric' + +// Distributed tracing types +export type { + SpanKind, + SpanStatusCode, + SpanAttributeValue, + SpanAttributes, + SpanTimeInput, + StartSpanOptions, + Span, + TracesConfig, + OtlpSpanKeyValue, + OtlpSpanEvent, + OtlpSpanStatus, + OtlpSpan, + OtlpTracesPayload, +} from './traces' diff --git a/packages/types/src/traces.ts b/packages/types/src/traces.ts new file mode 100644 index 0000000000..65f2d79c34 --- /dev/null +++ b/packages/types/src/traces.ts @@ -0,0 +1,286 @@ +import type { OtlpKeyValue } from './capture-log' + +/** + * Types for the distributed tracing API (`startSpan` / `withSpan` / `getActiveSpan`). + * + * Spans are exported as OpenTelemetry-shaped OTLP records to PostHog's tracing + * endpoint. PostHog does not depend on the OpenTelemetry SDK — these types are the + * SDK-facing surface, and the OTLP integer enums stay a wire-level concern. + */ + +/** + * What kind of work a span represents. Mirrors the OpenTelemetry span kinds. + * + * @default 'internal' + * + * @experimental Subject to change in a minor release. + */ +export type SpanKind = 'internal' | 'server' | 'client' | 'producer' | 'consumer' + +/** + * Outcome of the operation a span covers. A span that never has a status set is + * exported as `unset`, which is not the same as `ok`. + * + * @experimental Subject to change in a minor release. + */ +export type SpanStatusCode = 'ok' | 'error' + +/** + * A value that can be attached to a span, span event, or resource. + * + * The ingestion service flattens attribute values to strings for storage, so + * primitives are strongly preferred — nested arrays and objects survive only as + * serialized strings and cannot be filtered on. `null` and `undefined` drop the key. + * + * @experimental Subject to change in a minor release. + */ +export type SpanAttributeValue = + | string + | number + | boolean + | bigint + | SpanAttributeValue[] + | { [key: string]: SpanAttributeValue } + | null + | undefined + +export type SpanAttributes = Record + +/** + * A point in time, as a millisecond epoch number or a `Date`. + * + * @experimental Subject to change in a minor release. + */ +export type SpanTimeInput = number | Date + +/** + * Options accepted by `startSpan` and `withSpan`. + * + * @experimental Subject to change in a minor release. + */ +export interface StartSpanOptions { + /** + * What kind of work the span represents. + * + * @default 'internal' + */ + kind?: SpanKind + + /** + * Attributes to set at span start. User-supplied keys win over the + * SDK's auto-attached context attributes. + */ + attributes?: SpanAttributes + + /** + * Parent of this span: either a span handle, or a raw W3C `traceparent` + * string to continue a trace started by another service. + * + * When omitted the parent is the currently active span, or none. Only + * handles returned by this SDK are honoured; any other `Span` yields an + * inert span. + * + * @example Continue an inbound trace + * ```ts + * posthog.withSpan('POST /checkout', { parent: req.get('traceparent') }, handler) + * ``` + */ + parent?: Span | string + + /** + * The W3C `tracestate` value accompanying a `traceparent`-string `parent`. + * Ignored when `parent` is a span handle — those inherit the parent's + * tracestate. Preserved opaquely and passed on to children. + */ + tracestate?: string + + /** + * Backdate the span's start. Values outside the representable range fall + * back to the current time; starts more than 24 hours old are warned about, + * because the server clamps them to receive time. + */ + startTime?: SpanTimeInput +} + +/** + * A handle to a span in progress. + * + * Every method is safe to call at any time, including after `end()` and on + * no-op handles, so calling code never has to branch on whether tracing is on. + * + * @experimental Subject to change in a minor release. + */ +export interface Span { + /** Set a single attribute. Ignored after `end()`. */ + setAttribute(key: string, value: SpanAttributeValue): this + + /** Merge several attributes at once. Ignored after `end()`. */ + setAttributes(attributes: SpanAttributes): this + + /** + * Record a timestamped event within the span, e.g. a cache miss or a retry. + * Defaults to the current time. + */ + addEvent(name: string, attributes?: SpanAttributes, timestamp?: SpanTimeInput): this + + /** + * Set the span's outcome. Last write wins. + * + * When a `withSpan` callback throws, the SDK sets `error` automatically — + * unless the callback already set `ok`, which is treated as final. + */ + setStatus(status: SpanStatusCode, message?: string): this + + /** + * Record an exception on the span: sets status `error` and attaches an + * `exception` event carrying `exception.type` and `exception.message`. + * Does not end the span. + */ + recordException(error: unknown): this + + /** + * Replace the span's name. Useful when the low-cardinality name is only + * known after work begins — a route template resolving mid-request, say. + * + * Span names should be low-cardinality operation names (`GET /users/:id`), + * never interpolated with ids: PostHog aggregates operations by service and + * name, so variable values belong in attributes. + */ + updateName(name: string): this + + /** + * This span's W3C `traceparent` header value (`00---01`), + * for propagating the trace to another service. Returns `null` on a no-op + * span, so an id that was never recorded cannot propagate. + */ + traceparent(): string | null + + /** This span's W3C `tracestate` value, or `null` when it has none. */ + tracestate(): string | null + + /** + * End the span and queue it for export. Idempotent — later calls no-op. + * + * @param endTime - Override the recorded end. Invalid values fall back to + * the derived end time; an end before the start is corrected to the start. + */ + end(endTime?: SpanTimeInput): void +} + +/** + * Configuration for distributed tracing, passed as the `traces` client option. + * Tracing stays off until this object is supplied. + * + * @example + * ```ts + * const posthog = new PostHog('phc_...', { traces: { serviceName: 'checkout-api' } }) + * ``` + * + * @experimental Subject to change in a minor release. + */ +export interface TracesConfig { + /** + * Name of the service producing these spans, attached as the OTLP + * `service.name` resource attribute. PostHog groups operations by service + * and span name, so this is what makes spans attributable. + * + * @default 'unknown_service' + */ + serviceName?: string + + /** Service version, attached as OTLP `service.version`. */ + serviceVersion?: string + + /** + * Deployment environment (e.g. `'production'`, `'staging'`), attached as + * OTLP `deployment.environment`. + */ + environment?: string + + /** + * Extra OTLP resource attributes attached to every batch. + * + * A string `service.name`, `service.version` or `deployment.environment` here + * wins over the `serviceName` / `serviceVersion` / `environment` fields — set + * it either way. `telemetry.sdk.*` is SDK-controlled and always wins. + */ + resourceAttributes?: SpanAttributes + + /** + * How often queued spans are flushed, in milliseconds. Spans also flush when + * the queue reaches `maxExportBatchSize` and on `shutdown()`. + * + * @default 5000 + */ + flushIntervalMs?: number + + /** + * Maximum spans per outbound request, and the queue depth that triggers an + * immediate flush. On a 413 the SDK halves this, retries the same spans, then + * ramps back up. + * + * @default 512 + */ + maxExportBatchSize?: number + + /** + * Bound on the in-memory export queue. When it is full the incoming span is + * dropped rather than evicting a queued one, whose children may already have + * been exported. Never lower than `maxExportBatchSize`. + * + * @default 2048 + */ + maxQueueSize?: number +} + +// ============================================================================ +// OTLP wire types +// +// `AnyValue` and `KeyValue` are the same shapes the logs and metrics payloads +// use, and one shared encoder produces all three, so spans alias them rather +// than redeclaring them. The alias names remain so the span types below read as +// span types. +// ============================================================================ + +export type OtlpSpanKeyValue = OtlpKeyValue + +export interface OtlpSpanEvent { + name: string + timeUnixNano: string + attributes?: OtlpSpanKeyValue[] +} + +export interface OtlpSpanStatus { + /** unset 0, ok 1, error 2. */ + code: number + message?: string +} + +export interface OtlpSpan { + /** 32-char lowercase hex. */ + traceId: string + /** 16-char lowercase hex. */ + spanId: string + parentSpanId?: string + traceState?: string + name: string + /** unspecified 0, internal 1, server 2, client 3, producer 4, consumer 5. */ + kind: number + startTimeUnixNano: string + endTimeUnixNano: string + attributes?: OtlpSpanKeyValue[] + events?: OtlpSpanEvent[] + status?: OtlpSpanStatus + /** W3C trace flags in the low byte; the sampled bit is always set. */ + flags?: number +} + +export interface OtlpTracesPayload { + resourceSpans: Array<{ + resource: { attributes: OtlpSpanKeyValue[] } + scopeSpans: Array<{ + scope: { name: string; version?: string } + spans: OtlpSpan[] + }> + }> +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2ce37e96e6..8feff2acfa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -838,6 +838,9 @@ importers: '@posthog-tooling/tsconfig-base': specifier: workspace:* version: link:../../tooling/tsconfig-base + '@posthog/types': + specifier: workspace:^ + version: link:../types '@rslib/core': specifier: 'catalog:' version: 0.10.6(@microsoft/api-extractor@7.58.9(@types/node@20.19.9))(typescript@5.9.3) From 5f7b2ffc9d16801dd23de22c97d3caf333f97123 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Mon, 31 Aug 2026 21:51:23 -0400 Subject: [PATCH 02/24] docs(node): regenerate the references for the withSpan doc comment --- packages/node/references/posthog-node-references-latest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node/references/posthog-node-references-latest.json b/packages/node/references/posthog-node-references-latest.json index 8b8c6ecf76..58f79d4ad4 100644 --- a/packages/node/references/posthog-node-references-latest.json +++ b/packages/node/references/posthog-node-references-latest.json @@ -1448,7 +1448,7 @@ }, { "category": "Traces", - "description": "Runs a callback with a span active for its duration and ends the span for you — at return for a sync callback, at settle for an async one.\nSpans started inside the callback nest under it automatically. If the callback throws or rejects, the span records the exception and the original error is rethrown unchanged.\nSpans nest across `await` only on the Node runtime, which tracks the active span with `AsyncLocalStorage`. The edge build restores the active span when the callback returns its promise, so spans started after an `await` there begin a new trace.\n\nSubject to change in a minor release.", + "description": "Runs a callback with a span active for its duration and ends the span for you — at return for a sync callback, at settle for an async one. Takes an optional `StartSpanOptions` between the name and the callback: `withSpan(name, fn)` or `withSpan(name, options, fn)`.\nSpans started inside the callback nest under it automatically. If the callback throws or rejects, the span records the exception and the original error is rethrown unchanged. A callback that ends the span itself gets the rethrow but not the recording, since the span is already exported by then.\nSpans nest across `await` only on the Node runtime, which tracks the active span with `AsyncLocalStorage`. The edge build restores the active span when the callback returns its promise, so spans started after an `await` there begin a new trace.\n\nSubject to change in a minor release.", "details": null, "id": "withSpan", "showDocs": true, From 9ca920e8dff7e04c4af780eb4ddf41854078ee4f Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 1 Sep 2026 11:11:28 -0400 Subject: [PATCH 03/24] feat(node): bound live spans by count and age Adds maxLiveSpans and maxSpanAgeMs. Live accounting keeps a span id and a monotonic timestamp, never the span, so leaked handles stay collectable. --- .changeset/node-distributed-tracing.md | 2 + packages/core/src/traces/index.spec.ts | 64 +++++++++++++++++++ packages/core/src/traces/index.ts | 57 ++++++++++++++++- packages/core/src/traces/live-spans.spec.ts | 7 +- packages/core/src/traces/otlp.spec.ts | 2 + packages/core/src/traces/span.ts | 2 +- packages/core/src/traces/types.ts | 4 ++ .../src/__tests__/traces-defaults.spec.ts | 16 +++++ packages/node/src/traces-defaults.ts | 11 ++++ packages/types/src/traces.ts | 22 +++++++ 10 files changed, 182 insertions(+), 5 deletions(-) diff --git a/.changeset/node-distributed-tracing.md b/.changeset/node-distributed-tracing.md index 52aef9f86d..d0ddc4376a 100644 --- a/.changeset/node-distributed-tracing.md +++ b/.changeset/node-distributed-tracing.md @@ -6,4 +6,6 @@ Add distributed tracing to `posthog-node` — experimental. `withSpan`, `startSpan` and `getActiveSpan` record spans against a new `traces` client option; spans started inside a request context carry the distinct ID and session ID, and `parent` / `span.traceparent()` continue a W3C trace across services. +Code that starts spans and never ends them cannot grow the SDK's bookkeeping without limit: `traces.maxLiveSpans` (default 10000) caps how many spans may be open at once, and `traces.maxSpanAgeMs` (default one hour) stops accounting for one that stays open longer than that. `startSpan` returns an inert handle at the cap, and both kinds of drop are reported through the existing span-drop warning. + `IPostHog` gains these three members, so anything implementing that interface (hand-written test doubles, DI wrappers) needs them added, or can extend `PostHogBackendClient` instead. diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index cc5d2f21fe..3a9cd68c22 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -18,6 +18,8 @@ const resolveForTest = (partial?: Partial): ResolvedTraces flushIntervalMs: 5000, maxExportBatchSize: 512, maxQueueSize: 2048, + maxLiveSpans: 10000, + maxSpanAgeMs: 3600000, ...partial, }) @@ -1180,6 +1182,68 @@ describe('PostHogTraces', () => { }) }) + describe('live span bounds', () => { + it('returns an inert handle once maxLiveSpans spans are live', async () => { + const traces = createTraces({ maxLiveSpans: 2 }) + + traces.startSpan('live-a') + traces.startSpan('live-b') + const refused = traces.startSpan('refused') + refused.end() + await traces.flush() + + expect(sentSpans()).toHaveLength(0) + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('the live-span limit (2) was reached')) + }) + + it('frees the slot when a span ends', async () => { + const traces = createTraces({ maxLiveSpans: 1 }) + + traces.startSpan('first').end() + traces.startSpan('second').end() + await traces.flush() + + expect(sentSpans().map((s) => s.name)).toEqual(['first', 'second']) + }) + + it('never exports a span evicted for exceeding maxSpanAgeMs', async () => { + const traces = createTraces({ maxSpanAgeMs: 60_000 }) + const leaked = traces.startSpan('leaked') + + await jest.advanceTimersByTimeAsync(61_000) + // Eviction is lazy: the next startSpan sweeps. + traces.startSpan('later').end() + leaked.end() + await traces.flush() + + expect(sentSpans().map((s) => s.name)).toEqual(['later']) + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('still live after 60000ms')) + }) + + it('returns the slot on age eviction so a leak cannot disable tracing', async () => { + const traces = createTraces({ maxLiveSpans: 1, maxSpanAgeMs: 60_000 }) + traces.startSpan('leaked-forever') + + await jest.advanceTimersByTimeAsync(61_000) + traces.startSpan('after-the-leak').end() + await traces.flush() + + expect(sentSpans().map((s) => s.name)).toEqual(['after-the-leak']) + }) + + it('ages from startSpan, not from a caller-supplied startTime', async () => { + const traces = createTraces({ maxSpanAgeMs: 60_000 }) + // Backdated an hour: aging off the supplied time would evict it immediately. + const backdated = traces.startSpan('backdated', { startTime: Date.now() - 3_600_000 }) + + traces.startSpan('sweep-trigger').end() + backdated.end() + await traces.flush() + + expect(sentSpans().map((s) => s.name)).toEqual(['sweep-trigger', 'backdated']) + }) + }) + describe('reset', () => { it('abandons an in-flight pass instead of splicing spans it never sent', async () => { let release!: (outcome: SendTracesBatchOutcome) => void diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 0c837a5d75..79513d1b0b 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -8,7 +8,7 @@ import type { TraceSdkContext, TracesHost, } from './types' -import { NOOP_SPAN, PostHogSpan, describeError } from './span' +import { NOOP_SPAN, PostHogSpan, describeError, monotonicNow } from './span' import { newSpanId, newTraceId } from './ids' import { parseTraceparent, sanitizeTracestate } from './traceparent' import { assignUserAttributes, resolveStartTime, sanitizeName } from './sanitize' @@ -27,6 +27,11 @@ const MAX_FLUSH_BACKOFF_MS = 30_000 type SpanCallback = (span: Span) => T +/** Monotonic where the platform has one, wall clock otherwise. Both are ms, and a platform never switches. */ +function clockNow(): number { + return monotonicNow() ?? Date.now() +} + /** `instanceof` and property access both throw on a hostile proxy; `startSpan` must not. */ function isOwnSpan(value: unknown): value is PostHogSpan { try { @@ -76,6 +81,11 @@ export class PostHogTraces { private _headBatchSize = 0 // Bumped by reset(); a pass whose generation is stale abandons the queue. private _generation = 0 + // Live-span accounting: span id -> monotonic start. Ids and numbers only, + // never the span itself, so a handle the caller drops is still collectable + // and the bound can be generous. Insertion order is start order, so the + // oldest entries are at the front and eviction stops at the first live one. + private _liveSpans = new Map() constructor( private readonly _instance: TracesHost, @@ -112,13 +122,28 @@ export class PostHogTraces { const parent = this._resolveParent(options) + // Swept before the bound is read, so a process that has leaked its way to + // the bound recovers on the first `startSpan` after the leaks age out. + this._evictAgedSpans() + if (this._liveSpans.size >= this._config.maxLiveSpans) { + this._recordDrop( + 1, + `the live-span limit (${this._config.maxLiveSpans}) was reached — spans are being started and never ended` + ) + return NOOP_SPAN + } + const now = Date.now() const startTime = resolveStartTime(options?.startTime, now, this._logger) + const spanId = newSpanId() + // Read here rather than from the span: age is elapsed time since this call, + // so a backdated `startTime` neither ages a span early nor exempts it. + this._liveSpans.set(spanId, clockNow()) return new PostHogSpan( { traceId: parent?.traceId ?? newTraceId(), - spanId: newSpanId(), + spanId, parentSpanId: parent?.parentSpanId, traceState: parent?.traceState, name: sanitizeName(name, 'Span name', this._logger), @@ -223,6 +248,7 @@ export class PostHogTraces { reset(): void { this._clearFlushTimer() this._queue = [] + this._liveSpans.clear() this._flushPromise = null // Abandons any in-flight pass, which would otherwise splice out spans it never sent. this._generation++ @@ -311,7 +337,34 @@ export class PostHogTraces { } } + /** + * Drops live accounting for spans older than `maxSpanAgeMs`. An evicted span + * is never exported — its `end()` finds no entry — so one leak returns its + * slot instead of disabling tracing for the rest of the process. + */ + private _evictAgedSpans(): void { + const cutoff = clockNow() - this._config.maxSpanAgeMs + let evicted = 0 + for (const [spanId, startedAt] of this._liveSpans) { + // Insertion order is start order, so the first entry inside the bound ends the sweep. + if (startedAt > cutoff) { + break + } + this._liveSpans.delete(spanId) + evicted++ + } + if (evicted) { + this._recordDrop(evicted, `they were still live after ${this._config.maxSpanAgeMs}ms`) + } + } + private _onSpanEnd(record: SpanRecord): void { + // Deleted before any other gate, so an opted-out span still returns its slot. + if (!this._liveSpans.delete(record.spanId)) { + // Evicted for age while live: never exported, and already counted as a drop. + return + } + // Re-checked at end: opting out mid-trace must stop the span exporting. if (this._instance.isDisabled || this._instance.optedOut) { return diff --git a/packages/core/src/traces/live-spans.spec.ts b/packages/core/src/traces/live-spans.spec.ts index 64e6efa7dc..d4b780e7fe 100644 --- a/packages/core/src/traces/live-spans.spec.ts +++ b/packages/core/src/traces/live-spans.spec.ts @@ -3,8 +3,9 @@ import { SyncSpanContextManager } from './context' import type { ResolvedTracesConfig } from './types' import type { Logger } from '../types' -// The pipeline holds no reference to a span until that span ends, so a handle the -// caller drops is collectable like any other object. +// Live-span accounting keeps an id and a timestamp per span, never the span, so a +// handle the caller drops stays collectable. This probe is what stops a later +// change from turning that accounting into a registry of span objects. const gc = (globalThis as { gc?: () => void }).gc // `--expose-gc` is set by the `test:unit` script. A runner that invokes jest @@ -17,6 +18,8 @@ describe('live spans', () => { flushIntervalMs: 5000, maxExportBatchSize: 512, maxQueueSize: 2048, + maxLiveSpans: 10000, + maxSpanAgeMs: 3600000, } const createTraces = (): PostHogTraces => diff --git a/packages/core/src/traces/otlp.spec.ts b/packages/core/src/traces/otlp.spec.ts index a728cd12ac..798cf33242 100644 --- a/packages/core/src/traces/otlp.spec.ts +++ b/packages/core/src/traces/otlp.spec.ts @@ -118,6 +118,8 @@ describe('OTLP span encoding', () => { flushIntervalMs: 5000, maxExportBatchSize: 512, maxQueueSize: 2048, + maxLiveSpans: 10000, + maxSpanAgeMs: 3600000, ...partial, }) diff --git a/packages/core/src/traces/span.ts b/packages/core/src/traces/span.ts index dc2c336d61..d104920b36 100644 --- a/packages/core/src/traces/span.ts +++ b/packages/core/src/traces/span.ts @@ -9,7 +9,7 @@ import { isError } from '../utils' * A monotonic millisecond reading where the platform has one, so an NTP * correction mid-span can't produce a negative duration. */ -function monotonicNow(): number | undefined { +export function monotonicNow(): number | undefined { const perf = (globalThis as { performance?: { now?: () => number } }).performance return typeof perf?.now === 'function' ? perf.now() : undefined } diff --git a/packages/core/src/traces/types.ts b/packages/core/src/traces/types.ts index fb330fa760..6551d0bcac 100644 --- a/packages/core/src/traces/types.ts +++ b/packages/core/src/traces/types.ts @@ -98,4 +98,8 @@ export interface ResolvedTracesConfig extends TracesConfig { * dropped rather than queued ones, whose children may already have shipped. */ maxQueueSize: number + /** Bound on spans started but not yet ended. At the bound `startSpan` returns a no-op handle. */ + maxLiveSpans: number + /** How long a span may stay live before it stops being accounted for and can never export. */ + maxSpanAgeMs: number } diff --git a/packages/node/src/__tests__/traces-defaults.spec.ts b/packages/node/src/__tests__/traces-defaults.spec.ts index c8b7b5a225..9c6e2a8770 100644 --- a/packages/node/src/__tests__/traces-defaults.spec.ts +++ b/packages/node/src/__tests__/traces-defaults.spec.ts @@ -6,6 +6,22 @@ describe('resolveTracesConfig', () => { flushIntervalMs: 5000, maxExportBatchSize: 512, maxQueueSize: 2048, + maxLiveSpans: 10_000, + maxSpanAgeMs: 3_600_000, + }) + }) + + it('honours explicit live-span bounds', () => { + expect(resolveTracesConfig({ maxLiveSpans: 50, maxSpanAgeMs: 30_000 })).toMatchObject({ + maxLiveSpans: 50, + maxSpanAgeMs: 30_000, + }) + }) + + it.each([0, -1, Number.NaN])('falls back to the defaults for unusable live-span bounds (%p)', (value) => { + expect(resolveTracesConfig({ maxLiveSpans: value, maxSpanAgeMs: value })).toMatchObject({ + maxLiveSpans: 10_000, + maxSpanAgeMs: 3_600_000, }) }) diff --git a/packages/node/src/traces-defaults.ts b/packages/node/src/traces-defaults.ts index 2e1be86c67..9d96d3be5a 100644 --- a/packages/node/src/traces-defaults.ts +++ b/packages/node/src/traces-defaults.ts @@ -6,6 +6,15 @@ const DEFAULT_FLUSH_INTERVAL_MS = 5000 const DEFAULT_MAX_EXPORT_BATCH_SIZE = 512 const DEFAULT_MAX_QUEUE_SIZE = 2048 +// Live-span bounds. A server can legitimately hold thousands of spans open at +// once, and refusing a legitimate span is worse than tolerating a leak, so the +// count bound sits well above realistic concurrency — affordable because live +// accounting is an id and a timestamp per span, not the span. The age bound is +// an hour: production traces routinely run past ten minutes, and a span still +// open after an hour is a leak rather than slow work. +const DEFAULT_MAX_LIVE_SPANS = 10_000 +const DEFAULT_MAX_SPAN_AGE_MS = 3_600_000 + /** * Coerces a caller-supplied positive-integer option. `0`, a negative, or `NaN` * reaching the export loop would stall it. @@ -60,5 +69,7 @@ export function resolveTracesConfig(config: TracesConfig | undefined): ResolvedT maxExportBatchSize, // Never below the flush trigger, or the depth-based flush could never fire. maxQueueSize: Math.max(positiveInteger(config?.maxQueueSize, DEFAULT_MAX_QUEUE_SIZE), maxExportBatchSize), + maxLiveSpans: positiveInteger(config?.maxLiveSpans, DEFAULT_MAX_LIVE_SPANS), + maxSpanAgeMs: positiveInteger(config?.maxSpanAgeMs, DEFAULT_MAX_SPAN_AGE_MS), } } diff --git a/packages/types/src/traces.ts b/packages/types/src/traces.ts index 65f2d79c34..000e1171eb 100644 --- a/packages/types/src/traces.ts +++ b/packages/types/src/traces.ts @@ -231,6 +231,28 @@ export interface TracesConfig { * @default 2048 */ maxQueueSize?: number + + /** + * Bound on how many spans may be live (started but not ended) at once. At + * the bound `startSpan` returns an inert handle, so code that leaks spans + * cannot grow the SDK's bookkeeping without limit. The SDK tracks only an + * id and a timestamp per live span, never the span itself, so a high bound + * is inexpensive. + * + * @default 10000 + */ + maxLiveSpans?: number + + /** + * How long a span may stay live before the SDK stops accounting for it, in + * milliseconds. An evicted span is never exported, and its slot is returned + * so one leak cannot disable tracing for the rest of the process. Measured + * as monotonic elapsed time since `startSpan`, so a caller-supplied + * `startTime` neither ages a span early nor exempts it. + * + * @default 3600000 + */ + maxSpanAgeMs?: number } // ============================================================================ From bf5e7af75ed0bcee2c2284c9aa9602bb95022847 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 1 Sep 2026 12:18:27 -0400 Subject: [PATCH 04/24] fix(traces): pass an inbound trace through when tracing is off An inert span started with a `parent` header now echoes that header from `traceparent()` and is activated by `withSpan`, so a service that records nothing keeps a distributed trace whole instead of severing it. --- packages/core/src/index.ts | 2 +- packages/core/src/traces/index.spec.ts | 54 +++++++++++++++++++ packages/core/src/traces/index.ts | 17 +++--- packages/core/src/traces/span.ts | 42 +++++++++++++-- packages/core/src/traces/traceparent.spec.ts | 24 ++++++++- packages/core/src/traces/traceparent.ts | 26 ++++++++- .../posthog-node-references-latest.json | 2 +- packages/node/src/__tests__/traces.spec.ts | 29 ++++++++++ packages/node/src/client.ts | 31 +++++++++-- packages/types/src/traces.ts | 9 +++- 10 files changed, 214 insertions(+), 22 deletions(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d8045dc87d..e388cd6ddf 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -89,7 +89,7 @@ export type { } from './metrics/types' export { PostHogTraces } from './traces' export { SyncSpanContextManager } from './traces/context' -export { NOOP_SPAN } from './traces/span' +export { NOOP_SPAN, inertSpan } from './traces/span' export type { ResolvedTracesConfig, SpanContextManager, TraceSdkContext } from './traces/types' // Same barrel convention as logs and metrics for the user-facing tracing types. export type { diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 3a9cd68c22..4afa06a267 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -407,6 +407,60 @@ describe('PostHogTraces', () => { }) }) + // A service in the middle of a traced chain must not sever it just because it + // has no tracing of its own — OTel requires the API to carry the parent + // context through when no SDK is recording. + describe('trace context pass-through when tracing is off', () => { + const INBOUND_UNSAMPLED = `00-${TRACE_ID}-${REMOTE_SPAN_ID}-00` + + it('echoes an inbound traceparent, flags included, when the SDK is disabled', async () => { + const instance = createMockInstance({ isDisabled: true }) + const traces = createTraces({}, instance) + + const span = traces.startSpan('proxied', { parent: INBOUND_UNSAMPLED, tracestate: 'vendor=abc' }) + span.end() + await traces.flush() + + expect(span.traceparent()).toBe(INBOUND_UNSAMPLED) + expect(span.tracestate()).toBe('vendor=abc') + expect(sentSpans(instance)).toHaveLength(0) + }) + + it('echoes an inbound traceparent when the user has opted out', () => { + const traces = createTraces({}, createMockInstance({ optedOut: true })) + const span = traces.startSpan('proxied', { parent: `00-${TRACE_ID}-${REMOTE_SPAN_ID}-01` }) + + expect(span.traceparent()).toBe(`00-${TRACE_ID}-${REMOTE_SPAN_ID}-01`) + }) + + it('activates the pass-through handle so getActiveSpan can propagate it', () => { + const traces = createTraces({}, createMockInstance({ optedOut: true })) + + const propagated = traces.withSpan('proxied', { parent: INBOUND_UNSAMPLED }, () => + traces.getActiveSpan()?.traceparent() + ) + + expect(propagated).toBe(INBOUND_UNSAMPLED) + expect(traces.getActiveSpan()).toBeNull() + }) + + it('passes the inbound context through when the live-span limit refuses the span', () => { + const traces = createTraces({ maxLiveSpans: 1 }) + traces.startSpan('holds-the-only-slot') + + const refused = traces.startSpan('refused', { parent: INBOUND_UNSAMPLED }) + + expect(refused.traceparent()).toBe(INBOUND_UNSAMPLED) + }) + + it('has nothing to propagate without a usable parent', () => { + const traces = createTraces({}, createMockInstance({ optedOut: true })) + + expect(traces.startSpan('no-parent')).toBe(NOOP_SPAN) + expect(traces.startSpan('bad-parent', { parent: 'not-a-traceparent' })).toBe(NOOP_SPAN) + }) + }) + describe('auto-context', () => { it('attaches the distinct id and session id as the product join keys', async () => { context = { distinctId: 'user-123', sessionId: 'session-123' } diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 79513d1b0b..1246047948 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -8,7 +8,7 @@ import type { TraceSdkContext, TracesHost, } from './types' -import { NOOP_SPAN, PostHogSpan, describeError, monotonicNow } from './span' +import { NOOP_SPAN, PostHogSpan, describeError, inertSpan, monotonicNow } from './span' import { newSpanId, newTraceId } from './ids' import { parseTraceparent, sanitizeTracestate } from './traceparent' import { assignUserAttributes, resolveStartTime, sanitizeName } from './sanitize' @@ -105,7 +105,7 @@ export class PostHogTraces { */ startSpan(name: string, options?: StartSpanOptions): Span { if (this._instance.isDisabled || this._instance.optedOut) { - return NOOP_SPAN + return inertSpan(options) } const explicitParent = options?.parent @@ -115,8 +115,9 @@ export class PostHogTraces { this._logger.debug('Span parent is not a span from this SDK; returning an inert span') return NOOP_SPAN } - // Not a span at all — `req.headers.traceparent` is `string[]` when the header - // arrives twice. Ignored: falls back to the active span, or to a new trace. + // No `traceparent()` to read: `req.headers.traceparent` is `string[]` when the + // header arrives twice, and a span from another tracer exposes `spanContext()` + // instead. Ignored: falls back to the active span, or to a new trace. this._logger.debug('Ignoring an unusable span parent') } @@ -130,7 +131,7 @@ export class PostHogTraces { 1, `the live-span limit (${this._config.maxLiveSpans}) was reached — spans are being started and never ended` ) - return NOOP_SPAN + return inertSpan(options) } const now = Date.now() @@ -174,8 +175,10 @@ export class PostHogTraces { const span = this.startSpan(name, options) try { - // A no-op span is never activated, so `getActiveSpan()` inside the - // callback reads null — callbacks should use the handle they're given. + // The shared no-op is never activated, so `getActiveSpan()` inside the + // callback reads null — callbacks should use the handle they're given. A + // pass-through handle is activated, so `getActiveSpan()?.traceparent()` + // still propagates an inbound trace through a service with tracing off. const result = span === NOOP_SPAN ? fn(span) : this._contextManager.with(span, () => fn(span)) if (isPromise(result)) { diff --git a/packages/core/src/traces/span.ts b/packages/core/src/traces/span.ts index d104920b36..92828f7037 100644 --- a/packages/core/src/traces/span.ts +++ b/packages/core/src/traces/span.ts @@ -1,7 +1,7 @@ import type { Span, SpanAttributes, SpanAttributeValue, SpanKind, SpanStatusCode, SpanTimeInput } from '@posthog/types' import type { Logger } from '../types' import type { SpanEventRecord, SpanRecord } from './types' -import { formatTraceparent } from './traceparent' +import { formatTraceparent, normalizeTraceparent, sanitizeTracestate } from './traceparent' import { assignUserAttributes, clampEndTime, resolveSuppliedTime, sanitizeName } from './sanitize' import { isError } from '../utils' @@ -188,8 +188,8 @@ export class PostHogSpan implements Span { /** * An inert handle returned whenever tracing cannot run — traces unconfigured, * SDK disabled, user opted out. Supports the full surface so caller code never - * branches, is never activated, and returns `null` from `traceparent()` so an id - * that was never recorded cannot propagate. + * branches, and returns `null` from `traceparent()` so an id this SDK never + * recorded cannot propagate. */ export class NoopSpan implements Span { setAttribute(): this { @@ -223,6 +223,42 @@ export class NoopSpan implements Span { // concrete type would reject calls the interface allows. export const NOOP_SPAN: Span = /* @__PURE__ */ new NoopSpan() +/** + * An inert handle that carries an inbound trace context. Records nothing, and + * echoes the `traceparent` it was handed — including the caller's version and + * sampled flag — so a service with tracing off still forwards the trace it + * received rather than severing it. The ids it propagates are the upstream + * caller's own; this SDK invents none. + */ +export class PassThroughSpan extends NoopSpan { + constructor( + private readonly _traceparent: string, + private readonly _tracestate?: string + ) { + super() + } + + override traceparent(): string { + return this._traceparent + } + + override tracestate(): string | null { + return this._tracestate ?? null + } +} + +/** + * The handle to return when a span cannot be recorded: a pass-through when the + * caller supplied a usable `parent` header, the shared no-op otherwise. + */ +export function inertSpan(options?: { parent?: unknown; tracestate?: unknown }): Span { + const traceparent = normalizeTraceparent(options?.parent) + if (!traceparent) { + return NOOP_SPAN + } + return new PassThroughSpan(traceparent, sanitizeTracestate(options?.tracestate)) +} + /** * Extracts the OTel `exception.type` / `exception.message` pair from whatever was * thrown. Anything can be thrown in JS, so non-Errors are described by type. diff --git a/packages/core/src/traces/traceparent.spec.ts b/packages/core/src/traces/traceparent.spec.ts index 589ce273de..92901402c8 100644 --- a/packages/core/src/traces/traceparent.spec.ts +++ b/packages/core/src/traces/traceparent.spec.ts @@ -1,4 +1,4 @@ -import { formatTraceparent, parseTraceparent, sanitizeTracestate } from './traceparent' +import { formatTraceparent, normalizeTraceparent, parseTraceparent, sanitizeTracestate } from './traceparent' const TRACE_ID = '4bf92f3577b34da6a3ce929d0e0e4736' const SPAN_ID = '00f067aa0ba902b7' @@ -94,3 +94,25 @@ describe('tracestate character safety', () => { ) }) }) + +describe('normalizeTraceparent', () => { + it('carries version and flags through as received', () => { + expect(normalizeTraceparent(`00-${TRACE_ID}-${SPAN_ID}-00`)).toBe(`00-${TRACE_ID}-${SPAN_ID}-00`) + expect(normalizeTraceparent(`01-${TRACE_ID}-${SPAN_ID}-01`)).toBe(`01-${TRACE_ID}-${SPAN_ID}-01`) + }) + + it('canonicalises whitespace and case, and drops unknown trailing fields', () => { + expect(normalizeTraceparent(` 00-${TRACE_ID.toUpperCase()}-${SPAN_ID}-01-extra `)).toBe( + `00-${TRACE_ID}-${SPAN_ID}-01` + ) + }) + + it.each([ + ['a malformed header', 'not-a-traceparent'], + ['the invalid ff version', `ff-${TRACE_ID}-${SPAN_ID}-01`], + ['an all-zero trace id', `00-${'0'.repeat(32)}-${SPAN_ID}-01`], + ['a non-string', ['a', 'b']], + ])('rejects %s', (_name, value) => { + expect(normalizeTraceparent(value)).toBeUndefined() + }) +}) diff --git a/packages/core/src/traces/traceparent.ts b/packages/core/src/traces/traceparent.ts index 22466e74ed..b09bd436d6 100644 --- a/packages/core/src/traces/traceparent.ts +++ b/packages/core/src/traces/traceparent.ts @@ -18,6 +18,18 @@ const TRACEPARENT_RE = /^([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2 * and dropping the parentage would orphan our own spans. */ export function parseTraceparent(value: unknown): RemoteSpanContext | undefined { + const fields = matchTraceparent(value) + return fields && { traceId: fields.traceId, spanId: fields.spanId } +} + +interface TraceparentFields { + version: string + traceId: string + spanId: string + flags: string +} + +function matchTraceparent(value: unknown): TraceparentFields | undefined { if (typeof value !== 'string') { return undefined } @@ -25,14 +37,24 @@ export function parseTraceparent(value: unknown): RemoteSpanContext | undefined if (!match) { return undefined } - const [, version, traceId, spanId] = match + const [, version, traceId, spanId, flags] = match if (version === 'ff') { return undefined } if (!isValidTraceId(traceId) || !isValidSpanId(spanId)) { return undefined } - return { traceId, spanId } + return { version, traceId, spanId, flags } +} + +/** + * The canonical form of an inbound `traceparent`, or `undefined` when it is + * malformed. Version and flags are carried through as received, so a service + * that forwards this value continues the caller's trace exactly as sent. + */ +export function normalizeTraceparent(value: unknown): string | undefined { + const fields = matchTraceparent(value) + return fields && `${fields.version}-${fields.traceId}-${fields.spanId}-${fields.flags}` } /** diff --git a/packages/node/references/posthog-node-references-latest.json b/packages/node/references/posthog-node-references-latest.json index 58f79d4ad4..bf317fcc87 100644 --- a/packages/node/references/posthog-node-references-latest.json +++ b/packages/node/references/posthog-node-references-latest.json @@ -1303,7 +1303,7 @@ }, { "category": "Traces", - "description": "Starts a span without making it active — for work that can't wrap a callback. Prefer `withSpan`, which ends the span for you.\nAlways returns a handle, so calling code never has to branch: when the `traces` option is absent, the SDK is disabled, or the user has opted out, the handle is inert and nothing is exported.\n\nSubject to change in a minor release.", + "description": "Starts a span without making it active — for work that can't wrap a callback. Prefer `withSpan`, which ends the span for you.\nAlways returns a handle, so calling code never has to branch: when the `traces` option is absent, the SDK is disabled, or the user has opted out, the handle is inert and nothing is exported. An inert handle given a `parent` header still returns it from `traceparent()`, so a service with tracing off passes a distributed trace through instead of severing it.\n\nSubject to change in a minor release.", "details": null, "id": "startSpan", "showDocs": true, diff --git a/packages/node/src/__tests__/traces.spec.ts b/packages/node/src/__tests__/traces.spec.ts index 9f4723c6b2..8034c018dd 100644 --- a/packages/node/src/__tests__/traces.spec.ts +++ b/packages/node/src/__tests__/traces.spec.ts @@ -71,6 +71,35 @@ describe('PostHog traces', () => { expect(untraced.getActiveSpan()).toBeNull() await untraced.shutdown() }) + + it('passes an inbound trace through a service that has tracing off', async () => { + const untraced = createClient({ traces: undefined }) + const inbound = `00-${'4bf92f3577b34da6a3ce929d0e0e4736'}-00f067aa0ba902b7-00` + + const span = untraced.startSpan('proxied', { parent: inbound, tracestate: 'vendor=abc' }) + span.end() + await untraced.shutdown() + + // Echoed verbatim: the ids belong to the upstream caller, which recorded them. + expect(span.traceparent()).toBe(inbound) + expect(span.tracestate()).toBe('vendor=abc') + expect(traceRequests()).toHaveLength(0) + }) + + it('exposes a passed-through trace to getActiveSpan inside withSpan', async () => { + const untraced = createClient({ traces: undefined }) + const inbound = `00-${'4bf92f3577b34da6a3ce929d0e0e4736'}-00f067aa0ba902b7-01` + + const propagated = await untraced.withSpan('proxied', { parent: inbound }, async () => { + // After an await, so this also covers the AsyncLocalStorage path. + await Promise.resolve() + return untraced.getActiveSpan()?.traceparent() + }) + await untraced.shutdown() + + expect(propagated).toBe(inbound) + expect(untraced.getActiveSpan()).toBeNull() + }) }) describe('transport', () => { diff --git a/packages/node/src/client.ts b/packages/node/src/client.ts index 50a2950b72..7a9c251855 100644 --- a/packages/node/src/client.ts +++ b/packages/node/src/client.ts @@ -19,6 +19,7 @@ import { PostHogPersistedProperty, PostHogTraces, NOOP_SPAN, + inertSpan, Properties, resolveMetricsConfig, RetriableOptions, @@ -149,6 +150,7 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen protected readonly context?: IPostHogContext private _metrics?: PostHogMetrics private _traces?: PostHogTraces + private _spanContext?: SpanContextManager private readonly captureMode: CaptureMode private _v1Sender?: V1CaptureSender @@ -628,6 +630,18 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen return new SyncSpanContextManager() } + /** + * Active-span tracking, built on first use. Lives on the client rather than + * on the traces pipeline because a client with tracing off still activates a + * pass-through span, so it needs the same store the pipeline would use. + */ + private get _spanContextManager(): SpanContextManager { + if (!this._spanContext) { + this._spanContext = this.initializeSpanContextManager() + } + return this._spanContext + } + /** * The traces pipeline, built on first use. Returns `undefined` when the * `traces` client option is absent — tracing is off until configured. @@ -642,7 +656,7 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen resolveTracesConfig(this.options.traces), this._logger, () => this._tracingContext(), - this.initializeSpanContextManager(), + this._spanContextManager, // A handler that only traces still has to hold the invocation open. () => this.scheduleDebouncedFlush() ) @@ -666,7 +680,9 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen * * Always returns a handle, so calling code never has to branch: when the * `traces` option is absent, the SDK is disabled, or the user has opted out, - * the handle is inert and nothing is exported. + * the handle is inert and nothing is exported. An inert handle given a + * `parent` header still returns it from `traceparent()`, so a service with + * tracing off passes a distributed trace through instead of severing it. * * {@label Traces} * @@ -680,7 +696,7 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen * ``` */ startSpan(name: string, options?: StartSpanOptions): Span { - return this._tracesPipeline?.startSpan(name, options) ?? NOOP_SPAN + return this._tracesPipeline?.startSpan(name, options) ?? inertSpan(options) } /** @@ -720,7 +736,10 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen const pipeline = this._tracesPipeline if (!pipeline) { // Tracing off: still run the callback exactly once, with an inert handle. - return fn(NOOP_SPAN) + // A handle carrying an inbound `parent` is activated, so `getActiveSpan()` + // inside the callback can propagate the trace onward. + const span = inertSpan(options) + return span === NOOP_SPAN ? fn(span) : this._spanContextManager.with(span, () => fn(span)) } return options ? pipeline.withSpan(name, options, fn) : pipeline.withSpan(name, fn) } @@ -744,7 +763,9 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen * ``` */ getActiveSpan(): Span | null { - return this._tracesPipeline?.getActiveSpan() ?? null + // Read from the store directly rather than through the pipeline: the two + // share one manager, and with tracing off there is no pipeline to ask. + return this._spanContextManager.active() ?? null } /** diff --git a/packages/types/src/traces.ts b/packages/types/src/traces.ts index 000e1171eb..aedc80ee63 100644 --- a/packages/types/src/traces.ts +++ b/packages/types/src/traces.ts @@ -150,8 +150,13 @@ export interface Span { /** * This span's W3C `traceparent` header value (`00---01`), - * for propagating the trace to another service. Returns `null` on a no-op - * span, so an id that was never recorded cannot propagate. + * for propagating the trace to another service. + * + * When tracing is off, a span started with a `parent` header echoes that + * header back — version and sampled flag included — so a service that + * records nothing still keeps a distributed trace whole. With no `parent` + * there is no context to carry and this returns `null`, so an id this SDK + * never recorded cannot propagate. */ traceparent(): string | null From dc7aed3c39d37f4a7f4394cdaecca7d83afb88dc Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 1 Sep 2026 14:28:43 -0400 Subject: [PATCH 05/24] fix(traces): count spans dropped at the consent gates --- packages/core/src/traces/index.spec.ts | 25 +++++++++++++++++++++++++ packages/core/src/traces/index.ts | 3 +++ 2 files changed, 28 insertions(+) diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 4afa06a267..1af123fe3b 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -556,6 +556,18 @@ describe('PostHogTraces', () => { await traces.flush() expect(instance._sendTracesBatch).not.toHaveBeenCalled() }) + + it('counts a span dropped at the end-time gate', async () => { + const instance = createMockInstance() + const traces = createTraces({}, instance) + const span = traces.startSpan('checkout') + + instance.optedOut = true + span.end() + await traces.flush() + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('the user has opted out')) + }) }) describe('export', () => { @@ -931,6 +943,19 @@ describe('PostHogTraces', () => { expect(instance._sendTracesBatch).not.toHaveBeenCalled() }) + + it('counts spans discarded from the queue when consent is withdrawn', async () => { + const instance = createMockInstance() + const traces = createTraces({}, instance) + traces.startSpan('a').end() + traces.startSpan('b').end() + + instance.optedOut = true + await traces.flush() + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('2 span(s)')) + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('the user has opted out')) + }) }) describe('drop accounting', () => { diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 1246047948..0281556118 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -370,6 +370,7 @@ export class PostHogTraces { // Re-checked at end: opting out mid-trace must stop the span exporting. if (this._instance.isDisabled || this._instance.optedOut) { + this._recordDrop(1, 'the user has opted out') return } @@ -451,6 +452,8 @@ export class PostHogTraces { if (this._instance.isDisabled || this._instance.optedOut) { const discarded = this._queue.length this._queue = [] + this._recordDrop(discarded, 'the user has opted out') + this._warnAboutDrops() return discarded } From 92309f4e348586ea021ab5858fe53a5f0ec98c3f Mon Sep 17 00:00:00 2001 From: Anna Garcia <11654201+turnipdabeets@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:52:24 -0400 Subject: [PATCH 06/24] feat(node): emit os.name and os.version on spans (#4679) * feat(core): share the OTLP resource builder and emit os.name/os.version Logs, metrics and spans build resource attributes through one function instead of three copies. Node and browser now contribute the host OS, matching what react-native already sends. * fix(node): normalize os.name and guard the resourceAttributes merge * fix(traces): map the remaining node:os platforms in os.name Adds haiku, netbsd and cygwin so the table covers all eleven NodeJS.Platform values, and marks assignUserAttributes @internal. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DkmCQweddFe4pZLDMYqNoA --------- Co-authored-by: Claude Opus 5 (1M context) --- .changeset/otlp-os-resource-attributes.md | 6 +++ packages/core/src/index.ts | 1 + packages/core/src/traces/otlp.ts | 13 +---- packages/core/src/traces/sanitize.ts | 7 ++- packages/core/src/utils/otlp-resource.spec.ts | 25 +++++++--- packages/core/src/utils/otlp-resource.ts | 17 +++++-- packages/node/src/__tests__/host-os.spec.ts | 45 ++++++++++++++++++ .../src/__tests__/traces-defaults.spec.ts | 47 +++++++++++++++++++ packages/node/src/__tests__/traces.spec.ts | 26 +++++++++- packages/node/src/client.ts | 11 ++++- packages/node/src/entrypoints/index.node.ts | 5 ++ packages/node/src/host-os.node.ts | 22 +++++++++ packages/node/src/traces-defaults.ts | 16 +++++-- 13 files changed, 213 insertions(+), 28 deletions(-) create mode 100644 .changeset/otlp-os-resource-attributes.md create mode 100644 packages/node/src/__tests__/host-os.spec.ts create mode 100644 packages/node/src/host-os.node.ts diff --git a/.changeset/otlp-os-resource-attributes.md b/.changeset/otlp-os-resource-attributes.md new file mode 100644 index 0000000000..9743127a09 --- /dev/null +++ b/.changeset/otlp-os-resource-attributes.md @@ -0,0 +1,6 @@ +--- +'posthog-node': minor +'@posthog/core': patch +--- + +Add `os.name` and `os.version` resource attributes to the spans `posthog-node` sends, so traces can be filtered by platform. `os.name` is the human-readable name the other PostHog SDKs report (`macOS`, `Windows`, `Linux`) rather than the `node:os` identifier. Either key is omitted when the host cannot supply it, and `traces.resourceAttributes` still overrides both. diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e388cd6ddf..f8b93ec090 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -45,6 +45,7 @@ export { } from './logs/logs-utils' export { toOtlpAnyValue, toOtlpKeyValueList } from './utils/otlp-any-value' export { osResourceAttributes } from './utils/otlp-resource' +export { assignUserAttributes } from './traces/sanitize' export { PostHogLogs } from './logs' export type { BeforeSendLogFn, diff --git a/packages/core/src/traces/otlp.ts b/packages/core/src/traces/otlp.ts index a7fabf8368..75d66721c1 100644 --- a/packages/core/src/traces/otlp.ts +++ b/packages/core/src/traces/otlp.ts @@ -11,7 +11,7 @@ import type { Logger } from '../types' import type { ResolvedTracesConfig, SpanRecord } from './types' import { toOtlpKeyValueList } from '../utils/otlp-any-value' import { UNSERIALIZABLE_VALUE, sanitizeString } from '../utils/json-utils' -import { assignUserAttributes } from './sanitize' +import { buildOtlpResourceAttributes } from '../utils/otlp-resource' const SPAN_KIND_TO_OTLP: Record = { internal: 1, @@ -132,16 +132,7 @@ export function buildTracesResourceAttributes( sdkName: string, sdkVersion: string ): SpanAttributes { - return { - // Read through the shared guard: a throwing accessor here runs on every - // flush, before the pass's own error handling. - ...assignUserAttributes({}, config.resourceAttributes), - 'service.name': config.serviceName || 'unknown_service', - ...(config.environment && { 'deployment.environment': config.environment }), - ...(config.serviceVersion && { 'service.version': config.serviceVersion }), - 'telemetry.sdk.name': sdkName, - 'telemetry.sdk.version': sdkVersion, - } + return buildOtlpResourceAttributes(config, sdkName, sdkVersion) } /** diff --git a/packages/core/src/traces/sanitize.ts b/packages/core/src/traces/sanitize.ts index 1ba5b97bbd..47257deba7 100644 --- a/packages/core/src/traces/sanitize.ts +++ b/packages/core/src/traces/sanitize.ts @@ -114,8 +114,13 @@ export function resolveSuppliedTime( * * Read key by key rather than spread: a getter over a disposed resource or a * revoked proxy throws on the read itself, before the encoder's guards see it. + * + * @internal Exposed for cross-package use within this SDK; not part of the stable public API. */ -export function assignUserAttributes>(target: T, source: SpanAttributes | undefined): T { +export function assignUserAttributes>( + target: T, + source: Record | undefined +): T { if (!source) { return target } diff --git a/packages/core/src/utils/otlp-resource.spec.ts b/packages/core/src/utils/otlp-resource.spec.ts index a9b6d8891d..3ab1ea79de 100644 --- a/packages/core/src/utils/otlp-resource.spec.ts +++ b/packages/core/src/utils/otlp-resource.spec.ts @@ -2,6 +2,8 @@ import { buildResourceAttributes } from '../logs/logs-utils' import type { ResolvedPostHogLogsConfig } from '../logs/types' import { buildMetricsResourceAttributes } from '../metrics/metrics-utils' import type { ResolvedPostHogMetricsConfig } from '../metrics/types' +import { buildTracesResourceAttributes } from '../traces/otlp' +import type { ResolvedTracesConfig } from '../traces/types' import { normalizeOsName, osResourceAttributes } from './otlp-resource' const shared = { @@ -25,9 +27,10 @@ const conflicting = { }, } -const bothSignals = (partial: object): Record[] => [ +const allThree = (partial: object): Record[] => [ buildResourceAttributes(partial as ResolvedPostHogLogsConfig, 'posthog-node', '1.0.0'), buildMetricsResourceAttributes(partial as ResolvedPostHogMetricsConfig, 'posthog-node', '1.0.0'), + buildTracesResourceAttributes(partial as ResolvedTracesConfig, 'posthog-node', '1.0.0'), ] describe('shared OTLP resource attributes', () => { @@ -35,14 +38,16 @@ describe('shared OTLP resource attributes', () => { ['a fully populated config', shared], ['a config with conflicting user attributes', conflicting], ['an empty config', {}], - ])('produces the same attributes for logs and metrics given %s', (_label, config) => { - const [logs, metrics] = bothSignals(config) + ])('produces the same attributes for logs, metrics and traces given %s', (_label, config) => { + const [logs, metrics, traces] = allThree(config) expect(metrics).toEqual(logs) + expect(traces).toEqual(logs) expect(Object.keys(metrics)).toEqual(Object.keys(logs)) + expect(Object.keys(traces)).toEqual(Object.keys(logs)) }) it('layers the identity keys over user resource attributes', () => { - for (const attributes of bothSignals(conflicting)) { + for (const attributes of allThree(conflicting)) { expect(attributes).toEqual({ 'service.name': 'checkout', 'service.version': '2.1.0', @@ -55,7 +60,7 @@ describe('shared OTLP resource attributes', () => { }) it('keeps user resource attributes that do not collide', () => { - for (const attributes of bothSignals(shared)) { + for (const attributes of allThree(shared)) { expect(attributes).toEqual({ 'host.name': 'web-01', 'service.name': 'checkout', @@ -68,7 +73,7 @@ describe('shared OTLP resource attributes', () => { }) it('falls back to unknown_service and omits unset optional keys', () => { - for (const attributes of bothSignals({})) { + for (const attributes of allThree({})) { expect(attributes).toEqual({ 'service.name': 'unknown_service', 'telemetry.sdk.name': 'posthog-node', @@ -86,6 +91,12 @@ describe('osResourceAttributes', () => { ['linux', 'Linux'], ['android', 'Android'], ['freebsd', 'FreeBSD'], + ['openbsd', 'OpenBSD'], + ['netbsd', 'NetBSD'], + ['sunos', 'SunOS'], + ['aix', 'AIX'], + ['haiku', 'Haiku'], + ['cygwin', 'Windows'], // detectOS spellings ['Mac OS X', 'macOS'], ['iOS', 'iOS'], @@ -97,7 +108,7 @@ describe('osResourceAttributes', () => { }) it('passes an unmapped name through rather than dropping it', () => { - expect(normalizeOsName('Haiku')).toBe('Haiku') + expect(normalizeOsName('Plan 9')).toBe('Plan 9') expect(normalizeOsName('constructor')).toBe('constructor') }) diff --git a/packages/core/src/utils/otlp-resource.ts b/packages/core/src/utils/otlp-resource.ts index 0f19cbe19e..09ac1d9090 100644 --- a/packages/core/src/utils/otlp-resource.ts +++ b/packages/core/src/utils/otlp-resource.ts @@ -1,5 +1,7 @@ +import { assignUserAttributes } from '../traces/sanitize' + /** - * Shape the logs and metrics resolved configs share for resource + * Shape the logs, metrics and traces resolved configs share for resource * attribution. Generic over the attribute value type so each signal keeps its * own value union. */ @@ -11,7 +13,7 @@ export interface OtlpResourceConfig { } /** - * OTLP resource attributes shared by the logs and metrics envelopes. + * OTLP resource attributes shared by the logs, metrics and traces envelopes. * * User `resourceAttributes` are spread first, then SDK-controlled keys on top so * a stray user key can't clobber the ingestion-attribution ones; the dedicated @@ -26,7 +28,10 @@ export function buildOtlpResourceAttributes( sdkVersion: string ): Record { return { - ...config.resourceAttributes, + // Read key by key: a throwing accessor on a user-supplied attribute runs on + // every flush, before the pipeline's own error handling, and would otherwise + // stop the signal exporting entirely. + ...assignUserAttributes>({}, config.resourceAttributes), 'service.name': config.serviceName || 'unknown_service', ...(config.environment && { 'deployment.environment': config.environment }), ...(config.serviceVersion && { 'service.version': config.serviceVersion }), @@ -47,15 +52,19 @@ export function buildOtlpResourceAttributes( * platforms they cover. */ const OS_NAMES: Record = { - // node:os platform() + // node:os platform(), all eleven of them darwin: 'macOS', win32: 'Windows', + // Cygwin is a POSIX layer over Windows, so it belongs under the same filter. + cygwin: 'Windows', linux: 'Linux', android: 'Android', freebsd: 'FreeBSD', openbsd: 'OpenBSD', + netbsd: 'NetBSD', sunos: 'SunOS', aix: 'AIX', + haiku: 'Haiku', // detectOS 'Mac OS X': 'macOS', } diff --git a/packages/node/src/__tests__/host-os.spec.ts b/packages/node/src/__tests__/host-os.spec.ts new file mode 100644 index 0000000000..2fab37eb84 --- /dev/null +++ b/packages/node/src/__tests__/host-os.spec.ts @@ -0,0 +1,45 @@ +import { platform, release } from 'node:os' +import { hostOsResourceAttributes } from '../host-os.node' + +jest.mock('node:os', () => ({ platform: jest.fn(), release: jest.fn() })) + +const mockPlatform = platform as jest.Mock +const mockRelease = release as jest.Mock + +describe('hostOsResourceAttributes', () => { + it('reports the host OS', () => { + mockPlatform.mockReturnValue('linux') + mockRelease.mockReturnValue('6.1.0-27-amd64') + + expect(hostOsResourceAttributes()).toEqual({ 'os.name': 'Linux', 'os.version': '6.1.0-27-amd64' }) + }) + + it.each([ + ['darwin', 'macOS'], + ['win32', 'Windows'], + ['freebsd', 'FreeBSD'], + ])('reports %s as the OS name the other SDKs send, not the node:os identifier', (identifier, osName) => { + mockPlatform.mockReturnValue(identifier) + mockRelease.mockReturnValue('1.0.0') + + expect(hostOsResourceAttributes()['os.name']).toBe(osName) + }) + + it('omits a key node:os cannot supply rather than emitting it empty', () => { + mockPlatform.mockReturnValue('linux') + mockRelease.mockReturnValue('') + + expect(hostOsResourceAttributes()).toEqual({ 'os.name': 'Linux' }) + }) + + it('returns no attributes when node:os throws', () => { + mockPlatform.mockImplementation(() => { + throw new Error('unsupported') + }) + mockRelease.mockImplementation(() => { + throw new Error('unsupported') + }) + + expect(hostOsResourceAttributes()).toEqual({}) + }) +}) diff --git a/packages/node/src/__tests__/traces-defaults.spec.ts b/packages/node/src/__tests__/traces-defaults.spec.ts index 9c6e2a8770..b767799afa 100644 --- a/packages/node/src/__tests__/traces-defaults.spec.ts +++ b/packages/node/src/__tests__/traces-defaults.spec.ts @@ -87,6 +87,27 @@ describe('resolveTracesConfig', () => { it('floors an explicit maxQueueSize at the export batch size', () => { expect(resolveTracesConfig({ maxExportBatchSize: 512, maxQueueSize: 10 }).maxQueueSize).toBe(512) }) + + it('attaches the host resource attributes the entrypoint supplies', () => { + expect( + resolveTracesConfig(undefined, { 'os.name': 'linux', 'os.version': '6.1.0-27-amd64' }).resourceAttributes + ).toEqual({ 'os.name': 'linux', 'os.version': '6.1.0-27-amd64' }) + }) + + it('lets user resource attributes override the host ones', () => { + expect( + resolveTracesConfig( + { resourceAttributes: { 'os.name': 'my-os', 'os.version': '1.2.3' } }, + { 'os.name': 'linux', 'os.version': '6.1.0-27-amd64' } + ).resourceAttributes + ).toEqual({ 'os.name': 'my-os', 'os.version': '1.2.3' }) + }) + + it('resolves when the entrypoint supplies no host attributes', () => { + expect(resolveTracesConfig({ resourceAttributes: { 'host.name': 'worker-01' } }).resourceAttributes).toEqual({ + 'host.name': 'worker-01', + }) + }) }) describe('resourceAttributes guarding', () => { @@ -125,4 +146,30 @@ describe('resourceAttributes guarding', () => { expect(() => resolveTracesConfig({ resourceAttributes: hostile as never })).not.toThrow() }) + + it('does not throw when a non-identity accessor throws', () => { + const hostile = { 'service.name': 'checkout-api' } + Object.defineProperty(hostile, 'region', { + enumerable: true, + get() { + throw new Error('config getter exploded') + }, + }) + + expect(() => resolveTracesConfig({ resourceAttributes: hostile as never }, { 'os.name': 'Linux' })).not.toThrow() + }) + + it('keeps the readable attributes when one accessor throws', () => { + const hostile = { region: 'us' } + Object.defineProperty(hostile, 'tenant', { + enumerable: true, + get() { + throw new Error('config getter exploded') + }, + }) + + const resolved = resolveTracesConfig({ resourceAttributes: hostile as never }, { 'os.name': 'Linux' }) + + expect(resolved.resourceAttributes).toMatchObject({ 'os.name': 'Linux', region: 'us' }) + }) }) diff --git a/packages/node/src/__tests__/traces.spec.ts b/packages/node/src/__tests__/traces.spec.ts index 8034c018dd..dd37b982ee 100644 --- a/packages/node/src/__tests__/traces.spec.ts +++ b/packages/node/src/__tests__/traces.spec.ts @@ -1,7 +1,8 @@ +import { platform, release } from 'node:os' import { PostHog } from '@/entrypoints/index.node' import type { OtlpSpan, OtlpTracesPayload } from '@posthog/types' import { waitForPromises } from './utils' -import { isGzipSupported } from '@posthog/core' +import { isGzipSupported, osResourceAttributes } from '@posthog/core' jest.mock('../version', () => ({ version: '1.2.3' })) @@ -144,6 +145,29 @@ describe('PostHog traces', () => { value: { stringValue: 'posthog-node' }, }) }) + + it('sends the host OS as resource attributes', async () => { + posthog.startSpan('checkout').end() + await flushTraces() + + const attributes = sentPayloads()[0].resourceSpans[0].resource.attributes + expect(attributes).toContainEqual({ + key: 'os.name', + value: { stringValue: osResourceAttributes(platform(), release())['os.name'] }, + }) + expect(attributes).toContainEqual({ key: 'os.version', value: { stringValue: release() } }) + }) + + it('lets configured resourceAttributes override the host OS', async () => { + posthog = createClient({ traces: { serviceName: 'checkout-api', resourceAttributes: { 'os.name': 'my-os' } } }) + posthog.startSpan('checkout').end() + await flushTraces() + + expect(sentPayloads()[0].resourceSpans[0].resource.attributes).toContainEqual({ + key: 'os.name', + value: { stringValue: 'my-os' }, + }) + }) }) describe('span shape', () => { diff --git a/packages/node/src/client.ts b/packages/node/src/client.ts index 7a9c251855..08cb08b189 100644 --- a/packages/node/src/client.ts +++ b/packages/node/src/client.ts @@ -630,6 +630,15 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen return new SyncSpanContextManager() } + /** + * Runtime-detected OTLP resource attributes for every span. Overridden by the + * Node entrypoint with the host OS; the edge build contributes none, keeping + * `node:os` out of an edge bundle, which cannot resolve it. + */ + protected hostResourceAttributes(): Record { + return {} + } + /** * Active-span tracking, built on first use. Lives on the client rather than * on the traces pipeline because a client with tracing off still activates a @@ -653,7 +662,7 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen if (!this._traces) { this._traces = new PostHogTraces( this, - resolveTracesConfig(this.options.traces), + resolveTracesConfig(this.options.traces, this.hostResourceAttributes()), this._logger, () => this._tracingContext(), this._spanContextManager, diff --git a/packages/node/src/entrypoints/index.node.ts b/packages/node/src/entrypoints/index.node.ts index 96a9fed515..c456c50645 100644 --- a/packages/node/src/entrypoints/index.node.ts +++ b/packages/node/src/entrypoints/index.node.ts @@ -11,6 +11,7 @@ import type { SpanContextManager } from '@posthog/core' import { PostHogContext } from '../extensions/context/context' import { AsyncLocalStorageSpanContextManager } from '../extensions/context/span-context.node' import { gzipCompress } from '../gzip.node' +import { hostOsResourceAttributes } from '../host-os.node' export class PostHog extends PostHogBackendClient { getLibraryId(): string { @@ -29,6 +30,10 @@ export class PostHog extends PostHogBackendClient { return new AsyncLocalStorageSpanContextManager() } + protected override hostResourceAttributes(): Record { + return hostOsResourceAttributes() + } + protected override createErrorPropertiesBuilder(): CoreErrorTracking.ErrorPropertiesBuilder { return new CoreErrorTracking.ErrorPropertiesBuilder( [ diff --git a/packages/node/src/host-os.node.ts b/packages/node/src/host-os.node.ts new file mode 100644 index 0000000000..246bd2b429 --- /dev/null +++ b/packages/node/src/host-os.node.ts @@ -0,0 +1,22 @@ +import { platform, release } from 'node:os' +import { osResourceAttributes } from '@posthog/core' + +/** + * OTLP `os.name` / `os.version` for the machine running the SDK, so spans can + * be filtered by platform (e.g. "only the Linux workers") in PostHog. + * + * Node-only, like the other `.node` modules: importing `node:os` from a shared + * module would put it in the edge bundle. A failed read omits the key rather + * than throwing out of client construction. + */ +export function hostOsResourceAttributes(): Record { + let osName: string | undefined + let osVersion: string | undefined + try { + osName = platform() + osVersion = release() + } catch {} + // Through the shared builder, so a Node span reports the same `os.name` the + // browser, iOS and Android SDKs send rather than the `node:os` identifier. + return osResourceAttributes(osName, osVersion) +} diff --git a/packages/node/src/traces-defaults.ts b/packages/node/src/traces-defaults.ts index 9d96d3be5a..f4982505bc 100644 --- a/packages/node/src/traces-defaults.ts +++ b/packages/node/src/traces-defaults.ts @@ -1,3 +1,4 @@ +import { assignUserAttributes } from '@posthog/core' import type { ResolvedTracesConfig, TracesConfig } from '@posthog/core' // OpenTelemetry's BatchSpanProcessor defaults, which sit comfortably under the @@ -55,10 +56,19 @@ function withUsableIdentityKeys(attributes: TracesConfig['resourceAttributes']): /** * Resolves the public `traces` config into the shape core `PostHogTraces` consumes. * OTLP resource attributes take precedence over the named fields, matching the - * logs config — a user who sets `service.name` directly means it. + * logs config. `hostResourceAttributes` are runtime-detected by the entrypoint and + * merge first, so a user-supplied value of the same key wins. */ -export function resolveTracesConfig(config: TracesConfig | undefined): ResolvedTracesConfig { - const resourceAttributes = withUsableIdentityKeys(config?.resourceAttributes) +export function resolveTracesConfig( + config: TracesConfig | undefined, + hostResourceAttributes?: Record +): ResolvedTracesConfig { + // Copied key by key rather than spread: a throwing accessor on a user-supplied + // attribute would otherwise escape the first `startSpan`. + const resourceAttributes = assignUserAttributes( + { ...hostResourceAttributes }, + withUsableIdentityKeys(config?.resourceAttributes) + ) const maxExportBatchSize = positiveInteger(config?.maxExportBatchSize, DEFAULT_MAX_EXPORT_BATCH_SIZE) return { serviceName: (resourceAttributes?.['service.name'] as string | undefined) ?? config?.serviceName, From 0c640f27cfd772237dcfd263c644315090f68b23 Mon Sep 17 00:00:00 2001 From: Anna Garcia <11654201+turnipdabeets@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:15:27 -0400 Subject: [PATCH 07/24] feat(traces): propagate the inbound sampled flag and parent remoteness (#4728) * feat(traces): propagate the inbound sampled flag and parent remoteness A continued trace now carries the caller's trace-flags byte in both `traceparent()` and the exported span, rather than always sending `01`, so a downstream parent-based sampler is not handed a decision this SDK invented. Bits version 00 does not define are zeroed. Exported spans also set OTel's parent-remoteness bits. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MmYSjZ7Mr2UZp5acLDuEHe * docs(traces): say why a root span reports a known-not-remote parent Greptile read the known bit on a root span as asserting a local parent; the OTel Go and Java exporters set it there too. Names the existing test for what it covers and adds the local-parent case. --------- Co-authored-by: Claude Opus 5 (1M context) --- .changeset/traces-flags-fidelity.md | 7 ++++ packages/core/src/traces/index.spec.ts | 34 ++++++++++++++++ packages/core/src/traces/index.ts | 7 ++++ packages/core/src/traces/otlp.spec.ts | 30 ++++++++++++-- packages/core/src/traces/otlp.ts | 24 +++++++++-- packages/core/src/traces/span.spec.ts | 10 +++++ packages/core/src/traces/span.ts | 24 +++++++++-- packages/core/src/traces/traceparent.spec.ts | 43 ++++++++++++++++---- packages/core/src/traces/traceparent.ts | 36 ++++++++++++---- packages/core/src/traces/types.ts | 4 ++ packages/node/src/__tests__/traces.spec.ts | 14 +++++++ packages/types/src/traces.ts | 11 +++-- 12 files changed, 215 insertions(+), 29 deletions(-) create mode 100644 .changeset/traces-flags-fidelity.md diff --git a/.changeset/traces-flags-fidelity.md b/.changeset/traces-flags-fidelity.md new file mode 100644 index 0000000000..77dfd3c00e --- /dev/null +++ b/.changeset/traces-flags-fidelity.md @@ -0,0 +1,7 @@ +--- +'posthog-node': minor +'@posthog/core': minor +'@posthog/types': minor +--- + +Propagate the inbound W3C sampled flag on a continued trace instead of always sending `01`, so a downstream parent-based sampler sees the decision the head sampler made. Spans are still recorded and exported either way. Exported spans also carry OpenTelemetry's parent-remoteness bits, so a span that entered the service over HTTP is distinguishable from one started locally. diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 8256c2512a..20b412cfcc 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -192,6 +192,40 @@ describe('PostHogTraces', () => { expect(sentSpans()[0].traceId).toBe(TRACE_ID) }) + it('propagates the sampled-out flag onward rather than upgrading it to 01', async () => { + // A downstream parent-based sampler would otherwise record a trace its own + // head sampler had already rejected. + const traces = createTraces() + const span = traces.startSpan('handler', { parent: `00-${TRACE_ID}-${REMOTE_SPAN_ID}-00` }) + const child = traces.startSpan('inner', { parent: span }) + + expect(span.traceparent()!.startsWith(`00-${TRACE_ID}-`)).toBe(true) + expect(span.traceparent()!.endsWith('-00')).toBe(true) + // The whole chain agrees, not just the span that read the header. + expect(child.traceparent()!.endsWith('-00')).toBe(true) + + child.end() + span.end() + await traces.flush() + + // Recorded and exported all the same, with the wire agreeing with the header. + const byName = Object.fromEntries(sentSpans().map((sent) => [sent.name, sent.flags])) + expect(byName).toEqual({ handler: 0x300, inner: 0x100 }) + }) + + it('marks a header parent remote and a handle parent local', async () => { + const traces = createTraces() + const remote = traces.startSpan('handler', { parent: `00-${TRACE_ID}-${REMOTE_SPAN_ID}-01` }) + const local = traces.startSpan('inner', { parent: remote }) + + local.end() + remote.end() + await traces.flush() + + const byName = Object.fromEntries(sentSpans().map((span) => [span.name, span.flags])) + expect(byName).toEqual({ handler: 0x301, inner: 0x101 }) + }) + it('preserves tracestate opaquely and passes it to children', async () => { const traces = createTraces() const parent = traces.startSpan('handler', { diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 0281556118..7fd69b87db 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -53,6 +53,9 @@ interface ParentContext { traceId: string parentSpanId?: string traceState?: string + traceFlags?: string + /** True when the parent arrived as a `traceparent` header. */ + isRemote?: boolean } /** @@ -147,6 +150,8 @@ export class PostHogTraces { spanId, parentSpanId: parent?.parentSpanId, traceState: parent?.traceState, + traceFlags: parent?.traceFlags, + parentIsRemote: parent?.isRemote, name: sanitizeName(name, 'Span name', this._logger), kind: options?.kind ?? 'internal', // Auto-context first so user-supplied attributes win on collision. @@ -280,6 +285,8 @@ export class PostHogTraces { traceId: remote.traceId, parentSpanId: remote.spanId, traceState: sanitizeTracestate(options?.tracestate), + traceFlags: remote.flags, + isRemote: true, } } diff --git a/packages/core/src/traces/otlp.spec.ts b/packages/core/src/traces/otlp.spec.ts index 798cf33242..a36dded70d 100644 --- a/packages/core/src/traces/otlp.spec.ts +++ b/packages/core/src/traces/otlp.spec.ts @@ -12,6 +12,8 @@ const record = (overrides: Partial = {}): SpanRecord => ({ spanId: '00f067aa0ba902b7', name: 'checkout', kind: 'internal', + traceFlags: '01', + parentIsRemote: false, attributes: {}, events: [], startTime: 1_700_000_000_000, @@ -77,7 +79,7 @@ describe('OTLP span encoding', () => { kind: 1, startTimeUnixNano: '1700000000000000000', endTimeUnixNano: '1700000000080000000', - flags: 1, + flags: 0x101, }) }) @@ -108,8 +110,28 @@ describe('OTLP span encoding', () => { expect(span.events).toEqual([{ name: 'cache miss', timeUnixNano: '1700000000040000000' }]) }) - it('always sets the sampled trace flag', () => { - expect(buildOtlpSpan(record()).flags).toBe(1) + it('sets the sampled bit and marks a root span as known-not-remote', () => { + // A root span has no parent context to be remote, which the OTel Go and + // Java exporters also report as known-not-remote rather than unknown. + expect(buildOtlpSpan(record()).flags).toBe(0x101) + }) + + it('marks a local parent as known-not-remote', () => { + expect(buildOtlpSpan(record({ parentSpanId: 'b7ad6b7169203331' })).flags).toBe(0x101) + }) + + it('marks a parent that arrived as a header as remote', () => { + expect(buildOtlpSpan(record({ parentSpanId: 'b7ad6b7169203331', parentIsRemote: true })).flags).toBe(0x301) + }) + + it('propagates an inbound sampled-out flag rather than overriding it', () => { + // The span is still recorded and exported; what the wire says is the + // decision the head sampler made. + expect(buildOtlpSpan(record({ traceFlags: '00', parentIsRemote: true })).flags).toBe(0x300) + }) + + it('falls back to sampled when the flags byte is unusable', () => { + expect(buildOtlpSpan(record({ traceFlags: 'zz' })).flags).toBe(0x101) }) }) @@ -232,7 +254,7 @@ describe('OTLP span encoding', () => { kind: 2, startTimeUnixNano: '1700000000000000000', endTimeUnixNano: '1700000000080000000', - flags: 1, + flags: 0x101, attributes: [ { key: 'posthogDistinctId', value: { stringValue: 'user-123' } }, { key: 'sessionId', value: { stringValue: 'session-123' } }, diff --git a/packages/core/src/traces/otlp.ts b/packages/core/src/traces/otlp.ts index 75d66721c1..6da9bfb2e2 100644 --- a/packages/core/src/traces/otlp.ts +++ b/packages/core/src/traces/otlp.ts @@ -26,8 +26,26 @@ const SPAN_STATUS_TO_OTLP: Record = { error: 2, } -/** W3C trace flags: the sampled bit, always set because every captured span is recorded. */ -const TRACE_FLAGS_SAMPLED = 1 +/** W3C trace flags live in the low byte; the sampled bit is `0x01`. */ +const TRACE_FLAGS_SAMPLED = 0x01 +// OTel's span flags, above the W3C byte: one bit says the parent's remoteness is +// known, the other says it is remote. Both unset reads as "unknown", which this +// SDK never has to say — a string parent is remote, a handle parent is local, and +// a root span's parent context is empty, which is not remote. Setting the known +// bit on a root span is what the OTel Go and Java exporters do too. +const SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE = 0x100 +const SPAN_FLAGS_CONTEXT_IS_REMOTE = 0x200 + +/** + * The `flags` field for a span: its W3C trace-flags byte, plus OTel's + * parent-remoteness bits. Nothing reads the remoteness today, but a span + * exported without it can never be backfilled with it. + */ +function spanFlags(record: SpanRecord): number { + const traceFlags = parseInt(record.traceFlags, 16) + const w3c = Number.isFinite(traceFlags) ? traceFlags & 0xff : TRACE_FLAGS_SAMPLED + return w3c | SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE | (record.parentIsRemote ? SPAN_FLAGS_CONTEXT_IS_REMOTE : 0) +} /** * Every free-text string this encoder puts on the wire. A lone surrogate survives @@ -97,7 +115,7 @@ export function buildOtlpSpan(record: SpanRecord, logger?: Logger): OtlpSpan { kind: spanKindToOtlp(record.kind), startTimeUnixNano: msToUnixNanoString(record.startTime), endTimeUnixNano: msToUnixNanoString(record.endTime), - flags: TRACE_FLAGS_SAMPLED, + flags: spanFlags(record), } if (record.parentSpanId) { span.parentSpanId = record.parentSpanId diff --git a/packages/core/src/traces/span.spec.ts b/packages/core/src/traces/span.spec.ts index 989bb5202d..15ce17709d 100644 --- a/packages/core/src/traces/span.spec.ts +++ b/packages/core/src/traces/span.spec.ts @@ -309,11 +309,21 @@ describe('PostHogSpan', () => { expect(createSpan({ traceState: 'vendor=abc' }).tracestate()).toBe('vendor=abc') }) + it('propagates the trace flags it was started with', () => { + expect(createSpan({ traceFlags: '00' }).traceparent()).toBe(`00-${TRACE_ID}-${SPAN_ID}-00`) + expect(createSpan().traceparent()).toBe(`00-${TRACE_ID}-${SPAN_ID}-01`) + }) + + it('hands a child the flags it propagates, so the whole chain agrees', () => { + expect(createSpan({ traceFlags: '00' }).childContext().traceFlags).toBe('00') + }) + it('exposes a child context carrying its own span id as the parent', () => { expect(createSpan({ traceState: 'vendor=abc' }).childContext()).toEqual({ traceId: TRACE_ID, parentSpanId: SPAN_ID, traceState: 'vendor=abc', + traceFlags: '01', }) }) }) diff --git a/packages/core/src/traces/span.ts b/packages/core/src/traces/span.ts index 92828f7037..f8564ae179 100644 --- a/packages/core/src/traces/span.ts +++ b/packages/core/src/traces/span.ts @@ -1,7 +1,7 @@ import type { Span, SpanAttributes, SpanAttributeValue, SpanKind, SpanStatusCode, SpanTimeInput } from '@posthog/types' import type { Logger } from '../types' import type { SpanEventRecord, SpanRecord } from './types' -import { formatTraceparent, normalizeTraceparent, sanitizeTracestate } from './traceparent' +import { formatTraceparent, normalizeTraceparent, sanitizeTracestate, TRACE_FLAGS_SAMPLED } from './traceparent' import { assignUserAttributes, clampEndTime, resolveSuppliedTime, sanitizeName } from './sanitize' import { isError } from '../utils' @@ -19,6 +19,10 @@ export interface SpanInit { spanId: string parentSpanId?: string traceState?: string + /** The trace-flags byte to propagate; the inbound one when continuing a remote trace. */ + traceFlags?: string + /** True when the parent came from a `traceparent` header rather than a local handle. */ + parentIsRemote?: boolean name: string kind: SpanKind attributes: SpanAttributes @@ -33,6 +37,8 @@ export class PostHogSpan implements Span { private readonly _spanId: string private readonly _parentSpanId?: string private readonly _traceState?: string + private readonly _traceFlags: string + private readonly _parentIsRemote: boolean private readonly _startTime: number // Absent on backdated spans and on platforms with no monotonic source. private readonly _startMono?: number @@ -53,6 +59,8 @@ export class PostHogSpan implements Span { this._spanId = init.spanId this._parentSpanId = init.parentSpanId this._traceState = init.traceState + this._traceFlags = init.traceFlags ?? TRACE_FLAGS_SAMPLED + this._parentIsRemote = init.parentIsRemote ?? false this._name = init.name this._kind = init.kind this._attributes = init.attributes @@ -147,7 +155,7 @@ export class PostHogSpan implements Span { } traceparent(): string | null { - return formatTraceparent(this._traceId, this._spanId) + return formatTraceparent(this._traceId, this._spanId, this._traceFlags) } tracestate(): string | null { @@ -155,8 +163,14 @@ export class PostHogSpan implements Span { } /** Context a child span inherits when this handle is its parent. */ - childContext(): { traceId: string; parentSpanId: string; traceState?: string } { - return { traceId: this._traceId, parentSpanId: this._spanId, traceState: this._traceState } + childContext(): { traceId: string; parentSpanId: string; traceState?: string; traceFlags: string } { + return { + traceId: this._traceId, + parentSpanId: this._spanId, + traceState: this._traceState, + // A child of a continued trace keeps propagating the caller's decision. + traceFlags: this._traceFlags, + } } end(endTime?: SpanTimeInput): void { @@ -174,6 +188,8 @@ export class PostHogSpan implements Span { spanId: this._spanId, ...(this._parentSpanId && { parentSpanId: this._parentSpanId }), ...(this._traceState && { traceState: this._traceState }), + traceFlags: this._traceFlags, + parentIsRemote: this._parentIsRemote, name: this._name, kind: this._kind, ...(this._status && { status: this._status }), diff --git a/packages/core/src/traces/traceparent.spec.ts b/packages/core/src/traces/traceparent.spec.ts index 92901402c8..a6460e65ef 100644 --- a/packages/core/src/traces/traceparent.spec.ts +++ b/packages/core/src/traces/traceparent.spec.ts @@ -6,19 +6,39 @@ const SPAN_ID = '00f067aa0ba902b7' describe('traceparent', () => { describe('parseTraceparent', () => { it('parses a sampled header', () => { - expect(parseTraceparent(`00-${TRACE_ID}-${SPAN_ID}-01`)).toEqual({ traceId: TRACE_ID, spanId: SPAN_ID }) + expect(parseTraceparent(`00-${TRACE_ID}-${SPAN_ID}-01`)).toEqual({ + traceId: TRACE_ID, + spanId: SPAN_ID, + flags: '01', + }) + }) + + it('continues the trace even when the caller sampled it out, and keeps the flag', () => { + // Every captured span is recorded, so honouring an inbound `00` by + // dropping the parentage would orphan our own spans. The flag itself is + // kept, so what we propagate onward still says what the caller decided. + expect(parseTraceparent(`00-${TRACE_ID}-${SPAN_ID}-00`)).toEqual({ + traceId: TRACE_ID, + spanId: SPAN_ID, + flags: '00', + }) }) - it('continues the trace even when the caller sampled it out', () => { - // Every captured span is recorded, so honouring an inbound `00` would - // orphan our own spans rather than save anything. - expect(parseTraceparent(`00-${TRACE_ID}-${SPAN_ID}-00`)).toEqual({ traceId: TRACE_ID, spanId: SPAN_ID }) + it.each([ + ['a reserved bit alongside sampled', '05', '01'], + ['reserved bits with sampled unset', '04', '00'], + ['every bit set', 'ff', '01'], + ])('zeroes %s, which version 00 does not define', (_label, inbound, expected) => { + // We re-emit under version `00`, and W3C requires a vendor to zero every + // flag that version does not define rather than forward it. + expect(parseTraceparent(`00-${TRACE_ID}-${SPAN_ID}-${inbound}`)?.flags).toBe(expected) }) it('accepts a future version with extra fields', () => { expect(parseTraceparent(`01-${TRACE_ID}-${SPAN_ID}-01-something`)).toEqual({ traceId: TRACE_ID, spanId: SPAN_ID, + flags: '01', }) }) @@ -26,6 +46,7 @@ describe('traceparent', () => { expect(parseTraceparent(` 00-${TRACE_ID.toUpperCase()}-${SPAN_ID.toUpperCase()}-01 `)).toEqual({ traceId: TRACE_ID, spanId: SPAN_ID, + flags: '01', }) }) @@ -45,12 +66,20 @@ describe('traceparent', () => { }) describe('formatTraceparent', () => { - it('always sets the sampled flag', () => { + it('sets the sampled flag on a trace started here', () => { expect(formatTraceparent(TRACE_ID, SPAN_ID)).toBe(`00-${TRACE_ID}-${SPAN_ID}-01`) }) + it('propagates the flags byte it was given', () => { + expect(formatTraceparent(TRACE_ID, SPAN_ID, '00')).toBe(`00-${TRACE_ID}-${SPAN_ID}-00`) + }) + it('round-trips through the parser', () => { - expect(parseTraceparent(formatTraceparent(TRACE_ID, SPAN_ID))).toEqual({ traceId: TRACE_ID, spanId: SPAN_ID }) + expect(parseTraceparent(formatTraceparent(TRACE_ID, SPAN_ID))).toEqual({ + traceId: TRACE_ID, + spanId: SPAN_ID, + flags: '01', + }) }) }) diff --git a/packages/core/src/traces/traceparent.ts b/packages/core/src/traces/traceparent.ts index b09bd436d6..40fb622f3e 100644 --- a/packages/core/src/traces/traceparent.ts +++ b/packages/core/src/traces/traceparent.ts @@ -3,6 +3,8 @@ import { isValidSpanId, isValidTraceId } from './ids' export interface RemoteSpanContext { traceId: string spanId: string + /** The inbound trace-flags byte, e.g. `01` sampled, `00` sampled out. */ + flags: string } // `00-<32 hex>-<16 hex>-<2 hex>`. Version `ff` is invalid per the spec; other @@ -13,13 +15,23 @@ const TRACEPARENT_RE = /^([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2 * Parses an incoming `traceparent` header value, returning `undefined` for * anything malformed so a bad header starts a fresh root rather than throwing. * - * Incoming trace flags are deliberately ignored: we continue the trace even when - * the caller sampled it out (`00`), because PostHog records every captured span - * and dropping the parentage would orphan our own spans. + * A trace the caller sampled out (`00`) is still continued — PostHog records + * every captured span — but the inbound flag rides along, so what this SDK + * propagates onward says what the caller decided rather than overriding it. */ export function parseTraceparent(value: unknown): RemoteSpanContext | undefined { const fields = matchTraceparent(value) - return fields && { traceId: fields.traceId, spanId: fields.spanId } + return fields && { traceId: fields.traceId, spanId: fields.spanId, flags: definedFlags(fields.flags) } +} + +/** + * Keeps only the flags version `00` defines — the sampled bit. A span continuing + * this trace re-emits the byte under version `00`, and W3C requires a vendor to + * zero every flag that version does not define rather than forward one it cannot + * interpret. + */ +function definedFlags(flags: string): string { + return parseInt(flags, 16) & 0x01 ? TRACE_FLAGS_SAMPLED : TRACE_FLAGS_UNSAMPLED } interface TraceparentFields { @@ -57,12 +69,20 @@ export function normalizeTraceparent(value: unknown): string | undefined { return fields && `${fields.version}-${fields.traceId}-${fields.spanId}-${fields.flags}` } +/** The W3C sampled bit, set on a trace this SDK started. */ +export const TRACE_FLAGS_SAMPLED = '01' + +/** The same byte with the sampled bit clear, for a trace the caller sampled out. */ +const TRACE_FLAGS_UNSAMPLED = '00' + /** - * Builds the `traceparent` header value for a span. The sampled flag is always - * set, because a span we exported is by definition recorded. + * Builds the `traceparent` header value for a span. A span continuing a remote + * trace propagates the flags byte it was handed: a downstream parent-based + * sampler must see the decision the head sampler actually made, not one this + * SDK invented. A trace started here is sampled, because it is recorded. */ -export function formatTraceparent(traceId: string, spanId: string): string { - return `00-${traceId}-${spanId}-01` +export function formatTraceparent(traceId: string, spanId: string, flags: string = TRACE_FLAGS_SAMPLED): string { + return `00-${traceId}-${spanId}-${flags}` } // tracestate is a comma-separated list of at most 32 `key=value` members, and diff --git a/packages/core/src/traces/types.ts b/packages/core/src/traces/types.ts index 6551d0bcac..2b1e392570 100644 --- a/packages/core/src/traces/types.ts +++ b/packages/core/src/traces/types.ts @@ -64,6 +64,10 @@ export interface SpanRecord { spanId: string parentSpanId?: string traceState?: string + /** The W3C trace-flags byte this span propagates, e.g. `01` sampled. */ + traceFlags: string + /** True when the parent came from a `traceparent` header rather than a local handle. */ + parentIsRemote: boolean name: string kind: SpanKind status?: { code: SpanStatusCode; message?: string } diff --git a/packages/node/src/__tests__/traces.spec.ts b/packages/node/src/__tests__/traces.spec.ts index 1fb96bcfa7..79990cd842 100644 --- a/packages/node/src/__tests__/traces.spec.ts +++ b/packages/node/src/__tests__/traces.spec.ts @@ -293,6 +293,20 @@ describe('PostHog traces', () => { expect(span.parentSpanId).toBe(spanId) }) + it('hands the next service the flag the caller sent, not an upgraded one', async () => { + const inbound = `00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00` + let propagated: string | null = null + + await posthog.withSpan('handler', { parent: inbound }, async () => { + propagated = posthog.getActiveSpan()!.traceparent() + }) + await flushTraces() + + expect(propagated).toEqual(expect.stringMatching(/-00$/)) + // Recorded and exported, with the remote-parent bits set. + expect(sentSpans()[0].flags).toBe(0x300) + }) + it('produces a traceparent for the next service', async () => { let traceparent: string | null = null posthog.withSpan('POST /checkout', () => { diff --git a/packages/types/src/traces.ts b/packages/types/src/traces.ts index aedc80ee63..8e6239fd9b 100644 --- a/packages/types/src/traces.ts +++ b/packages/types/src/traces.ts @@ -149,8 +149,10 @@ export interface Span { updateName(name: string): this /** - * This span's W3C `traceparent` header value (`00---01`), - * for propagating the trace to another service. + * This span's W3C `traceparent` header value (`00---`), + * for propagating the trace to another service. A span continuing a remote + * trace propagates the flags byte it was handed, so a downstream sampler + * sees the decision the head sampler made; a trace started here is sampled. * * When tracing is off, a span started with a `parent` header echoes that * header back — version and sampled flag included — so a service that @@ -298,7 +300,10 @@ export interface OtlpSpan { attributes?: OtlpSpanKeyValue[] events?: OtlpSpanEvent[] status?: OtlpSpanStatus - /** W3C trace flags in the low byte; the sampled bit is always set. */ + /** + * W3C trace flags in the low byte — the sampled bit as this span propagates + * it — plus OTel's parent-remoteness bits (`0x100` known, `0x200` remote). + */ flags?: number } From d7ffe61591dbb9d235779347703987f0addf8916 Mon Sep 17 00:00:00 2001 From: Anna Garcia <11654201+turnipdabeets@users.noreply.github.com> Date: Fri, 4 Sep 2026 07:50:01 -0400 Subject: [PATCH 08/24] refactor(traces): keep the traces plumbing off the public API (#4773) Moves resolveTracesConfig into core beside resolveMetricsConfig and assignUserAttributes into utils, so packages/node no longer reaches for either, and replaces the duplicated activation rule with runWithActiveSpan. Claude-Session: https://claude.ai/code/session_01L8eFZB35NExUZyq5hGgh43 Co-authored-by: Claude Opus 5 (1M context) --- packages/core/src/index.ts | 4 +-- .../src/traces/config.spec.ts} | 2 +- .../src/traces/config.ts} | 5 +-- packages/core/src/traces/index.ts | 11 +++--- packages/core/src/traces/sanitize.ts | 36 ------------------- packages/core/src/traces/span.ts | 17 +++++++-- packages/core/src/utils/json-utils.ts | 33 +++++++++++++++++ packages/core/src/utils/otlp-resource.ts | 2 +- packages/node/src/client.ts | 7 ++-- 9 files changed, 62 insertions(+), 55 deletions(-) rename packages/{node/src/__tests__/traces-defaults.spec.ts => core/src/traces/config.spec.ts} (99%) rename packages/{node/src/traces-defaults.ts => core/src/traces/config.ts} (95%) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f8b93ec090..8a3aad8015 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -45,7 +45,6 @@ export { } from './logs/logs-utils' export { toOtlpAnyValue, toOtlpKeyValueList } from './utils/otlp-any-value' export { osResourceAttributes } from './utils/otlp-resource' -export { assignUserAttributes } from './traces/sanitize' export { PostHogLogs } from './logs' export type { BeforeSendLogFn, @@ -90,7 +89,8 @@ export type { } from './metrics/types' export { PostHogTraces } from './traces' export { SyncSpanContextManager } from './traces/context' -export { NOOP_SPAN, inertSpan } from './traces/span' +export { inertSpan, runWithActiveSpan } from './traces/span' +export { resolveTracesConfig } from './traces/config' export type { ResolvedTracesConfig, SpanContextManager, TraceSdkContext } from './traces/types' // Same barrel convention as logs and metrics for the user-facing tracing types. export type { diff --git a/packages/node/src/__tests__/traces-defaults.spec.ts b/packages/core/src/traces/config.spec.ts similarity index 99% rename from packages/node/src/__tests__/traces-defaults.spec.ts rename to packages/core/src/traces/config.spec.ts index b767799afa..902e5f35f5 100644 --- a/packages/node/src/__tests__/traces-defaults.spec.ts +++ b/packages/core/src/traces/config.spec.ts @@ -1,4 +1,4 @@ -import { resolveTracesConfig } from '../traces-defaults' +import { resolveTracesConfig } from './config' describe('resolveTracesConfig', () => { it('applies the documented defaults', () => { diff --git a/packages/node/src/traces-defaults.ts b/packages/core/src/traces/config.ts similarity index 95% rename from packages/node/src/traces-defaults.ts rename to packages/core/src/traces/config.ts index f4982505bc..4cec80a594 100644 --- a/packages/node/src/traces-defaults.ts +++ b/packages/core/src/traces/config.ts @@ -1,5 +1,6 @@ -import { assignUserAttributes } from '@posthog/core' -import type { ResolvedTracesConfig, TracesConfig } from '@posthog/core' +import { assignUserAttributes } from '../utils/json-utils' +import type { ResolvedTracesConfig } from './types' +import type { TracesConfig } from '@posthog/types' // OpenTelemetry's BatchSpanProcessor defaults, which sit comfortably under the // server's 2 MB body cap. diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 7fd69b87db..89b42f1a14 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -8,10 +8,11 @@ import type { TraceSdkContext, TracesHost, } from './types' -import { NOOP_SPAN, PostHogSpan, describeError, inertSpan, monotonicNow } from './span' +import { NOOP_SPAN, PostHogSpan, describeError, inertSpan, monotonicNow, runWithActiveSpan } from './span' import { newSpanId, newTraceId } from './ids' import { parseTraceparent, sanitizeTracestate } from './traceparent' -import { assignUserAttributes, resolveStartTime, sanitizeName } from './sanitize' +import { resolveStartTime, sanitizeName } from './sanitize' +import { assignUserAttributes } from '../utils/json-utils' import { buildOtlpSpan, buildOtlpTracesPayload, buildTracesResourceAttributes } from './otlp' import { isPromise, safeSetTimeout } from '../utils' @@ -180,11 +181,7 @@ export class PostHogTraces { const span = this.startSpan(name, options) try { - // The shared no-op is never activated, so `getActiveSpan()` inside the - // callback reads null — callbacks should use the handle they're given. A - // pass-through handle is activated, so `getActiveSpan()?.traceparent()` - // still propagates an inbound trace through a service with tracing off. - const result = span === NOOP_SPAN ? fn(span) : this._contextManager.with(span, () => fn(span)) + const result = runWithActiveSpan(this._contextManager, span, fn) if (isPromise(result)) { return result.then( diff --git a/packages/core/src/traces/sanitize.ts b/packages/core/src/traces/sanitize.ts index 47257deba7..bd31398941 100644 --- a/packages/core/src/traces/sanitize.ts +++ b/packages/core/src/traces/sanitize.ts @@ -6,7 +6,6 @@ import type { Logger } from '../types' import type { SpanAttributes, SpanTimeInput } from '@posthog/types' -import { UNSERIALIZABLE_VALUE } from '../utils/json-utils' const FALLBACK_SPAN_NAME = 'unknown' @@ -108,38 +107,3 @@ export function resolveSuppliedTime( } return supplied } - -/** - * Copies caller-supplied attributes onto `target`, own enumerable keys only. - * - * Read key by key rather than spread: a getter over a disposed resource or a - * revoked proxy throws on the read itself, before the encoder's guards see it. - * - * @internal Exposed for cross-package use within this SDK; not part of the stable public API. - */ -export function assignUserAttributes>( - target: T, - source: Record | undefined -): T { - if (!source) { - return target - } - let keys: string[] = [] - try { - keys = Object.keys(source) - } catch { - keys = [] - } - for (const key of keys) { - let value: unknown - try { - value = source[key] - } catch { - value = UNSERIALIZABLE_VALUE - } - // defineProperty, not assignment: `attributes['__proto__'] = v` hits the - // prototype setter and the attribute vanishes. - Object.defineProperty(target, key, { value, enumerable: true, writable: true, configurable: true }) - } - return target -} diff --git a/packages/core/src/traces/span.ts b/packages/core/src/traces/span.ts index f8564ae179..6336c00cbb 100644 --- a/packages/core/src/traces/span.ts +++ b/packages/core/src/traces/span.ts @@ -1,8 +1,9 @@ import type { Span, SpanAttributes, SpanAttributeValue, SpanKind, SpanStatusCode, SpanTimeInput } from '@posthog/types' import type { Logger } from '../types' -import type { SpanEventRecord, SpanRecord } from './types' +import type { SpanContextManager, SpanEventRecord, SpanRecord } from './types' import { formatTraceparent, normalizeTraceparent, sanitizeTracestate, TRACE_FLAGS_SAMPLED } from './traceparent' -import { assignUserAttributes, clampEndTime, resolveSuppliedTime, sanitizeName } from './sanitize' +import { clampEndTime, resolveSuppliedTime, sanitizeName } from './sanitize' +import { assignUserAttributes } from '../utils/json-utils' import { isError } from '../utils' /** @@ -275,6 +276,18 @@ export function inertSpan(options?: { parent?: unknown; tracestate?: unknown }): return new PassThroughSpan(traceparent, sanitizeTracestate(options?.tracestate)) } +/** + * Runs `fn` with `span` active, which every scoped helper does the same way. + * + * The shared no-op is never activated, so `getActiveSpan()` inside the callback + * reads null — callbacks should use the handle they're given. A pass-through + * handle is activated, so `getActiveSpan()?.traceparent()` still propagates an + * inbound trace through a service with tracing off. + */ +export function runWithActiveSpan(contextManager: SpanContextManager, span: Span, fn: (span: Span) => T): T { + return span === NOOP_SPAN ? fn(span) : contextManager.with(span, () => fn(span)) +} + /** * Extracts the OTel `exception.type` / `exception.message` pair from whatever was * thrown. Anything can be thrown in JS, so non-Errors are described by type. diff --git a/packages/core/src/utils/json-utils.ts b/packages/core/src/utils/json-utils.ts index 95007246d8..bf92ecc80f 100644 --- a/packages/core/src/utils/json-utils.ts +++ b/packages/core/src/utils/json-utils.ts @@ -153,3 +153,36 @@ export function toJsonSafeValue(value: unknown): unknown { return convert(value, 0) } + +/** + * Copies caller-supplied attributes onto `target`, own enumerable keys only. + * + * Read key by key rather than spread: a getter over a disposed resource or a + * revoked proxy throws on the read itself, before the encoder's guards see it. + */ +export function assignUserAttributes>( + target: T, + source: Record | undefined +): T { + if (!source) { + return target + } + let keys: string[] = [] + try { + keys = Object.keys(source) + } catch { + keys = [] + } + for (const key of keys) { + let value: unknown + try { + value = source[key] + } catch { + value = UNSERIALIZABLE_VALUE + } + // defineProperty, not assignment: `attributes['__proto__'] = v` hits the + // prototype setter and the attribute vanishes. + Object.defineProperty(target, key, { value, enumerable: true, writable: true, configurable: true }) + } + return target +} diff --git a/packages/core/src/utils/otlp-resource.ts b/packages/core/src/utils/otlp-resource.ts index 09ac1d9090..50963e8488 100644 --- a/packages/core/src/utils/otlp-resource.ts +++ b/packages/core/src/utils/otlp-resource.ts @@ -1,4 +1,4 @@ -import { assignUserAttributes } from '../traces/sanitize' +import { assignUserAttributes } from './json-utils' /** * Shape the logs, metrics and traces resolved configs share for resource diff --git a/packages/node/src/client.ts b/packages/node/src/client.ts index 08cb08b189..adf99bee1a 100644 --- a/packages/node/src/client.ts +++ b/packages/node/src/client.ts @@ -18,10 +18,11 @@ import { PostHogMetrics, PostHogPersistedProperty, PostHogTraces, - NOOP_SPAN, inertSpan, Properties, resolveMetricsConfig, + resolveTracesConfig, + runWithActiveSpan, RetriableOptions, raceWithTimeout, safeSetTimeout, @@ -29,7 +30,6 @@ import { uuidv7, } from '@posthog/core' import type { Metrics, Span, SpanContextManager, StartSpanOptions, TraceSdkContext } from '@posthog/core' -import { resolveTracesConfig } from './traces-defaults' import { AllFlagsOptions, EventMessage, @@ -747,8 +747,7 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen // Tracing off: still run the callback exactly once, with an inert handle. // A handle carrying an inbound `parent` is activated, so `getActiveSpan()` // inside the callback can propagate the trace onward. - const span = inertSpan(options) - return span === NOOP_SPAN ? fn(span) : this._spanContextManager.with(span, () => fn(span)) + return runWithActiveSpan(this._spanContextManager, inertSpan(options), fn) } return options ? pipeline.withSpan(name, options, fn) : pipeline.withSpan(name, fn) } From 44b53a14348f4a731d5c87ef1bd445e7711ade0e Mon Sep 17 00:00:00 2001 From: Anna Garcia <11654201+turnipdabeets@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:42:52 -0400 Subject: [PATCH 09/24] fix(traces): reject a traceparent W3C requires a vendor to ignore (#4776) * fix(traces): restart the trace on a version 00 traceparent with extra fields W3C defines version 00 as exactly `trace-id "-" parent-id "-" trace-flags` and scopes the tolerate-trailing-fields rule to a higher version, so a version 00 header carrying anything more is malformed. Accepting it continued a trace that a conformant peer restarts, splitting the trace across the two services. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018AAPCyRnC5HbukEZV9hbai * fix(traces): stop folding an inbound traceparent to lowercase W3C spells every traceparent field lowercase hex and requires a vendor to ignore a header whose ids are not, so folding the case continued a trace that a conformant peer restarts. Surrounding whitespace is still trimmed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018AAPCyRnC5HbukEZV9hbai * chore: changeset for the traceparent conformance fixes Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018AAPCyRnC5HbukEZV9hbai * chore: declare only @posthog/core in the traceparent changeset No file under packages/node changed, and the entry now reads as one line for someone skimming release notes rather than carrying its rationale. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018AAPCyRnC5HbukEZV9hbai * chore(traces): drop the traceparent comments that restate the code Leaves the W3C provenance on each rejection and takes out the sentences that repeat the regex, the trim, or the test name. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018AAPCyRnC5HbukEZV9hbai --------- Co-authored-by: Claude Opus 5 (1M context) --- .changeset/traceparent-w3c-strictness.md | 5 +++++ packages/core/src/traces/traceparent.spec.ts | 22 ++++++++++++++------ packages/core/src/traces/traceparent.ts | 18 ++++++++++++---- 3 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 .changeset/traceparent-w3c-strictness.md diff --git a/.changeset/traceparent-w3c-strictness.md b/.changeset/traceparent-w3c-strictness.md new file mode 100644 index 0000000000..950193e06e --- /dev/null +++ b/.changeset/traceparent-w3c-strictness.md @@ -0,0 +1,5 @@ +--- +'@posthog/core': patch +--- + +Ignore an inbound `traceparent` that W3C requires a vendor to reject — a version `00` header carrying fields beyond `trace-id`, `parent-id` and `trace-flags`, or one whose ids are uppercase hex — and start a fresh trace instead. diff --git a/packages/core/src/traces/traceparent.spec.ts b/packages/core/src/traces/traceparent.spec.ts index a6460e65ef..872fea75cd 100644 --- a/packages/core/src/traces/traceparent.spec.ts +++ b/packages/core/src/traces/traceparent.spec.ts @@ -42,14 +42,24 @@ describe('traceparent', () => { }) }) - it('normalizes case and surrounding whitespace', () => { - expect(parseTraceparent(` 00-${TRACE_ID.toUpperCase()}-${SPAN_ID.toUpperCase()}-01 `)).toEqual({ + it('rejects version 00 with extra fields, which only a higher version may carry', () => { + expect(parseTraceparent(`00-${TRACE_ID}-${SPAN_ID}-01-something`)).toBeUndefined() + expect(parseTraceparent(`00-${TRACE_ID}-${SPAN_ID}-01-`)).toBeUndefined() + }) + + it('trims surrounding whitespace', () => { + expect(parseTraceparent(` 00-${TRACE_ID}-${SPAN_ID}-01 `)).toEqual({ traceId: TRACE_ID, spanId: SPAN_ID, flags: '01', }) }) + it('rejects uppercase hex, which W3C requires a vendor to ignore', () => { + expect(parseTraceparent(`00-${TRACE_ID.toUpperCase()}-${SPAN_ID}-01`)).toBeUndefined() + expect(parseTraceparent(`00-${TRACE_ID}-${SPAN_ID.toUpperCase()}-01`)).toBeUndefined() + }) + it.each([ ['garbage', 'garbage'], ['an empty string', ''], @@ -130,16 +140,16 @@ describe('normalizeTraceparent', () => { expect(normalizeTraceparent(`01-${TRACE_ID}-${SPAN_ID}-01`)).toBe(`01-${TRACE_ID}-${SPAN_ID}-01`) }) - it('canonicalises whitespace and case, and drops unknown trailing fields', () => { - expect(normalizeTraceparent(` 00-${TRACE_ID.toUpperCase()}-${SPAN_ID}-01-extra `)).toBe( - `00-${TRACE_ID}-${SPAN_ID}-01` - ) + it("trims surrounding whitespace, and drops a higher version's trailing fields", () => { + expect(normalizeTraceparent(` 01-${TRACE_ID}-${SPAN_ID}-01-extra `)).toBe(`01-${TRACE_ID}-${SPAN_ID}-01`) }) it.each([ ['a malformed header', 'not-a-traceparent'], ['the invalid ff version', `ff-${TRACE_ID}-${SPAN_ID}-01`], ['an all-zero trace id', `00-${'0'.repeat(32)}-${SPAN_ID}-01`], + ['version 00 with trailing fields', `00-${TRACE_ID}-${SPAN_ID}-01-extra`], + ['an uppercase trace id', `00-${TRACE_ID.toUpperCase()}-${SPAN_ID}-01`], ['a non-string', ['a', 'b']], ])('rejects %s', (_name, value) => { expect(normalizeTraceparent(value)).toBeUndefined() diff --git a/packages/core/src/traces/traceparent.ts b/packages/core/src/traces/traceparent.ts index 40fb622f3e..826cd22456 100644 --- a/packages/core/src/traces/traceparent.ts +++ b/packages/core/src/traces/traceparent.ts @@ -7,8 +7,9 @@ export interface RemoteSpanContext { flags: string } -// `00-<32 hex>-<16 hex>-<2 hex>`. Version `ff` is invalid per the spec; other -// unknown versions are forwards-compatible, so we parse the first four fields only. +// Version `ff` is invalid per the spec, and a higher version may append fields +// after the first four, so the trailing group captures them rather than failing +// the match. const TRACEPARENT_RE = /^([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})(-.*)?$/ /** @@ -45,14 +46,23 @@ function matchTraceparent(value: unknown): TraceparentFields | undefined { if (typeof value !== 'string') { return undefined } - const match = TRACEPARENT_RE.exec(value.trim().toLowerCase()) + // W3C spells every field lowercase hex and requires a vendor to ignore a + // `traceparent` whose ids are not, so folding the case here would continue a + // trace that a conformant peer restarts. + const match = TRACEPARENT_RE.exec(value.trim()) if (!match) { return undefined } - const [, version, traceId, spanId, flags] = match + const [, version, traceId, spanId, flags, trailing] = match if (version === 'ff') { return undefined } + // Version `00` is defined as exactly `trace-id "-" parent-id "-" trace-flags`. + // W3C scopes the tolerate-what-you-don't-know rule to a *higher* version, so a + // version `00` header with anything appended is malformed. + if (version === '00' && trailing) { + return undefined + } if (!isValidTraceId(traceId) || !isValidSpanId(spanId)) { return undefined } From 24d51e30022362e9cc63775248bedfc3b0ffd250 Mon Sep 17 00:00:00 2001 From: Anna Garcia <11654201+turnipdabeets@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:01:06 -0400 Subject: [PATCH 10/24] fix(traces): keep inbound context and flag a future span start (#4790) * fix(traces): keep inbound context and flag a future span start Four conformance gaps an independent review of the traces stack found, each reproduced before it was changed. A `traceparent` above version `00` was rebuilt from the four fields this SDK reads, so a peer that understands the version's extra fields received a header still labelled with it but missing them. It is now forwarded whole. A handle returned while tracing was off carried the inbound trace context, but passing it back as `parent` produced a bare no-op, so everything below it started a fresh trace. The child stays inert, as the spec requires, and now echoes that context. A `startTime` in the future was accepted silently and cost the span its duration; ingestion stores it un-clamped. It now warns, matching how a deep backdate is handled. A batch dropped as non-retriable or too large left the consecutive-failure count standing, which held the depth trigger off and the flush interval at its ceiling until an unrelated send succeeded. A removed batch is progress, so the count clears with it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MTHb6UUMhVJSWC2cWBzSxc * Update packages/core/src/traces/sanitize.ts Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../traces-context-and-time-fidelity.md | 5 + packages/core/src/traces/index.spec.ts | 98 ++++++++++++++++++- packages/core/src/traces/index.ts | 10 +- packages/core/src/traces/sanitize.ts | 4 + packages/core/src/traces/span.ts | 19 +++- packages/core/src/traces/traceparent.spec.ts | 8 +- packages/core/src/traces/traceparent.ts | 11 ++- 7 files changed, 142 insertions(+), 13 deletions(-) create mode 100644 .changeset/traces-context-and-time-fidelity.md diff --git a/.changeset/traces-context-and-time-fidelity.md b/.changeset/traces-context-and-time-fidelity.md new file mode 100644 index 0000000000..c2cab61516 --- /dev/null +++ b/.changeset/traces-context-and-time-fidelity.md @@ -0,0 +1,5 @@ +--- +'@posthog/core': patch +--- + +Forward an inbound `traceparent` whose version is above `00` whole, including the fields that version adds, rather than trimming it to the four this SDK reads. Keep the inbound context when a handle returned with tracing off is passed back as `parent`, so a child no longer starts a fresh trace. Warn when a span's `startTime` is in the future, which costs it its duration. diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 20b412cfcc..a6d5aeb4a5 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -1,6 +1,6 @@ import { PostHogTraces } from './index' import { SyncSpanContextManager } from './context' -import { NOOP_SPAN } from './span' +import { NOOP_SPAN, inertSpan } from './span' import type { OtlpSpan, OtlpTracesPayload, @@ -171,6 +171,16 @@ describe('PostHogTraces', () => { expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('24 hours')) expect(sentSpans()).toHaveLength(1) }) + + it('warns when a start is in the future, which costs the span its duration', async () => { + const traces = createTraces() + traces.startSpan('ahead', { startTime: Date.now() + 60 * 60 * 1000 }).end() + await traces.flush() + + expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('in the future')) + const [span] = sentSpans() + expect(span.endTimeUnixNano).toBe(span.startTimeUnixNano) + }) }) describe('trace continuation', () => { @@ -493,6 +503,49 @@ describe('PostHogTraces', () => { expect(traces.startSpan('no-parent')).toBe(NOOP_SPAN) expect(traces.startSpan('bad-parent', { parent: 'not-a-traceparent' })).toBe(NOOP_SPAN) }) + + it('keeps the inbound context when a pass-through handle is used as a parent', () => { + // The child is inert either way; what it must not do is drop the context + // and leave everything downstream of it on a fresh trace. + const traces = createTraces({}, createMockInstance({ optedOut: true })) + const parent = traces.startSpan('proxied', { parent: INBOUND_UNSAMPLED, tracestate: 'vendor=abc' }) + + const child = traces.startSpan('child', { parent }) + + expect(child.traceparent()).toBe(INBOUND_UNSAMPLED) + expect(child.tracestate()).toBe('vendor=abc') + expect(sentSpans()).toHaveLength(0) + }) + + it('records nothing for a child of a pass-through once tracing is back on', async () => { + // Spec: a child of an inert handle is inert, so this must propagate without + // enqueueing a span, even though this instance is recording. + const traces = createTraces() + const parent = inertSpan({ parent: INBOUND_UNSAMPLED }) + + const child = traces.startSpan('child', { parent }) + child.end() + await traces.flush() + + expect(child.traceparent()).toBe(INBOUND_UNSAMPLED) + expect(sentSpans()).toHaveLength(0) + }) + + it('yields a no-op for a child of a no-op', () => { + const traces = createTraces() + expect(traces.startSpan('child', { parent: NOOP_SPAN })).toBe(NOOP_SPAN) + }) + + it('survives a parent whose traceparent throws', () => { + const traces = createTraces() + const hostile = { + traceparent: () => { + throw new Error('nope') + }, + } + + expect(traces.startSpan('child', { parent: hostile as never })).toBe(NOOP_SPAN) + }) }) describe('auto-context', () => { @@ -992,6 +1045,49 @@ describe('PostHogTraces', () => { }) }) + describe('flush backoff', () => { + it('resumes the depth trigger after a non-retriable drop clears the backlog', async () => { + // A 503 burst raises the consecutive-failure count, which disables the + // depth trigger. Dropping the poison batch is progress, so the count has to + // clear with it or the queue stays on the slow timer while the endpoint is + // healthy. + mockInstance._sendTracesBatch + .mockResolvedValueOnce({ kind: 'retry-later', error: new Error('503') }) + .mockResolvedValueOnce({ kind: 'fatal', error: new Error('400') }) + .mockResolvedValue({ kind: 'ok' }) + + const traces = createTraces({ maxExportBatchSize: 1, flushIntervalMs: 10_000 }) + traces.startSpan('poison').end() + await flushMicrotasks() + await traces.flush() + mockInstance._sendTracesBatch.mockClear() + + // Depth trigger only fires again if the failure count was cleared. + traces.startSpan('after').end() + await flushMicrotasks() + + expect(mockInstance._sendTracesBatch).toHaveBeenCalled() + }) + + it('resumes the depth trigger after a too-large single-span drop', async () => { + mockInstance._sendTracesBatch + .mockResolvedValueOnce({ kind: 'retry-later', error: new Error('503') }) + .mockResolvedValueOnce({ kind: 'too-large' }) + .mockResolvedValue({ kind: 'ok' }) + + const traces = createTraces({ maxExportBatchSize: 1, flushIntervalMs: 10_000 }) + traces.startSpan('huge').end() + await flushMicrotasks() + await traces.flush() + mockInstance._sendTracesBatch.mockClear() + + traces.startSpan('after').end() + await flushMicrotasks() + + expect(mockInstance._sendTracesBatch).toHaveBeenCalled() + }) + }) + describe('drop accounting', () => { it('still warns about queue-full drops while the endpoint is failing', async () => { mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('down') }) diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 89b42f1a14..3776416872 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -8,7 +8,7 @@ import type { TraceSdkContext, TracesHost, } from './types' -import { NOOP_SPAN, PostHogSpan, describeError, inertSpan, monotonicNow, runWithActiveSpan } from './span' +import { PostHogSpan, describeError, inertSpan, monotonicNow, runWithActiveSpan } from './span' import { newSpanId, newTraceId } from './ids' import { parseTraceparent, sanitizeTracestate } from './traceparent' import { resolveStartTime, sanitizeName } from './sanitize' @@ -115,9 +115,11 @@ export class PostHogTraces { const explicitParent = options?.parent if (explicitParent && typeof explicitParent !== 'string' && !isOwnSpan(explicitParent)) { if (looksLikeSpan(explicitParent)) { - // A child of a no-op is itself a no-op, never an orphan with invented ids. + // Inert like its parent, never an orphan with invented ids — but a + // pass-through parent's inbound context carries to the child rather than + // the trace ending here. this._logger.debug('Span parent is not a span from this SDK; returning an inert span') - return NOOP_SPAN + return inertSpan(options) } // No `traceparent()` to read: `req.headers.traceparent` is `string[]` when the // header arrives twice, and a span from another tracer exposes `spanContext()` @@ -523,6 +525,7 @@ export class PostHogTraces { remaining -= 1 removed += 1 this._recordDrop(1, 'the ingestion endpoint rejected it as too large') + this._consecutiveFlushFailures = 0 this._headBatchFailures = 0 continue } @@ -560,6 +563,7 @@ export class PostHogTraces { this._queue.splice(0, size) remaining -= size removed += size + this._consecutiveFlushFailures = 0 this._headBatchFailures = 0 this._recordDrop(size, 'the ingestion endpoint rejected the batch') } diff --git a/packages/core/src/traces/sanitize.ts b/packages/core/src/traces/sanitize.ts index bd31398941..258620cf53 100644 --- a/packages/core/src/traces/sanitize.ts +++ b/packages/core/src/traces/sanitize.ts @@ -75,6 +75,10 @@ export function resolveStartTime(value: SpanTimeInput | undefined, now: number, logger?.debug( 'Span startTime is more than 24 hours in the past; the server will clamp it to receive time and keep the original in $originalTimestamp' ) + } else if (supplied > now) { + // Warned rather than clamped, matching the deep-backdate rule: the value is + // the caller's. The duration is what suffers, since the end clamps to it. + logger?.debug('Span startTime is in the future; the span may export with a zero duration') } return supplied } diff --git a/packages/core/src/traces/span.ts b/packages/core/src/traces/span.ts index 6336c00cbb..a93ea62d31 100644 --- a/packages/core/src/traces/span.ts +++ b/packages/core/src/traces/span.ts @@ -269,11 +269,26 @@ export class PassThroughSpan extends NoopSpan { * caller supplied a usable `parent` header, the shared no-op otherwise. */ export function inertSpan(options?: { parent?: unknown; tracestate?: unknown }): Span { - const traceparent = normalizeTraceparent(options?.parent) + const parent = options?.parent + // A handle parent reports its own context, read behind a guard because a + // foreign handle's accessor may throw. A no-op reports none and stays a no-op. + const inbound = typeof parent === 'string' || parent == null ? parent : readHandle(parent, 'traceparent') + const traceparent = normalizeTraceparent(inbound) if (!traceparent) { return NOOP_SPAN } - return new PassThroughSpan(traceparent, sanitizeTracestate(options?.tracestate)) + const tracestate = + typeof parent === 'string' || parent == null ? options?.tracestate : readHandle(parent, 'tracestate') + return new PassThroughSpan(traceparent, sanitizeTracestate(tracestate)) +} + +function readHandle(parent: unknown, method: 'traceparent' | 'tracestate'): unknown { + try { + const fn = (parent as Span)[method] + return typeof fn === 'function' ? fn.call(parent) : undefined + } catch { + return undefined + } } /** diff --git a/packages/core/src/traces/traceparent.spec.ts b/packages/core/src/traces/traceparent.spec.ts index 872fea75cd..2b65face90 100644 --- a/packages/core/src/traces/traceparent.spec.ts +++ b/packages/core/src/traces/traceparent.spec.ts @@ -140,8 +140,12 @@ describe('normalizeTraceparent', () => { expect(normalizeTraceparent(`01-${TRACE_ID}-${SPAN_ID}-01`)).toBe(`01-${TRACE_ID}-${SPAN_ID}-01`) }) - it("trims surrounding whitespace, and drops a higher version's trailing fields", () => { - expect(normalizeTraceparent(` 01-${TRACE_ID}-${SPAN_ID}-01-extra `)).toBe(`01-${TRACE_ID}-${SPAN_ID}-01`) + it("keeps a higher version's trailing fields, so a peer that reads them still can", () => { + expect(normalizeTraceparent(`01-${TRACE_ID}-${SPAN_ID}-01-extra`)).toBe(`01-${TRACE_ID}-${SPAN_ID}-01-extra`) + }) + + it('trims surrounding whitespace', () => { + expect(normalizeTraceparent(` 00-${TRACE_ID}-${SPAN_ID}-01 `)).toBe(`00-${TRACE_ID}-${SPAN_ID}-01`) }) it.each([ diff --git a/packages/core/src/traces/traceparent.ts b/packages/core/src/traces/traceparent.ts index 826cd22456..1593ffecf0 100644 --- a/packages/core/src/traces/traceparent.ts +++ b/packages/core/src/traces/traceparent.ts @@ -70,13 +70,14 @@ function matchTraceparent(value: unknown): TraceparentFields | undefined { } /** - * The canonical form of an inbound `traceparent`, or `undefined` when it is - * malformed. Version and flags are carried through as received, so a service - * that forwards this value continues the caller's trace exactly as sent. + * The inbound `traceparent` as received, or `undefined` when it is malformed. + * + * Returned whole rather than rebuilt: a version above `00` may append fields + * this SDK does not read, and rebuilding would forward a header still labelled + * with that version but missing what the version defines. */ export function normalizeTraceparent(value: unknown): string | undefined { - const fields = matchTraceparent(value) - return fields && `${fields.version}-${fields.traceId}-${fields.spanId}-${fields.flags}` + return matchTraceparent(value) && (value as string).trim() } /** The W3C sampled bit, set on a trace this SDK started. */ From 9d10ad44e4a58b7fa4bea555fa958208b686ae61 Mon Sep 17 00:00:00 2001 From: Anna Garcia <11654201+turnipdabeets@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:45:22 -0400 Subject: [PATCH 11/24] feat(node): beforeSpanSend hook and per-span limits (#4584) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(node): beforeSpanSend hook and per-span limits Adds a scrubbing/drop hook on finished spans, caps user attributes, events and attribute-value length per span, and records exception stacktraces. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0138vKReknFeXLbvMyUX7fPZ * fix(traces): take the propagation fields from the span, not the hook Restoring traceFlags and parentIsRemote onto a hook's return value throws when the hook freezes it, dropping the span. Read them from the pre-hook snapshot in the rebuild, the same treatment the dropped counts already get. * refactor(traces): derive the internal span record from the hook-visible one The two declarations repeated every shared field. Declaring only what a hook may not rewrite keeps the halves from drifting; no behaviour change. * fix(traces): close four gaps in the per-span attribute bound - a back-reference is handed back as the encoder's marker, not the raw ancestor, which a copied parent let the encoder walk one level unbounded - arrays are built from the walked range rather than sliced, so an accessor past the encoder cap cannot cost every item its bound - the key walk stops where the encoder stops emitting, instead of running every getter on a wide object - bounded values are always copied, so a caller mutating what it passed to setAttribute cannot change what ships Also warn when a non-function beforeSpanSend entry is dropped: it is the redaction point, and silently filtering nothing ships what it would remove. * fix(traces): keep a span whose hook froze a rebuilt record Restoring the identity onto the hook's return value throws when the hook freezes it, and a throwing hook drops the span — so a rebuilding, freezing scrubber lost every span carrying a parent id while roots still exported. The rebuild takes all four identity fields from the pre-hook snapshot, and the write-back that lets a later hook in the chain read true ids is now best-effort. A forged id is ignored with the documented debug warning instead of costing the span. Also leave a Date unbounded, matching the encoder's own Date branch: the value walk turned it into an ISO string and then cut it, shipping a corrupted timestamp at a low maxAttributeValueLength. * fix(traces): report an inert beforeSpanSend entry at critical posthog-node gates every level except critical behind debug: true, so the warning an operator most needs — a redaction hook that is silently doing nothing — never reached them in the default configuration. * test(traces): cover the hook shape that actually drops the propagation fields The existing tests spread the record, which carries traceFlags through whether or not the rebuild reads it from the span, so both passed against the bug. Name the public fields instead, the shape the hook-visible type invites. Typing the rebuild so every field must be named turns a future field on either half of SpanRecord into a compile error there. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013ERsAtgPJqKF46yur8bm6K * fix(traces): keep earliest-set attributes and bound span names applySpanLimits walked Object.keys, which hoists integer-like keys ahead of insertion order, so a beforeSpanSend hook adding `attributes['0']` evicted an attribute the caller had set before it ran. The spec makes the cap earliest-set-wins, so the walk now follows the span's own pre-hook key order. Span names and event names were the only caller-controlled free text left unbounded: a name built from a URL shipped whole while the status message beside it obeyed maxAttributeValueLength, and one large enough 413s the batch. Also: a discriminating test for the maxEventsPerSpan resolver, which a mutation survived; TSDoc on the hook-visible SpanRecord members; and the changeset now says how large the exception reserve is instead of calling it small. * fix(traces): charge every value against the attribute traversal budget Leaves skipped the node charge, so only containers spent the budget. A value whose siblings share a subtree is re-walked once per path that reaches it, and without a leaf charge that costs budget * items string copies where the encoder stops at budget: a 38 MB shared graph took 245 ms inside setAttribute and left 422 MB on the span record until flush. Now 4 ms and 19 MB. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013ERsAtgPJqKF46yur8bm6K * fix(traces): stop a hook-deleted prototype key from coming back orderedKeys tested `key in attributes`, which walks the prototype chain, so an attribute named `constructor` or `toString` survived the hook deleting it and read back as the inherited member. At the cap those ghosts took the slots: a hook returning a scrubbed bag exported ["toString","valueOf"] and dropped the one attribute it meant to keep. Three more from the same review: - The new name bound cut the SDK's own `exception` event name below a bound of 9, so the reserve stopped recognising it and dropped the event it exists to keep. `eventNameBound` floors it, at both sanitize call sites. - `_startFlush` installed its in-flight slot only after `_flushInner` had run its synchronous prefix, which this PR lengthened by bounding the resource attributes. A getter there that ends a span re-entered with no pass recorded and re-sent the head batch — 3070 times in a probe, not once. The pass now starts a microtask later, after the slot is installed. - A `false` entry from `[featureEnabled && scrub]` no longer reports an inert redaction hook, and the maxAttributeValueLength doc lists what it now bounds. * fix(traces): stop nullish leaves spending the truncation budget The encoder drops a nullish value without charging its budget, so charging one here made this walk the stricter of the two and broke the invariant the rest of the walk relies on: a value with 10,000 null leaves ahead of a large string exhausted this budget while the encoder still had room, and the string shipped whole — 2 MB under a bound of 8, with no backstop on either side. Nullish leaves are free again; the shared-subtree cost that charging fixed stays fixed. Also from the same review: - `_startFlush` samples the generation before the microtask, so a `reset()` in that window marks the pending pass stale rather than letting it drain the post-reset queue alongside the pass `reset()` started. - `orderedKeys` uses `propertyIsEnumerable`, the predicate the encoder itself uses, so a key a hook hid by making it non-enumerable cannot take a cap slot. - The `beforeSpanSend` filter reports only on a value meant to be a hook, so `[items.length && scrub]` and `[name && scrub]` are quiet too. * fix(traces): wait for the span export before a flush settles `_flushEventsAndSpans` combined the two flushes with `Promise.all`, which rejects the moment the event flush does. A serverless host treats the returned promise as the end of the invocation, so an event endpoint failing mid-request let the platform freeze the handler with the span POST still open. `allSettled` waits for both and still surfaces the events rejection to the caller. `reset()` also discarded whatever was queued without a word, while the only line the operator had seen was the export failure promising a retry on a flush that will never come. It now names the count at `critical`, the one level posthog-node does not gate behind `debug`. * docs(traces): tighten the truncation comments and the traces changesets Budget parity was explained in four places; keep it in the function doc. Rewrite the three changesets as one-line, outcome-first entries. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013ERsAtgPJqKF46yur8bm6K * fix(traces): recheck consent between span batches A drain sends one batch per loop iteration, so the user could opt out while a batch was in flight and the batches behind it would still export. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01L8eFZB35NExUZyq5hGgh43 * fix(traces): materialize a toJSON that resolves to nothing Keeping the object left the encoder to probe toJSON a second time, so a serializer that answered null under the bound could answer with a megabyte over it. Stores the string the encoder builds from the same result instead, which leaves the wire unchanged and gives it nothing left to re-probe. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MTHb6UUMhVJSWC2cWBzSxc * fix(traces): take the default for a fractional numeric option Flooring made maxAttributesPerSpan: 1.5 resolve to 1, capping a span an order of magnitude below what the caller wrote and saying nothing. Every numeric traces option now falls back the way the spec describes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MTHb6UUMhVJSWC2cWBzSxc * fix(traces): drop a beforeSpanSend result missing a required field Carrying attributes and events was the whole shape check, so a hook returning only those two exported a span named unknown at a fallback time with no join keys, silently. A record missing any field the public SpanRecord requires is now a counted drop, as the rest of the hook contract already is. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MTHb6UUMhVJSWC2cWBzSxc * fix(traces): give a later beforeSpanSend hook the real span identity An earlier hook that forged an id and froze what it returned refused the restoring writes, so the next hook in the chain sampled on the forged id. Hands it a corrected view built from the record's own descriptors, which keeps the prototype and the keys the hook returned. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MTHb6UUMhVJSWC2cWBzSxc * test(traces): pin the toJSON bound through buildOtlpSpan The second call is the encoder's, so the guarantee is worth asserting past the record. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MTHb6UUMhVJSWC2cWBzSxc * fix(traces): reserve exception slots by provenance, not event name The reserve is for what the SDK records on your behalf, so an event named `exception` by the caller was claiming it too. Marks SDK-recorded events with an internal symbol the hook cannot see and the wire cannot carry, and keys both enforcement points on that. Removes eventNameBound with it: the name no longer decides anything, so it no longer needs a floor. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MTHb6UUMhVJSWC2cWBzSxc * fix(traces): make maxEventsPerSpan an absolute cap Drops the four reserved slots for exception events. The reserve bought a case we cannot show occurs — a span holding 128 events that then throws — at the cost of the most intricate code in this change, which had already carried one bug. The cap now matches the spec's number exactly. A span that fills its events and then throws keeps its error status and reports the loss through droppedEventsCount, so the case is measurable once traces ships and the reserve can be added back additively if it turns out to matter. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MTHb6UUMhVJSWC2cWBzSxc * docs(traces): drop the exception reserve from the span-limits changeset The reserve was removed and the event cap is now absolute, but the changeset still promised it. Also names the dropped counters, which are how a caller sees that a span was truncated. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01J1XjgPzJ4zbHBcYmEDymA2 --------- Co-authored-by: Claude Opus 5 (1M context) --- .changeset/node-before-span-send.md | 7 + .changeset/node-exception-stacktrace.md | 7 + .changeset/node-span-limits.md | 7 + packages/core/src/index.ts | 3 + packages/core/src/traces/config.spec.ts | 96 +- packages/core/src/traces/config.ts | 55 +- packages/core/src/traces/index.spec.ts | 1110 +++++++++++++++++ packages/core/src/traces/index.ts | 377 +++++- packages/core/src/traces/live-spans.spec.ts | 4 + packages/core/src/traces/otlp.spec.ts | 4 + packages/core/src/traces/otlp.ts | 11 + packages/core/src/traces/sanitize.ts | 7 +- packages/core/src/traces/span.spec.ts | 613 ++++++++- packages/core/src/traces/span.ts | 473 ++++++- packages/core/src/traces/types.ts | 37 +- packages/node/src/__tests__/traces.spec.ts | 119 ++ .../src/__tests__/waituntil-flush.spec.ts | 36 + packages/node/src/client.ts | 13 +- packages/node/src/exports.ts | 2 + packages/types/src/index.ts | 2 + packages/types/src/traces.ts | 109 +- 21 files changed, 3001 insertions(+), 91 deletions(-) create mode 100644 .changeset/node-before-span-send.md create mode 100644 .changeset/node-exception-stacktrace.md create mode 100644 .changeset/node-span-limits.md diff --git a/.changeset/node-before-span-send.md b/.changeset/node-before-span-send.md new file mode 100644 index 0000000000..8d2afd5521 --- /dev/null +++ b/.changeset/node-before-span-send.md @@ -0,0 +1,7 @@ +--- +'posthog-node': minor +'@posthog/core': minor +'@posthog/types': minor +--- + +Add a `traces.beforeSpanSend` hook to edit a finished span before it is queued, or drop it by returning `null` — a hook that throws, or returns anything that is not a span record, also drops the span. diff --git a/.changeset/node-exception-stacktrace.md b/.changeset/node-exception-stacktrace.md new file mode 100644 index 0000000000..e4cfa02136 --- /dev/null +++ b/.changeset/node-exception-stacktrace.md @@ -0,0 +1,7 @@ +--- +'posthog-node': minor +'@posthog/core': minor +'@posthog/types': minor +--- + +Attach `exception.stacktrace` to the exception events recorded by `recordException` and by a throwing `withSpan` callback — remove it in `traces.beforeSpanSend` to keep your server's file paths out of PostHog. diff --git a/.changeset/node-span-limits.md b/.changeset/node-span-limits.md new file mode 100644 index 0000000000..fbe06127fc --- /dev/null +++ b/.changeset/node-span-limits.md @@ -0,0 +1,7 @@ +--- +'posthog-node': minor +'@posthog/core': minor +'@posthog/types': minor +--- + +Cap spans at 128 user attributes, 128 events and 8192 characters per string, configurable with `traces.maxAttributesPerSpan`, `traces.maxEventsPerSpan` and `traces.maxAttributeValueLength`. The earliest entries are kept, and a span that lost any reports how many as `droppedAttributesCount` and `droppedEventsCount`. The event cap is absolute, so an `exception` event the SDK records for you spends an ordinary slot. diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 8a3aad8015..dfe61ff460 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -92,6 +92,9 @@ export { SyncSpanContextManager } from './traces/context' export { inertSpan, runWithActiveSpan } from './traces/span' export { resolveTracesConfig } from './traces/config' export type { ResolvedTracesConfig, SpanContextManager, TraceSdkContext } from './traces/types' +// The `beforeSpanSend` shapes come straight from @posthog/types: hooks see the +// public record, not core's internal one, which also carries `traceState`. +export type { SpanRecord, BeforeSpanSendFn } from '@posthog/types' // Same barrel convention as logs and metrics for the user-facing tracing types. export type { Span, diff --git a/packages/core/src/traces/config.spec.ts b/packages/core/src/traces/config.spec.ts index 902e5f35f5..5237a0d52d 100644 --- a/packages/core/src/traces/config.spec.ts +++ b/packages/core/src/traces/config.spec.ts @@ -1,16 +1,53 @@ +import { createMockLogger } from '@/testing' import { resolveTracesConfig } from './config' describe('resolveTracesConfig', () => { + it.each([ + ['zero', 0], + ['negative', -1], + ['not a number', NaN], + // Floored, these read as 1, 2 and 3 — caps an order of magnitude below what + // the caller wrote, applied silently. + ['a fraction', 1.5], + ['a large fraction', 200.5], + ['infinity', Infinity], + ])('falls back to the default per-span caps when given %s', (_label, value) => { + const resolved = resolveTracesConfig({ + maxAttributesPerSpan: value, + maxEventsPerSpan: value, + maxAttributeValueLength: value, + }) + expect(resolved.maxAttributesPerSpan).toBe(128) + expect(resolved.maxEventsPerSpan).toBe(128) + expect(resolved.maxAttributeValueLength).toBe(8192) + }) + + it('honours explicit per-span caps', () => { + // Without this the resolver can ignore maxEventsPerSpan entirely and every + // other test still passes, because they all assert the default. + expect(resolveTracesConfig({ maxAttributesPerSpan: 5, maxEventsPerSpan: 7 })).toMatchObject({ + maxAttributesPerSpan: 5, + maxEventsPerSpan: 7, + }) + }) + it('applies the documented defaults', () => { expect(resolveTracesConfig(undefined)).toMatchObject({ flushIntervalMs: 5000, maxExportBatchSize: 512, maxQueueSize: 2048, + maxAttributesPerSpan: 128, + maxEventsPerSpan: 128, + maxAttributeValueLength: 8192, maxLiveSpans: 10_000, maxSpanAgeMs: 3_600_000, }) }) + it('honours an explicit attribute value bound', () => { + expect(resolveTracesConfig({ maxAttributeValueLength: 256 }).maxAttributeValueLength).toBe(256) + }) + it('honours explicit live-span bounds', () => { expect(resolveTracesConfig({ maxLiveSpans: 50, maxSpanAgeMs: 30_000 })).toMatchObject({ maxLiveSpans: 50, @@ -65,14 +102,23 @@ describe('resolveTracesConfig', () => { } ) - it('floors a fractional batch size to an integer', () => { - expect(resolveTracesConfig({ maxExportBatchSize: 10.9 }).maxExportBatchSize).toBe(10) + it('takes the default for a fractional batch size rather than flooring it', () => { + // Every numeric knob resolves the same way, so a fraction is a value the + // caller did not mean rather than one to round down behind their back. + expect(resolveTracesConfig({ maxExportBatchSize: 10.9 }).maxExportBatchSize).toBe(512) }) - it.each([0, -1, Number.NaN])('falls back to the default for an unusable flushIntervalMs (%p)', (value) => { + it.each([0, -1, 1.5, Number.NaN])('falls back to the default for an unusable flushIntervalMs (%p)', (value) => { expect(resolveTracesConfig({ flushIntervalMs: value }).flushIntervalMs).toBe(5000) }) + it.each([0.5, 10_000.5])('falls back to the defaults for fractional live-span bounds (%p)', (value) => { + expect(resolveTracesConfig({ maxLiveSpans: value, maxSpanAgeMs: value })).toMatchObject({ + maxLiveSpans: 10_000, + maxSpanAgeMs: 3_600_000, + }) + }) + it('keeps the queue at least as large as the export batch', () => { // A queue smaller than the flush trigger would stop the depth-based flush // from ever firing. @@ -135,6 +181,50 @@ describe('resourceAttributes guarding', () => { expect(resolved.resourceAttributes).toEqual({ region: 'us' }) }) + it('ignores a beforeSpanSend entry that is not a function', () => { + // A plain-JS caller passing the wrong shape would otherwise have every span + // dropped by a hook that throws on call, with tracing silently off. + const scrub = (span: any): any => span + const resolved = resolveTracesConfig({ beforeSpanSend: ['not a function', scrub] as never }) + + expect(resolved.beforeSpanSend).toEqual([scrub]) + }) + + it('warns about a dropped hook, since the redaction it was configured for is gone', () => { + const logger = createMockLogger() + const scrub = (span: any): any => span + + resolveTracesConfig({ beforeSpanSend: ['not a function', scrub] as never }, undefined, logger) + + // `critical`, not `warn`: every other level is gated behind `debug: true`. + expect(logger.critical).toHaveBeenCalledWith(expect.stringContaining('ignoring 1 of 2 entries')) + }) + + it('stays quiet for a conditionally disabled hook', () => { + // `[featureEnabled && scrub]` is ordinary JS; nothing was configured to + // redact, so claiming redaction is broken would be false alarm. + const logger = createMockLogger() + + const resolved = resolveTracesConfig({ beforeSpanSend: [false, null, undefined] as never }, undefined, logger) + + expect(resolved.beforeSpanSend).toEqual([]) + expect(logger.critical).not.toHaveBeenCalled() + }) + + it('stays quiet when every hook is callable', () => { + const logger = createMockLogger() + + resolveTracesConfig({ beforeSpanSend: [(span: any): any => span] }, undefined, logger) + + expect(logger.critical).not.toHaveBeenCalled() + }) + + it('resolves to no hooks when beforeSpanSend is the wrong type entirely', () => { + const resolved = resolveTracesConfig({ beforeSpanSend: { scrub: true } as never }) + + expect(resolved.beforeSpanSend).toEqual([]) + }) + it('does not throw when an identity accessor throws', () => { const hostile = {} Object.defineProperty(hostile, 'service.name', { diff --git a/packages/core/src/traces/config.ts b/packages/core/src/traces/config.ts index 4cec80a594..dc60973a43 100644 --- a/packages/core/src/traces/config.ts +++ b/packages/core/src/traces/config.ts @@ -1,12 +1,22 @@ import { assignUserAttributes } from '../utils/json-utils' import type { ResolvedTracesConfig } from './types' -import type { TracesConfig } from '@posthog/types' +import type { BeforeSpanSendFn, TracesConfig } from '@posthog/types' +import type { Logger } from '../types' // OpenTelemetry's BatchSpanProcessor defaults, which sit comfortably under the // server's 2 MB body cap. const DEFAULT_FLUSH_INTERVAL_MS = 5000 const DEFAULT_MAX_EXPORT_BATCH_SIZE = 512 const DEFAULT_MAX_QUEUE_SIZE = 2048 +// OpenTelemetry's per-span defaults. +const DEFAULT_MAX_ATTRIBUTES_PER_SPAN = 128 +const DEFAULT_MAX_EVENTS_PER_SPAN = 128 +// OpenTelemetry leaves the value length unlimited, which is what lets one +// multi-MB attribute make a span too large for the endpoint to accept — and an +// oversized span is dropped whole. 8 KB holds a deep stack trace and any +// realistic header, query string or payload excerpt, and keeps a span at the +// attribute cap under 1 MB, comfortably inside the 2 MB body cap. +const DEFAULT_MAX_ATTRIBUTE_VALUE_LENGTH = 8192 // Live-span bounds. A server can legitimately hold thousands of spans open at // once, and refusing a legitimate span is worse than tolerating a leak, so the @@ -20,9 +30,13 @@ const DEFAULT_MAX_SPAN_AGE_MS = 3_600_000 /** * Coerces a caller-supplied positive-integer option. `0`, a negative, or `NaN` * reaching the export loop would stall it. + * + * A fraction takes the default rather than being floored: these are documented + * as positive integers, and silently reading `maxAttributesPerSpan: 1.5` as `1` + * caps a span an order of magnitude below what the caller wrote. */ function positiveInteger(value: number | undefined, fallback: number): number { - return typeof value === 'number' && Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback + return typeof value === 'number' && Number.isInteger(value) && value >= 1 ? value : fallback } const IDENTITY_KEYS = ['service.name', 'service.version', 'deployment.environment'] as const @@ -54,6 +68,36 @@ function withUsableIdentityKeys(attributes: TracesConfig['resourceAttributes']): } } +/** + * Keeps only the callable hooks. Anything else is dropped rather than called: an + * untyped caller passing the wrong shape would otherwise have every span dropped + * by a hook that throws, leaving tracing silently off. + * + * Dropping one is reported rather than thrown on. `beforeSpanSend` is where + * redaction lives, so a configuration that silently filters nothing ships the + * values it was meant to remove — but a client constructor that throws takes the + * application down with it, which is the worse of the two. `critical`, because + * every other level is gated behind `debug: true`, and a redaction hook that is + * quietly inert is exactly what an operator has to hear about without opting in. + */ +function resolveBeforeSpanSend(beforeSpanSend: TracesConfig['beforeSpanSend'], logger?: Logger): BeforeSpanSendFn[] { + if (!beforeSpanSend) { + return [] + } + // `[featureEnabled && scrub]` is ordinary JS, and a caller who wrote it did not + // configure a hook at all — only a value that was meant to be one is worth + // shouting about. + const supplied = [beforeSpanSend].flat().filter((hook) => Boolean(hook)) + const hooks = supplied.filter((hook): hook is BeforeSpanSendFn => typeof hook === 'function') + if (hooks.length !== supplied.length) { + logger?.critical( + `beforeSpanSend: ignoring ${supplied.length - hooks.length} of ${supplied.length} entries that are not functions. ` + + 'Spans export without them, so whatever they were redacting is not redacted.' + ) + } + return hooks +} + /** * Resolves the public `traces` config into the shape core `PostHogTraces` consumes. * OTLP resource attributes take precedence over the named fields, matching the @@ -62,7 +106,8 @@ function withUsableIdentityKeys(attributes: TracesConfig['resourceAttributes']): */ export function resolveTracesConfig( config: TracesConfig | undefined, - hostResourceAttributes?: Record + hostResourceAttributes?: Record, + logger?: Logger ): ResolvedTracesConfig { // Copied key by key rather than spread: a throwing accessor on a user-supplied // attribute would otherwise escape the first `startSpan`. @@ -76,6 +121,10 @@ export function resolveTracesConfig( serviceVersion: (resourceAttributes?.['service.version'] as string | undefined) ?? config?.serviceVersion, environment: (resourceAttributes?.['deployment.environment'] as string | undefined) ?? config?.environment, resourceAttributes, + beforeSpanSend: resolveBeforeSpanSend(config?.beforeSpanSend, logger), + maxAttributesPerSpan: positiveInteger(config?.maxAttributesPerSpan, DEFAULT_MAX_ATTRIBUTES_PER_SPAN), + maxEventsPerSpan: positiveInteger(config?.maxEventsPerSpan, DEFAULT_MAX_EVENTS_PER_SPAN), + maxAttributeValueLength: positiveInteger(config?.maxAttributeValueLength, DEFAULT_MAX_ATTRIBUTE_VALUE_LENGTH), flushIntervalMs: positiveInteger(config?.flushIntervalMs, DEFAULT_FLUSH_INTERVAL_MS), maxExportBatchSize, // Never below the flush trigger, or the depth-based flush could never fire. diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index a6d5aeb4a5..0712ea99f8 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -6,6 +6,7 @@ import type { OtlpTracesPayload, ResolvedTracesConfig, SendTracesBatchOutcome, + SpanRecord, TraceSdkContext, } from './types' import type { Logger } from '../types' @@ -18,6 +19,10 @@ const resolveForTest = (partial?: Partial): ResolvedTraces flushIntervalMs: 5000, maxExportBatchSize: 512, maxQueueSize: 2048, + beforeSpanSend: [], + maxAttributesPerSpan: 128, + maxEventsPerSpan: 128, + maxAttributeValueLength: 8192, maxLiveSpans: 10000, maxSpanAgeMs: 3600000, ...partial, @@ -376,6 +381,7 @@ describe('PostHogTraces', () => { attributes: [ { key: 'exception.type', value: { stringValue: 'TypeError' } }, { key: 'exception.message', value: { stringValue: 'boom' } }, + { key: 'exception.stacktrace', value: { stringValue: expect.stringContaining('TypeError: boom') } }, ], }) }) @@ -657,6 +663,1048 @@ describe('PostHogTraces', () => { }) }) + describe('reset', () => { + it('says so when it discards queued spans', async () => { + // Terminal loss: there is no next flush to retry on, and the export + // failure the caller already saw promises one. + const instance = createMockInstance({ + _sendTracesBatch: vi.fn(() => Promise.resolve({ kind: 'retry-later' as const, error: new Error('down') })), + }) + const traces = createTraces({}, instance) + traces.startSpan('a').end() + traces.startSpan('b').end() + await traces.flush() + + traces.reset() + + expect(logger.critical).toHaveBeenCalledWith(expect.stringContaining('Discarding 2 span(s)')) + }) + + it('stays quiet when nothing was queued', () => { + const traces = createTraces() + + traces.reset() + + expect(logger.critical).not.toHaveBeenCalled() + }) + }) + + describe('flush reentrancy', () => { + it('does not re-send the head batch when a span ends during the flush prefix', async () => { + // `_flushInner` runs synchronously as far as its first await, and it reads + // the resource attributes in that window. A getter there that ends a span + // used to re-enter the flush with no pass yet recorded, and the same head + // batch went out again on every pass — thousands of times, unbounded. + const resourceAttributes: Record = {} + Object.defineProperty(resourceAttributes, 'tenant', { + enumerable: true, + // Reads `traces` only when a flush runs, which is after it is assigned. + get: () => { + traces.startSpan('late').end() + return 'acme' + }, + }) + const instance = createMockInstance() + const traces = createTraces({ maxExportBatchSize: 2, resourceAttributes }, instance) + + traces.startSpan('a').end() + traces.startSpan('b').end() + await traces.flush() + await traces.flush() + + expect(sentPayloads(instance)).toHaveLength(2) + expect(sentSpans(instance).map((s) => s.name)).toEqual(['a', 'b', 'late']) + }) + }) + + describe('beforeSpanSend', () => { + const endOneSpan = (beforeSpanSend: any): PostHogTraces => { + const traces = createTraces({ beforeSpanSend: [beforeSpanSend].flat() }) + traces.startSpan('checkout', { attributes: { userId: 42 } }).end() + return traces + } + + it('drops a span when the hook returns null', async () => { + await endOneSpan(() => null).flush() + expect(sentSpans()).toHaveLength(0) + }) + + it('drops the span when the hook throws', async () => { + await endOneSpan(() => { + throw new Error('scrubber broke') + }).flush() + + expect(sentSpans()).toHaveLength(0) + expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('beforeSpanSend failed'), expect.anything()) + }) + + it('counts a span the hook dropped', async () => { + await endOneSpan(() => null).flush() + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('beforeSpanSend dropped it')) + }) + + it('counts a span dropped because the hook threw', async () => { + // A permanently broken scrubber otherwise drops every span with the drop + // counter reading zero. + await endOneSpan(() => { + throw new Error('scrubber broke') + }).flush() + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('beforeSpanSend failed')) + }) + + it('hands the hook plain values, not the OTLP encoding', () => { + const seen: unknown[] = [] + endOneSpan((span: SpanRecord) => { + seen.push(span.attributes.userId) + return span + }) + + expect(seen).toEqual([42]) + }) + + it('keeps the original ids when a hook rewrites them', async () => { + const traces = createTraces({ + beforeSpanSend: [ + (span: any) => { + span.traceId = '0'.repeat(32) + span.spanId = '1'.repeat(16) + return span + }, + ], + }) + const started = traces.startSpan('checkout') + const originalTraceId = started.traceparent()!.split('-')[1] + started.end() + await traces.flush() + + const [span] = sentSpans() + expect(span.traceId).toBe(originalTraceId) + expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('identity field')) + }) + + it('exports a span whose record the hook froze', async () => { + // A defensive hook may freeze what it returns. Assigning to a frozen + // property throws even when the value is the one already there, so the + // post-hook pass works on a copy — otherwise every span the hook saw is + // dropped by the fail-closed branch, with only a debug line to say so. + const traces = createTraces({ + beforeSpanSend: [(span: SpanRecord) => Object.freeze({ ...span, attributes: { route: '/checkout' } })], + }) + const span = traces.startSpan('checkout') + + expect(() => span.end()).not.toThrow() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.attributes!.map((attribute) => attribute.key)).toEqual(['route']) + }) + + it('exports a span whose attributes the hook froze', async () => { + const traces = createTraces({ + maxAttributesPerSpan: 1, + beforeSpanSend: [ + (span: SpanRecord) => ({ ...span, attributes: Object.freeze({ route: '/checkout', extra: 1 }) as never }), + ], + }) + traces.startSpan('checkout').end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.attributes!.map((attribute) => attribute.key)).toEqual(['route']) + expect(sent.droppedAttributesCount).toBe(1) + }) + + it('rejects a timestamp the server could not decode', async () => { + const instance = createMockInstance() + const traces = createTraces( + { beforeSpanSend: [(span: SpanRecord) => ({ ...span, startTime: span.startTime * 1e6 })] }, + instance + ) + traces.startSpan('poison').end() + await traces.flush() + + const [span] = sentSpans(instance) + expect(span.startTimeUnixNano.length).toBeLessThanOrEqual(19) + }) + + it('keeps tracestate a rebuilding hook would have dropped', async () => { + const instance = createMockInstance() + const traces = createTraces( + { beforeSpanSend: [(span: SpanRecord) => ({ ...span, traceState: undefined }) as SpanRecord] }, + instance + ) + traces.startSpan('child', { parent: `00-${TRACE_ID}-${REMOTE_SPAN_ID}-01`, tracestate: 'vendor=abc' }).end() + await traces.flush() + + expect(sentSpans(instance)[0].traceState).toBe('vendor=abc') + }) + + it('keeps the trace flags and parent remoteness a rebuilding hook would have dropped', async () => { + const instance = createMockInstance() + const traces = createTraces({ beforeSpanSend: [(span: SpanRecord) => ({ ...span }) as SpanRecord] }, instance) + traces.startSpan('child', { parent: `00-${TRACE_ID}-${REMOTE_SPAN_ID}-00` }).end() + await traces.flush() + + // Sampled-out inbound flag, plus both remoteness bits for a header parent. + expect(sentSpans(instance)[0].flags).toBe(0x300) + }) + + it('keeps them when the hook builds its record from the fields it can see', async () => { + // Spreading carries the propagation fields through even though no public + // type declares them; naming the public fields is what drops them. + const instance = createMockInstance() + const traces = createTraces( + { + beforeSpanSend: [ + (span: SpanRecord) => + ({ + traceId: span.traceId, + spanId: span.spanId, + parentSpanId: span.parentSpanId, + name: span.name, + kind: span.kind, + status: span.status, + attributes: span.attributes, + events: span.events, + startTime: span.startTime, + endTime: span.endTime, + }) as SpanRecord, + ], + }, + instance + ) + traces.startSpan('child', { parent: `00-${TRACE_ID}-${REMOTE_SPAN_ID}-00` }).end() + await traces.flush() + + expect(sentSpans(instance)[0].flags).toBe(0x300) + }) + + it('keeps them when the rebuilding hook also freezes what it returns', async () => { + // Restoring these onto the returned record would throw here, and a throwing + // hook drops the span. + const instance = createMockInstance() + const traces = createTraces( + { beforeSpanSend: [(span: SpanRecord) => Object.freeze({ ...span }) as SpanRecord] }, + instance + ) + traces.startSpan('child', { parent: `00-${TRACE_ID}-${REMOTE_SPAN_ID}-00` }).end() + await traces.flush() + + expect(sentSpans(instance).map((s) => s.flags)).toEqual([0x300]) + }) + + it('does not resurrect a prototype-named attribute the hook removed', async () => { + // `key in attributes` walks the prototype chain, so a deleted `constructor` + // read back as the inherited function and shipped as [Function]. + const instance = createMockInstance() + const traces = createTraces( + { + beforeSpanSend: [ + (span: SpanRecord) => { + delete (span.attributes as Record).constructor + return span + }, + ], + }, + instance + ) + const span = traces.startSpan('ghost') + span.setAttribute('constructor', 'user-value') + span.setAttribute('safe', 'ok') + span.end() + await traces.flush() + + expect(sentSpans(instance)[0].attributes.map((a) => a.key)).toEqual(['safe']) + }) + + it('does not let prototype-named ghosts evict what the hook kept', async () => { + // Worse than resurrection: at the cap the ghosts won the slots and the + // attribute the hook deliberately kept was the one dropped. + const instance = createMockInstance() + const traces = createTraces( + { + maxAttributesPerSpan: 2, + beforeSpanSend: [(span: SpanRecord) => ({ ...span, attributes: { onlyThis: 'yes' } }) as SpanRecord], + }, + instance + ) + const span = traces.startSpan('ghosts') + span.setAttribute('toString', 1) + span.setAttribute('valueOf', 2) + span.end() + await traces.flush() + + expect(sentSpans(instance)[0].attributes.map((a) => a.key)).toEqual(['onlyThis']) + }) + + it('keeps the earliest-set attributes when the hook adds an integer-like key', async () => { + // Object.keys hoists integer-like keys whatever the write order, so a key + // the hook added last outranked one the caller set before it ran. + const instance = createMockInstance() + const traces = createTraces( + { + maxAttributesPerSpan: 3, + beforeSpanSend: [ + (span: SpanRecord) => { + span.attributes['0'] = 'added-last' + return span + }, + ], + }, + instance + ) + const span = traces.startSpan('ordered') + span.setAttribute('alpha', 1) + span.setAttribute('beta', 2) + span.setAttribute('gamma', 3) + span.end() + await traces.flush() + + expect(sentSpans(instance)[0].attributes.map((a) => a.key)).toEqual(['alpha', 'beta', 'gamma']) + }) + + it('ignores a forged identity from a frozen hook rather than dropping the span', async () => { + // Writing the id back onto a frozen return throws, and a throwing hook + // drops the span, so forging plus freezing used to lose every span. + const instance = createMockInstance() + const traces = createTraces( + { beforeSpanSend: [(span: SpanRecord) => Object.freeze({ ...span, traceId: '0'.repeat(32) }) as SpanRecord] }, + instance + ) + traces.startSpan('forged').end() + await traces.flush() + + expect(sentSpans(instance)).toHaveLength(1) + expect(sentSpans(instance)[0].traceId).not.toBe('0'.repeat(32)) + expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('keeping the original ids')) + }) + + it('exports a child span whose record a frozen hook rebuilt without the parent id', async () => { + // The shape that loses children but keeps roots: a rebuilt record has no + // parentSpanId to match, so restoring it wrote to a frozen object. + const instance = createMockInstance() + const traces = createTraces( + { + beforeSpanSend: [ + (span: SpanRecord) => + Object.freeze({ + traceId: span.traceId, + spanId: span.spanId, + name: span.name, + kind: span.kind, + attributes: span.attributes, + events: span.events, + startTime: span.startTime, + endTime: span.endTime, + }) as SpanRecord, + ], + }, + instance + ) + const root = traces.startSpan('root') + traces.startSpan('child', { parent: root }).end() + root.end() + await traces.flush() + + expect( + sentSpans(instance) + .map((s) => s.name) + .sort() + ).toEqual(['child', 'root']) + }) + + it('runs hooks left to right and stops at the first null', async () => { + const order: string[] = [] + await endOneSpan([ + (span: SpanRecord) => { + order.push('first') + return span + }, + () => { + order.push('second') + return null + }, + (span: SpanRecord) => { + order.push('third') + return span + }, + ]).flush() + + expect(order).toEqual(['first', 'second']) + expect(sentSpans()).toHaveLength(0) + }) + + it('exports the edits a hook made', async () => { + await endOneSpan((span: SpanRecord) => { + delete span.attributes.userId + span.name = 'redacted' + return span + }).flush() + + const [span] = sentSpans() + expect(span.name).toBe('redacted') + expect(span.attributes?.find((attribute) => attribute.key === 'userId')).toBeUndefined() + }) + }) + + describe('beforeSpanSend validity', () => { + it('sanitises an event the hook pushed without a timestamp', async () => { + const traces = createTraces({ + beforeSpanSend: [ + (span) => { + span.events.push({ name: 'audited' } as never) + return span + }, + ], + }) + traces.startSpan('checkout').end() + await traces.flush() + + const [event] = sentSpans()[0].events! + expect(event.name).toBe('audited') + expect(event.timeUnixNano).toMatch(/^\d+$/) + }) + + it('clamps an out-of-range timestamp on a hook-supplied event', async () => { + const traces = createTraces({ + beforeSpanSend: [ + (span) => { + span.events.push({ name: 'audited', timestamp: -1 } as never) + return span + }, + ], + }) + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans()[0].events![0].timeUnixNano).toMatch(/^\d+$/) + }) + + it('bounds a status message the hook rewrote', async () => { + const traces = createTraces({ + maxAttributeValueLength: 4, + beforeSpanSend: [ + (span) => { + span.status = { code: 'error', message: 'abcdefgh' } + return span + }, + ], + }) + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans()[0].status).toEqual({ code: 2, message: 'abcd' }) + }) + + it('keeps the original status when the hook writes an unknown code', async () => { + // An unknown code maps to nothing and encodes as an empty status object, + // which silently loses an error the span really had. + const traces = createTraces({ + beforeSpanSend: [(span) => ({ ...span, status: { code: 'ERROR' as never, message: 'boom' } })], + }) + const span = traces.startSpan('checkout') + span.setStatus('error', 'boom') + span.end() + await traces.flush() + + expect(sentSpans()[0].status).toEqual({ code: 2, message: 'boom' }) + }) + + it('ignores a dropped count the hook invented', async () => { + const traces = createTraces({ + beforeSpanSend: [(span) => ({ ...span, droppedAttributesCount: 'lots' as never })], + }) + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans()[0].droppedAttributesCount).toBeUndefined() + }) + + it('lets a hook scrub the auto-context keys', async () => { + // The exemption is from the count cap only. A hook is the documented + // scrubbing point, so it has to be able to remove the join keys as well. + context = { distinctId: 'user-1', sessionId: 'session-1' } + const traces = createTraces({ + beforeSpanSend: [ + (span) => { + delete span.attributes.posthogDistinctId + delete span.attributes.sessionId + return span + }, + ], + }) + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans()[0].attributes).toBeUndefined() + }) + + it('keeps the error status when the event cap costs the exception event', async () => { + // The status is set independently of the event, so a span whose exception + // event did not fit still exports as failed and still counts the loss. + // That pair is what makes the case findable once traces is live. + const traces = createTraces({ + maxEventsPerSpan: 1, + beforeSpanSend: [ + (span) => { + span.events.push({ name: 'audited', timestamp: Date.now() }) + return span + }, + ], + }) + const span = traces.startSpan('checkout') + span.addEvent('step') + span.recordException(new Error('boom')) + span.end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.events!.map((event) => event.name)).toEqual(['step']) + expect(sent.droppedEventsCount).toBe(2) + expect(sent.status).toEqual({ code: 2, message: 'boom' }) + }) + + it('keeps the original status when the hook mutates the code in place', async () => { + // The hook is documented as editing the record in place, so snapshotting a + // reference to `status` would restore the mutation onto itself. + const traces = createTraces({ + beforeSpanSend: [ + (span) => { + ;(span.status as { code: string }).code = 'ERROR' + return span + }, + ], + }) + const span = traces.startSpan('checkout') + span.setStatus('error', 'boom') + span.end() + await traces.flush() + + expect(sentSpans()[0].status).toEqual({ code: 2, message: 'boom' }) + }) + + it('exports a span whose record is a class instance with prototype getters', async () => { + // A spread copies own properties only, so `events` behind a prototype + // getter arrived undefined and the fail-closed branch ate every span. + class Wrapped { + constructor(private readonly _inner: SpanRecord) {} + get traceId(): string { + return this._inner.traceId + } + get spanId(): string { + return this._inner.spanId + } + get name(): string { + return this._inner.name + } + get kind(): SpanRecord['kind'] { + return this._inner.kind + } + get attributes(): SpanRecord['attributes'] { + return this._inner.attributes + } + get events(): SpanRecord['events'] { + return this._inner.events + } + get startTime(): number { + return this._inner.startTime + } + get endTime(): number { + return this._inner.endTime + } + } + const traces = createTraces({ + beforeSpanSend: [(span: SpanRecord) => new Wrapped(span) as unknown as SpanRecord], + }) + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans().map((span) => span.name)).toEqual(['checkout']) + }) + + it('survives a hook that leaves a hole in the events array', async () => { + const traces = createTraces({ + beforeSpanSend: [ + (span) => { + span.events.length = 2 + return span + }, + ], + }) + const span = traces.startSpan('checkout') + span.addEvent('step') + span.end() + await traces.flush() + + expect(sentSpans()[0].events!.map((event) => event.name)).toEqual(['step']) + }) + + it('keeps the span-side dropped count when the hook overwrites the counter', async () => { + const traces = createTraces({ + maxEventsPerSpan: 1, + beforeSpanSend: [ + (span) => { + ;(span as unknown as { droppedEventsCount: unknown }).droppedEventsCount = 'lots' + span.events.push({ name: 'audited', timestamp: Date.now() }) + return span + }, + ], + }) + const span = traces.startSpan('checkout') + span.addEvent('step-0') + span.addEvent('step-1') + span.end() + await traces.flush() + + // One dropped at the span, one by the post-hook re-apply. + expect(sentSpans()[0].droppedEventsCount).toBe(2) + }) + + it('drops only the event the hook made unreadable', async () => { + const traces = createTraces({ + beforeSpanSend: [ + (span) => { + span.events = [span.events[0], null as never, span.events[1]] + return span + }, + ], + }) + const span = traces.startSpan('checkout') + span.addEvent('step-0') + span.addEvent('step-1') + span.end() + await traces.flush() + + expect(sentSpans()[0].events!.map((event) => event.name)).toEqual(['step-0', 'step-1']) + }) + + it.each([ + ['attributes replaced with null', (span: SpanRecord) => ({ ...span, attributes: null as never })], + ['attributes replaced with an array', (span: SpanRecord) => ({ ...span, attributes: ['a'] as never })], + ['events replaced with null', (span: SpanRecord) => ({ ...span, events: null as never })], + ['an async hook returning a promise', (span: SpanRecord) => Promise.resolve(span) as never], + // Carrying both collections was the whole shape check, so these reached + // the wire as a span named `unknown` at a fallback time with no join keys. + ['only the two collections', () => ({ attributes: {}, events: [] }) as never], + ['no name', (span: SpanRecord) => ({ ...span, name: undefined as never })], + ['no kind', (span: SpanRecord) => ({ ...span, kind: undefined as never })], + ['no start time', (span: SpanRecord) => ({ ...span, startTime: undefined as never })], + ['no end time', (span: SpanRecord) => ({ ...span, endTime: undefined as never })], + ])('drops the span when the hook returns %s', async (_label, beforeSpanSend) => { + // Repairing these would export a nameless span carrying no join keys. + const traces = createTraces({ beforeSpanSend: [beforeSpanSend] }) + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans()).toHaveLength(0) + }) + + it('counts an incomplete hook result as a drop rather than losing it silently', async () => { + const traces = createTraces({ beforeSpanSend: [() => ({ attributes: {}, events: [] }) as never] }) + traces.startSpan('checkout').end() + await traces.flush() + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('beforeSpanSend returned an unusable record')) + }) + + it('still exports a span whose required fields the hook left in place', async () => { + // The shape check reads presence, so an ordinary scrub is untouched by it. + const traces = createTraces({ + beforeSpanSend: [ + (span) => { + delete span.attributes.secret + return span + }, + ], + }) + traces.startSpan('checkout', { attributes: { secret: 'shh', keep: 1 } }).end() + await traces.flush() + + expect(sentSpans()).toHaveLength(1) + expect(sentSpans()[0].name).toBe('checkout') + expect(sentSpans()[0].attributes!.map((a) => a.key)).toContain('keep') + expect(sentSpans()[0].attributes!.map((a) => a.key)).not.toContain('secret') + }) + + it('gives a later hook the real identity after an earlier one froze a forged record', async () => { + // The export reads the snapshot either way, but a hook that samples or + // routes on an id must not see one an earlier hook invented. + const seen: { traceId: string; spanId: string }[] = [] + const traces = createTraces({ + beforeSpanSend: [ + (span) => Object.freeze({ ...span, traceId: '0'.repeat(32), spanId: 'f'.repeat(16) }), + (span) => { + seen.push({ traceId: span.traceId, spanId: span.spanId }) + return span + }, + ], + }) + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans()).toHaveLength(1) + expect(seen[0].traceId).toBe(sentSpans()[0].traceId) + expect(seen[0].spanId).toBe(sentSpans()[0].spanId) + expect(seen[0].traceId).not.toBe('0'.repeat(32)) + }) + + it('leaves the rest of a frozen forged record readable to the next hook', async () => { + // The corrected view is built from the record's own descriptors, so a hook + // reading anything but identity sees exactly what the previous one returned. + let seen: SpanRecord | undefined + const traces = createTraces({ + beforeSpanSend: [ + (span) => Object.freeze({ ...span, name: 'renamed', traceId: '0'.repeat(32) }), + (span) => { + seen = span + return span + }, + ], + }) + traces.startSpan('checkout', { attributes: { keep: 1 } }).end() + await traces.flush() + + expect(seen!.name).toBe('renamed') + expect(seen!.attributes.keep).toBe(1) + expect(Object.keys(seen!)).toContain('name') + expect(sentSpans()[0].name).toBe('renamed') + }) + + it('applies the event cap to what a hook leaves behind', async () => { + // A hook can append events or rewrite them, neither of which goes through + // `addEvent`, so the cap has to be re-applied to whatever it returns. + const traces = createTraces({ + maxEventsPerSpan: 1, + beforeSpanSend: [ + (span) => { + span.events.push({ name: 'exception', timestamp: Date.now() }) + span.events.push({ name: 'appended', timestamp: Date.now() }) + return span + }, + ], + }) + const span = traces.startSpan('checkout') + span.addEvent('step-0') + span.end() + await traces.flush() + + expect(sentSpans()[0].events!.map((event) => event.name)).toEqual(['step-0']) + }) + + it('exports the span when the hook status message refuses to stringify', async () => { + // The encoder downstream only marks the field, so coercing here must not + // be the thing that costs the span. + const traces = createTraces({ + maxAttributeValueLength: 4, + beforeSpanSend: [ + (span) => { + span.status = { + code: 'error', + message: { + toString() { + throw new Error('nope') + }, + } as never, + } + return span + }, + ], + }) + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans()).toHaveLength(1) + expect(sentSpans()[0].status?.code).toBe(2) + }) + + it('bounds a non-string status message the hook wrote', async () => { + const traces = createTraces({ + maxAttributeValueLength: 4, + beforeSpanSend: [ + (span) => { + span.status = { code: 'error', message: { toString: () => 'abcdefgh' } as never } + return span + }, + ], + }) + traces.startSpan('checkout').end() + await traces.flush() + + expect(sentSpans()[0].status).toEqual({ code: 2, message: 'abcd' }) + }) + + it('does not spend cap budget on a value the hook blanked', async () => { + const traces = createTraces({ + maxAttributesPerSpan: 2, + beforeSpanSend: [ + (span) => { + span.attributes.secret = null + span.attributes.scrubbed = true + return span + }, + ], + }) + traces.startSpan('checkout', { attributes: { secret: 'sk-live', route: '/checkout' } }).end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.attributes!.map((attribute) => attribute.key)).toEqual(['route', 'scrubbed']) + expect(sent.droppedAttributesCount).toBeUndefined() + }) + }) + + describe('span limits', () => { + it('keeps the earliest attributes and counts the rest', async () => { + const traces = createTraces({ maxAttributesPerSpan: 3 }) + const span = traces.startSpan('checkout') + for (let i = 0; i < 5; i++) { + span.setAttribute(`key-${i}`, i) + } + span.end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.attributes!.map((attribute) => attribute.key)).toEqual(['key-0', 'key-1', 'key-2']) + expect(sent.droppedAttributesCount).toBe(2) + }) + + it('re-applies the attribute cap to what beforeSpanSend added', async () => { + const traces = createTraces({ + maxAttributesPerSpan: 2, + beforeSpanSend: [ + (span) => { + for (let i = 0; i < 5; i++) { + span.attributes[`added-${i}`] = i + } + return span + }, + ], + }) + traces.startSpan('checkout', { attributes: { kept: true } }).end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.attributes!.map((attribute) => attribute.key)).toEqual(['kept', 'added-0']) + expect(sent.droppedAttributesCount).toBe(4) + }) + + it('re-applies the event cap to what beforeSpanSend added', async () => { + const traces = createTraces({ + maxEventsPerSpan: 1, + beforeSpanSend: [ + (span) => { + span.events.push({ name: 'added', timestamp: span.startTime }) + return span + }, + ], + }) + const span = traces.startSpan('checkout') + span.addEvent('original') + span.end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.events!.map((event) => event.name)).toEqual(['original']) + expect(sent.droppedEventsCount).toBe(1) + }) + + it('keeps the auto-context keys when beforeSpanSend pushes past the cap', async () => { + context = { distinctId: 'alice', sessionId: 'session-1' } + const traces = createTraces({ + maxAttributesPerSpan: 1, + beforeSpanSend: [ + (span) => { + span.attributes.late = true + return span + }, + ], + }) + traces.startSpan('checkout', { attributes: { early: true } }).end() + await traces.flush() + + const keys = sentSpans()[0].attributes!.map((attribute) => attribute.key) + expect(keys).toEqual(expect.arrayContaining(['posthogDistinctId', 'sessionId', 'early'])) + expect(keys).not.toContain('late') + }) + + it('re-applies the value bound to what beforeSpanSend wrote', async () => { + const traces = createTraces({ + maxAttributeValueLength: 8, + beforeSpanSend: [ + (span) => { + span.attributes.enriched = 'y'.repeat(5000) + span.events.push({ name: 'added', timestamp: span.startTime, attributes: { blob: 'z'.repeat(5000) } }) + return span + }, + ], + }) + traces.startSpan('checkout').end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.attributes!.find((attribute) => attribute.key === 'enriched')!.value).toEqual({ + stringValue: 'yyyyyyyy', + }) + expect(sent.events!.at(-1)!.attributes!.find((attribute) => attribute.key === 'blob')!.value).toEqual({ + stringValue: 'zzzzzzzz', + }) + }) + + it('does not invent a dropped count when beforeSpanSend only removes', async () => { + const traces = createTraces({ + maxAttributesPerSpan: 5, + beforeSpanSend: [ + (span) => { + delete span.attributes.secret + return span + }, + ], + }) + traces.startSpan('checkout', { attributes: { secret: 'x', kept: true } }).end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.attributes!.map((attribute) => attribute.key)).toEqual(['kept']) + expect(sent.droppedAttributesCount).toBeUndefined() + }) + + it('never evicts the auto-context keys', async () => { + context = { distinctId: 'alice', sessionId: 'session-1' } + const traces = createTraces({ maxAttributesPerSpan: 1 }) + const span = traces.startSpan('checkout') + for (let i = 0; i < 5; i++) { + span.setAttribute(`key-${i}`, i) + } + span.end() + await traces.flush() + + const keys = sentSpans()[0].attributes!.map((attribute) => attribute.key) + expect(keys).toEqual(expect.arrayContaining(['posthogDistinctId', 'sessionId', 'key-0'])) + expect(keys).not.toContain('key-1') + }) + + it('caps events and counts the rest', async () => { + const traces = createTraces({ maxEventsPerSpan: 2 }) + const span = traces.startSpan('checkout') + for (let i = 0; i < 4; i++) { + span.addEvent(`event-${i}`) + } + span.end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.events!.map((event) => event.name)).toEqual(['event-0', 'event-1']) + expect(sent.droppedEventsCount).toBe(2) + }) + + it('lets a caller overwrite an attribute it already set while at the cap', async () => { + const traces = createTraces({ maxAttributesPerSpan: 1 }) + const span = traces.startSpan('checkout') + span.setAttribute('plan', 'free') + span.setAttribute('plan', 'pro') + span.end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.attributes).toEqual([{ key: 'plan', value: { stringValue: 'pro' } }]) + expect(sent.droppedAttributesCount).toBeUndefined() + }) + + it('omits the counters when nothing was dropped', async () => { + const traces = createTraces() + traces.startSpan('checkout', { attributes: { plan: 'pro' } }).end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent).not.toHaveProperty('droppedAttributesCount') + expect(sent).not.toHaveProperty('droppedEventsCount') + }) + + it('counts a parsed __proto__ key against the cap instead of smuggling it through', async () => { + // JSON.parse produces an own `__proto__` key; a plain object store would + // swap its prototype and leak every nested key past the cap. + const traces = createTraces({ maxAttributesPerSpan: 2 }) + const parsed = JSON.parse('{"__proto__": {"leaked": 1}, "orderId": "abc"}') + traces.startSpan('checkout', { attributes: parsed }).end() + await traces.flush() + + const keys = sentSpans()[0].attributes!.map((attribute) => attribute.key) + expect(keys).not.toContain('leaked') + expect(keys).toContain('orderId') + }) + + it('does not let reserved property names bypass the cap', async () => { + const traces = createTraces({ maxAttributesPerSpan: 1 }) + const span = traces.startSpan('checkout') + span.setAttribute('kept', 1) + span.setAttribute('toString', 'nope') + span.setAttribute('constructor', 'nope') + span.end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.attributes!.map((attribute) => attribute.key)).toEqual(['kept']) + expect(sent.droppedAttributesCount).toBe(2) + }) + + it('does not spend cap budget on values that are dropped at encode time', async () => { + const traces = createTraces({ maxAttributesPerSpan: 2 }) + const span = traces.startSpan('checkout') + span.setAttribute('skipped-a', undefined) + span.setAttribute('skipped-b', null) + span.setAttribute('orderId', 'abc-123') + span.end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.attributes!.map((attribute) => attribute.key)).toEqual(['orderId']) + expect(sent.droppedAttributesCount).toBeUndefined() + }) + + it('still caps a key first seen with an optional value', async () => { + const traces = createTraces({ maxAttributesPerSpan: 2 }) + const span = traces.startSpan('checkout') + for (let i = 0; i < 5; i++) { + span.setAttribute(`field-${i}`, undefined) + } + for (let i = 0; i < 5; i++) { + span.setAttribute(`field-${i}`, i) + } + span.end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.attributes).toHaveLength(2) + expect(sent.droppedAttributesCount).toBe(3) + }) + + it('clears a key that is set back to null', async () => { + const traces = createTraces({ maxAttributesPerSpan: 2 }) + const span = traces.startSpan('checkout') + span.setAttribute('orderId', 'abc-123') + span.setAttribute('orderId', null) + span.setAttribute('a', 1) + span.setAttribute('b', 2) + span.end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.attributes!.map((attribute) => attribute.key)).toEqual(['a', 'b']) + expect(sent.droppedAttributesCount).toBeUndefined() + }) + + it('caps attributes supplied at start', async () => { + const traces = createTraces({ maxAttributesPerSpan: 2 }) + traces.startSpan('checkout', { attributes: { a: 1, b: 2, c: 3 } }).end() + await traces.flush() + + const [sent] = sentSpans() + expect(sent.attributes!.map((attribute) => attribute.key)).toEqual(['a', 'b']) + expect(sent.droppedAttributesCount).toBe(1) + }) + }) + describe('export', () => { it('flushes when the queue reaches the batch size', async () => { const traces = createTraces({ maxExportBatchSize: 2 }) @@ -1043,6 +2091,54 @@ describe('PostHogTraces', () => { expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('2 span(s)')) expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('the user has opted out')) }) + + it('stops draining a backlog when optOut() lands while a batch is in flight', async () => { + const instance = createMockInstance() + instance._sendTracesBatch = vi.fn(() => + Promise.resolve({ kind: 'retry-later' as const, error: new Error('down') }) + ) + const traces = createTraces({ maxExportBatchSize: 2 }, instance) + context = { distinctId: 'alice', sessionId: 'session-1' } + for (let i = 0; i < 6; i++) { + traces.startSpan(`span-${i}`).end() + await flushMicrotasks() + } + expect(instance._sendTracesBatch).toHaveBeenCalledTimes(1) + + instance._sendTracesBatch.mockImplementation(() => + Promise.resolve().then(() => { + instance.optedOut = true + return { kind: 'ok' as const } + }) + ) + await traces.flush() + + expect(instance._sendTracesBatch).toHaveBeenCalledTimes(2) + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('4 span(s)')) + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('the user has opted out')) + }) + + it('stops draining a backlog when the client is disabled while a batch is in flight', async () => { + const instance = createMockInstance() + instance._sendTracesBatch = vi.fn(() => + Promise.resolve({ kind: 'retry-later' as const, error: new Error('down') }) + ) + const traces = createTraces({ maxExportBatchSize: 2 }, instance) + for (let i = 0; i < 6; i++) { + traces.startSpan(`span-${i}`).end() + await flushMicrotasks() + } + + instance._sendTracesBatch.mockImplementation(() => + Promise.resolve().then(() => { + instance.isDisabled = true + return { kind: 'ok' as const } + }) + ) + await traces.flush() + + expect(instance._sendTracesBatch).toHaveBeenCalledTimes(2) + }) }) describe('flush backoff', () => { @@ -1187,6 +2283,20 @@ describe('PostHogTraces', () => { expect(sentSpans().map((span) => span.name)).toEqual(['checkout']) }) + it('bounds a long resource attribute value', async () => { + // Resource attributes are caller-supplied like span attributes, and they + // ride on every batch rather than on one span. + const traces = createTraces({ + maxAttributeValueLength: 4, + resourceAttributes: { 'host.name': 'abcdefgh' } as never, + }) + traces.startSpan('checkout').end() + await traces.flush() + + const resource = sentPayloads()[0].resourceSpans[0].resource!.attributes + expect(resource.find((attribute) => attribute.key === 'host.name')?.value).toEqual({ stringValue: 'abcd' }) + }) + it('does not throw on a Date-like object with no Date slot', () => { const traces = createTraces() const fakeDate = Object.create(Date.prototype) diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 3776416872..5d67e00d81 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -1,17 +1,26 @@ -import type { Span, SpanAttributes, StartSpanOptions } from '@posthog/types' +import type { Span, SpanAttributes, SpanRecord as HookSpanRecord, StartSpanOptions } from '@posthog/types' import type { Logger } from '../types' import type { OtlpSpan, ResolvedTracesConfig, SpanContextManager, + SpanEventRecord, SpanRecord, TraceSdkContext, TracesHost, } from './types' -import { PostHogSpan, describeError, inertSpan, monotonicNow, runWithActiveSpan } from './span' +import { + PostHogSpan, + applySpanLimits, + describeError, + inertSpan, + monotonicNow, + runWithActiveSpan, + truncateAttributes, +} from './span' import { newSpanId, newTraceId } from './ids' import { parseTraceparent, sanitizeTracestate } from './traceparent' -import { resolveStartTime, sanitizeName } from './sanitize' +import { clampEndTime, resolveStartTime, resolveSuppliedTime, sanitizeName, toEpochMs } from './sanitize' import { assignUserAttributes } from '../utils/json-utils' import { buildOtlpSpan, buildOtlpTracesPayload, buildTracesResourceAttributes } from './otlp' import { isPromise, safeSetTimeout } from '../utils' @@ -50,6 +59,79 @@ function looksLikeSpan(value: unknown): boolean { } } +/** + * The rebuilt record with every field named, optional ones included. A field + * added to either half of `SpanRecord` is a compile error at the rebuild until + * it says whether a hook may set that field or the span keeps its own value. + */ +type RebuiltSpanRecord = { [K in keyof Required]: SpanRecord[K] } + +interface SpanIdentity { + traceId: string + spanId: string + parentSpanId?: string + traceState?: string +} + +/** + * Whether a `beforeSpanSend` return value still carries every field the public + * `SpanRecord` declares as required. An array is rejected for `attributes`: it + * would encode as `{ "0": ... }` rather than fail. + * + * Presence, not usability: a field that is there but holds the wrong type is a + * hook editing a real record badly, and the sanitising below is what answers + * that. A field that is absent means the hook returned something that was never + * a span record, and the fallbacks would dress it up as one. + */ +function isSpanRecordShape(record: SpanRecord): boolean { + return ( + !!record.attributes && + typeof record.attributes === 'object' && + !Array.isArray(record.attributes) && + Array.isArray(record.events) && + record.name !== undefined && + record.kind !== undefined && + record.startTime !== undefined && + record.endTime !== undefined + ) +} + +/** Writes `value` onto `record` only when it isn't already there. */ +function restoreField(record: SpanIdentity, field: K, value: SpanIdentity[K]): void { + if (record[field] !== value) { + record[field] = value + } +} + +/** + * A stand-in for a record whose identity could not be written back, carrying the + * original ids and everything else the hook returned. + * + * Built from the descriptors rather than spread so a class instance keeps its + * prototype — `instanceof` and a field exposed as a prototype getter both still + * answer — and so `Object.keys` reads what it read before. Only the four + * identity descriptors are replaced, which is what makes the copy writable where + * the original was frozen. + */ +function withRestoredIdentity(hooked: HookSpanRecord, original: SpanIdentity): HookSpanRecord { + try { + const descriptors = Object.getOwnPropertyDescriptors(hooked) as Record + for (const field of ['traceId', 'spanId', 'parentSpanId', 'traceState'] as const) { + descriptors[field] = { + value: original[field], + enumerable: true, + writable: true, + configurable: true, + } + } + return Object.create(Object.getPrototypeOf(hooked) as object | null, descriptors) as HookSpanRecord + } catch { + // A hostile descriptor read. The export still uses the snapshot, so this + // costs the next hook a correct id rather than the span. + return hooked + } +} + interface ParentContext { traceId: string parentSpanId?: string @@ -147,6 +229,8 @@ export class PostHogTraces { // so a backdated `startTime` neither ages a span early nor exempts it. this._liveSpans.set(spanId, clockNow()) + const autoAttributes = this._autoContextAttributes() + return new PostHogSpan( { traceId: parent?.traceId ?? newTraceId(), @@ -155,14 +239,18 @@ export class PostHogTraces { traceState: parent?.traceState, traceFlags: parent?.traceFlags, parentIsRemote: parent?.isRemote, - name: sanitizeName(name, 'Span name', this._logger), + name: sanitizeName(name, 'Span name', this._config.maxAttributeValueLength, this._logger), kind: options?.kind ?? 'internal', // Auto-context first so user-supplied attributes win on collision. - attributes: assignUserAttributes(this._autoContextAttributes(), options?.attributes), + attributes: assignUserAttributes({ ...autoAttributes }, options?.attributes), + autoAttributeKeys: Object.keys(autoAttributes), + maxAttributes: this._config.maxAttributesPerSpan, + maxEvents: this._config.maxEventsPerSpan, + maxAttributeValueLength: this._config.maxAttributeValueLength, startTime, backdated: startTime !== now, }, - (record) => this._onSpanEnd(record), + (record, autoKeys) => this._onSpanEnd(record, autoKeys), this._logger ) } @@ -239,14 +327,25 @@ export class PostHogTraces { private _startFlush(): Promise { this._clearFlushTimer() - const promise = this._flushInner().finally(() => { - // Only clear the slot this call installed: a `reset()` mid-flight may - // already have installed a newer one. - if (this._flushPromise === promise) { - this._flushPromise = null - } - this._armFlushTimerIfQueued() - }) + // Deferred by a microtask so the slot below is installed before the pass + // reads anything: `_flushInner` runs synchronously as far as its first + // await, and a resource-attribute getter or `toJSON` that ends a span in + // that window would otherwise re-enter here, find no pass in flight, and + // send the same head batch again — without bound. + // Sampled before the microtask, not inside `_flushInner`: a `reset()` landing + // in the window would otherwise be invisible to this pass, which would then + // drain the post-reset queue alongside the pass `reset()` started. + const startedAtGeneration = this._generation + const promise = Promise.resolve() + .then(() => (startedAtGeneration === this._generation ? this._flushInner() : 0)) + .finally(() => { + // Only clear the slot this call installed: a `reset()` mid-flight may + // already have installed a newer one. + if (this._flushPromise === promise) { + this._flushPromise = null + } + this._armFlushTimerIfQueued() + }) this._flushPromise = promise return promise } @@ -254,6 +353,16 @@ export class PostHogTraces { /** Clears the queue and timer. Used on shutdown and between tests. */ reset(): void { this._clearFlushTimer() + if (this._queue.length) { + // Critical, and said here rather than counted: this is the last chance to + // say anything about these spans, the drop warning is gated behind `debug` + // on some hosts, and the only other line the caller sees is the export + // failure promising a retry on a flush that will never come. + this._logger.critical( + `Discarding ${this._queue.length} span(s) that were still queued when tracing was shut down. ` + + 'Raise the shutdown timeout or flush earlier if they matter.' + ) + } this._queue = [] this._liveSpans.clear() this._flushPromise = null @@ -339,8 +448,12 @@ export class PostHogTraces { if (!(span instanceof PostHogSpan)) { return } - const { type, message } = describeError(error) - span.addEvent('exception', { 'exception.type': type, 'exception.message': message }) + const { type, message, stack } = describeError(error) + span.addEvent('exception', { + 'exception.type': type, + 'exception.message': message, + ...(stack && { 'exception.stacktrace': stack }), + }) if (!span.statusIsExplicitlyOk) { span.setStatus('error', message) } @@ -367,9 +480,9 @@ export class PostHogTraces { } } - private _onSpanEnd(record: SpanRecord): void { - // Deleted before any other gate, so an opted-out span still returns its slot. - if (!this._liveSpans.delete(record.spanId)) { + private _onSpanEnd(incoming: SpanRecord, autoKeys: ReadonlySet): void { + // Deleted before any other gate, so a span dropped later still returns its slot. + if (!this._liveSpans.delete(incoming.spanId)) { // Evicted for age while live: never exported, and already counted as a drop. return } @@ -380,6 +493,11 @@ export class PostHogTraces { return } + const record = this._runBeforeSpanSend(incoming, autoKeys) + if (!record) { + return + } + if (this._queue.length >= this._config.maxQueueSize) { // Drop the incoming span, not queued ones: those are completed parents whose // children may already have shipped. @@ -407,6 +525,183 @@ export class PostHogTraces { } } + /** + * Runs the `beforeSpanSend` chain, returning the span to enqueue or `null` to + * drop it. + * + * A throwing hook drops the span: the hook is the documented scrubbing point, + * so a broken scrubber must not let the unscrubbed record through. Identity + * fields are restored afterwards, since rewriting them orphans shipped children. + */ + private _runBeforeSpanSend(record: SpanRecord, autoKeys: ReadonlySet): SpanRecord | null { + if (!this._config.beforeSpanSend.length) { + return record + } + + // Snapshotted before any hook runs: a hook that mutates in place would + // otherwise leave nothing to restore from. + const identity = { + traceId: record.traceId, + spanId: record.spanId, + parentSpanId: record.parentSpanId, + traceState: record.traceState, + } + const originalTimes = { startTime: record.startTime, endTime: record.endTime } + // Snapshotted with the rest: the hook mutates the record in place, so reading + // these back afterwards reads whatever the hook left there. + const originalDropped = { + attributes: record.droppedAttributesCount, + events: record.droppedEventsCount, + } + // Read here rather than restored onto the hook's return value: writing them + // back would throw on a frozen record, and neither is on the record a hook + // is handed, so a rebuilding hook always arrives without them. + // The order the span itself wrote them in, so the caps below can keep the + // earliest-set entries even when a hook adds an integer-like key. + const keysBeforeHook = Object.keys(record.attributes) + const originalPropagation = { + traceFlags: record.traceFlags, + parentIsRemote: record.parentIsRemote, + } + // Copied, not referenced: the hook is documented as mutating the record in + // place, and a reference would restore the mutation onto itself. + const originalStatus = record.status && { ...record.status } + let hooked: HookSpanRecord = record + let current = record + try { + for (const hook of this._config.beforeSpanSend) { + const result = hook(hooked) + if (!result) { + this._recordDrop(1, 'beforeSpanSend dropped it') + return null + } + hooked = this._keepSpanIdentity(result, identity) + } + + // Rebuilt field by field before anything below writes to it. The hook's + // return value may be frozen, where every write here would throw, or a + // class instance whose fields are prototype getters a spread would miss. + // Naming them also bounds what can reach the wire. + const rebuilt: RebuiltSpanRecord = { + // All four from the snapshot, never from the hook's return value. A hook + // that forges an id has it ignored, which is the documented behaviour, + // and one that also freezes what it returns keeps its span: writing the + // id back onto a frozen object throws, and a throw here drops the span. + traceId: identity.traceId, + spanId: identity.spanId, + parentSpanId: identity.parentSpanId, + traceState: identity.traceState, + name: hooked.name, + kind: hooked.kind, + status: hooked.status, + attributes: hooked.attributes, + events: hooked.events, + startTime: hooked.startTime, + endTime: hooked.endTime, + // Taken from the span for the same reason as the dropped counts: no + // public type declares them, so a rebuilding hook returns without them + // and a `?? fallback` here would export a sampled-out trace as sampled. + traceFlags: originalPropagation.traceFlags, + parentIsRemote: originalPropagation.parentIsRemote, + // Taken from the span, not from the hook's return value: these are SDK + // bookkeeping that no public type declares, so a hook overwriting them + // must not erase what the span actually dropped. + droppedAttributesCount: originalDropped.attributes, + droppedEventsCount: originalDropped.events, + } + current = rebuilt + // A value missing a required field is not a span record — an `async` hook + // returns a Promise, truthy and `undefined` for every field. Filling the + // gaps in would export a span named `unknown` at a fallback time carrying + // no person or session, joinable to nothing and silent about it. + if (!isSpanRecordShape(current)) { + this._logger.debug('beforeSpanSend did not return a span record; dropping the span') + this._recordDrop(1, 'beforeSpanSend returned an unusable record') + return null + } + + // Re-applied to whatever the hook returned: one undecodable timestamp 400s + // the whole request, taking unrelated spans with it. + current.name = sanitizeName(current.name, 'Span name', this._config.maxAttributeValueLength, this._logger) + // A status the hook rewrote never went through `setStatus`. An unknown code + // encodes as an empty status object, which loses an error the span really had. + if (current.status && current.status.code !== 'ok' && current.status.code !== 'error') { + this._logger.debug('beforeSpanSend set an unknown span status; keeping the original') + current.status = originalStatus + } + current.startTime = toEpochMs(current.startTime) ?? originalTimes.startTime + current.endTime = clampEndTime(toEpochMs(current.endTime) ?? originalTimes.endTime, current.startTime) + // Events a hook pushed never went through `addEvent`, so they carry no + // sanitised name or timestamp; an unvalidated one encodes as `NaN000NaN` + // and the ingestion service refuses the whole batch. + const sanitizedEvents: SpanEventRecord[] = [] + for (const event of current.events) { + try { + sanitizedEvents.push({ + ...event, + name: sanitizeName(event.name, 'Span event name', this._config.maxAttributeValueLength, this._logger), + timestamp: resolveSuppliedTime(event.timestamp, current.startTime, 'event timestamp', this._logger), + }) + } catch { + // A hook can leave a `null` in the array or a throwing accessor on an + // event. That costs the event; the rest of the span still ships. + this._logger.debug('beforeSpanSend left an unreadable span event; dropping it') + } + } + current.events = sanitizedEvents + applySpanLimits( + current, + autoKeys, + this._config.maxAttributesPerSpan, + this._config.maxEventsPerSpan, + this._config.maxAttributeValueLength, + keysBeforeHook + ) + return current + } catch (error) { + // Covers the hook and everything done to its return value: a frozen or + // hostile record must not throw out of `end()` into application code. + this._logger.debug('beforeSpanSend failed; dropping the span rather than exporting it unscrubbed', error) + this._recordDrop(1, 'beforeSpanSend failed') + return null + } + } + + /** + * Restores the fields a hook must not change. Runs per hook so a later hook in + * the chain cannot sample on an id an earlier one forged. + */ + private _keepSpanIdentity(hooked: HookSpanRecord, original: SpanIdentity): HookSpanRecord { + if ( + hooked.traceId !== original.traceId || + hooked.spanId !== original.spanId || + hooked.parentSpanId !== original.parentSpanId + ) { + this._logger.debug('beforeSpanSend changed a span identity field; keeping the original ids') + } + // Only the fields that actually differ are written back. Assigning a value + // to a frozen property throws even when it is the value already there, and + // a hook that freezes the record it returns would otherwise drop every span. + // Best-effort, for the next hook in the chain only: the record this builds + // is not what gets exported. A frozen return refuses every write, and the + // span must survive that. + try { + restoreField(hooked, 'traceId', original.traceId) + restoreField(hooked, 'spanId', original.spanId) + restoreField(hooked, 'parentSpanId', original.parentSpanId) + // A hook that rebuilds the record instead of spreading it would otherwise + // drop tracestate, which is not part of the record the hook is handed. + restoreField(hooked, 'traceState', original.traceState) + } catch { + // Frozen, so the writes above were refused and this record still carries + // whatever identity the hook forged. The export reads the snapshot either + // way, but the next hook in the chain reads this — and would sample or + // route on a forged id, which identity immutability exists to prevent. + return withRestoredIdentity(hooked, original) + } + return hooked + } + private _recordDrop(count: number, reason: string): void { this._droppedSinceWarning += count this._dropReasons.add(reason) @@ -446,27 +741,38 @@ export class PostHogTraces { return encoded } + /** + * Discards the queue when consent has been withdrawn, returning how many spans + * it dropped. Spans carry `posthogDistinctId` and `sessionId`, so anything still + * queued when the user opts out must not be exported. + */ + private _discardQueueIfConsentWithdrawn(): number { + if (!this._instance.isDisabled && !this._instance.optedOut) { + return 0 + } + const discarded = this._queue.length + this._queue = [] + this._recordDrop(discarded, 'the user has opted out') + this._warnAboutDrops() + return discarded + } + /** Returns how many spans it removed from the queue, sent or dropped. */ private async _flushInner(): Promise { if (!this._queue.length) { return 0 } - // Consent can flip between a span being queued and this pass running. Spans - // carry `posthogDistinctId` and `sessionId`, so anything still queued when - // the user opts out must be discarded rather than exported. - if (this._instance.isDisabled || this._instance.optedOut) { - const discarded = this._queue.length - this._queue = [] - this._recordDrop(discarded, 'the user has opted out') - this._warnAboutDrops() - return discarded + const discardedBeforeDrain = this._discardQueueIfConsentWithdrawn() + if (discardedBeforeDrain) { + return discardedBeforeDrain } - const resourceAttributes = buildTracesResourceAttributes( - this._config, - this._instance.getLibraryId(), - this._instance.getLibraryVersion() + // Bounded like span attributes: resource attributes are caller-supplied too, + // and they ride on every batch rather than on one span. + const resourceAttributes = truncateAttributes( + buildTracesResourceAttributes(this._config, this._instance.getLibraryId(), this._instance.getLibraryVersion()), + this._config.maxAttributeValueLength ) const scopeName = this._instance.getLibraryId() const scopeVersion = this._instance.getLibraryVersion() @@ -478,6 +784,13 @@ export class PostHogTraces { try { while (remaining > 0 && this._queue.length > 0) { + // Re-checked per batch: a send suspends, so the user can opt out while one + // batch is in flight and the batches behind it would still export. + const discardedMidDrain = this._discardQueueIfConsentWithdrawn() + if (discardedMidDrain) { + return removed + discardedMidDrain + } + // Floor at one, or a non-positive batch size loops forever on an empty batch. const cap = this._headBatchFailures > 0 diff --git a/packages/core/src/traces/live-spans.spec.ts b/packages/core/src/traces/live-spans.spec.ts index 895fda1c79..47ab14c5d9 100644 --- a/packages/core/src/traces/live-spans.spec.ts +++ b/packages/core/src/traces/live-spans.spec.ts @@ -20,6 +20,10 @@ describe('live spans', () => { maxQueueSize: 2048, maxLiveSpans: 10000, maxSpanAgeMs: 3600000, + beforeSpanSend: [], + maxAttributesPerSpan: 128, + maxEventsPerSpan: 128, + maxAttributeValueLength: 8192, } const createTraces = (): PostHogTraces => diff --git a/packages/core/src/traces/otlp.spec.ts b/packages/core/src/traces/otlp.spec.ts index a36dded70d..641171df2d 100644 --- a/packages/core/src/traces/otlp.spec.ts +++ b/packages/core/src/traces/otlp.spec.ts @@ -140,6 +140,10 @@ describe('OTLP span encoding', () => { flushIntervalMs: 5000, maxExportBatchSize: 512, maxQueueSize: 2048, + beforeSpanSend: [], + maxAttributesPerSpan: 128, + maxEventsPerSpan: 128, + maxAttributeValueLength: 8192, maxLiveSpans: 10000, maxSpanAgeMs: 3600000, ...partial, diff --git a/packages/core/src/traces/otlp.ts b/packages/core/src/traces/otlp.ts index 6da9bfb2e2..686b7e8e86 100644 --- a/packages/core/src/traces/otlp.ts +++ b/packages/core/src/traces/otlp.ts @@ -9,6 +9,7 @@ import type { } from '@posthog/types' import type { Logger } from '../types' import type { ResolvedTracesConfig, SpanRecord } from './types' +import { nonNegativeCount } from './span' import { toOtlpKeyValueList } from '../utils/otlp-any-value' import { UNSERIALIZABLE_VALUE, sanitizeString } from '../utils/json-utils' import { buildOtlpResourceAttributes } from '../utils/otlp-resource' @@ -130,6 +131,16 @@ export function buildOtlpSpan(record: SpanRecord, logger?: Logger): OtlpSpan { if (record.events.length) { span.events = record.events.map((event) => toOtlpEvent(event, logger)) } + // Coerced: a `beforeSpanSend` hook can write anything onto the record, and a + // non-integer here is refused for the whole request. + const droppedAttributes = nonNegativeCount(record.droppedAttributesCount) + if (droppedAttributes) { + span.droppedAttributesCount = droppedAttributes + } + const droppedEvents = nonNegativeCount(record.droppedEventsCount) + if (droppedEvents) { + span.droppedEventsCount = droppedEvents + } if (record.status) { span.status = { code: SPAN_STATUS_TO_OTLP[record.status.code], diff --git a/packages/core/src/traces/sanitize.ts b/packages/core/src/traces/sanitize.ts index 258620cf53..701286f4db 100644 --- a/packages/core/src/traces/sanitize.ts +++ b/packages/core/src/traces/sanitize.ts @@ -23,9 +23,12 @@ const DEEP_BACKDATE_WARNING_MS = 24 * 60 * 60 * 1000 * replaced rather than dropped, so a mis-instrumented call site loses its name, * not its span. `label` names what is being sanitized in the warning. */ -export function sanitizeName(name: unknown, label: string, logger?: Logger): string { +export function sanitizeName(name: unknown, label: string, maxLength: number, logger?: Logger): string { if (typeof name === 'string' && name.trim()) { - return name + // Bounded like a status message and an attribute value: a name built from a + // URL or a payload is caller-controlled too, and one large enough takes the + // span past the ingestion body limit, which drops it whole. + return name.length > maxLength ? name.slice(0, maxLength) : name } logger?.debug(`${label} must be a non-empty string; using "${FALLBACK_SPAN_NAME}"`) return FALLBACK_SPAN_NAME diff --git a/packages/core/src/traces/span.spec.ts b/packages/core/src/traces/span.spec.ts index 15ce17709d..2039adc43a 100644 --- a/packages/core/src/traces/span.spec.ts +++ b/packages/core/src/traces/span.spec.ts @@ -1,8 +1,10 @@ -import { NOOP_SPAN, PostHogSpan, describeError } from './span' +import { NOOP_SPAN, PostHogSpan, describeError, truncateAttributeValue } from './span' +import { buildOtlpSpan } from './otlp' import type { SpanInit } from './span' import type { SpanRecord } from './types' import type { Logger } from '../types' import { createMockLogger } from '@/testing' +import { MAX_JSON_SAFE_VALUE_ITEMS, MAX_JSON_SAFE_VALUE_NODES } from '../utils/json-utils' const TRACE_ID = '4bf92f3577b34da6a3ce929d0e0e4736' const SPAN_ID = '00f067aa0ba902b7' @@ -21,6 +23,10 @@ describe('PostHogSpan', () => { attributes: {}, startTime: Date.now(), backdated: false, + autoAttributeKeys: [], + maxAttributes: 128, + maxEvents: 128, + maxAttributeValueLength: 8192, ...init, }, (record) => ended.push(record), @@ -166,6 +172,75 @@ describe('PostHogSpan', () => { expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('OK')) }) + describe('event cap', () => { + const fillEvents = (span: PostHogSpan, count: number): void => { + for (let i = 0; i < count; i++) { + span.addEvent(`step-${i}`) + } + } + + it('drops an exception event on a span that has filled its events', () => { + // The cap is absolute, so an exception arriving last is dropped like any + // other event. The span keeps its `error` status and `droppedEventsCount` + // reports the loss, which is what makes the case findable in production. + const span = createSpan({ maxEvents: 2 }) + fillEvents(span, 2) + span.recordException(new Error('boom')) + span.end() + + expect(ended[0].events.map((event) => event.name)).toEqual(['step-0', 'step-1']) + expect(ended[0].status).toEqual({ code: 'error', message: 'boom' }) + expect(ended[0].droppedEventsCount).toBe(1) + }) + + it('records an exception like any other event while the cap has room', () => { + const span = createSpan({ maxEvents: 128 }) + for (let i = 0; i < 20; i++) { + span.recordException(new Error(`boom-${i}`)) + } + span.end() + + expect(ended[0].events).toHaveLength(20) + expect(ended[0].droppedEventsCount).toBeUndefined() + }) + + it('does not let an exception-named event bypass the cap', () => { + // The cap counts events, not names: nothing about the name `exception` + // buys a slot, whoever wrote it. + const span = createSpan({ maxEvents: 2 }) + for (let i = 0; i < 7; i++) { + span.addEvent('exception', { mine: i }) + } + span.end() + + expect(ended[0].events).toHaveLength(2) + expect(ended[0].droppedEventsCount).toBe(5) + }) + + it('drops ordinary events past the cap', () => { + const span = createSpan({ maxEvents: 2 }) + fillEvents(span, 5) + span.end() + + expect(ended[0].events.map((event) => event.name)).toEqual(['step-0', 'step-1']) + expect(ended[0].droppedEventsCount).toBe(3) + }) + }) + + describe('attribute store hygiene', () => { + it('does not copy a polluted Object.prototype key into the span', () => { + ;(Object.prototype as any).polluted = 'yes' + try { + const span = createSpan({ attributes: { real: 1 } }) + span.end() + + expect(Object.keys(ended[0].attributes)).toEqual(['real']) + } finally { + delete (Object.prototype as any).polluted + } + }) + }) + describe('poison attributes', () => { const withThrowingGetter = (): any => { const attributes: any = { ok: 1 } @@ -224,10 +299,471 @@ describe('PostHogSpan', () => { expect(ended[0].events).toEqual([ expect.objectContaining({ name: 'exception', - attributes: { 'exception.type': 'TypeError', 'exception.message': 'boom' }, + attributes: expect.objectContaining({ 'exception.type': 'TypeError', 'exception.message': 'boom' }), }), ]) }) + + it('attaches the stack as exception.stacktrace', () => { + const span = createSpan() + span.recordException(new TypeError('boom')) + span.end() + + const stack = ended[0].events[0].attributes?.['exception.stacktrace'] + expect(stack).toEqual(expect.stringContaining('TypeError: boom')) + }) + + it('bounds the stack by maxAttributeValueLength', () => { + const span = createSpan({ maxAttributeValueLength: 40 }) + const error = new Error('boom') + error.stack = `Error: boom\n${' at somewhere deep\n'.repeat(500)}` + + span.recordException(error) + span.end() + + expect(ended[0].events[0].attributes?.['exception.stacktrace']).toHaveLength(40) + }) + + it('records an exception with no stack without inventing one', () => { + const span = createSpan() + span.recordException('just a string') + span.end() + + expect(ended[0].events[0].attributes).not.toHaveProperty('exception.stacktrace') + }) + }) + + describe('maxAttributeValueLength', () => { + it('truncates a long string attribute without counting it as dropped', () => { + const span = createSpan({ maxAttributeValueLength: 10 }) + span.setAttribute('payload', 'x'.repeat(5000)) + span.end() + + expect(ended[0].attributes.payload).toBe('xxxxxxxxxx') + // The count is for whole entries; a trimmed value is still exported. + expect(ended[0].droppedAttributesCount).toBeUndefined() + }) + + it('truncates the strings inside an array attribute, and leaves other types alone', () => { + const span = createSpan({ maxAttributeValueLength: 3 }) + span.setAttributes({ tags: ['abcdef', 'ab'], count: 1234567, flag: true }) + span.end() + + expect(ended[0].attributes).toMatchObject({ tags: ['abc', 'ab'], count: 1234567, flag: true }) + }) + + it('truncates strings nested inside an object value', () => { + // `setAttribute('payload', { body: res.body })` is the natural way to attach + // a response, and an unbounded one is what pushes a span past the endpoint. + const span = createSpan({ maxAttributeValueLength: 4 }) + span.setAttribute('payload', { body: 'abcdefgh', status: 200 }) + span.end() + + expect(ended[0].attributes.payload).toEqual({ body: 'abcd', status: 200 }) + }) + + it('truncates strings nested inside an array value', () => { + const span = createSpan({ maxAttributeValueLength: 4 }) + span.setAttribute('rows', [{ body: 'abcdefgh' }, ['abcdefgh']]) + span.end() + + expect(ended[0].attributes.rows).toEqual([{ body: 'abcd' }, ['abcd']]) + }) + + it('bounds the work a shared subtree costs, not just a cyclic one', () => { + // Siblings pointing at one object are re-walked once per path, so this + // reaches the same leaves ten million times. Only the node budget stops it. + const leaf: string[] = [] + for (let i = 0; i < 1000; i++) { + leaf.push('x'.repeat(2000)) + } + const mid = Array.from({ length: 1000 }, () => leaf) + const shared = Array.from({ length: 10 }, () => mid) + + const bounded = truncateAttributeValue(shared, 8) + + // Counting what the walk shortened, not what the result can reach: past + // the budget the original is handed back by reference. + const cap = MAX_JSON_SAFE_VALUE_NODES * 2 + let shortened = 0 + const stack: unknown[] = [bounded] + while (stack.length && shortened <= cap) { + const value = stack.pop() + if (typeof value === 'string') { + if (value.length === 8) { + shortened++ + } + } else if (Array.isArray(value) && value !== leaf && value !== mid) { + stack.push(...value) + } + } + expect(shortened).toBeLessThanOrEqual(cap) + }) + + it('terminates on a self-referencing value', () => { + const cyclic: any = { body: 'abcdefgh' } + cyclic.self = cyclic + const span = createSpan({ maxAttributeValueLength: 4 }) + + expect(() => { + span.setAttribute('payload', cyclic) + span.end() + }).not.toThrow() + }) + + it('charges a throwing accessor to its own key, still bounding its siblings', () => { + // A lazy ORM relation next to a large field is the shape that matters: if + // the throw abandons the whole walk, the large field ships at full length. + const span = createSpan({ maxAttributeValueLength: 4 }) + const hostile = { + ok: 'abcdefgh', + get boom() { + throw new Error('getter exploded') + }, + } + + expect(() => { + span.setAttribute('payload', hostile as any) + span.end() + }).not.toThrow() + + const payload = ended[0].attributes.payload as Record + expect(payload.ok).toBe('abcd') + expect(payload.boom).toBe('[Unserializable]') + }) + + it('does not walk into a value whose toJSON redacts it', () => { + // Copying the object's own keys would hand the encoder a plain object it + // no longer recognises as self-describing, putting the internals of a + // value that redacts itself on the wire. + class Redacted { + constructor(public secret: string) {} + toJSON(): null { + return null + } + } + const span = createSpan({ maxAttributeValueLength: 10 }) + + span.setAttribute('payload', { inner: new Redacted('S'.repeat(50)) } as any) + span.end() + + // The string the encoder builds from the same `null`, so the wire is + // unchanged, and the secret is nowhere in what the span kept. + expect((ended[0].attributes.payload as any).inner).toBe('null') + expect(JSON.stringify(ended[0].attributes)).not.toContain('S') + }) + + it('materializes a toJSON that resolves to nothing, so a second call cannot answer differently', () => { + // Keeping the object itself left the encoder to probe toJSON again. A + // serializer that answered `null` here could answer with a megabyte + // there, past the bound entirely. + let calls = 0 + const stateful = { + toJSON: () => { + calls++ + return calls === 1 ? null : 'x'.repeat(100) + }, + } + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.setAttribute('doc', stateful as any) + span.end() + + expect(calls).toBe(1) + expect(ended[0].attributes.doc).toBe('null') + }) + + it('keeps a stateful toJSON bounded through to the encoded span', () => { + // End to end, because the second call is the encoder's: what the span + // stored has to leave it nothing to call. + let calls = 0 + const stateful = { + toJSON: () => { + calls++ + return calls === 1 ? null : 'x'.repeat(100) + }, + } + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.setAttribute('doc', stateful as any) + span.end() + const encoded = buildOtlpSpan(ended[0]) + + expect(calls).toBe(1) + expect(encoded.attributes?.find((attribute) => attribute.key === 'doc')).toEqual({ + key: 'doc', + value: { stringValue: 'null' }, + }) + }) + + it('describes a toJSON resolving to undefined the way the encoder would', () => { + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.setAttribute('ghost', { toJSON: () => undefined } as any) + span.end() + + // Not trimmed to the bound: this is the SDK's own marker, like + // `[Circular]`, and `unde` reads as nothing at all. + expect(ended[0].attributes.ghost).toBe('undefined') + }) + + it('bounds event attributes too', () => { + const span = createSpan({ maxAttributeValueLength: 4 }) + span.addEvent('cache.miss', { key: 'abcdefgh' }) + span.end() + + expect(ended[0].events[0].attributes).toEqual({ key: 'abcd' }) + }) + + it('walks a value that points at itself twice only once', () => { + // Depth alone is not a bound. Two back-references at each level cost + // 2 ** 20 visits, which is a quarter-second inside the caller's own + // `setAttribute` call, and a third reference is minutes. + let reads = 0 + const cyclic: any = { + get body() { + reads++ + return 'abcdefgh' + }, + } + cyclic.self1 = cyclic + cyclic.self2 = cyclic + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.setAttribute('payload', cyclic) + span.end() + + expect(reads).toBe(1) + }) + + it('bounds the nodes an acyclic value costs, not just its depth', () => { + // Siblings sharing a subtree are not a cycle, so the ancestor set does not + // catch them: 3 ** 12 visits without a node budget. + let reads = 0 + let level: any = { + get body() { + reads++ + return 'abcdefgh' + }, + } + for (let i = 0; i < 12; i++) { + level = { a: level, b: level, c: level } + } + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.setAttribute('payload', level) + span.end() + + expect(reads).toBeLessThanOrEqual(10_000) + }) + + it('bounds the value a toJSON produces, which is what the encoder puts on the wire', () => { + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.setAttribute('doc', { toJSON: () => 'abcdefgh' } as any) + span.end() + + expect(ended[0].attributes.doc).toBe('abcd') + }) + + it('resolves toJSON exactly once, so a second call cannot dodge the bound', () => { + // Returning the original object when nothing needed shortening left the + // encoder to call toJSON again — a value that answered differently the + // second time reached the wire unbounded. + let calls = 0 + const doc = { + toJSON: () => { + calls++ + return calls === 1 ? 'ab' : 'x'.repeat(4000) + }, + } + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.setAttribute('doc', doc as any) + span.end() + + expect(calls).toBe(1) + expect(ended[0].attributes.doc).toBe('ab') + }) + + it('bounds a string that follows a large collection of nulls', () => { + // The encoder drops a nullish value without spending its budget, so a walk + // that charges for one runs out first and leaves the string after it + // unbounded on both sides — a 2 MB value under a bound of 8. + const span = createSpan({ maxAttributeValueLength: 8 }) + + span.setAttribute('payload', { + rows: Array.from({ length: 400 }, () => + Object.fromEntries(Array.from({ length: 50 }, (_unused, index) => [`c${index}`, null])) + ), + html: 'X'.repeat(50000), + }) + span.end() + + expect((ended[0].attributes.payload as any).html).toHaveLength(8) + }) + + it('bounds a string that follows a large collection', () => { + // The traversal budget is spent on containers, not leaves: a big array + // used to exhaust it and leave every later string at full length. + const span = createSpan({ maxAttributeValueLength: 8 }) + + span.setAttribute('payload', { + rows: Array.from({ length: 20000 }, (_, index) => index), + html: 'X'.repeat(50000), + }) + span.end() + + expect((ended[0].attributes.payload as any).html).toHaveLength(8) + }) + + it('keeps a nested __proto__ key as an ordinary entry', () => { + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.setAttribute('payload', JSON.parse('{"__proto__": {"body": "abcdefgh"}}')) + span.end() + + const payload = ended[0].attributes.payload as Record + expect(Object.keys(payload)).toEqual(['__proto__']) + expect(Object.getOwnPropertyDescriptor(payload, '__proto__')?.value).toEqual({ body: 'abcd' }) + }) + + it('bounds an event name the SDK records like any other', () => { + const span = createSpan({ maxAttributeValueLength: 8, maxEvents: 4 }) + + span.recordException(new Error('boom')) + span.end() + + expect(ended[0].events[0].name).toBe('exceptio') + expect(ended[0].events[0].attributes?.['exception.type']).toBe('Error') + }) + + it('bounds a span name and an event name, like a status message', () => { + // A name built from a URL is caller-controlled, and one large enough takes + // the span past the ingestion body limit. + const span = createSpan({ maxAttributeValueLength: 12 }) + + span.updateName('abcdefghijklmnop') + span.addEvent('abcdefghijklmnop') + span.end() + + expect(ended[0].name).toBe('abcdefghijkl') + expect(ended[0].events[0].name).toBe('abcdefghijkl') + }) + + it('bounds a status message', () => { + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.setStatus('error', 'abcdefgh') + span.end() + + expect(ended[0].status).toEqual({ code: 'error', message: 'abcd' }) + }) + + it('bounds the status message recordException sets, like the event attribute', () => { + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.recordException(new Error('abcdefgh')) + span.end() + + expect(ended[0].status?.message).toBe('abcd') + expect(ended[0].events[0].attributes?.['exception.message']).toBe('abcd') + }) + + it('replaces a back-reference with the marker rather than the value itself', () => { + // Handing the raw ancestor back left it inside a copied parent, where the + // encoder's own cycle detection no longer recognised it and walked one + // more level of its strings at full length. + const cyclic: any = { body: 'abcdefgh' } + cyclic.self = cyclic + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.setAttribute('payload', cyclic) + span.end() + + expect(ended[0].attributes.payload).toEqual({ body: 'abcd', self: '[Circular]' }) + }) + + it('still bounds an array whose accessor past the encoder cap throws', () => { + // `slice()` read the whole array to copy it, so one throwing accessor + // beyond the encoder's cap cost every item in range its bound. + const rows: unknown[] = ['abcdefgh'] + for (let index = 1; index < MAX_JSON_SAFE_VALUE_ITEMS + 200; index++) { + rows.push('x') + } + Object.defineProperty(rows, MAX_JSON_SAFE_VALUE_ITEMS + 100, { + get: () => { + throw new Error('lazy relation') + }, + enumerable: true, + configurable: true, + }) + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.setAttribute('rows', rows as any) + span.end() + + expect((ended[0].attributes.rows as unknown[])[0]).toBe('abcd') + }) + + it('stops reading keys where the encoder stops emitting them', () => { + // Every key was read even though the encoder emits at most the cap, so a + // wide object charged `setAttribute` for getters that never ship. + let reads = 0 + const wide: Record = {} + for (let index = 0; index < MAX_JSON_SAFE_VALUE_ITEMS * 2; index++) { + Object.defineProperty(wide, `k${index}`, { + get: () => { + reads++ + return 'v' + }, + enumerable: true, + configurable: true, + }) + } + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.setAttribute('payload', wide) + span.end() + + expect(reads).toBe(MAX_JSON_SAFE_VALUE_ITEMS) + }) + + it('copies a nested value the caller goes on to mutate', () => { + // A value that needed no truncation was attached as it came, so the span + // held caller-owned state and shipped whatever it was changed to. + const nested = { body: 'ok' } + const span = createSpan({ maxAttributeValueLength: 4 }) + + span.setAttribute('payload', { nested }) + nested.body = 'abcdefgh' + span.end() + + expect(ended[0].attributes.payload).toEqual({ nested: { body: 'ok' } }) + }) + + it('leaves a Date whole rather than truncating its timestamp', () => { + // The encoder emits a Date from its own branch ahead of any `toJSON`, so + // bounding it here shipped a cut-off timestamp instead of a shorter one. + const span = createSpan({ maxAttributeValueLength: 10 }) + + span.setAttribute('when', new Date('2020-01-02T03:04:05.000Z') as never) + span.end() + + expect(ended[0].attributes.when).toEqual(new Date('2020-01-02T03:04:05.000Z')) + }) + + it('bounds an SDK-attached value, which is exempt from the count cap only', () => { + const span = createSpan({ + maxAttributeValueLength: 4, + attributes: { posthogDistinctId: 'user-12345' }, + autoAttributeKeys: ['posthogDistinctId'], + }) + span.setAttribute('posthogDistinctId', 'user-12345') + span.end() + + expect(ended[0].attributes.posthogDistinctId).toBe('user') + }) }) describe('timestamps', () => { @@ -349,6 +885,61 @@ describe('NoopSpan', () => { }) }) +describe('attribute store', () => { + it('hands out a record whose attributes behave like an ordinary object', () => { + const ended: SpanRecord[] = [] + const span = new PostHogSpan( + { + traceId: TRACE_ID, + spanId: SPAN_ID, + name: 'checkout', + kind: 'internal', + attributes: { plan: 'pro' }, + startTime: Date.now(), + backdated: false, + autoAttributeKeys: [], + maxAttributes: 128, + maxEvents: 128, + maxAttributeValueLength: 8192, + }, + (record) => ended.push(record) + ) + span.end() + + const { attributes } = ended[0] + expect(Object.getPrototypeOf(attributes)).toBe(Object.prototype) + expect(Object.prototype.hasOwnProperty.call(attributes, 'plan')).toBe(true) + expect(() => JSON.stringify(attributes)).not.toThrow() + }) + + it('keeps a parsed __proto__ key as an ordinary attribute', () => { + const ended: SpanRecord[] = [] + const span = new PostHogSpan( + { + traceId: TRACE_ID, + spanId: SPAN_ID, + name: 'checkout', + kind: 'internal', + attributes: JSON.parse('{"__proto__": {"leaked": 1}, "orderId": "abc"}'), + startTime: Date.now(), + backdated: false, + autoAttributeKeys: [], + maxAttributes: 128, + maxEvents: 128, + maxAttributeValueLength: 8192, + }, + (record) => ended.push(record) + ) + span.end() + + const keys: string[] = [] + for (const key in ended[0].attributes) { + keys.push(key) + } + expect(keys).not.toContain('leaked') + }) +}) + describe('describeError', () => { it.each([ ['an Error', new Error('boom'), { type: 'Error', message: 'boom' }], @@ -357,7 +948,23 @@ describe('describeError', () => { ['an object with a message', { name: 'CustomError', message: 'oops' }, { type: 'CustomError', message: 'oops' }], ['an object without a name', { message: 'oops' }, { type: 'Object', message: 'oops' }], ])('describes %s', (_name, error, expected) => { - expect(describeError(error)).toEqual(expected) + expect(describeError(error)).toMatchObject(expected) + }) + + it('carries the stack where the thrown value has one, and nothing where it does not', () => { + expect(describeError(new Error('boom')).stack).toEqual(expect.stringContaining('Error: boom')) + expect(describeError('just a string').stack).toBeUndefined() + expect(describeError({ message: 'oops' }).stack).toBeUndefined() + }) + + it('survives a throwing stack accessor', () => { + const hostile = { + message: 'oops', + get stack() { + throw new Error('nope') + }, + } + expect(describeError(hostile)).toEqual({ type: 'Object', message: 'oops' }) }) it('describes a thrown primitive', () => { diff --git a/packages/core/src/traces/span.ts b/packages/core/src/traces/span.ts index a93ea62d31..3124945c9d 100644 --- a/packages/core/src/traces/span.ts +++ b/packages/core/src/traces/span.ts @@ -3,8 +3,15 @@ import type { Logger } from '../types' import type { SpanContextManager, SpanEventRecord, SpanRecord } from './types' import { formatTraceparent, normalizeTraceparent, sanitizeTracestate, TRACE_FLAGS_SAMPLED } from './traceparent' import { clampEndTime, resolveSuppliedTime, sanitizeName } from './sanitize' -import { assignUserAttributes } from '../utils/json-utils' -import { isError } from '../utils' +import { isArray, isError, isNullish } from '../utils' +import { + CIRCULAR_VALUE, + MAX_JSON_SAFE_VALUE_DEPTH, + MAX_JSON_SAFE_VALUE_ITEMS, + MAX_JSON_SAFE_VALUE_NODES, + UNSERIALIZABLE_VALUE, + assignUserAttributes, +} from '../utils/json-utils' /** * A monotonic millisecond reading where the platform has one, so an NTP @@ -31,6 +38,11 @@ export interface SpanInit { startTime: number /** True when the caller supplied an explicit `startTime`. */ backdated: boolean + /** Keys the SDK attached itself. Exempt from the attribute cap and never evicted. */ + autoAttributeKeys: string[] + maxAttributes: number + maxEvents: number + maxAttributeValueLength: number } export class PostHogSpan implements Span { @@ -50,10 +62,18 @@ export class PostHogSpan implements Span { private _events: SpanEventRecord[] = [] private _status?: { code: SpanStatusCode; message?: string } private _ended = false + private readonly _autoKeys: Set + private readonly _maxAttributes: number + private readonly _maxEvents: number + private readonly _maxAttributeValueLength: number + private _userAttributeCount = 0 + private _userEventCount = 0 + private _droppedAttributes = 0 + private _droppedEvents = 0 constructor( init: SpanInit, - private readonly _onEnd: (record: SpanRecord) => void, + private readonly _onEnd: (record: SpanRecord, autoKeys: ReadonlySet) => void, private readonly _logger?: Logger ) { this._traceId = init.traceId @@ -64,7 +84,19 @@ export class PostHogSpan implements Span { this._parentIsRemote = init.parentIsRemote ?? false this._name = init.name this._kind = init.kind - this._attributes = init.attributes + this._autoKeys = new Set(init.autoAttributeKeys) + this._maxAttributes = init.maxAttributes + this._maxEvents = init.maxEvents + this._maxAttributeValueLength = init.maxAttributeValueLength + // Null-prototype: a `__proto__` key would otherwise swap this object's prototype + // instead of becoming an entry, and `toString` and friends would read as + // already-present. + this._attributes = Object.create(null) as SpanAttributes + // Object.keys, not for...in: the latter walks the prototype chain, so a + // polluted `Object.prototype` key would become an attribute of every span. + for (const key of Object.keys(init.attributes)) { + this._writeAttribute(key, init.attributes[key]) + } this._startTime = init.startTime this._startMono = init.backdated ? undefined : monotonicNow() } @@ -92,27 +124,72 @@ export class PostHogSpan implements Span { return true } + /** + * Writes an attribute unless the span is already at its user-attribute cap. + * + * Overwriting a key already on the span always succeeds — the cap counts + * distinct user keys, not writes — and SDK-attached keys never count toward + * it, so a span at the cap still carries its person and session ids. + */ + private _writeAttribute(key: string, value: SpanAttributeValue): void { + // Nullish removes the key rather than occupying it: storing one would spend no + // budget and make every later write to that key free, exceeding the cap. + if (isNullish(value)) { + if (key in this._attributes && !this._autoKeys.has(key)) { + this._userAttributeCount-- + } + delete this._attributes[key] + return + } + // The cap is checked before the value is bounded: walking a value the span is + // about to drop is the dominant cost of a span that overflows its cap. + if (!this._autoKeys.has(key) && !(key in this._attributes)) { + if (this._userAttributeCount >= this._maxAttributes) { + this._droppedAttributes++ + return + } + this._userAttributeCount++ + } + this._attributes[key] = truncateAttributeValue(value, this._maxAttributeValueLength) + } + setAttribute(key: string, value: SpanAttributeValue): this { if (this._mutable('setAttribute')) { - Object.defineProperty(this._attributes, key, { value, enumerable: true, writable: true, configurable: true }) + this._writeAttribute(key, value) } return this } setAttributes(attributes: SpanAttributes): this { if (this._mutable('setAttributes')) { - assignUserAttributes(this._attributes, attributes) + // Read through the shared guard first — own enumerable keys only, and a + // throwing getter costs its own key — then write each through the cap. + const safe: SpanAttributes = assignUserAttributes({}, attributes) + for (const key of Object.keys(safe)) { + this._writeAttribute(key, safe[key]) + } } return this } addEvent(name: string, attributes?: SpanAttributes, timestamp?: SpanTimeInput): this { if (this._mutable('addEvent')) { + // An exception the SDK records spends an ordinary slot like any other + // event. A span that fills its events and then throws therefore keeps its + // `error` status but loses the exception detail, which `droppedEventsCount` + // reports — enough to find the case in production if it turns out to occur. + if (this._userEventCount >= this._maxEvents) { + this._droppedEvents++ + return this + } + this._userEventCount++ this._events.push({ - name: sanitizeName(name, 'Span event name', this._logger), + name: sanitizeName(name, 'Span event name', this._maxAttributeValueLength, this._logger), timestamp: resolveSuppliedTime(timestamp, this._now(), 'event timestamp', this._logger), // Copied so a caller reusing one object across events can't mutate a recorded one. - ...(attributes && { attributes: assignUserAttributes({}, attributes) }), + ...(attributes && { + attributes: truncateAttributes(assignUserAttributes({}, attributes), this._maxAttributeValueLength), + }), }) } return this @@ -124,7 +201,12 @@ export class PostHogSpan implements Span { this._logger?.debug(`Ignoring unknown span status "${String(status)}"; expected "ok" or "error"`) return this } - this._status = { code: status, ...(message && { message }) } + // Bounded like an attribute value: a status message is one more string the + // caller controls, and one large enough takes the span past the body limit. + this._status = { + code: status, + ...(message && { message: truncateString(message, this._maxAttributeValueLength) }), + } } return this } @@ -138,10 +220,11 @@ export class PostHogSpan implements Span { if (!this._mutable('recordException')) { return this } - const { type, message } = describeError(error) - this.addEvent('exception', { + const { type, message, stack } = describeError(error) + this.addEvent(EXCEPTION_EVENT_NAME, { 'exception.type': type, 'exception.message': message, + ...(stack && { 'exception.stacktrace': stack }), }) // recordException is itself an explicit call, so it follows last-write-wins // rather than deferring to an earlier `ok`. @@ -150,7 +233,7 @@ export class PostHogSpan implements Span { updateName(name: string): this { if (this._mutable('updateName')) { - this._name = sanitizeName(name, 'Span name', this._logger) + this._name = sanitizeName(name, 'Span name', this._maxAttributeValueLength, this._logger) } return this } @@ -184,22 +267,147 @@ export class PostHogSpan implements Span { const derived = this._now() const resolved = resolveSuppliedTime(endTime, derived, 'end time', this._logger) - this._onEnd({ - traceId: this._traceId, - spanId: this._spanId, - ...(this._parentSpanId && { parentSpanId: this._parentSpanId }), - ...(this._traceState && { traceState: this._traceState }), - traceFlags: this._traceFlags, - parentIsRemote: this._parentIsRemote, - name: this._name, - kind: this._kind, - ...(this._status && { status: this._status }), - attributes: this._attributes, - events: this._events, - startTime: this._startTime, - endTime: clampEndTime(resolved, this._startTime), + this._onEnd( + { + traceId: this._traceId, + spanId: this._spanId, + ...(this._parentSpanId && { parentSpanId: this._parentSpanId }), + ...(this._traceState && { traceState: this._traceState }), + traceFlags: this._traceFlags, + parentIsRemote: this._parentIsRemote, + name: this._name, + kind: this._kind, + ...(this._status && { status: this._status }), + // Copied out with an ordinary prototype: the store is null-prototype, but a + // record handed to user code should behave like a normal object. + attributes: { ...this._attributes }, + events: this._events, + startTime: this._startTime, + endTime: clampEndTime(resolved, this._startTime), + ...(this._droppedAttributes && { droppedAttributesCount: this._droppedAttributes }), + ...(this._droppedEvents && { droppedEventsCount: this._droppedEvents }), + }, + this._autoKeys + ) + } +} + +const EXCEPTION_EVENT_NAME = 'exception' + +/** A value as its string form, or the encoder's marker when it refuses to produce one. */ +function safeString(value: unknown): string { + try { + return typeof value === 'string' ? value : String(value) + } catch { + return UNSERIALIZABLE_VALUE + } +} + +/** A caller-visible counter read back as a number, or 0 for anything else. */ +export function nonNegativeCount(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0 +} + +/** + * Re-applies the per-span caps to a record a `beforeSpanSend` hook has already + * seen. The hook writes to the plain record, not through the span's own guarded + * writer, so an enriching hook would otherwise push a span past the cap it was + * trimmed to and back into the 413 path the cap exists to avoid. + * + * Earliest-set entries win, matching the span-side rule; SDK-attached keys are + * exempt. Counts add to whatever the span already dropped. + */ +/** + * The record's keys with the ones the span itself set first, in that order. + * + * `Object.keys` hoists integer-like keys to the front whatever the write order, + * so a hook adding `attributes['0']` would otherwise outrank an attribute the + * caller set before the hook ran — and the cap is documented as earliest-set-wins. + */ +function orderedKeys(attributes: SpanAttributes, keysBeforeHook: readonly string[]): string[] { + if (!keysBeforeHook.length) { + return Object.keys(attributes) + } + // The encoder's own predicate: `in` would walk the prototype chain, so a key + // the caller set that collides with Object.prototype survives the hook deleting + // it and reads back as the inherited member, and `hasOwnProperty` would keep a + // key the hook hid by making it non-enumerable, which the encoder never emits. + const beforeHook = keysBeforeHook.filter((key) => Object.prototype.propertyIsEnumerable.call(attributes, key)) + const seen = new Set(beforeHook) + return [...beforeHook, ...Object.keys(attributes).filter((key) => !seen.has(key))] +} + +export function applySpanLimits( + record: SpanRecord, + autoKeys: ReadonlySet, + maxAttributes: number, + maxEvents: number, + maxAttributeValueLength: number, + keysBeforeHook: readonly string[] = [] +): void { + let kept = 0 + let droppedAttributes = 0 + // Built fresh rather than edited in place: a hook is free to return a record + // whose attributes it froze, and a `delete` on one throws. + const attributes: SpanAttributes = {} + for (const key of orderedKeys(record.attributes, keysBeforeHook)) { + const value = record.attributes[key] + // Matches `_writeAttribute`: the encoder drops these, so a hook that blanks a + // value rather than deleting the key must not evict a real attribute. + if (isNullish(value)) { + continue + } + if (!autoKeys.has(key)) { + if (kept >= maxAttributes) { + droppedAttributes++ + continue + } + kept++ + } + Object.defineProperty(attributes, key, { + value: truncateAttributeValue(value, maxAttributeValueLength), + enumerable: true, + writable: true, + configurable: true, }) } + record.attributes = attributes + if (droppedAttributes) { + // Coerced, not trusted: a hook can put anything in the counter, and a + // non-number there would erase the count the span itself accumulated. + record.droppedAttributesCount = nonNegativeCount(record.droppedAttributesCount) + droppedAttributes + } + + // Walked rather than sliced: a hook can append events or rewrite their + // attributes, neither of which goes through `addEvent`, so each one still + // needs its attributes bounded on the way past. + let keptEvents = 0 + let droppedEvents = 0 + const events: SpanEventRecord[] = [] + for (const event of record.events) { + if (keptEvents >= maxEvents) { + droppedEvents++ + continue + } + keptEvents++ + if (event.attributes) { + event.attributes = truncateAttributes({ ...event.attributes }, maxAttributeValueLength) + } + events.push(event) + } + record.events = events + if (droppedEvents) { + record.droppedEventsCount = nonNegativeCount(record.droppedEventsCount) + droppedEvents + } + if (record.status?.message) { + // Coerced first: a non-string would reach the encoder to be stringified at + // full length. Guarded, because a throwing `toString` here would cost the + // span, where the encoder downstream only marks the field. + record.status = { + ...record.status, + message: truncateString(safeString(record.status.message), maxAttributeValueLength), + } + } } /** @@ -264,6 +472,21 @@ export class PassThroughSpan extends NoopSpan { } } +/** + * The `stack` of whatever was thrown, as OTel's `exception.stacktrace`. Reads + * the property behind its own guard: a getter on a hostile object throws, and a + * thrown string has no stack at all. The value is bounded like any other + * attribute, by `maxAttributeValueLength`. + */ +function readStack(error: unknown): { stack?: string } { + try { + const stack = (error as { stack?: unknown }).stack + return typeof stack === 'string' && stack ? { stack } : {} + } catch { + return {} + } +} + /** * The handle to return when a span cannot be recorded: a pass-through when the * caller supplied a usable `parent` header, the shared no-op otherwise. @@ -291,6 +514,197 @@ function readHandle(parent: unknown, method: 'traceparent' | 'tracestate'): unkn } } +/** + * The same depth, node and item caps `encodeAnyValue` uses, but spent per + * attribute rather than per bag: the encoder allocates one budget for a whole + * attribute map, this walk allocates one per value. That makes the encoder's + * budget the stricter of the two — whatever this walk hands back unbounded, the + * encoder has already stopped short of — at the cost of a wide span paying for + * a walk whose results the encoder then discards. + */ +interface TruncateState { + /** Containers on the current path, so a back-reference stops the walk. */ + ancestors: WeakSet + remainingNodes: number +} + +/** + * Bounds every string reachable from an attribute value to `maxLength` + * characters, including the strings nested inside arrays and objects. Numbers + * and booleans are bounded already. + * + * An unbounded value is the one thing the per-span caps do not stop: a single + * multi-MB attribute makes the whole span too large for the ingestion endpoint, + * and the 413 path then drops that span whole. `setAttribute('payload', { body })` + * is the usual way one arrives, so the bound has to reach inside the value. + * + * Returns the value it was given when nothing needed shortening, so the common + * case allocates nothing. + */ +function truncateString(value: string, maxLength: number): string { + return value.length > maxLength ? value.slice(0, maxLength) : value +} + +export function truncateAttributeValue(value: SpanAttributeValue, maxLength: number): SpanAttributeValue { + return truncateValue(value, maxLength, { ancestors: new WeakSet(), remainingNodes: MAX_JSON_SAFE_VALUE_NODES }, 0) +} + +/** + * Walks under the same depth cap, node budget and ancestor set as + * `encodeAnyValue`, charging the same values. The encoder spends one budget + * across the whole attribute bag where this spends one per value, so it runs out + * no later and marks whatever this walk returned whole. + * + * Depth alone does not bound this: a value whose children point back at their + * siblings costs `fanout ** depth` visits, which is minutes of synchronous work + * inside the caller's own `setAttribute` call. + */ +function truncateValue( + value: SpanAttributeValue, + maxLength: number, + state: TruncateState, + depth: number +): SpanAttributeValue { + if (value === null || typeof value !== 'object') { + // Free, as it is in the encoder, which drops a nullish leaf without charging. + if (isNullish(value)) { + return value + } + // A shared subtree is re-walked once per path reaching it, so a leaf that + // skips the charge lets one value cost `budget * items` string copies. + if (state.remainingNodes <= 0) { + return value + } + state.remainingNodes-- + return typeof value === 'string' ? truncateString(value, maxLength) : value + } + if (state.ancestors.has(value)) { + // The marker the encoder would produce, not the value itself. Handing the + // raw ancestor back puts it inside a *copied* parent, where the encoder's + // own cycle detection no longer recognises it and walks one more level of + // its strings at full length. + return CIRCULAR_VALUE + } + if (state.remainingNodes <= 0 || depth >= MAX_JSON_SAFE_VALUE_DEPTH) { + return value + } + state.remainingNodes-- + state.ancestors.add(value) + try { + // A Date is emitted by the encoder from its own branch, ahead of any + // `toJSON` probe, so bounding it here would ship a truncated timestamp + // rather than a shorter one. + if (value instanceof Date) { + return value + } + // The representation the value defines for itself is what the encoder puts + // on the wire, so it is what has to be bounded — a `toJSON` returning a + // megabyte of text is invisible to a walk over the object's own keys. + const resolved = resolveToJson(value) + if (resolved.selfDescribed) { + // Resolving to nothing is the value's answer. Walking its keys anyway + // would build a plain object the encoder no longer treats as + // self-describing, putting the internals of a redacted value on the wire. + // Stored as the string the encoder builds from that same nullish result + // rather than as the value itself: the encoder probes `toJSON` a second + // time, so one that answers `null` here is free to answer with a megabyte + // there, past the bound this walk exists to apply. Left unbounded like the + // other markers — nine characters at most, and trimming it to `unde` would + // only make it unreadable. + return isNullish(resolved.value) + ? String(resolved.value) + : truncateValue(resolved.value, maxLength, state, depth + 1) + } + if (isArray(value)) { + // Only the items the encoder will emit are walked; it stops at the same + // cap, so bounding the rest is work spent on values that never ship. + const walked = Math.min(value.length, MAX_JSON_SAFE_VALUE_ITEMS) + // Accumulated rather than copied from the value: `slice()` reads every + // element, accessors past the cap included, and one of those throwing + // would reach the outer catch and cost the whole array its bound. + const boundedItems: SpanAttributeValue[] = [] + for (let index = 0; index < walked; index++) { + try { + boundedItems.push(truncateValue(value[index], maxLength, state, depth + 1)) + } catch { + // A throwing accessor costs its own item, as it does in the encoder. + boundedItems.push(UNSERIALIZABLE_VALUE) + } + } + // Carried so the encoder still marks what it cut. + if (value.length > walked) { + boundedItems.length = value.length + } + return boundedItems + } + const bounded: SpanAttributes = {} + // Counted the way the encoder counts, so the walk stops where its output + // does: a key it skips costs no slot, and reading past the last one it can + // emit is getter work on values that never ship. + let emittable = 0 + for (const key of Object.keys(value)) { + if (emittable >= MAX_JSON_SAFE_VALUE_ITEMS) { + break + } + let boundedItem: SpanAttributeValue + try { + // Read once: re-reading to compare would run a getter a second time. + boundedItem = truncateValue((value as SpanAttributes)[key], maxLength, state, depth + 1) + } catch { + // A throwing accessor costs its own key. Reaching the walk's own catch + // would abandon the whole value unbounded, which is how a lazy ORM + // relation next to a large field puts that field on the wire whole. + boundedItem = UNSERIALIZABLE_VALUE + } + if (key && !isNullish(boundedItem)) { + emittable++ + } + // defineProperty, not assignment: a nested `__proto__` key would otherwise + // swap the copy's prototype and vanish. + Object.defineProperty(bounded, key, { + value: boundedItem, + enumerable: true, + writable: true, + configurable: true, + }) + } + return bounded + } catch { + // Whatever is left — a hostile `Object.keys`, a `slice` that throws — costs + // this value its bound rather than the span. Per-key reads are guarded + // above, so a single bad property does not reach here. + return value + } finally { + // Siblings pointing at the same object are duplication, not a cycle. + state.ancestors.delete(value) + } +} + +/** + * The value's own serialized form. `selfDescribed` is false when it defines no + * `toJSON`, or when reading one throws — both fall through to the plain walk, + * as they do in the encoder. + */ +function resolveToJson(value: object): { selfDescribed: boolean; value?: SpanAttributeValue } { + try { + const toJSON = (value as { toJSON?: unknown }).toJSON + if (typeof toJSON === 'function') { + return { selfDescribed: true, value: toJSON.call(value) as SpanAttributeValue } + } + } catch { + // Falls through to the plain walk. + } + return { selfDescribed: false } +} + +/** `truncateAttributeValue` across an attribute bag, in place. */ +export function truncateAttributes(attributes: SpanAttributes, maxLength: number): SpanAttributes { + for (const key of Object.keys(attributes)) { + attributes[key] = truncateAttributeValue(attributes[key], maxLength) + } + return attributes +} + /** * Runs `fn` with `span` active, which every scoped helper does the same way. * @@ -307,10 +721,11 @@ export function runWithActiveSpan(contextManager: SpanContextManager, span: S * Extracts the OTel `exception.type` / `exception.message` pair from whatever was * thrown. Anything can be thrown in JS, so non-Errors are described by type. */ -export function describeError(error: unknown): { type: string; message: string } { +export function describeError(error: unknown): { type: string; message: string; stack?: string } { try { + const stack = readStack(error) if (isError(error)) { - return { type: error.name || 'Error', message: error.message || '' } + return { type: error.name || 'Error', message: error.message || '', ...stack } } if (typeof error === 'string') { return { type: 'string', message: error } @@ -318,7 +733,7 @@ export function describeError(error: unknown): { type: string; message: string } if (error && typeof error === 'object') { const maybe = error as { name?: unknown; message?: unknown } if (typeof maybe.message === 'string') { - return { type: typeof maybe.name === 'string' ? maybe.name : 'Object', message: maybe.message } + return { type: typeof maybe.name === 'string' ? maybe.name : 'Object', message: maybe.message, ...stack } } } return { type: typeof error, message: String(error) } diff --git a/packages/core/src/traces/types.ts b/packages/core/src/traces/types.ts index 2b1e392570..7360c70a26 100644 --- a/packages/core/src/traces/types.ts +++ b/packages/core/src/traces/types.ts @@ -7,6 +7,7 @@ export type { SpanTimeInput, StartSpanOptions, TracesConfig, + BeforeSpanSendFn, OtlpSpan, OtlpSpanEvent, OtlpSpanKeyValue, @@ -14,7 +15,16 @@ export type { OtlpTracesPayload, } from '@posthog/types' -import type { OtlpTracesPayload, Span, SpanAttributes, SpanKind, SpanStatusCode, TracesConfig } from '@posthog/types' +import type { + BeforeSpanSendFn, + OtlpTracesPayload, + Span, + SpanAttributes, + SpanKind, + SpanRecord as HookSpanRecord, + SpanStatusCode, + TracesConfig, +} from '@posthog/types' /** Same tagged outcome shape as `SendLogsBatchOutcome` — one policy for all three signals. */ export type SendTracesBatchOutcome = @@ -56,26 +66,19 @@ export interface SpanEventRecord { } /** - * A completed span in plain, pre-encoding form: strings for kind and status, a - * plain attribute map, ms-epoch timestamps. + * A finished span as the SDK carries it, which is the hook-visible record plus + * the fields no hook may rewrite. Declaring only the additions keeps the shared + * half from drifting; a field added here rather than to the public record is a + * field `beforeSpanSend` cannot see, and so cannot corrupt. */ -export interface SpanRecord { - traceId: string - spanId: string - parentSpanId?: string +export interface SpanRecord extends HookSpanRecord { traceState?: string /** The W3C trace-flags byte this span propagates, e.g. `01` sampled. */ traceFlags: string /** True when the parent came from a `traceparent` header rather than a local handle. */ parentIsRemote: boolean - name: string - kind: SpanKind - status?: { code: SpanStatusCode; message?: string } - attributes: SpanAttributes - events: SpanEventRecord[] - /** ms epoch. */ - startTime: number - endTime: number + droppedAttributesCount?: number + droppedEventsCount?: number } /** @@ -102,6 +105,10 @@ export interface ResolvedTracesConfig extends TracesConfig { * dropped rather than queued ones, whose children may already have shipped. */ maxQueueSize: number + beforeSpanSend: BeforeSpanSendFn[] + maxAttributesPerSpan: number + maxEventsPerSpan: number + maxAttributeValueLength: number /** Bound on spans started but not yet ended. At the bound `startSpan` returns a no-op handle. */ maxLiveSpans: number /** How long a span may stay live before it stops being accounted for and can never export. */ diff --git a/packages/node/src/__tests__/traces.spec.ts b/packages/node/src/__tests__/traces.spec.ts index 79990cd842..964bc075ab 100644 --- a/packages/node/src/__tests__/traces.spec.ts +++ b/packages/node/src/__tests__/traces.spec.ts @@ -31,6 +31,9 @@ describe('PostHog traces', () => { const attributeOf = (span: OtlpSpan, key: string): any => span.attributes?.find((a) => a.key === key)?.value + const attributeOfEvent = (event: NonNullable[number], key: string): any => + event.attributes?.find((a: any) => a.key === key)?.value + // Traces run their own flush cycle; this advances it without calling flush(). const DEFAULT_TRACES_FLUSH_INTERVAL_MS = 5000 const flushTraces = async (): Promise => { @@ -333,6 +336,35 @@ describe('PostHog traces', () => { const [span] = sentSpans() expect(span.status).toEqual({ code: 2, message: 'boom' }) expect(span.events?.[0].name).toBe('exception') + expect(attributeOfEvent(span.events![0], 'exception.stacktrace')).toEqual({ + stringValue: expect.stringContaining('TypeError: boom'), + }) + }) + + it('beforeSpanSend can scrub a stacktrace', async () => { + const scrubbed = createClient({ + traces: { + serviceName: 'checkout-api', + beforeSpanSend: (span: any) => { + for (const event of span.events) { + if (event.attributes?.['exception.stacktrace']) { + event.attributes['exception.stacktrace'] = '[redacted]' + } + } + return span + }, + }, + }) + + expect(() => + scrubbed.withSpan('job', () => { + throw new TypeError('boom') + }) + ).toThrow('boom') + await scrubbed.shutdown() + + const [span] = sentSpans() + expect(attributeOfEvent(span.events![0], 'exception.stacktrace')).toEqual({ stringValue: '[redacted]' }) }) }) @@ -505,6 +537,93 @@ describe('PostHog traces', () => { }) }) + describe('beforeSpanSend', () => { + it('scrubs attributes before they leave the process', async () => { + const client = createClient({ + traces: { + serviceName: 'svc', + beforeSpanSend: (span: any) => { + delete span.attributes.password + return span + }, + }, + }) + client.startSpan('login', { attributes: { password: 'hunter2', ok: true } }).end() + await client.shutdown() + + const [span] = sentSpans() + expect(span.attributes?.find((a) => a.key === 'password')).toBeUndefined() + expect(span.attributes?.find((a) => a.key === 'ok')).toBeDefined() + }) + + it('runs an array of hooks through the client option', async () => { + const client = createClient({ + traces: { + serviceName: 'svc', + beforeSpanSend: [ + (span: any) => { + span.attributes.first = true + return span + }, + (span: any) => { + span.attributes.second = true + return span + }, + ], + }, + }) + client.startSpan('checkout').end() + await client.shutdown() + + const keys = sentSpans()[0].attributes?.map((a) => a.key) + expect(keys).toEqual(expect.arrayContaining(['first', 'second'])) + }) + + it('drops a span the hook rejects', async () => { + const client = createClient({ + traces: { + serviceName: 'svc', + beforeSpanSend: (span: any) => (span.attributes['http.route'] === '/health' ? null : span), + }, + }) + client.startSpan('GET /health', { attributes: { 'http.route': '/health' } }).end() + client.startSpan('GET /orders', { attributes: { 'http.route': '/orders' } }).end() + await client.shutdown() + + expect(sentSpans().map((s) => s.name)).toEqual(['GET /orders']) + }) + }) + + describe('span limits', () => { + it('caps attributes and reports how many were dropped', async () => { + const client = createClient({ traces: { serviceName: 'svc', maxAttributesPerSpan: 2 } }) + const span = client.startSpan('checkout') + span.setAttribute('a', 1) + span.setAttribute('b', 2) + span.setAttribute('c', 3) + span.end() + await client.shutdown() + + const [sent] = sentSpans() + expect(sent.attributes?.map((a) => a.key)).toEqual(['a', 'b']) + expect(sent.droppedAttributesCount).toBe(1) + }) + + it('defaults to the OpenTelemetry cap of 128', async () => { + const client = createClient({ traces: { serviceName: 'svc' } }) + const span = client.startSpan('checkout') + for (let i = 0; i < 130; i++) { + span.setAttribute(`key-${i}`, i) + } + span.end() + await client.shutdown() + + const [sent] = sentSpans() + expect(sent.attributes).toHaveLength(128) + expect(sent.droppedAttributesCount).toBe(2) + }) + }) + describe('shutdown', () => { it('is bounded by the shutdown timeout when the transport hangs', async () => { const client = createClient() diff --git a/packages/node/src/__tests__/waituntil-flush.spec.ts b/packages/node/src/__tests__/waituntil-flush.spec.ts index e16c36cb8c..6cb4855988 100644 --- a/packages/node/src/__tests__/waituntil-flush.spec.ts +++ b/packages/node/src/__tests__/waituntil-flush.spec.ts @@ -12,6 +12,42 @@ function getFlushedBatches(): any[][] { .map((c) => JSON.parse((c[1] as any).body).batch) } +describe('flush combines events and spans', () => { + it('waits for the span export even when the event flush rejects', async () => { + // `Promise.all` settled on the event rejection, so a serverless host could + // freeze the invocation with the span request still open. + vi.useRealTimers() + const order: string[] = [] + mockedFetch.mockImplementation(async (url: any) => { + if (String(url).includes('/i/v1/traces')) { + order.push('traces-start') + await new Promise((resolve) => setTimeout(resolve, 30)) + order.push('traces-done') + return { status: 200, text: () => Promise.resolve('{}'), json: () => Promise.resolve({}) } as any + } + throw new Error('events endpoint down') + }) + const posthog = new PostHog('key', { + host: 'http://example.com', + flushAt: 1, + fetchRetryCount: 0, + traces: { serviceName: 'svc' }, + disableCompression: true, + }) + posthog.capture({ distinctId: 'u', event: 'e' }) + posthog.startSpan('s').end() + + await expect(posthog.flush()).rejects.toThrow() + order.push('flush-returned') + + // Exact, not an index comparison: before the fix `traces-done` is absent + // when flush returns, and `indexOf` gives -1, which passes any `lessThan`. + expect(order).toEqual(['traces-start', 'traces-done', 'flush-returned']) + await posthog.shutdown() + vi.useFakeTimers() + }) +}) + describe('waitUntil debounced flush', () => { vi.useFakeTimers() diff --git a/packages/node/src/client.ts b/packages/node/src/client.ts index adf99bee1a..92a9cd99c4 100644 --- a/packages/node/src/client.ts +++ b/packages/node/src/client.ts @@ -1,6 +1,7 @@ import { version } from './version' import { + allSettled, FeatureFlagValue, getEventUuid, isBlockedUA, @@ -298,7 +299,15 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen if (!this._traces) { return events } - return Promise.all([events, this._traces.flush().catch(() => {})]).then(() => undefined) + // Settled, not `all`: `all` rejects the moment the event flush does, and a + // serverless host that treats this promise as the end of the invocation can + // freeze it with the span request still open. The events rejection is still + // the one the caller sees. + return allSettled([events, this._traces.flush().catch(() => {})]).then(([eventsResult]) => { + if (eventsResult.status === 'rejected') { + throw eventsResult.reason + } + }) } override async flush(): Promise { @@ -662,7 +671,7 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen if (!this._traces) { this._traces = new PostHogTraces( this, - resolveTracesConfig(this.options.traces, this.hostResourceAttributes()), + resolveTracesConfig(this.options.traces, this.hostResourceAttributes(), this._logger), this._logger, () => this._tracingContext(), this._spanContextManager, diff --git a/packages/node/src/exports.ts b/packages/node/src/exports.ts index 23e4897a93..35950883a4 100644 --- a/packages/node/src/exports.ts +++ b/packages/node/src/exports.ts @@ -24,6 +24,8 @@ export type { SpanStatusCode, SpanTimeInput, StartSpanOptions, + SpanRecord, + BeforeSpanSendFn, TracesConfig, } from '@posthog/core' diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 04e2974cce..4d550f07f0 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -138,6 +138,8 @@ export type { SpanTimeInput, StartSpanOptions, Span, + SpanRecord, + BeforeSpanSendFn, TracesConfig, OtlpSpanKeyValue, OtlpSpanEvent, diff --git a/packages/types/src/traces.ts b/packages/types/src/traces.ts index 8e6239fd9b..eaf84e2309 100644 --- a/packages/types/src/traces.ts +++ b/packages/types/src/traces.ts @@ -133,8 +133,10 @@ export interface Span { /** * Record an exception on the span: sets status `error` and attaches an - * `exception` event carrying `exception.type` and `exception.message`. - * Does not end the span. + * `exception` event carrying `exception.type`, `exception.message` and, + * where the thrown value has one, `exception.stacktrace`. The stack is + * truncated to `maxAttributeValueLength` like any other attribute value, + * and `beforeSpanSend` sees it before it is exported. Does not end the span. */ recordException(error: unknown): this @@ -174,6 +176,43 @@ export interface Span { end(endTime?: SpanTimeInput): void } +/** + * A completed span as `beforeSpanSend` sees it: plain values, not the OTLP wire + * encoding — `userId: 42` reads as `42`, not `{ intValue: "42" }`. + * + * @experimental Subject to change in a minor release. + */ +export interface SpanRecord { + /** + * Assignment to any of the three identity fields is ignored with a debug + * warning: rewriting ids orphans children that already shipped. + */ + readonly traceId: string + readonly spanId: string + /** Absent on a root span. */ + readonly parentSpanId?: string + name: string + kind: SpanKind + status?: { code: SpanStatusCode; message?: string } + /** Editable in place; this is where to redact. */ + attributes: SpanAttributes + events: { name: string; /** Millisecond epoch. */ timestamp: number; attributes?: SpanAttributes }[] + /** Millisecond epoch. */ + startTime: number + /** Millisecond epoch. */ + endTime: number +} + +/** + * Inspects, edits or drops a finished span. Return `null` to drop it. + * + * The hook runs synchronously as part of `end()`; a returned promise is not + * awaited and the span is dropped. + * + * @experimental Subject to change in a minor release. + */ +export type BeforeSpanSendFn = (span: SpanRecord) => SpanRecord | null + /** * Configuration for distributed tracing, passed as the `traces` client option. * Tracing stays off until this object is supplied. @@ -239,6 +278,68 @@ export interface TracesConfig { */ maxQueueSize?: number + /** + * Runs on every finished span before it is queued. Edit the span in place, + * or return `null` to drop it. An array runs left to right, and the first + * hook to return `null` stops the chain. + * + * This is the place to scrub sensitive attributes, so a hook that throws + * drops the span rather than exporting an unscrubbed one. + * + * @example Drop health checks and redact a header + * ```ts + * traces: { + * beforeSpanSend: (span) => { + * if (span.attributes['http.route'] === '/health') return null + * delete span.attributes['http.request.header.authorization'] + * return span + * }, + * } + * ``` + */ + beforeSpanSend?: BeforeSpanSendFn | BeforeSpanSendFn[] + + /** + * Maximum user-supplied attributes on a single span. Attributes the SDK + * attaches itself — `posthogDistinctId`, `sessionId` and friends — are + * exempt and are never evicted, because they are what links a span to a + * person and a session. + * + * On overflow the earliest-set attributes are kept and later ones are + * dropped, with the number dropped reported on the exported span. + * + * @default 128 + */ + maxAttributesPerSpan?: number + + /** + * Maximum events on a single span. On overflow the earliest events are kept + * and later ones are dropped, with the number dropped reported on the + * exported span. + * + * The cap is absolute: an `exception` event the SDK records on your behalf + * spends an ordinary slot like any other, so a span that fills its events + * and then throws keeps its `error` status but not the exception detail. + * Raise the cap on spans that record many events and can also fail. + * + * @default 128 + */ + maxEventsPerSpan?: number + + /** + * Maximum length of a string attribute value. Longer values are truncated, + * and the bound reaches every string the value contains, including the ones + * nested inside arrays and objects. It applies to span attributes, event + * attributes, span names, event names, status messages and resource + * attributes alike — including `exception.stacktrace`. + * + * The bound is what keeps one large value from making a span too large for + * the ingestion endpoint, which drops an oversized span whole. + * + * @default 8192 + */ + maxAttributeValueLength?: number + /** * Bound on how many spans may be live (started but not ended) at once. At * the bound `startSpan` returns an inert handle, so code that leaks spans @@ -305,6 +406,10 @@ export interface OtlpSpan { * it — plus OTel's parent-remoteness bits (`0x100` known, `0x200` remote). */ flags?: number + /** User attributes dropped by `maxAttributesPerSpan`. Omitted when none were. */ + droppedAttributesCount?: number + /** Events dropped by `maxEventsPerSpan`. Omitted when none were. */ + droppedEventsCount?: number } export interface OtlpTracesPayload { From 815d53d8870a6f17052fa02bb2038922e83e1a82 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 15:37:51 -0400 Subject: [PATCH 12/24] feat(traces): cap the attributes on a span event maxAttributesPerSpan does not reach inside events, so a span's width was bounded but its events' was not. Adds maxAttributesPerEvent, default 128, reported per event as the OTLP dropped_attributes_count. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018AAPCyRnC5HbukEZV9hbai --- .changeset/node-span-limits.md | 2 +- packages/core/src/traces/config.spec.ts | 8 ++- packages/core/src/traces/config.ts | 2 + packages/core/src/traces/index.spec.ts | 21 ++++++++ packages/core/src/traces/index.ts | 2 + packages/core/src/traces/otlp.spec.ts | 24 +++++++++ packages/core/src/traces/otlp.ts | 4 ++ packages/core/src/traces/span.spec.ts | 70 +++++++++++++++++++++++++ packages/core/src/traces/span.ts | 59 +++++++++++++++++++-- packages/core/src/traces/types.ts | 4 ++ packages/types/src/traces.ts | 15 ++++++ 11 files changed, 205 insertions(+), 6 deletions(-) diff --git a/.changeset/node-span-limits.md b/.changeset/node-span-limits.md index fbe06127fc..9c8a660f82 100644 --- a/.changeset/node-span-limits.md +++ b/.changeset/node-span-limits.md @@ -4,4 +4,4 @@ '@posthog/types': minor --- -Cap spans at 128 user attributes, 128 events and 8192 characters per string, configurable with `traces.maxAttributesPerSpan`, `traces.maxEventsPerSpan` and `traces.maxAttributeValueLength`. The earliest entries are kept, and a span that lost any reports how many as `droppedAttributesCount` and `droppedEventsCount`. The event cap is absolute, so an `exception` event the SDK records for you spends an ordinary slot. +Cap spans at 128 user attributes, 128 events, 128 attributes per event and 8192 characters per string, configurable with `traces.maxAttributesPerSpan`, `traces.maxEventsPerSpan`, `traces.maxAttributesPerEvent` and `traces.maxAttributeValueLength`. The earliest entries are kept, and a span that lost any reports how many as `droppedAttributesCount` and `droppedEventsCount`. The event cap is absolute, so an `exception` event the SDK records for you spends an ordinary slot. diff --git a/packages/core/src/traces/config.spec.ts b/packages/core/src/traces/config.spec.ts index 5237a0d52d..30ef7c623a 100644 --- a/packages/core/src/traces/config.spec.ts +++ b/packages/core/src/traces/config.spec.ts @@ -15,19 +15,24 @@ describe('resolveTracesConfig', () => { const resolved = resolveTracesConfig({ maxAttributesPerSpan: value, maxEventsPerSpan: value, + maxAttributesPerEvent: value, maxAttributeValueLength: value, }) expect(resolved.maxAttributesPerSpan).toBe(128) expect(resolved.maxEventsPerSpan).toBe(128) + expect(resolved.maxAttributesPerEvent).toBe(128) expect(resolved.maxAttributeValueLength).toBe(8192) }) it('honours explicit per-span caps', () => { // Without this the resolver can ignore maxEventsPerSpan entirely and every // other test still passes, because they all assert the default. - expect(resolveTracesConfig({ maxAttributesPerSpan: 5, maxEventsPerSpan: 7 })).toMatchObject({ + expect( + resolveTracesConfig({ maxAttributesPerSpan: 5, maxEventsPerSpan: 7, maxAttributesPerEvent: 9 }) + ).toMatchObject({ maxAttributesPerSpan: 5, maxEventsPerSpan: 7, + maxAttributesPerEvent: 9, }) }) @@ -38,6 +43,7 @@ describe('resolveTracesConfig', () => { maxQueueSize: 2048, maxAttributesPerSpan: 128, maxEventsPerSpan: 128, + maxAttributesPerEvent: 128, maxAttributeValueLength: 8192, maxLiveSpans: 10_000, maxSpanAgeMs: 3_600_000, diff --git a/packages/core/src/traces/config.ts b/packages/core/src/traces/config.ts index dc60973a43..0d999997e8 100644 --- a/packages/core/src/traces/config.ts +++ b/packages/core/src/traces/config.ts @@ -11,6 +11,7 @@ const DEFAULT_MAX_QUEUE_SIZE = 2048 // OpenTelemetry's per-span defaults. const DEFAULT_MAX_ATTRIBUTES_PER_SPAN = 128 const DEFAULT_MAX_EVENTS_PER_SPAN = 128 +const DEFAULT_MAX_ATTRIBUTES_PER_EVENT = 128 // OpenTelemetry leaves the value length unlimited, which is what lets one // multi-MB attribute make a span too large for the endpoint to accept — and an // oversized span is dropped whole. 8 KB holds a deep stack trace and any @@ -124,6 +125,7 @@ export function resolveTracesConfig( beforeSpanSend: resolveBeforeSpanSend(config?.beforeSpanSend, logger), maxAttributesPerSpan: positiveInteger(config?.maxAttributesPerSpan, DEFAULT_MAX_ATTRIBUTES_PER_SPAN), maxEventsPerSpan: positiveInteger(config?.maxEventsPerSpan, DEFAULT_MAX_EVENTS_PER_SPAN), + maxAttributesPerEvent: positiveInteger(config?.maxAttributesPerEvent, DEFAULT_MAX_ATTRIBUTES_PER_EVENT), maxAttributeValueLength: positiveInteger(config?.maxAttributeValueLength, DEFAULT_MAX_ATTRIBUTE_VALUE_LENGTH), flushIntervalMs: positiveInteger(config?.flushIntervalMs, DEFAULT_FLUSH_INTERVAL_MS), maxExportBatchSize, diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 0712ea99f8..696620e79f 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -22,6 +22,7 @@ const resolveForTest = (partial?: Partial): ResolvedTraces beforeSpanSend: [], maxAttributesPerSpan: 128, maxEventsPerSpan: 128, + maxAttributesPerEvent: 128, maxAttributeValueLength: 8192, maxLiveSpans: 10000, maxSpanAgeMs: 3600000, @@ -1509,6 +1510,26 @@ describe('PostHogTraces', () => { expect(sent.droppedEventsCount).toBe(1) }) + it('re-applies the event attribute cap to what beforeSpanSend widened', async () => { + const traces = createTraces({ + maxAttributesPerEvent: 2, + beforeSpanSend: [ + (span) => { + span.events[0].attributes = { a: 1, b: 2, c: 3, d: 4 } + return span + }, + ], + }) + const span = traces.startSpan('checkout') + span.addEvent('query', { a: 1 }) + span.end() + await traces.flush() + + const event = sentSpans()[0].events![0] + expect(event.attributes!.map((attribute) => attribute.key)).toEqual(['a', 'b']) + expect(event.droppedAttributesCount).toBe(2) + }) + it('keeps the auto-context keys when beforeSpanSend pushes past the cap', async () => { context = { distinctId: 'alice', sessionId: 'session-1' } const traces = createTraces({ diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 5d67e00d81..2766535ab9 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -246,6 +246,7 @@ export class PostHogTraces { autoAttributeKeys: Object.keys(autoAttributes), maxAttributes: this._config.maxAttributesPerSpan, maxEvents: this._config.maxEventsPerSpan, + maxAttributesPerEvent: this._config.maxAttributesPerEvent, maxAttributeValueLength: this._config.maxAttributeValueLength, startTime, backdated: startTime !== now, @@ -654,6 +655,7 @@ export class PostHogTraces { autoKeys, this._config.maxAttributesPerSpan, this._config.maxEventsPerSpan, + this._config.maxAttributesPerEvent, this._config.maxAttributeValueLength, keysBeforeHook ) diff --git a/packages/core/src/traces/otlp.spec.ts b/packages/core/src/traces/otlp.spec.ts index 641171df2d..e09d57f061 100644 --- a/packages/core/src/traces/otlp.spec.ts +++ b/packages/core/src/traces/otlp.spec.ts @@ -110,6 +110,30 @@ describe('OTLP span encoding', () => { expect(span.events).toEqual([{ name: 'cache miss', timeUnixNano: '1700000000040000000' }]) }) + it('carries an event attribute drop as the proto counter', () => { + const span = buildOtlpSpan( + record({ + events: [{ name: 'query', timestamp: 1_700_000_000_040, droppedAttributesCount: 3 }], + }) + ) + expect(span.events?.[0]).toMatchObject({ name: 'query', droppedAttributesCount: 3 }) + }) + + it.each([ + ['none were dropped', 0], + ['a hook wrote a negative', -2], + ['a hook wrote a non-number', 'lots' as unknown as number], + ])('omits the event drop counter when %s', (_label, dropped) => { + // Coerced like the span's own counters: a non-integer here is refused for + // the whole request, taking unrelated spans with it. + const span = buildOtlpSpan( + record({ + events: [{ name: 'query', timestamp: 1_700_000_000_040, droppedAttributesCount: dropped }], + }) + ) + expect(span.events?.[0].droppedAttributesCount).toBeUndefined() + }) + it('sets the sampled bit and marks a root span as known-not-remote', () => { // A root span has no parent context to be remote, which the OTel Go and // Java exporters also report as known-not-remote rather than unknown. diff --git a/packages/core/src/traces/otlp.ts b/packages/core/src/traces/otlp.ts index 686b7e8e86..7d397f977c 100644 --- a/packages/core/src/traces/otlp.ts +++ b/packages/core/src/traces/otlp.ts @@ -105,6 +105,10 @@ function toOtlpEvent(event: SpanRecord['events'][number], logger?: Logger): Otlp encoded.attributes = attributes } } + const dropped = nonNegativeCount(event.droppedAttributesCount) + if (dropped) { + encoded.droppedAttributesCount = dropped + } return encoded } diff --git a/packages/core/src/traces/span.spec.ts b/packages/core/src/traces/span.spec.ts index 2039adc43a..289d5eaf55 100644 --- a/packages/core/src/traces/span.spec.ts +++ b/packages/core/src/traces/span.spec.ts @@ -26,6 +26,7 @@ describe('PostHogSpan', () => { autoAttributeKeys: [], maxAttributes: 128, maxEvents: 128, + maxAttributesPerEvent: 128, maxAttributeValueLength: 8192, ...init, }, @@ -227,6 +228,73 @@ describe('PostHogSpan', () => { }) }) + describe('event attribute cap', () => { + it('keeps the first attributes and reports the rest as dropped', () => { + const span = createSpan({ maxAttributesPerEvent: 2 }) + span.addEvent('query', { a: 1, b: 2, c: 3, d: 4 }) + span.end() + + expect(ended[0].events[0].attributes).toEqual({ a: 1, b: 2 }) + expect(ended[0].events[0].droppedAttributesCount).toBe(2) + }) + + it('leaves the count off an event that lost nothing', () => { + const span = createSpan({ maxAttributesPerEvent: 2 }) + span.addEvent('query', { a: 1, b: 2 }) + span.end() + + expect(ended[0].events[0].droppedAttributesCount).toBeUndefined() + }) + + it('bounds an exception event like any other', () => { + // The SDK's own `exception.*` attributes are width the caller sees too, so + // they spend the cap rather than being exempt from it. + const span = createSpan({ maxAttributesPerEvent: 1 }) + span.recordException(new Error('boom')) + span.end() + + expect(Object.keys(ended[0].events[0].attributes ?? {})).toEqual(['exception.type']) + expect(ended[0].events[0].droppedAttributesCount).toBe(2) + }) + + it('does not read a value past the cap', () => { + // The cap is spent before the value is bounded, so a wide bag does not pay + // for getters on entries that are about to be dropped. + const read: string[] = [] + const watched: any = {} + for (const key of ['a', 'b', 'c']) { + Object.defineProperty(watched, key, { + enumerable: true, + get() { + read.push(key) + return key + }, + }) + } + + const span = createSpan({ maxAttributesPerEvent: 2 }) + span.addEvent('query', watched) + span.end() + + expect(read).toEqual(['a', 'b']) + expect(ended[0].events[0].droppedAttributesCount).toBe(1) + }) + + it('counts the span attribute cap separately from an event cap', () => { + // maxAttributesPerSpan does not reach inside events, which is the gap this + // cap closes: a span at its own cap can still carry full-width events. + const span = createSpan({ maxAttributes: 1, maxAttributesPerEvent: 3 }) + span.setAttributes({ kept: 1, dropped: 2 }) + span.addEvent('query', { a: 1, b: 2, c: 3 }) + span.end() + + expect(ended[0].attributes).toEqual({ kept: 1 }) + expect(ended[0].droppedAttributesCount).toBe(1) + expect(ended[0].events[0].attributes).toEqual({ a: 1, b: 2, c: 3 }) + expect(ended[0].events[0].droppedAttributesCount).toBeUndefined() + }) + }) + describe('attribute store hygiene', () => { it('does not copy a polluted Object.prototype key into the span', () => { ;(Object.prototype as any).polluted = 'yes' @@ -900,6 +968,7 @@ describe('attribute store', () => { autoAttributeKeys: [], maxAttributes: 128, maxEvents: 128, + maxAttributesPerEvent: 128, maxAttributeValueLength: 8192, }, (record) => ended.push(record) @@ -926,6 +995,7 @@ describe('attribute store', () => { autoAttributeKeys: [], maxAttributes: 128, maxEvents: 128, + maxAttributesPerEvent: 128, maxAttributeValueLength: 8192, }, (record) => ended.push(record) diff --git a/packages/core/src/traces/span.ts b/packages/core/src/traces/span.ts index 3124945c9d..930857be8e 100644 --- a/packages/core/src/traces/span.ts +++ b/packages/core/src/traces/span.ts @@ -42,6 +42,7 @@ export interface SpanInit { autoAttributeKeys: string[] maxAttributes: number maxEvents: number + maxAttributesPerEvent: number maxAttributeValueLength: number } @@ -65,6 +66,7 @@ export class PostHogSpan implements Span { private readonly _autoKeys: Set private readonly _maxAttributes: number private readonly _maxEvents: number + private readonly _maxAttributesPerEvent: number private readonly _maxAttributeValueLength: number private _userAttributeCount = 0 private _userEventCount = 0 @@ -87,6 +89,7 @@ export class PostHogSpan implements Span { this._autoKeys = new Set(init.autoAttributeKeys) this._maxAttributes = init.maxAttributes this._maxEvents = init.maxEvents + this._maxAttributesPerEvent = init.maxAttributesPerEvent this._maxAttributeValueLength = init.maxAttributeValueLength // Null-prototype: a `__proto__` key would otherwise swap this object's prototype // instead of becoming an entry, and `toString` and friends would read as @@ -183,12 +186,15 @@ export class PostHogSpan implements Span { return this } this._userEventCount++ + // Copied so a caller reusing one object across events can't mutate a recorded one. + const bounded = + attributes && boundAttributes(attributes, this._maxAttributesPerEvent, this._maxAttributeValueLength) this._events.push({ name: sanitizeName(name, 'Span event name', this._maxAttributeValueLength, this._logger), timestamp: resolveSuppliedTime(timestamp, this._now(), 'event timestamp', this._logger), - // Copied so a caller reusing one object across events can't mutate a recorded one. - ...(attributes && { - attributes: truncateAttributes(assignUserAttributes({}, attributes), this._maxAttributeValueLength), + ...(bounded && { + attributes: bounded.attributes, + ...(bounded.dropped && { droppedAttributesCount: bounded.dropped }), }), }) } @@ -342,6 +348,7 @@ export function applySpanLimits( autoKeys: ReadonlySet, maxAttributes: number, maxEvents: number, + maxAttributesPerEvent: number, maxAttributeValueLength: number, keysBeforeHook: readonly string[] = [] ): void { @@ -391,7 +398,13 @@ export function applySpanLimits( } keptEvents++ if (event.attributes) { - event.attributes = truncateAttributes({ ...event.attributes }, maxAttributeValueLength) + // A hook can widen an event as freely as it can add one, and neither goes + // through `addEvent`. + const bounded = boundAttributes(event.attributes, maxAttributesPerEvent, maxAttributeValueLength) + event.attributes = bounded.attributes + if (bounded.dropped) { + event.droppedAttributesCount = nonNegativeCount(event.droppedAttributesCount) + bounded.dropped + } } events.push(event) } @@ -697,6 +710,44 @@ function resolveToJson(value: object): { selfDescribed: boolean; value?: SpanAtt return { selfDescribed: false } } +/** + * A copy of a caller-supplied attribute bag holding at most `max` entries, each + * value bounded to `maxLength`, plus how many entries were refused. + * + * Keys past the cap are never read, so a wide object does not pay for the getters + * on values that are about to be dropped — the order `_writeAttribute` uses for + * the same reason. + */ +function boundAttributes( + source: SpanAttributes, + max: number, + maxLength: number +): { attributes: SpanAttributes; dropped: number } { + let keys: string[] + try { + keys = Object.keys(source) + } catch { + // A hostile own-keys trap costs the bag, not the event carrying it. + return { attributes: {}, dropped: 0 } + } + const attributes: SpanAttributes = {} + const kept = Math.min(keys.length, max) + for (let index = 0; index < kept; index++) { + const key = keys[index] + let value: SpanAttributeValue + try { + value = truncateAttributeValue(source[key], maxLength) + } catch { + // A throwing getter costs its own key, as it does in `assignUserAttributes`. + value = UNSERIALIZABLE_VALUE + } + // defineProperty, not assignment: `attributes['__proto__'] = v` hits the + // prototype setter and the attribute vanishes. + Object.defineProperty(attributes, key, { value, enumerable: true, writable: true, configurable: true }) + } + return { attributes, dropped: keys.length - kept } +} + /** `truncateAttributeValue` across an attribute bag, in place. */ export function truncateAttributes(attributes: SpanAttributes, maxLength: number): SpanAttributes { for (const key of Object.keys(attributes)) { diff --git a/packages/core/src/traces/types.ts b/packages/core/src/traces/types.ts index 7360c70a26..9e333c2b2c 100644 --- a/packages/core/src/traces/types.ts +++ b/packages/core/src/traces/types.ts @@ -63,6 +63,7 @@ export interface SpanEventRecord { /** ms epoch. */ timestamp: number attributes?: SpanAttributes + droppedAttributesCount?: number } /** @@ -72,6 +73,8 @@ export interface SpanEventRecord { * field `beforeSpanSend` cannot see, and so cannot corrupt. */ export interface SpanRecord extends HookSpanRecord { + /** The hook-visible event plus the count no hook may rewrite. */ + events: SpanEventRecord[] traceState?: string /** The W3C trace-flags byte this span propagates, e.g. `01` sampled. */ traceFlags: string @@ -108,6 +111,7 @@ export interface ResolvedTracesConfig extends TracesConfig { beforeSpanSend: BeforeSpanSendFn[] maxAttributesPerSpan: number maxEventsPerSpan: number + maxAttributesPerEvent: number maxAttributeValueLength: number /** Bound on spans started but not yet ended. At the bound `startSpan` returns a no-op handle. */ maxLiveSpans: number diff --git a/packages/types/src/traces.ts b/packages/types/src/traces.ts index eaf84e2309..350dc5ac36 100644 --- a/packages/types/src/traces.ts +++ b/packages/types/src/traces.ts @@ -326,6 +326,19 @@ export interface TracesConfig { */ maxEventsPerSpan?: number + /** + * Maximum attributes on a single span event. On overflow the first + * `maxAttributesPerEvent` are kept and later ones are dropped, with the + * number dropped reported on the exported event. + * + * `maxAttributesPerSpan` counts a span's own attributes and does not reach + * inside its events, so this is what bounds an event's width — including the + * `exception.*` attributes the SDK records for you. + * + * @default 128 + */ + maxAttributesPerEvent?: number + /** * Maximum length of a string attribute value. Longer values are truncated, * and the bound reaches every string the value contains, including the ones @@ -378,6 +391,8 @@ export interface OtlpSpanEvent { name: string timeUnixNano: string attributes?: OtlpSpanKeyValue[] + /** Attributes dropped by `maxAttributesPerEvent`. Omitted when none were. */ + droppedAttributesCount?: number } export interface OtlpSpanStatus { From 479689047d125d9f2de3b91e82d1b1d58aeb7f97 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 16:22:56 -0400 Subject: [PATCH 13/24] fix(traces): stop a nullish value spending an event's attribute slot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The encoder drops a nullish attribute, so charging one against maxAttributesPerEvent let a blanked value evict a real one — the rule _writeAttribute and the span half of applySpanLimits already follow. Also corrects a comment claiming a hook cannot rewrite the per-event drop count, and the two ResolvedTracesConfig fixtures that stopped typechecking. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018AAPCyRnC5HbukEZV9hbai --- .changeset/node-span-limits.md | 2 +- .../rn-flags-disable-and-update-flags.md | 7 +++++ packages/core/src/traces/config.spec.ts | 4 +-- packages/core/src/traces/config.ts | 1 + packages/core/src/traces/live-spans.spec.ts | 1 + packages/core/src/traces/otlp.spec.ts | 1 + packages/core/src/traces/span.spec.ts | 28 +++++++++++++++++++ packages/core/src/traces/span.ts | 26 +++++++++++------ packages/core/src/traces/types.ts | 2 +- 9 files changed, 60 insertions(+), 12 deletions(-) create mode 100644 .changeset/rn-flags-disable-and-update-flags.md diff --git a/.changeset/node-span-limits.md b/.changeset/node-span-limits.md index 9c8a660f82..85ccb33595 100644 --- a/.changeset/node-span-limits.md +++ b/.changeset/node-span-limits.md @@ -4,4 +4,4 @@ '@posthog/types': minor --- -Cap spans at 128 user attributes, 128 events, 128 attributes per event and 8192 characters per string, configurable with `traces.maxAttributesPerSpan`, `traces.maxEventsPerSpan`, `traces.maxAttributesPerEvent` and `traces.maxAttributeValueLength`. The earliest entries are kept, and a span that lost any reports how many as `droppedAttributesCount` and `droppedEventsCount`. The event cap is absolute, so an `exception` event the SDK records for you spends an ordinary slot. +Cap spans at 128 user attributes, 128 events, 128 attributes per event and 8192 characters per string, configurable with `traces.maxAttributesPerSpan`, `traces.maxEventsPerSpan`, `traces.maxAttributesPerEvent` and `traces.maxAttributeValueLength`. The earliest entries are kept, and a span that lost any reports how many as `droppedAttributesCount` and `droppedEventsCount`, as does an event that lost attributes. The event cap is absolute, so an `exception` event the SDK records for you spends an ordinary slot. diff --git a/.changeset/rn-flags-disable-and-update-flags.md b/.changeset/rn-flags-disable-and-update-flags.md new file mode 100644 index 0000000000..73879e9713 --- /dev/null +++ b/.changeset/rn-flags-disable-and-update-flags.md @@ -0,0 +1,7 @@ +--- +'@posthog/core': minor +'posthog-react-native': minor +'posthog-js-lite': minor +--- + +feat(flags): add the `advancedDisableFeatureFlags` option and a public `updateFlags(flags, payloads?, { merge? })` method, matching the web SDK's `advanced_disable_feature_flags` and `updateFlags`. With the option set, `reloadFeatureFlags()` and the reloads triggered by `identify()`, `group()`, `setPersonPropertiesForFlags()` and `reset()` become no-ops, and any flags request that still goes out for remote config or surveys carries `disable_flags: true` so the server skips flag evaluation. `updateFlags` supplies locally evaluated flag values (with payloads) at runtime: values persist, `getFeatureFlag()`/`getFeatureFlagPayload()` read them back, and `onFeatureFlags` listeners fire — so React Native session replay gated on a linked flag re-evaluates when flags are pushed in. diff --git a/packages/core/src/traces/config.spec.ts b/packages/core/src/traces/config.spec.ts index 30ef7c623a..03a07b6a64 100644 --- a/packages/core/src/traces/config.spec.ts +++ b/packages/core/src/traces/config.spec.ts @@ -11,7 +11,7 @@ describe('resolveTracesConfig', () => { ['a fraction', 1.5], ['a large fraction', 200.5], ['infinity', Infinity], - ])('falls back to the default per-span caps when given %s', (_label, value) => { + ])('falls back to the default caps when given %s', (_label, value) => { const resolved = resolveTracesConfig({ maxAttributesPerSpan: value, maxEventsPerSpan: value, @@ -24,7 +24,7 @@ describe('resolveTracesConfig', () => { expect(resolved.maxAttributeValueLength).toBe(8192) }) - it('honours explicit per-span caps', () => { + it('honours explicit per-span and per-event caps', () => { // Without this the resolver can ignore maxEventsPerSpan entirely and every // other test still passes, because they all assert the default. expect( diff --git a/packages/core/src/traces/config.ts b/packages/core/src/traces/config.ts index 0d999997e8..a34b05f648 100644 --- a/packages/core/src/traces/config.ts +++ b/packages/core/src/traces/config.ts @@ -11,6 +11,7 @@ const DEFAULT_MAX_QUEUE_SIZE = 2048 // OpenTelemetry's per-span defaults. const DEFAULT_MAX_ATTRIBUTES_PER_SPAN = 128 const DEFAULT_MAX_EVENTS_PER_SPAN = 128 +// OpenTelemetry's per-event default, which is the same number. const DEFAULT_MAX_ATTRIBUTES_PER_EVENT = 128 // OpenTelemetry leaves the value length unlimited, which is what lets one // multi-MB attribute make a span too large for the endpoint to accept — and an diff --git a/packages/core/src/traces/live-spans.spec.ts b/packages/core/src/traces/live-spans.spec.ts index 47ab14c5d9..3e9815bafe 100644 --- a/packages/core/src/traces/live-spans.spec.ts +++ b/packages/core/src/traces/live-spans.spec.ts @@ -23,6 +23,7 @@ describe('live spans', () => { beforeSpanSend: [], maxAttributesPerSpan: 128, maxEventsPerSpan: 128, + maxAttributesPerEvent: 128, maxAttributeValueLength: 8192, } diff --git a/packages/core/src/traces/otlp.spec.ts b/packages/core/src/traces/otlp.spec.ts index e09d57f061..4c926668f3 100644 --- a/packages/core/src/traces/otlp.spec.ts +++ b/packages/core/src/traces/otlp.spec.ts @@ -167,6 +167,7 @@ describe('OTLP span encoding', () => { beforeSpanSend: [], maxAttributesPerSpan: 128, maxEventsPerSpan: 128, + maxAttributesPerEvent: 128, maxAttributeValueLength: 8192, maxLiveSpans: 10000, maxSpanAgeMs: 3600000, diff --git a/packages/core/src/traces/span.spec.ts b/packages/core/src/traces/span.spec.ts index 289d5eaf55..2733952119 100644 --- a/packages/core/src/traces/span.spec.ts +++ b/packages/core/src/traces/span.spec.ts @@ -280,6 +280,34 @@ describe('PostHogSpan', () => { expect(ended[0].events[0].droppedAttributesCount).toBe(1) }) + it('does not let a nullish value spend a slot', () => { + // The encoder drops these, so a caller who blanked a value rather than + // omitting the key must not cost the event a real attribute. Same rule the + // span half of the cap already follows. + const span = createSpan({ maxAttributesPerEvent: 2 }) + span.addEvent('query', { blanked: undefined, cleared: null, real: 1, second: 2 }) + span.end() + + expect(ended[0].events[0].attributes).toEqual({ real: 1, second: 2 }) + expect(ended[0].events[0].droppedAttributesCount).toBeUndefined() + }) + + it('survives an attribute bag whose own keys cannot be read', () => { + const hostile = new Proxy( + {}, + { + ownKeys() { + throw new Error('ownKeys exploded') + }, + } + ) + const span = createSpan() + expect(() => span.addEvent('query', hostile)).not.toThrow() + expect(() => span.end()).not.toThrow() + + expect(ended[0].events[0].attributes).toEqual({}) + }) + it('counts the span attribute cap separately from an event cap', () => { // maxAttributesPerSpan does not reach inside events, which is the gap this // cap closes: a span at its own cap can still carry full-width events. diff --git a/packages/core/src/traces/span.ts b/packages/core/src/traces/span.ts index 930857be8e..16be19e1e4 100644 --- a/packages/core/src/traces/span.ts +++ b/packages/core/src/traces/span.ts @@ -712,11 +712,10 @@ function resolveToJson(value: object): { selfDescribed: boolean; value?: SpanAtt /** * A copy of a caller-supplied attribute bag holding at most `max` entries, each - * value bounded to `maxLength`, plus how many entries were refused. + * value bounded to `maxLength`, plus how many entries the cap refused. * - * Keys past the cap are never read, so a wide object does not pay for the getters - * on values that are about to be dropped — the order `_writeAttribute` uses for - * the same reason. + * Once the cap is spent the remaining keys are counted without being read, so a + * wide object does not pay for the getters on values it is about to drop. */ function boundAttributes( source: SpanAttributes, @@ -731,9 +730,13 @@ function boundAttributes( return { attributes: {}, dropped: 0 } } const attributes: SpanAttributes = {} - const kept = Math.min(keys.length, max) - for (let index = 0; index < kept; index++) { - const key = keys[index] + let kept = 0 + let dropped = 0 + for (const key of keys) { + if (kept >= max) { + dropped++ + continue + } let value: SpanAttributeValue try { value = truncateAttributeValue(source[key], maxLength) @@ -741,11 +744,18 @@ function boundAttributes( // A throwing getter costs its own key, as it does in `assignUserAttributes`. value = UNSERIALIZABLE_VALUE } + // Nullish spends no slot, matching `_writeAttribute` and the span half of + // `applySpanLimits`: the encoder drops these, so a caller who blanked a value + // rather than omitting the key must not lose a real attribute to it. + if (isNullish(value)) { + continue + } + kept++ // defineProperty, not assignment: `attributes['__proto__'] = v` hits the // prototype setter and the attribute vanishes. Object.defineProperty(attributes, key, { value, enumerable: true, writable: true, configurable: true }) } - return { attributes, dropped: keys.length - kept } + return { attributes, dropped } } /** `truncateAttributeValue` across an attribute bag, in place. */ diff --git a/packages/core/src/traces/types.ts b/packages/core/src/traces/types.ts index 9e333c2b2c..83e995f3fe 100644 --- a/packages/core/src/traces/types.ts +++ b/packages/core/src/traces/types.ts @@ -73,7 +73,7 @@ export interface SpanEventRecord { * field `beforeSpanSend` cannot see, and so cannot corrupt. */ export interface SpanRecord extends HookSpanRecord { - /** The hook-visible event plus the count no hook may rewrite. */ + /** The hook-visible event plus the SDK's own per-event drop count. */ events: SpanEventRecord[] traceState?: string /** The W3C trace-flags byte this span propagates, e.g. `01` sampled. */ From 1ce3bc3b46a95ac7dc05dd262f333193d216a2d0 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 16:29:51 -0400 Subject: [PATCH 14/24] chore: drop an unrelated changeset committed by mistake rn-flags-disable-and-update-flags belongs to separate RN work; it is preserved in the stash it came from. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018AAPCyRnC5HbukEZV9hbai --- .changeset/rn-flags-disable-and-update-flags.md | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 .changeset/rn-flags-disable-and-update-flags.md diff --git a/.changeset/rn-flags-disable-and-update-flags.md b/.changeset/rn-flags-disable-and-update-flags.md deleted file mode 100644 index 73879e9713..0000000000 --- a/.changeset/rn-flags-disable-and-update-flags.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@posthog/core': minor -'posthog-react-native': minor -'posthog-js-lite': minor ---- - -feat(flags): add the `advancedDisableFeatureFlags` option and a public `updateFlags(flags, payloads?, { merge? })` method, matching the web SDK's `advanced_disable_feature_flags` and `updateFlags`. With the option set, `reloadFeatureFlags()` and the reloads triggered by `identify()`, `group()`, `setPersonPropertiesForFlags()` and `reset()` become no-ops, and any flags request that still goes out for remote config or surveys carries `disable_flags: true` so the server skips flag evaluation. `updateFlags` supplies locally evaluated flag values (with payloads) at runtime: values persist, `getFeatureFlag()`/`getFeatureFlagPayload()` read them back, and `onFeatureFlags` listeners fire — so React Native session replay gated on a linked flag re-evaluates when flags are pushed in. From 701bba2aa27ac603361ce0af35763daa721b871b Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 16:49:15 -0400 Subject: [PATCH 15/24] docs(traces): fold the traces changesets into one, and tidy stray comments Six PRs merged into this branch each carried their own changeset for what ships as a single unreleased feature. Adds a changeset for the shared OTLP resource-attribute hardening, which reaches metrics too. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XBUa53f3QqECUyxnkSiWGA --- .../metrics-resource-attribute-getters.md | 6 ++++ .changeset/node-before-span-send.md | 7 ---- .changeset/node-distributed-tracing.md | 6 ++-- .changeset/node-exception-stacktrace.md | 7 ---- .changeset/node-span-limits.md | 7 ---- .changeset/otlp-os-resource-attributes.md | 6 ---- .changeset/traceparent-w3c-strictness.md | 5 --- .../traces-context-and-time-fidelity.md | 5 --- .changeset/traces-flags-fidelity.md | 7 ---- packages/core/src/index.ts | 1 - packages/core/src/traces/config.ts | 9 ++--- packages/core/src/traces/index.ts | 16 ++++----- packages/core/src/traces/otlp.ts | 6 +--- packages/core/src/traces/span.ts | 36 ++++++++----------- 14 files changed, 35 insertions(+), 89 deletions(-) create mode 100644 .changeset/metrics-resource-attribute-getters.md delete mode 100644 .changeset/node-before-span-send.md delete mode 100644 .changeset/node-exception-stacktrace.md delete mode 100644 .changeset/node-span-limits.md delete mode 100644 .changeset/otlp-os-resource-attributes.md delete mode 100644 .changeset/traceparent-w3c-strictness.md delete mode 100644 .changeset/traces-context-and-time-fidelity.md delete mode 100644 .changeset/traces-flags-fidelity.md diff --git a/.changeset/metrics-resource-attribute-getters.md b/.changeset/metrics-resource-attribute-getters.md new file mode 100644 index 0000000000..0b311bb4fc --- /dev/null +++ b/.changeset/metrics-resource-attribute-getters.md @@ -0,0 +1,6 @@ +--- +'posthog-node': patch +'@posthog/core': patch +--- + +Stop a throwing getter in `metrics.resourceAttributes` from breaking every metrics export — the key is recorded as `[Unserializable]` instead. diff --git a/.changeset/node-before-span-send.md b/.changeset/node-before-span-send.md deleted file mode 100644 index 8d2afd5521..0000000000 --- a/.changeset/node-before-span-send.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'posthog-node': minor -'@posthog/core': minor -'@posthog/types': minor ---- - -Add a `traces.beforeSpanSend` hook to edit a finished span before it is queued, or drop it by returning `null` — a hook that throws, or returns anything that is not a span record, also drops the span. diff --git a/.changeset/node-distributed-tracing.md b/.changeset/node-distributed-tracing.md index d0ddc4376a..7c87ac54d2 100644 --- a/.changeset/node-distributed-tracing.md +++ b/.changeset/node-distributed-tracing.md @@ -4,8 +4,8 @@ '@posthog/types': minor --- -Add distributed tracing to `posthog-node` — experimental. `withSpan`, `startSpan` and `getActiveSpan` record spans against a new `traces` client option; spans started inside a request context carry the distinct ID and session ID, and `parent` / `span.traceparent()` continue a W3C trace across services. +Add distributed tracing to `posthog-node` — experimental. `startSpan`, `withSpan` and `getActiveSpan` record spans against a new `traces` client option, and return a working handle even before it is set, so calling code never branches on whether tracing is on. Spans started inside a request context carry the distinct ID and session ID; `parent` and `span.traceparent()` continue a W3C trace across services, inbound sampled flag included; `flush()` drains queued spans alongside events. -Code that starts spans and never ends them cannot grow the SDK's bookkeeping without limit: `traces.maxLiveSpans` (default 10000) caps how many spans may be open at once, and `traces.maxSpanAgeMs` (default one hour) stops accounting for one that stays open longer than that. `startSpan` returns an inert handle at the cap, and both kinds of drop are reported through the existing span-drop warning. +Configure it through `traces`: `serviceName`, `serviceVersion`, `environment` and `resourceAttributes` for attribution (spans also report the host's `os.name` and `os.version`), `beforeSpanSend` to edit or drop a finished span, `flushIntervalMs` / `maxExportBatchSize` / `maxQueueSize` for export, and `maxAttributesPerSpan` (128), `maxEventsPerSpan` (128), `maxAttributeValueLength` (8192), `maxLiveSpans` (10000) and `maxSpanAgeMs` (one hour) to bound what instrumentation can accumulate. `span.recordException()` and a throwing `withSpan` callback attach `exception.type`, `exception.message` and `exception.stacktrace` — drop the stack in `beforeSpanSend` to keep your server's file paths out of PostHog. -`IPostHog` gains these three members, so anything implementing that interface (hand-written test doubles, DI wrappers) needs them added, or can extend `PostHogBackendClient` instead. +`IPostHog` gains `startSpan`, `withSpan` and `getActiveSpan`, so anything implementing that interface (hand-written test doubles, DI wrappers) needs them added, or can extend `PostHogBackendClient` instead. diff --git a/.changeset/node-exception-stacktrace.md b/.changeset/node-exception-stacktrace.md deleted file mode 100644 index e4cfa02136..0000000000 --- a/.changeset/node-exception-stacktrace.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'posthog-node': minor -'@posthog/core': minor -'@posthog/types': minor ---- - -Attach `exception.stacktrace` to the exception events recorded by `recordException` and by a throwing `withSpan` callback — remove it in `traces.beforeSpanSend` to keep your server's file paths out of PostHog. diff --git a/.changeset/node-span-limits.md b/.changeset/node-span-limits.md deleted file mode 100644 index fbe06127fc..0000000000 --- a/.changeset/node-span-limits.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'posthog-node': minor -'@posthog/core': minor -'@posthog/types': minor ---- - -Cap spans at 128 user attributes, 128 events and 8192 characters per string, configurable with `traces.maxAttributesPerSpan`, `traces.maxEventsPerSpan` and `traces.maxAttributeValueLength`. The earliest entries are kept, and a span that lost any reports how many as `droppedAttributesCount` and `droppedEventsCount`. The event cap is absolute, so an `exception` event the SDK records for you spends an ordinary slot. diff --git a/.changeset/otlp-os-resource-attributes.md b/.changeset/otlp-os-resource-attributes.md deleted file mode 100644 index 9743127a09..0000000000 --- a/.changeset/otlp-os-resource-attributes.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'posthog-node': minor -'@posthog/core': patch ---- - -Add `os.name` and `os.version` resource attributes to the spans `posthog-node` sends, so traces can be filtered by platform. `os.name` is the human-readable name the other PostHog SDKs report (`macOS`, `Windows`, `Linux`) rather than the `node:os` identifier. Either key is omitted when the host cannot supply it, and `traces.resourceAttributes` still overrides both. diff --git a/.changeset/traceparent-w3c-strictness.md b/.changeset/traceparent-w3c-strictness.md deleted file mode 100644 index 950193e06e..0000000000 --- a/.changeset/traceparent-w3c-strictness.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@posthog/core': patch ---- - -Ignore an inbound `traceparent` that W3C requires a vendor to reject — a version `00` header carrying fields beyond `trace-id`, `parent-id` and `trace-flags`, or one whose ids are uppercase hex — and start a fresh trace instead. diff --git a/.changeset/traces-context-and-time-fidelity.md b/.changeset/traces-context-and-time-fidelity.md deleted file mode 100644 index c2cab61516..0000000000 --- a/.changeset/traces-context-and-time-fidelity.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@posthog/core': patch ---- - -Forward an inbound `traceparent` whose version is above `00` whole, including the fields that version adds, rather than trimming it to the four this SDK reads. Keep the inbound context when a handle returned with tracing off is passed back as `parent`, so a child no longer starts a fresh trace. Warn when a span's `startTime` is in the future, which costs it its duration. diff --git a/.changeset/traces-flags-fidelity.md b/.changeset/traces-flags-fidelity.md deleted file mode 100644 index 77dfd3c00e..0000000000 --- a/.changeset/traces-flags-fidelity.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'posthog-node': minor -'@posthog/core': minor -'@posthog/types': minor ---- - -Propagate the inbound W3C sampled flag on a continued trace instead of always sending `01`, so a downstream parent-based sampler sees the decision the head sampler made. Spans are still recorded and exported either way. Exported spans also carry OpenTelemetry's parent-remoteness bits, so a span that entered the service over HTTP is distinguishable from one started locally. diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index dfe61ff460..d55a08390f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -95,7 +95,6 @@ export type { ResolvedTracesConfig, SpanContextManager, TraceSdkContext } from ' // The `beforeSpanSend` shapes come straight from @posthog/types: hooks see the // public record, not core's internal one, which also carries `traceState`. export type { SpanRecord, BeforeSpanSendFn } from '@posthog/types' -// Same barrel convention as logs and metrics for the user-facing tracing types. export type { Span, SpanAttributes, diff --git a/packages/core/src/traces/config.ts b/packages/core/src/traces/config.ts index dc60973a43..d0c6143fde 100644 --- a/packages/core/src/traces/config.ts +++ b/packages/core/src/traces/config.ts @@ -73,12 +73,9 @@ function withUsableIdentityKeys(attributes: TracesConfig['resourceAttributes']): * untyped caller passing the wrong shape would otherwise have every span dropped * by a hook that throws, leaving tracing silently off. * - * Dropping one is reported rather than thrown on. `beforeSpanSend` is where - * redaction lives, so a configuration that silently filters nothing ships the - * values it was meant to remove — but a client constructor that throws takes the - * application down with it, which is the worse of the two. `critical`, because - * every other level is gated behind `debug: true`, and a redaction hook that is - * quietly inert is exactly what an operator has to hear about without opting in. + * Reported at `critical` rather than thrown on: a constructor that throws takes + * the application down, but every other log level is gated behind `debug: true`, + * and an inert redaction hook ships the values it was meant to remove. */ function resolveBeforeSpanSend(beforeSpanSend: TracesConfig['beforeSpanSend'], logger?: Logger): BeforeSpanSendFn[] { if (!beforeSpanSend) { diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 5d67e00d81..fca29ebaee 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -553,12 +553,12 @@ export class PostHogTraces { attributes: record.droppedAttributesCount, events: record.droppedEventsCount, } - // Read here rather than restored onto the hook's return value: writing them - // back would throw on a frozen record, and neither is on the record a hook - // is handed, so a rebuilding hook always arrives without them. // The order the span itself wrote them in, so the caps below can keep the // earliest-set entries even when a hook adds an integer-like key. const keysBeforeHook = Object.keys(record.attributes) + // Read here rather than restored onto the hook's return value: writing them + // back would throw on a frozen record, and neither is on the record a hook + // is handed, so a rebuilding hook always arrives without them. const originalPropagation = { traceFlags: record.traceFlags, parentIsRemote: record.parentIsRemote, @@ -679,12 +679,10 @@ export class PostHogTraces { ) { this._logger.debug('beforeSpanSend changed a span identity field; keeping the original ids') } - // Only the fields that actually differ are written back. Assigning a value - // to a frozen property throws even when it is the value already there, and - // a hook that freezes the record it returns would otherwise drop every span. - // Best-effort, for the next hook in the chain only: the record this builds - // is not what gets exported. A frozen return refuses every write, and the - // span must survive that. + // Only the fields that actually differ are written back: assigning to a + // frozen property throws even when the value is the one already there, and + // a hook that freezes what it returns would otherwise drop every span. The + // record this builds is for the next hook in the chain, not for the export. try { restoreField(hooked, 'traceId', original.traceId) restoreField(hooked, 'spanId', original.spanId) diff --git a/packages/core/src/traces/otlp.ts b/packages/core/src/traces/otlp.ts index 686b7e8e86..4d93be4a14 100644 --- a/packages/core/src/traces/otlp.ts +++ b/packages/core/src/traces/otlp.ts @@ -37,11 +37,7 @@ const TRACE_FLAGS_SAMPLED = 0x01 const SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE = 0x100 const SPAN_FLAGS_CONTEXT_IS_REMOTE = 0x200 -/** - * The `flags` field for a span: its W3C trace-flags byte, plus OTel's - * parent-remoteness bits. Nothing reads the remoteness today, but a span - * exported without it can never be backfilled with it. - */ +/** The `flags` field for a span: its W3C trace-flags byte, plus OTel's parent-remoteness bits. */ function spanFlags(record: SpanRecord): number { const traceFlags = parseInt(record.traceFlags, 16) const w3c = Number.isFinite(traceFlags) ? traceFlags & 0xff : TRACE_FLAGS_SAMPLED diff --git a/packages/core/src/traces/span.ts b/packages/core/src/traces/span.ts index 3124945c9d..9d657c0aee 100644 --- a/packages/core/src/traces/span.ts +++ b/packages/core/src/traces/span.ts @@ -308,15 +308,6 @@ export function nonNegativeCount(value: unknown): number { return typeof value === 'number' && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0 } -/** - * Re-applies the per-span caps to a record a `beforeSpanSend` hook has already - * seen. The hook writes to the plain record, not through the span's own guarded - * writer, so an enriching hook would otherwise push a span past the cap it was - * trimmed to and back into the 413 path the cap exists to avoid. - * - * Earliest-set entries win, matching the span-side rule; SDK-attached keys are - * exempt. Counts add to whatever the span already dropped. - */ /** * The record's keys with the ones the span itself set first, in that order. * @@ -337,6 +328,15 @@ function orderedKeys(attributes: SpanAttributes, keysBeforeHook: readonly string return [...beforeHook, ...Object.keys(attributes).filter((key) => !seen.has(key))] } +/** + * Re-applies the per-span caps to a record a `beforeSpanSend` hook has already + * seen. The hook writes to the plain record, not through the span's own guarded + * writer, so an enriching hook would otherwise push a span past the cap it was + * trimmed to and back into the 413 path the cap exists to avoid. + * + * Earliest-set entries win, matching the span-side rule; SDK-attached keys are + * exempt. Counts add to whatever the span already dropped. + */ export function applySpanLimits( record: SpanRecord, autoKeys: ReadonlySet, @@ -514,20 +514,18 @@ function readHandle(parent: unknown, method: 'traceparent' | 'tracestate'): unkn } } -/** - * The same depth, node and item caps `encodeAnyValue` uses, but spent per - * attribute rather than per bag: the encoder allocates one budget for a whole - * attribute map, this walk allocates one per value. That makes the encoder's - * budget the stricter of the two — whatever this walk hands back unbounded, the - * encoder has already stopped short of — at the cost of a wide span paying for - * a walk whose results the encoder then discards. - */ +/** One walk's budget, allocated per attribute value. */ interface TruncateState { /** Containers on the current path, so a back-reference stops the walk. */ ancestors: WeakSet + /** Nodes this walk may still visit. */ remainingNodes: number } +function truncateString(value: string, maxLength: number): string { + return value.length > maxLength ? value.slice(0, maxLength) : value +} + /** * Bounds every string reachable from an attribute value to `maxLength` * characters, including the strings nested inside arrays and objects. Numbers @@ -541,10 +539,6 @@ interface TruncateState { * Returns the value it was given when nothing needed shortening, so the common * case allocates nothing. */ -function truncateString(value: string, maxLength: number): string { - return value.length > maxLength ? value.slice(0, maxLength) : value -} - export function truncateAttributeValue(value: SpanAttributeValue, maxLength: number): SpanAttributeValue { return truncateValue(value, maxLength, { ancestors: new WeakSet(), remainingNodes: MAX_JSON_SAFE_VALUE_NODES }, 0) } From a54592581d5710331d4151d420f264196d3f5e35 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 16:53:05 -0400 Subject: [PATCH 16/24] docs(traces): cut the tracing changeset to a single line Config surface, limits and the IPostHog note are PR-body and docs material, not release notes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XBUa53f3QqECUyxnkSiWGA --- .changeset/node-distributed-tracing.md | 6 +----- .changeset/rn-flags-disable-and-update-flags.md | 7 +++++++ 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 .changeset/rn-flags-disable-and-update-flags.md diff --git a/.changeset/node-distributed-tracing.md b/.changeset/node-distributed-tracing.md index 7c87ac54d2..222606bce6 100644 --- a/.changeset/node-distributed-tracing.md +++ b/.changeset/node-distributed-tracing.md @@ -4,8 +4,4 @@ '@posthog/types': minor --- -Add distributed tracing to `posthog-node` — experimental. `startSpan`, `withSpan` and `getActiveSpan` record spans against a new `traces` client option, and return a working handle even before it is set, so calling code never branches on whether tracing is on. Spans started inside a request context carry the distinct ID and session ID; `parent` and `span.traceparent()` continue a W3C trace across services, inbound sampled flag included; `flush()` drains queued spans alongside events. - -Configure it through `traces`: `serviceName`, `serviceVersion`, `environment` and `resourceAttributes` for attribution (spans also report the host's `os.name` and `os.version`), `beforeSpanSend` to edit or drop a finished span, `flushIntervalMs` / `maxExportBatchSize` / `maxQueueSize` for export, and `maxAttributesPerSpan` (128), `maxEventsPerSpan` (128), `maxAttributeValueLength` (8192), `maxLiveSpans` (10000) and `maxSpanAgeMs` (one hour) to bound what instrumentation can accumulate. `span.recordException()` and a throwing `withSpan` callback attach `exception.type`, `exception.message` and `exception.stacktrace` — drop the stack in `beforeSpanSend` to keep your server's file paths out of PostHog. - -`IPostHog` gains `startSpan`, `withSpan` and `getActiveSpan`, so anything implementing that interface (hand-written test doubles, DI wrappers) needs them added, or can extend `PostHogBackendClient` instead. +Add experimental distributed tracing to `posthog-node`: `startSpan`, `withSpan` and `getActiveSpan` record spans against a new `traces` client option. diff --git a/.changeset/rn-flags-disable-and-update-flags.md b/.changeset/rn-flags-disable-and-update-flags.md new file mode 100644 index 0000000000..73879e9713 --- /dev/null +++ b/.changeset/rn-flags-disable-and-update-flags.md @@ -0,0 +1,7 @@ +--- +'@posthog/core': minor +'posthog-react-native': minor +'posthog-js-lite': minor +--- + +feat(flags): add the `advancedDisableFeatureFlags` option and a public `updateFlags(flags, payloads?, { merge? })` method, matching the web SDK's `advanced_disable_feature_flags` and `updateFlags`. With the option set, `reloadFeatureFlags()` and the reloads triggered by `identify()`, `group()`, `setPersonPropertiesForFlags()` and `reset()` become no-ops, and any flags request that still goes out for remote config or surveys carries `disable_flags: true` so the server skips flag evaluation. `updateFlags` supplies locally evaluated flag values (with payloads) at runtime: values persist, `getFeatureFlag()`/`getFeatureFlagPayload()` read them back, and `onFeatureFlags` listeners fire — so React Native session replay gated on a linked flag re-evaluates when flags are pushed in. From 6fc61fadbc951c73f7df929304cf37e3b4b2fb40 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 16:53:39 -0400 Subject: [PATCH 17/24] chore: drop the unrelated RN changeset again rn-flags-disable-and-update-flags belongs to separate RN work; it was swept back in by a broad `git add`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XBUa53f3QqECUyxnkSiWGA --- .changeset/rn-flags-disable-and-update-flags.md | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 .changeset/rn-flags-disable-and-update-flags.md diff --git a/.changeset/rn-flags-disable-and-update-flags.md b/.changeset/rn-flags-disable-and-update-flags.md deleted file mode 100644 index 73879e9713..0000000000 --- a/.changeset/rn-flags-disable-and-update-flags.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@posthog/core': minor -'posthog-react-native': minor -'posthog-js-lite': minor ---- - -feat(flags): add the `advancedDisableFeatureFlags` option and a public `updateFlags(flags, payloads?, { merge? })` method, matching the web SDK's `advanced_disable_feature_flags` and `updateFlags`. With the option set, `reloadFeatureFlags()` and the reloads triggered by `identify()`, `group()`, `setPersonPropertiesForFlags()` and `reset()` become no-ops, and any flags request that still goes out for remote config or surveys carries `disable_flags: true` so the server skips flag evaluation. `updateFlags` supplies locally evaluated flag values (with payloads) at runtime: values persist, `getFeatureFlag()`/`getFeatureFlagPayload()` read them back, and `onFeatureFlags` listeners fire — so React Native session replay gated on a linked flag re-evaluates when flags are pushed in. From a9e76dcfa84adf4170902fdfb37b59f688e30e34 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 17:47:29 -0400 Subject: [PATCH 18/24] refactor(traces): keep the per-event attribute cap internal The cap is not one of the knobs the traces spec enumerates, so it stays a fixed 128 instead of a public `maxAttributesPerEvent` option. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HQy8eimnfV3jVQKbzELc65 --- packages/core/src/traces/config.spec.ts | 16 ++++++++-------- packages/core/src/traces/config.ts | 7 +++++-- packages/types/src/traces.ts | 19 +++++-------------- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/packages/core/src/traces/config.spec.ts b/packages/core/src/traces/config.spec.ts index 03a07b6a64..eaef6082bc 100644 --- a/packages/core/src/traces/config.spec.ts +++ b/packages/core/src/traces/config.spec.ts @@ -11,31 +11,31 @@ describe('resolveTracesConfig', () => { ['a fraction', 1.5], ['a large fraction', 200.5], ['infinity', Infinity], - ])('falls back to the default caps when given %s', (_label, value) => { + ])('falls back to the default per-span caps when given %s', (_label, value) => { const resolved = resolveTracesConfig({ maxAttributesPerSpan: value, maxEventsPerSpan: value, - maxAttributesPerEvent: value, maxAttributeValueLength: value, }) expect(resolved.maxAttributesPerSpan).toBe(128) expect(resolved.maxEventsPerSpan).toBe(128) - expect(resolved.maxAttributesPerEvent).toBe(128) expect(resolved.maxAttributeValueLength).toBe(8192) }) - it('honours explicit per-span and per-event caps', () => { + it('honours explicit per-span caps', () => { // Without this the resolver can ignore maxEventsPerSpan entirely and every // other test still passes, because they all assert the default. - expect( - resolveTracesConfig({ maxAttributesPerSpan: 5, maxEventsPerSpan: 7, maxAttributesPerEvent: 9 }) - ).toMatchObject({ + expect(resolveTracesConfig({ maxAttributesPerSpan: 5, maxEventsPerSpan: 7 })).toMatchObject({ maxAttributesPerSpan: 5, maxEventsPerSpan: 7, - maxAttributesPerEvent: 9, }) }) + it('keeps the per-event attribute cap fixed', () => { + // The cap is internal, so an untyped caller naming it gets the default. + expect(resolveTracesConfig({ maxAttributesPerEvent: 9 } as any).maxAttributesPerEvent).toBe(128) + }) + it('applies the documented defaults', () => { expect(resolveTracesConfig(undefined)).toMatchObject({ flushIntervalMs: 5000, diff --git a/packages/core/src/traces/config.ts b/packages/core/src/traces/config.ts index fd81bfc34f..cf7faca372 100644 --- a/packages/core/src/traces/config.ts +++ b/packages/core/src/traces/config.ts @@ -11,7 +11,10 @@ const DEFAULT_MAX_QUEUE_SIZE = 2048 // OpenTelemetry's per-span defaults. const DEFAULT_MAX_ATTRIBUTES_PER_SPAN = 128 const DEFAULT_MAX_EVENTS_PER_SPAN = 128 -// OpenTelemetry's per-event default, which is the same number. +// OpenTelemetry's per-event limit, which is the same number. Fixed rather than +// configurable: `maxAttributesPerSpan` and `maxEventsPerSpan` already give a +// caller room to shape a span, and this one only has to stop an event holding +// an unbounded bag. const DEFAULT_MAX_ATTRIBUTES_PER_EVENT = 128 // OpenTelemetry leaves the value length unlimited, which is what lets one // multi-MB attribute make a span too large for the endpoint to accept — and an @@ -123,7 +126,7 @@ export function resolveTracesConfig( beforeSpanSend: resolveBeforeSpanSend(config?.beforeSpanSend, logger), maxAttributesPerSpan: positiveInteger(config?.maxAttributesPerSpan, DEFAULT_MAX_ATTRIBUTES_PER_SPAN), maxEventsPerSpan: positiveInteger(config?.maxEventsPerSpan, DEFAULT_MAX_EVENTS_PER_SPAN), - maxAttributesPerEvent: positiveInteger(config?.maxAttributesPerEvent, DEFAULT_MAX_ATTRIBUTES_PER_EVENT), + maxAttributesPerEvent: DEFAULT_MAX_ATTRIBUTES_PER_EVENT, maxAttributeValueLength: positiveInteger(config?.maxAttributeValueLength, DEFAULT_MAX_ATTRIBUTE_VALUE_LENGTH), flushIntervalMs: positiveInteger(config?.flushIntervalMs, DEFAULT_FLUSH_INTERVAL_MS), maxExportBatchSize, diff --git a/packages/types/src/traces.ts b/packages/types/src/traces.ts index 350dc5ac36..1881574ef1 100644 --- a/packages/types/src/traces.ts +++ b/packages/types/src/traces.ts @@ -120,6 +120,10 @@ export interface Span { /** * Record a timestamped event within the span, e.g. a cache miss or a retry. * Defaults to the current time. + * + * One event carries at most 128 attributes; further keys are dropped and + * counted on the exported event. Use `maxEventsPerSpan` to bound how many + * events a span carries. */ addEvent(name: string, attributes?: SpanAttributes, timestamp?: SpanTimeInput): this @@ -326,19 +330,6 @@ export interface TracesConfig { */ maxEventsPerSpan?: number - /** - * Maximum attributes on a single span event. On overflow the first - * `maxAttributesPerEvent` are kept and later ones are dropped, with the - * number dropped reported on the exported event. - * - * `maxAttributesPerSpan` counts a span's own attributes and does not reach - * inside its events, so this is what bounds an event's width — including the - * `exception.*` attributes the SDK records for you. - * - * @default 128 - */ - maxAttributesPerEvent?: number - /** * Maximum length of a string attribute value. Longer values are truncated, * and the bound reaches every string the value contains, including the ones @@ -391,7 +382,7 @@ export interface OtlpSpanEvent { name: string timeUnixNano: string attributes?: OtlpSpanKeyValue[] - /** Attributes dropped by `maxAttributesPerEvent`. Omitted when none were. */ + /** Attributes dropped by the SDK's per-event attribute cap. Omitted when none were. */ droppedAttributesCount?: number } From 833a165ed572a65572eb7ea4d2ea02eee5141fb3 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 09:41:29 -0400 Subject: [PATCH 19/24] fix(traces): clamp drop counts to uint32 and correct the size comments A `beforeSpanSend` hook can write a count past the OTLP field's range onto an event, which is refused for the whole request rather than the one span. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjT26chjUzuPSyJZqh4AWQ --- packages/core/src/traces/config.ts | 10 +++++---- packages/core/src/traces/span.spec.ts | 29 +++++++++++++++++++++++++++ packages/core/src/traces/span.ts | 15 ++++++++++++-- packages/core/src/traces/types.ts | 8 ++++++++ 4 files changed, 56 insertions(+), 6 deletions(-) diff --git a/packages/core/src/traces/config.ts b/packages/core/src/traces/config.ts index cf7faca372..8a46bf88b7 100644 --- a/packages/core/src/traces/config.ts +++ b/packages/core/src/traces/config.ts @@ -4,7 +4,7 @@ import type { BeforeSpanSendFn, TracesConfig } from '@posthog/types' import type { Logger } from '../types' // OpenTelemetry's BatchSpanProcessor defaults, which sit comfortably under the -// server's 2 MB body cap. +// server's request body cap. const DEFAULT_FLUSH_INTERVAL_MS = 5000 const DEFAULT_MAX_EXPORT_BATCH_SIZE = 512 const DEFAULT_MAX_QUEUE_SIZE = 2048 @@ -18,9 +18,11 @@ const DEFAULT_MAX_EVENTS_PER_SPAN = 128 const DEFAULT_MAX_ATTRIBUTES_PER_EVENT = 128 // OpenTelemetry leaves the value length unlimited, which is what lets one // multi-MB attribute make a span too large for the endpoint to accept — and an -// oversized span is dropped whole. 8 KB holds a deep stack trace and any -// realistic header, query string or payload excerpt, and keeps a span at the -// attribute cap under 1 MB, comfortably inside the 2 MB body cap. +// oversized span is dropped whole. 8 KB bounds a single string: it holds a deep +// stack trace and any realistic header, query string or payload excerpt, and +// keeps a span's own attributes under 1 MB at the attribute cap. A span's total +// size is the product of these caps; the body limit itself is enforced at the +// batch boundary, which is where the whole payload can be weighed at once. const DEFAULT_MAX_ATTRIBUTE_VALUE_LENGTH = 8192 // Live-span bounds. A server can legitimately hold thousands of spans open at diff --git a/packages/core/src/traces/span.spec.ts b/packages/core/src/traces/span.spec.ts index 2733952119..ad814772b7 100644 --- a/packages/core/src/traces/span.spec.ts +++ b/packages/core/src/traces/span.spec.ts @@ -1,5 +1,6 @@ import { NOOP_SPAN, PostHogSpan, describeError, truncateAttributeValue } from './span' import { buildOtlpSpan } from './otlp' +import { resolveTracesConfig } from './config' import type { SpanInit } from './span' import type { SpanRecord } from './types' import type { Logger } from '../types' @@ -308,6 +309,34 @@ describe('PostHogSpan', () => { expect(ended[0].events[0].attributes).toEqual({}) }) + it('drops the attribute past the shipped default and nothing before it', () => { + // Ties the default the SDK actually ships to the behaviour at its boundary: + // the other cases here pick small caps, so neither half moves the other. + const limit = resolveTracesConfig(undefined).maxAttributesPerEvent + const atLimit = Object.fromEntries(Array.from({ length: limit }, (_, index) => [`k${index}`, index])) + + const span = createSpan({ maxAttributesPerEvent: limit }) + span.addEvent('at-limit', atLimit) + span.addEvent('over-limit', { ...atLimit, extra: 1 }) + span.end() + + expect(ended[0].events[0].attributes).toEqual(atLimit) + expect(ended[0].events[0].droppedAttributesCount).toBeUndefined() + expect(ended[0].events[1].attributes).toEqual(atLimit) + expect(ended[0].events[1].droppedAttributesCount).toBe(1) + }) + + it('clamps a hook-written drop count to what the wire field holds', () => { + // The count is a uint32 on the wire, and a value over it is refused for the + // whole request rather than the one span that carried it. + const span = createSpan() + span.addEvent('query', { a: 1 }) + span.end() + ended[0].events[0].droppedAttributesCount = Number.MAX_SAFE_INTEGER + + expect(buildOtlpSpan(ended[0], logger).events?.[0].droppedAttributesCount).toBe(0xffff_ffff) + }) + it('counts the span attribute cap separately from an event cap', () => { // maxAttributesPerSpan does not reach inside events, which is the gap this // cap closes: a span at its own cap can still carry full-width events. diff --git a/packages/core/src/traces/span.ts b/packages/core/src/traces/span.ts index caf4f42535..07b3850625 100644 --- a/packages/core/src/traces/span.ts +++ b/packages/core/src/traces/span.ts @@ -300,6 +300,9 @@ export class PostHogSpan implements Span { const EXCEPTION_EVENT_NAME = 'exception' +/** The widest value the OTLP `dropped_*_count` fields, declared `uint32`, can carry. */ +const MAX_UINT32 = 0xffff_ffff + /** A value as its string form, or the encoder's marker when it refuses to produce one. */ function safeString(value: unknown): string { try { @@ -309,9 +312,17 @@ function safeString(value: unknown): string { } } -/** A caller-visible counter read back as a number, or 0 for anything else. */ +/** + * A caller-visible counter read back as a number, or 0 for anything else. + * Clamped to the `uint32` the OTLP field is declared as: a `beforeSpanSend` hook + * can write a larger number onto an event, and one that overflows the field is + * refused for the whole request. + */ export function nonNegativeCount(value: unknown): number { - return typeof value === 'number' && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0 + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) { + return 0 + } + return Math.min(Math.floor(value), MAX_UINT32) } /** diff --git a/packages/core/src/traces/types.ts b/packages/core/src/traces/types.ts index 83e995f3fe..cf6ee19b9c 100644 --- a/packages/core/src/traces/types.ts +++ b/packages/core/src/traces/types.ts @@ -63,6 +63,14 @@ export interface SpanEventRecord { /** ms epoch. */ timestamp: number attributes?: SpanAttributes + /** + * How many of this event's attributes the cap discarded. + * + * Unlike the span-level counts, this one is carried on an object a + * `beforeSpanSend` hook holds: the public event type omits it, so a hook that + * rebuilds its events returns them without it. Events have no identity to + * match a rebuilt array back against, so what a hook drops here stays dropped. + */ droppedAttributesCount?: number } From 40278277f4759c53e4ccddafea70f28c282434ad Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 09:52:34 -0400 Subject: [PATCH 20/24] docs(traces): mark cross-package plumbing internal and correct the parent contract A span from another tracer reports through `spanContext()`, so it is ignored rather than yielding an inert span; a test now pins that. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjT26chjUzuPSyJZqh4AWQ --- packages/core/src/traces/config.ts | 2 ++ packages/core/src/traces/context.ts | 2 ++ packages/core/src/traces/index.spec.ts | 21 +++++++++++++++++++++ packages/core/src/traces/index.ts | 2 ++ packages/core/src/traces/span.ts | 4 ++++ packages/core/src/traces/types.ts | 4 ++++ packages/types/src/traces.ts | 9 ++++++--- 7 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/core/src/traces/config.ts b/packages/core/src/traces/config.ts index d0c6143fde..2b947598fe 100644 --- a/packages/core/src/traces/config.ts +++ b/packages/core/src/traces/config.ts @@ -100,6 +100,8 @@ function resolveBeforeSpanSend(beforeSpanSend: TracesConfig['beforeSpanSend'], l * OTLP resource attributes take precedence over the named fields, matching the * logs config. `hostResourceAttributes` are runtime-detected by the entrypoint and * merge first, so a user-supplied value of the same key wins. + * + * @internal Exposed for cross-package use within this SDK; not part of the stable public API. */ export function resolveTracesConfig( config: TracesConfig | undefined, diff --git a/packages/core/src/traces/context.ts b/packages/core/src/traces/context.ts index 8744a8c335..60f3324527 100644 --- a/packages/core/src/traces/context.ts +++ b/packages/core/src/traces/context.ts @@ -8,6 +8,8 @@ import type { SpanContextManager } from './types' * * The fallback for runtimes with no ambient async context; Node injects an * `AsyncLocalStorage`-backed manager instead. `parent` is the escape hatch. + * + * @internal Exposed for cross-package use within this SDK; not part of the stable public API. */ export class SyncSpanContextManager implements SpanContextManager { private _active: Span | undefined diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 0712ea99f8..1020ccc1cb 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -10,6 +10,7 @@ import type { TraceSdkContext, } from './types' import type { Logger } from '../types' +import type { Span } from '@posthog/types' import { createMockLogger } from '@/testing' const TRACE_ID = '4bf92f3577b34da6a3ce929d0e0e4736' @@ -277,6 +278,26 @@ describe('PostHogTraces', () => { expect(span.parentSpanId).toBeUndefined() }) + it('ignores a span from another tracer, which reports through spanContext', async () => { + // OTel's shape: no `traceparent()` to read, so it parents to the active + // span rather than continuing a trace the SDK cannot read the ids of. + const otelSpan = { + spanContext: () => ({ traceId: TRACE_ID, spanId: REMOTE_SPAN_ID, traceFlags: 1 }), + } + + const traces = createTraces() + traces.withSpan('handler', () => { + traces.startSpan('child', { parent: otelSpan as unknown as Span }).end() + }) + await traces.flush() + + const child = sentSpans().find((s) => s.name === 'child')! + const handler = sentSpans().find((s) => s.name === 'handler')! + expect(child.traceId).toBe(handler.traceId) + expect(child.traceId).not.toBe(TRACE_ID) + expect(child.parentSpanId).toBe(handler.spanId) + }) + it('parents to the active span when the parent is not a span', async () => { const traces = createTraces() traces.withSpan('handler', () => { diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index fca29ebaee..30227e96f1 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -145,6 +145,8 @@ interface ParentContext { * The traces pipeline: span creation, active-span parenting, and OTLP export. * Separate from the analytics-events pipeline — own queue, endpoint and flush * cycle — mirroring logs and metrics. + * + * @internal Exposed for cross-package use within this SDK; not part of the stable public API. */ export class PostHogTraces { private _queue: SpanRecord[] = [] diff --git a/packages/core/src/traces/span.ts b/packages/core/src/traces/span.ts index 9d657c0aee..d35e616a00 100644 --- a/packages/core/src/traces/span.ts +++ b/packages/core/src/traces/span.ts @@ -490,6 +490,8 @@ function readStack(error: unknown): { stack?: string } { /** * The handle to return when a span cannot be recorded: a pass-through when the * caller supplied a usable `parent` header, the shared no-op otherwise. + * + * @internal Exposed for cross-package use within this SDK; not part of the stable public API. */ export function inertSpan(options?: { parent?: unknown; tracestate?: unknown }): Span { const parent = options?.parent @@ -706,6 +708,8 @@ export function truncateAttributes(attributes: SpanAttributes, maxLength: number * reads null — callbacks should use the handle they're given. A pass-through * handle is activated, so `getActiveSpan()?.traceparent()` still propagates an * inbound trace through a service with tracing off. + * + * @internal Exposed for cross-package use within this SDK; not part of the stable public API. */ export function runWithActiveSpan(contextManager: SpanContextManager, span: Span, fn: (span: Span) => T): T { return span === NOOP_SPAN ? fn(span) : contextManager.with(span, () => fn(span)) diff --git a/packages/core/src/traces/types.ts b/packages/core/src/traces/types.ts index 7360c70a26..80329d5763 100644 --- a/packages/core/src/traces/types.ts +++ b/packages/core/src/traces/types.ts @@ -46,6 +46,8 @@ export interface TracesHost { * PostHog context snapshotted onto every span at start, so traces join back to * persons and sessions. Each SDK fills the fields that apply to it; absent * fields add no attribute. Internal to `@posthog/core`. + * + * @internal Exposed for cross-package use within this SDK; not part of the stable public API. */ export interface TraceSdkContext { distinctId?: string @@ -96,6 +98,8 @@ export interface SpanContextManager { /** * Fields `PostHogTraces` needs resolved at runtime. The host SDK applies its own * defaults and hands the resolved config to the constructor. + * + * @internal Exposed for cross-package use within this SDK; not part of the stable public API. */ export interface ResolvedTracesConfig extends TracesConfig { flushIntervalMs: number diff --git a/packages/types/src/traces.ts b/packages/types/src/traces.ts index eaf84e2309..b0f93bc82f 100644 --- a/packages/types/src/traces.ts +++ b/packages/types/src/traces.ts @@ -76,9 +76,12 @@ export interface StartSpanOptions { * Parent of this span: either a span handle, or a raw W3C `traceparent` * string to continue a trace started by another service. * - * When omitted the parent is the currently active span, or none. Only - * handles returned by this SDK are honoured; any other `Span` yields an - * inert span. + * When omitted the parent is the currently active span, or none. A recording + * handle from this SDK parents normally, and an inert one yields an inert + * span carrying the same context. A span from another tracer, which reports + * its context through `spanContext()` rather than `traceparent()`, is + * ignored: the span parents to the active span or starts a new trace. Pass + * that tracer's `traceparent` string to continue its trace. * * @example Continue an inbound trace * ```ts From 2581b66ef21b477030d42fbe369baec1b8b4938b Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 10:17:17 -0400 Subject: [PATCH 21/24] fix(traces): keep the inbound trace context through nested spans An inbound `traceparent` now parents a span that names no parent, on both the recording and inert paths, so a nested span no longer starts a new trace. Also accepts a one-element header array and trims an over-long `tracestate` instead of dropping it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjT26chjUzuPSyJZqh4AWQ --- .changeset/node-distributed-tracing.md | 2 +- packages/core/src/traces/index.spec.ts | 43 +++++++++++++- packages/core/src/traces/index.ts | 61 +++++++++++++------- packages/core/src/traces/span.ts | 17 ++++-- packages/core/src/traces/traceparent.spec.ts | 14 ++++- packages/core/src/traces/traceparent.ts | 34 ++++++++--- packages/node/src/client.ts | 4 +- 7 files changed, 134 insertions(+), 41 deletions(-) diff --git a/.changeset/node-distributed-tracing.md b/.changeset/node-distributed-tracing.md index 222606bce6..d2e9393b7d 100644 --- a/.changeset/node-distributed-tracing.md +++ b/.changeset/node-distributed-tracing.md @@ -4,4 +4,4 @@ '@posthog/types': minor --- -Add experimental distributed tracing to `posthog-node`: `startSpan`, `withSpan` and `getActiveSpan` record spans against a new `traces` client option. +Add experimental distributed tracing to `posthog-node`: `startSpan`, `withSpan` and `getActiveSpan` record spans against a new `traces` client option. A service with tracing off still forwards an inbound `traceparent`, including from spans nested inside the one that received it, so a distributed trace is not severed. A `traceparent` may be passed as the one-element array `req.headersDistinct` gives. diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 1020ccc1cb..28c412b675 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -15,6 +15,7 @@ import { createMockLogger } from '@/testing' const TRACE_ID = '4bf92f3577b34da6a3ce929d0e0e4736' const REMOTE_SPAN_ID = '00f067aa0ba902b7' +const DUPLICATED_HEADERS = [`00-${TRACE_ID}-${REMOTE_SPAN_ID}-01`, `00-${TRACE_ID}-${REMOTE_SPAN_ID}-00`] const resolveForTest = (partial?: Partial): ResolvedTracesConfig => ({ flushIntervalMs: 5000, @@ -268,11 +269,22 @@ describe('PostHogTraces', () => { expect(span.parentSpanId).toBeUndefined() }) - it('starts a fresh root when the parent is not a span, as a duplicated header is', async () => { + it('continues the trace when the header arrives as a one-element array', async () => { + // What `headersDistinct.traceparent` hands over for a single inbound header. const traces = createTraces() traces.startSpan('handler', { parent: [`00-${TRACE_ID}-${REMOTE_SPAN_ID}-01`] as unknown as string }).end() await traces.flush() + const [span] = sentSpans() + expect(span.traceId).toBe(TRACE_ID) + expect(span.parentSpanId).toBe(REMOTE_SPAN_ID) + }) + + it('starts a fresh root when the parent is not a span, as two inbound headers are', async () => { + const traces = createTraces() + traces.startSpan('handler', { parent: DUPLICATED_HEADERS as unknown as string }).end() + await traces.flush() + const [span] = sentSpans() expect(span.traceId).not.toBe(TRACE_ID) expect(span.parentSpanId).toBeUndefined() @@ -301,7 +313,7 @@ describe('PostHogTraces', () => { it('parents to the active span when the parent is not a span', async () => { const traces = createTraces() traces.withSpan('handler', () => { - traces.startSpan('child', { parent: [`00-${TRACE_ID}-${REMOTE_SPAN_ID}-01`] as unknown as string }).end() + traces.startSpan('child', { parent: DUPLICATED_HEADERS as unknown as string }).end() }) await traces.flush() @@ -531,6 +543,33 @@ describe('PostHogTraces', () => { expect(traces.startSpan('bad-parent', { parent: 'not-a-traceparent' })).toBe(NOOP_SPAN) }) + it('keeps the inbound context in a nested span that names no parent', () => { + const traces = createTraces({}, createMockInstance({ optedOut: true })) + + const propagated = traces.withSpan('outer', { parent: INBOUND_UNSAMPLED }, () => + traces.withSpan('inner', (span) => span.traceparent()) + ) + + expect(propagated).toBe(INBOUND_UNSAMPLED) + }) + + it('parents a recorded span to an active pass-through handle', async () => { + // Tracing is on, but the span that received the header was inert, so its + // child is the first recorded span of the inbound trace. + const foreign = { traceparent: () => INBOUND_UNSAMPLED, tracestate: () => 'vendor=abc' } + const traces = createTraces() + + traces.withSpan('proxied', { parent: foreign as unknown as Span }, () => { + traces.startSpan('child').end() + }) + await traces.flush() + + const child = sentSpans().find((span) => span.name === 'child')! + expect(child.traceId).toBe(TRACE_ID) + expect(child.parentSpanId).toBe(REMOTE_SPAN_ID) + expect(child.traceState).toBe('vendor=abc') + }) + it('keeps the inbound context when a pass-through handle is used as a parent', () => { // The child is inert either way; what it must not do is drop the context // and leave everything downstream of it on a fresh trace. diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 30227e96f1..72457d426e 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -10,6 +10,7 @@ import type { TracesHost, } from './types' import { + PassThroughSpan, PostHogSpan, applySpanLimits, describeError, @@ -19,7 +20,7 @@ import { truncateAttributes, } from './span' import { newSpanId, newTraceId } from './ids' -import { parseTraceparent, sanitizeTracestate } from './traceparent' +import { parseTraceparent, sanitizeTracestate, traceparentHeader } from './traceparent' import { clampEndTime, resolveStartTime, resolveSuppliedTime, sanitizeName, toEpochMs } from './sanitize' import { assignUserAttributes } from '../utils/json-utils' import { buildOtlpSpan, buildOtlpTracesPayload, buildTracesResourceAttributes } from './otlp' @@ -51,6 +52,21 @@ function isOwnSpan(value: unknown): value is PostHogSpan { } } +/** A `traceparent` header as the parent context it describes, or nothing if it is malformed. */ +function remoteContext(header: string, tracestate: string | undefined): ParentContext | undefined { + const remote = parseTraceparent(header) + if (!remote) { + return undefined + } + return { + traceId: remote.traceId, + parentSpanId: remote.spanId, + traceState: sanitizeTracestate(tracestate), + traceFlags: remote.flags, + isRemote: true, + } +} + function looksLikeSpan(value: unknown): boolean { try { return typeof (value as Span).traceparent === 'function' @@ -193,25 +209,26 @@ export class PostHogTraces { */ startSpan(name: string, options?: StartSpanOptions): Span { if (this._instance.isDisabled || this._instance.optedOut) { - return inertSpan(options) + return inertSpan(options, this._contextManager.active()) } - const explicitParent = options?.parent + const explicitParent = traceparentHeader(options?.parent) if (explicitParent && typeof explicitParent !== 'string' && !isOwnSpan(explicitParent)) { if (looksLikeSpan(explicitParent)) { // Inert like its parent, never an orphan with invented ids — but a // pass-through parent's inbound context carries to the child rather than // the trace ending here. this._logger.debug('Span parent is not a span from this SDK; returning an inert span') - return inertSpan(options) + return inertSpan(options, this._contextManager.active()) } - // No `traceparent()` to read: `req.headers.traceparent` is `string[]` when the - // header arrives twice, and a span from another tracer exposes `spanContext()` - // instead. Ignored: falls back to the active span, or to a new trace. + // No `traceparent()` to read: a span from another tracer exposes + // `spanContext()` instead, and `headersDistinct.traceparent` is a `string[]` + // holding more than one inbound value. Ignored: falls back to the active + // span, or to a new trace. this._logger.debug('Ignoring an unusable span parent') } - const parent = this._resolveParent(options) + const parent = this._resolveParent(explicitParent, options) // Swept before the bound is read, so a process that has leaked its way to // the bound recovers on the first `startSpan` after the leaks age out. @@ -221,7 +238,7 @@ export class PostHogTraces { 1, `the live-span limit (${this._config.maxLiveSpans}) was reached — spans are being started and never ended` ) - return inertSpan(options) + return inertSpan(options, this._contextManager.active()) } const now = Date.now() @@ -382,22 +399,13 @@ export class PostHogTraces { * Resolves a span's parent: an explicit `parent`, then the active span, then a * fresh root. A no-op explicit parent is rejected earlier, in `startSpan`. */ - private _resolveParent(options?: StartSpanOptions): ParentContext | undefined { - const explicit = options?.parent - + private _resolveParent(explicit: unknown, options?: StartSpanOptions): ParentContext | undefined { if (typeof explicit === 'string') { - const remote = parseTraceparent(explicit) + const remote = remoteContext(explicit, options?.tracestate) if (!remote) { this._logger.debug('Ignoring malformed traceparent; starting a new trace') - return undefined - } - return { - traceId: remote.traceId, - parentSpanId: remote.spanId, - traceState: sanitizeTracestate(options?.tracestate), - traceFlags: remote.flags, - isRemote: true, } + return remote } if (isOwnSpan(explicit)) { @@ -407,7 +415,16 @@ export class PostHogTraces { } const active = this._contextManager.active() - return isOwnSpan(active) ? active.childContext() : undefined + if (isOwnSpan(active)) { + return active.childContext() + } + // A pass-through handle is active when an earlier span in this trace could + // not be recorded. Its context still parents this one, so the inbound trace + // survives a span the SDK declined rather than ending there. + if (active instanceof PassThroughSpan) { + return remoteContext(active.traceparent(), active.tracestate() ?? undefined) + } + return undefined } /** diff --git a/packages/core/src/traces/span.ts b/packages/core/src/traces/span.ts index d35e616a00..2d3347c90d 100644 --- a/packages/core/src/traces/span.ts +++ b/packages/core/src/traces/span.ts @@ -1,7 +1,13 @@ import type { Span, SpanAttributes, SpanAttributeValue, SpanKind, SpanStatusCode, SpanTimeInput } from '@posthog/types' import type { Logger } from '../types' import type { SpanContextManager, SpanEventRecord, SpanRecord } from './types' -import { formatTraceparent, normalizeTraceparent, sanitizeTracestate, TRACE_FLAGS_SAMPLED } from './traceparent' +import { + formatTraceparent, + normalizeTraceparent, + sanitizeTracestate, + traceparentHeader, + TRACE_FLAGS_SAMPLED, +} from './traceparent' import { clampEndTime, resolveSuppliedTime, sanitizeName } from './sanitize' import { isArray, isError, isNullish } from '../utils' import { @@ -488,13 +494,14 @@ function readStack(error: unknown): { stack?: string } { } /** - * The handle to return when a span cannot be recorded: a pass-through when the - * caller supplied a usable `parent` header, the shared no-op otherwise. + * The handle to return when a span cannot be recorded: a pass-through when a + * context is available, the shared no-op otherwise. With no explicit `parent` + * the active handle supplies it, so an inbound trace survives nesting. * * @internal Exposed for cross-package use within this SDK; not part of the stable public API. */ -export function inertSpan(options?: { parent?: unknown; tracestate?: unknown }): Span { - const parent = options?.parent +export function inertSpan(options?: { parent?: unknown; tracestate?: unknown }, active?: Span): Span { + const parent = traceparentHeader(options?.parent) ?? active // A handle parent reports its own context, read behind a guard because a // foreign handle's accessor may throw. A no-op reports none and stays a no-op. const inbound = typeof parent === 'string' || parent == null ? parent : readHandle(parent, 'traceparent') diff --git a/packages/core/src/traces/traceparent.spec.ts b/packages/core/src/traces/traceparent.spec.ts index 2b65face90..4621e69223 100644 --- a/packages/core/src/traces/traceparent.spec.ts +++ b/packages/core/src/traces/traceparent.spec.ts @@ -107,11 +107,21 @@ describe('traceparent', () => { ['a member without a value', 'vendor'], ['a non-string', 42], ['undefined', undefined], - ['more than 32 members', Array.from({ length: 33 }, (_v, i) => `k${i}=v`).join(',')], - ['an overlong value', `vendor=${'a'.repeat(600)}`], + ['a single member longer than the whole limit', `vendor=${'a'.repeat(600)}`], ])('discards %s', (_name, value) => { expect(sanitizeTracestate(value)).toBeUndefined() }) + + it('keeps the first 32 members of a longer list', () => { + const members = Array.from({ length: 33 }, (_v, i) => `k${i}=v`) + expect(sanitizeTracestate(members.join(','))).toBe(members.slice(0, 32).join(',')) + }) + + it('keeps the members that fit inside the length limit', () => { + // 102 characters each, so the fifth crosses 512 and the first four stay. + const members = Array.from({ length: 6 }, (_v, i) => `k${i}=${'a'.repeat(100)}`) + expect(sanitizeTracestate(members.join(','))).toBe(members.slice(0, 4).join(',')) + }) }) }) diff --git a/packages/core/src/traces/traceparent.ts b/packages/core/src/traces/traceparent.ts index 1593ffecf0..cee0e310da 100644 --- a/packages/core/src/traces/traceparent.ts +++ b/packages/core/src/traces/traceparent.ts @@ -80,6 +80,15 @@ export function normalizeTraceparent(value: unknown): string | undefined { return matchTraceparent(value) && (value as string).trim() } +/** + * A `traceparent` as the string it is, unwrapping the one-element array Node's + * `headersDistinct` hands over. A longer array is two different inbound values, + * and picking either would be a guess. + */ +export function traceparentHeader(value: unknown): unknown { + return Array.isArray(value) && value.length === 1 ? value[0] : value +} + /** The W3C sampled bit, set on a trace this SDK started. */ export const TRACE_FLAGS_SAMPLED = '01' @@ -103,15 +112,19 @@ const TRACESTATE_MAX_LENGTH = 512 /** * Validates an incoming `tracestate` far enough to know it is safe to echo back. - * An invalid one is discarded without invalidating its traceparent, so a - * malformed vendor entry never costs us the trace continuation. + * A malformed one is discarded without invalidating its traceparent, so a bad + * vendor entry never costs us the trace continuation. + * + * Over-long input is trimmed rather than discarded: W3C makes the limits a + * reason to drop members from the end, so the entries that fit are still valid + * state the next hop can use. */ export function sanitizeTracestate(value: unknown): string | undefined { if (typeof value !== 'string') { return undefined } const trimmed = value.trim() - if (!trimmed || trimmed.length > TRACESTATE_MAX_LENGTH) { + if (!trimmed) { return undefined } // W3C restricts tracestate to printable ASCII plus HTAB as optional whitespace. @@ -121,9 +134,6 @@ export function sanitizeTracestate(value: unknown): string | undefined { return undefined } const members = trimmed.split(',') - if (members.length > TRACESTATE_MAX_MEMBERS) { - return undefined - } for (const member of members) { // An empty member is tolerated by the spec (list optional-white-space), but // a member without a `=` is not a key/value pair at all. @@ -131,5 +141,15 @@ export function sanitizeTracestate(value: unknown): string | undefined { return undefined } } - return trimmed + const kept: string[] = [] + let length = 0 + for (const member of members.slice(0, TRACESTATE_MAX_MEMBERS)) { + const separator = kept.length ? 1 : 0 + if (length + separator + member.length > TRACESTATE_MAX_LENGTH) { + break + } + kept.push(member) + length += separator + member.length + } + return kept.length ? kept.join(',') : undefined } diff --git a/packages/node/src/client.ts b/packages/node/src/client.ts index 5b83a9524f..8194d73224 100644 --- a/packages/node/src/client.ts +++ b/packages/node/src/client.ts @@ -714,7 +714,7 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen * ``` */ startSpan(name: string, options?: StartSpanOptions): Span { - return this._tracesPipeline?.startSpan(name, options) ?? inertSpan(options) + return this._tracesPipeline?.startSpan(name, options) ?? inertSpan(options, this._spanContextManager.active()) } /** @@ -756,7 +756,7 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen // Tracing off: still run the callback exactly once, with an inert handle. // A handle carrying an inbound `parent` is activated, so `getActiveSpan()` // inside the callback can propagate the trace onward. - return runWithActiveSpan(this._spanContextManager, inertSpan(options), fn) + return runWithActiveSpan(this._spanContextManager, inertSpan(options, this._spanContextManager.active()), fn) } return options ? pipeline.withSpan(name, options, fn) : pipeline.withSpan(name, fn) } From b2aaa72d5b99c15fea4bcd7c1e8b836abac4a8c0 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 10:45:35 -0400 Subject: [PATCH 22/24] feat(traces): report a span's limit drops once, event attributes included Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjT26chjUzuPSyJZqh4AWQ --- packages/core/src/traces/index.spec.ts | 28 ++++++++++++++++++++++++++ packages/core/src/traces/index.ts | 21 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 9a88f86867..e6b6c60011 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -876,6 +876,34 @@ describe('PostHogTraces', () => { expect(sent.droppedAttributesCount).toBe(1) }) + it('reports every limit drop once per span, hook drops included', () => { + const traces = createTraces({ + maxAttributesPerSpan: 1, + maxEventsPerSpan: 1, + beforeSpanSend: [ + (span: SpanRecord) => ({ ...span, attributes: { ...span.attributes, added: 1, alsoAdded: 2 } }), + ], + }) + + const span = traces.startSpan('checkout', { attributes: { route: '/checkout' } }) + span.addEvent('first', { a: 1 }) + span.addEvent('second') + span.end() + + const messages = logger.debug.mock.calls.map(([message]) => String(message)) + expect(messages.filter((message) => message.includes('Span limits discarded'))).toEqual([ + 'Span limits discarded data from "checkout": 2 attributes, 1 events, 0 event attributes', + ]) + }) + + it('stays quiet for a span that lost nothing', () => { + const traces = createTraces() + traces.startSpan('checkout', { attributes: { route: '/checkout' } }).end() + + const messages = logger.debug.mock.calls.map(([message]) => String(message)) + expect(messages.some((message) => message.includes('Span limits discarded'))).toBe(false) + }) + it('rejects a timestamp the server could not decode', async () => { const instance = createMockInstance() const traces = createTraces( diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 557c3014ba..bf25098deb 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -517,6 +517,7 @@ export class PostHogTraces { if (!record) { return } + this._reportLimitDrops(record) if (this._queue.length >= this._config.maxQueueSize) { // Drop the incoming span, not queued ones: those are completed parents whose @@ -545,6 +546,26 @@ export class PostHogTraces { } } + /** + * One diagnostic per span when its limits discarded anything, which is what + * OTel asks for. Counted after the post-hook pass, so drops a `beforeSpanSend` + * hook caused are included. + */ + private _reportLimitDrops(record: SpanRecord): void { + const attributes = record.droppedAttributesCount ?? 0 + const events = record.droppedEventsCount ?? 0 + let eventAttributes = 0 + for (const event of record.events) { + eventAttributes += event.droppedAttributesCount ?? 0 + } + if (attributes || events || eventAttributes) { + this._logger.debug( + `Span limits discarded data from "${record.name}": ` + + `${attributes} attributes, ${events} events, ${eventAttributes} event attributes` + ) + } + } + /** * Runs the `beforeSpanSend` chain, returning the span to enqueue or `null` to * drop it. From c8f9a66eae5e8c5f1cea494035ce5f33344485c6 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 10:56:40 -0400 Subject: [PATCH 23/24] fix(traces): follow Jon's revised tracestate and parent-doc review More than 32 tracestate members is malformed, not oversized, so it is rejected again; a valid over-long header drops members over 128 characters first, then from the right. Restores the original `parent` docs, whose fix was withdrawn. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjT26chjUzuPSyJZqh4AWQ --- packages/core/src/traces/traceparent.spec.ts | 21 ++++++++-- packages/core/src/traces/traceparent.ts | 41 ++++++++++++++------ packages/node/src/__tests__/traces.spec.ts | 18 +++++++++ packages/types/src/traces.ts | 9 ++--- 4 files changed, 67 insertions(+), 22 deletions(-) diff --git a/packages/core/src/traces/traceparent.spec.ts b/packages/core/src/traces/traceparent.spec.ts index 4621e69223..7ad5d73246 100644 --- a/packages/core/src/traces/traceparent.spec.ts +++ b/packages/core/src/traces/traceparent.spec.ts @@ -107,21 +107,34 @@ describe('traceparent', () => { ['a member without a value', 'vendor'], ['a non-string', 42], ['undefined', undefined], + [ + 'more than 32 members, which the list grammar does not admit', + Array.from({ length: 33 }, (_v, i) => `k${i}=v`).join(','), + ], ['a single member longer than the whole limit', `vendor=${'a'.repeat(600)}`], ])('discards %s', (_name, value) => { expect(sanitizeTracestate(value)).toBeUndefined() }) - it('keeps the first 32 members of a longer list', () => { - const members = Array.from({ length: 33 }, (_v, i) => `k${i}=v`) - expect(sanitizeTracestate(members.join(','))).toBe(members.slice(0, 32).join(',')) + it('drops the largest members first when a valid list is too long', () => { + // W3C names members over 128 characters as the ones to drop first, so the + // small entries survive even though they sit to the right of the big one. + const big = `big=${'a'.repeat(200)}` + const small = Array.from({ length: 4 }, (_v, i) => `k${i}=${'b'.repeat(80)}`) + + expect(sanitizeTracestate([big, ...small].join(','))).toBe(small.join(',')) }) - it('keeps the members that fit inside the length limit', () => { + it('drops from the right once no member is oversized', () => { // 102 characters each, so the fifth crosses 512 and the first four stay. const members = Array.from({ length: 6 }, (_v, i) => `k${i}=${'a'.repeat(100)}`) expect(sanitizeTracestate(members.join(','))).toBe(members.slice(0, 4).join(',')) }) + + it('leaves a valid header inside the limit exactly as received', () => { + const members = Array.from({ length: 32 }, (_v, i) => `k${i}=v`) + expect(sanitizeTracestate(members.join(','))).toBe(members.join(',')) + }) }) }) diff --git a/packages/core/src/traces/traceparent.ts b/packages/core/src/traces/traceparent.ts index cee0e310da..8a615de026 100644 --- a/packages/core/src/traces/traceparent.ts +++ b/packages/core/src/traces/traceparent.ts @@ -113,11 +113,13 @@ const TRACESTATE_MAX_LENGTH = 512 /** * Validates an incoming `tracestate` far enough to know it is safe to echo back. * A malformed one is discarded without invalidating its traceparent, so a bad - * vendor entry never costs us the trace continuation. + * vendor entry never costs us the trace continuation. More than 32 members is + * malformed: W3C's list grammar admits no more. * - * Over-long input is trimmed rather than discarded: W3C makes the limits a - * reason to drop members from the end, so the entries that fit are still valid - * state the next hop can use. + * A valid header over the length W3C asks us to propagate is trimmed instead, + * by whole members. Members over 128 characters go first — W3C names those as + * the ones to drop — and the rest from the right, so the entries nearest the + * caller survive. */ export function sanitizeTracestate(value: unknown): string | undefined { if (typeof value !== 'string') { @@ -134,6 +136,9 @@ export function sanitizeTracestate(value: unknown): string | undefined { return undefined } const members = trimmed.split(',') + if (members.length > TRACESTATE_MAX_MEMBERS) { + return undefined + } for (const member of members) { // An empty member is tolerated by the spec (list optional-white-space), but // a member without a `=` is not a key/value pair at all. @@ -141,15 +146,27 @@ export function sanitizeTracestate(value: unknown): string | undefined { return undefined } } - const kept: string[] = [] - let length = 0 - for (const member of members.slice(0, TRACESTATE_MAX_MEMBERS)) { - const separator = kept.length ? 1 : 0 - if (length + separator + member.length > TRACESTATE_MAX_LENGTH) { - break + if (trimmed.length <= TRACESTATE_MAX_LENGTH) { + return trimmed + } + return trimToLength(members) +} + +// W3C's own guidance for which members to drop when a list is too long. +const TRACESTATE_LARGE_MEMBER_LENGTH = 128 + +/** The members that fit, dropping the largest first and then from the right. */ +function trimToLength(members: string[]): string | undefined { + const kept = [...members] + const joinedLength = (): number => kept.reduce((total, member) => total + member.length, 0) + kept.length - 1 + + for (let index = kept.length - 1; index >= 0 && joinedLength() > TRACESTATE_MAX_LENGTH; index--) { + if (kept[index].length > TRACESTATE_LARGE_MEMBER_LENGTH) { + kept.splice(index, 1) } - kept.push(member) - length += separator + member.length + } + while (kept.length && joinedLength() > TRACESTATE_MAX_LENGTH) { + kept.pop() } return kept.length ? kept.join(',') : undefined } diff --git a/packages/node/src/__tests__/traces.spec.ts b/packages/node/src/__tests__/traces.spec.ts index 964bc075ab..a27b233b34 100644 --- a/packages/node/src/__tests__/traces.spec.ts +++ b/packages/node/src/__tests__/traces.spec.ts @@ -104,6 +104,24 @@ describe('PostHog traces', () => { expect(propagated).toBe(inbound) expect(untraced.getActiveSpan()).toBeNull() }) + + it('gives a nested span the inbound context, across an await', async () => { + const untraced = createClient({ traces: undefined }) + const inbound = `00-${'4bf92f3577b34da6a3ce929d0e0e4736'}-00f067aa0ba902b7-01` + + const propagated = await untraced.withSpan('outer', { parent: inbound }, async () => { + await Promise.resolve() + return untraced.withSpan('inner', async (span) => { + await Promise.resolve() + return span.traceparent() + }) + }) + await untraced.shutdown() + + // The handle the callback is given, not just `getActiveSpan()`: a nested + // span that names no parent must still carry the trace onward. + expect(propagated).toBe(inbound) + }) }) describe('transport', () => { diff --git a/packages/types/src/traces.ts b/packages/types/src/traces.ts index b0f93bc82f..eaf84e2309 100644 --- a/packages/types/src/traces.ts +++ b/packages/types/src/traces.ts @@ -76,12 +76,9 @@ export interface StartSpanOptions { * Parent of this span: either a span handle, or a raw W3C `traceparent` * string to continue a trace started by another service. * - * When omitted the parent is the currently active span, or none. A recording - * handle from this SDK parents normally, and an inert one yields an inert - * span carrying the same context. A span from another tracer, which reports - * its context through `spanContext()` rather than `traceparent()`, is - * ignored: the span parents to the active span or starts a new trace. Pass - * that tracer's `traceparent` string to continue its trace. + * When omitted the parent is the currently active span, or none. Only + * handles returned by this SDK are honoured; any other `Span` yields an + * inert span. * * @example Continue an inbound trace * ```ts From 43425f86d9b9d8f66e17aae0714159e69edd536f Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 13:34:22 -0400 Subject: [PATCH 24/24] docs(changeset): name the span hook and per-span caps that ship with tracing #4584 folded into this branch, so `beforeSpanSend` and the caps go out in the same minor as `startSpan`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RWoFx4BctNmXnpnKS79Qgz --- .changeset/node-distributed-tracing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/node-distributed-tracing.md b/.changeset/node-distributed-tracing.md index d2e9393b7d..8d0d1ff0fb 100644 --- a/.changeset/node-distributed-tracing.md +++ b/.changeset/node-distributed-tracing.md @@ -4,4 +4,4 @@ '@posthog/types': minor --- -Add experimental distributed tracing to `posthog-node`: `startSpan`, `withSpan` and `getActiveSpan` record spans against a new `traces` client option. A service with tracing off still forwards an inbound `traceparent`, including from spans nested inside the one that received it, so a distributed trace is not severed. A `traceparent` may be passed as the one-element array `req.headersDistinct` gives. +Add experimental distributed tracing to `posthog-node`: `startSpan`, `withSpan` and `getActiveSpan` record spans against a new `traces` client option. A service with tracing off still forwards an inbound `traceparent`, including from spans nested inside the one that received it, so a distributed trace is not severed. A `traceparent` may be passed as the one-element array `req.headersDistinct` gives. A `beforeSpanSend` hook sees every span before it is exported and may edit or drop it, and `maxAttributesPerSpan`, `maxEventsPerSpan`, `maxAttributeValueLength`, `maxLiveSpans` and `maxSpanAgeMs` bound what a single span and a single process may hold.