diff --git a/.changeset/otlp-bigint-int64.md b/.changeset/otlp-bigint-int64.md new file mode 100644 index 0000000000..0afa2d4ca0 --- /dev/null +++ b/.changeset/otlp-bigint-int64.md @@ -0,0 +1,8 @@ +--- +'@posthog/core': patch +'posthog-js': patch +'posthog-react-native': patch +'posthog-node': patch +--- + +Change `bigint` attributes on logs, metrics and spans to send as an int64 rather than as a string. diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 66e7b09420..8219875000 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -42,9 +42,8 @@ export { buildResourceAttributes, getOtlpSeverityNumber, getOtlpSeverityText, - toOtlpAnyValue, - toOtlpKeyValueList, } from './logs/logs-utils' +export { toOtlpAnyValue, toOtlpKeyValueList } from './utils/otlp-any-value' export { PostHogLogs } from './logs' export type { BeforeSendLogFn, diff --git a/packages/core/src/logs/logs-utils.spec.ts b/packages/core/src/logs/logs-utils.spec.ts index b3072d42a5..95dff8a02b 100644 --- a/packages/core/src/logs/logs-utils.spec.ts +++ b/packages/core/src/logs/logs-utils.spec.ts @@ -1,13 +1,6 @@ -import type { CaptureLogOptions, LogAttributeValue, LogSeverityLevel } from '@posthog/types' +import type { CaptureLogOptions, LogSeverityLevel } from '@posthog/types' import type { LogSdkContext } from './types' -import { - buildOtlpLogRecord, - buildOtlpLogsPayload, - getOtlpSeverityNumber, - getOtlpSeverityText, - toOtlpAnyValue, - toOtlpKeyValueList, -} from './logs-utils' +import { buildOtlpLogRecord, buildOtlpLogsPayload, getOtlpSeverityNumber, getOtlpSeverityText } from './logs-utils' const browserSdkContext: LogSdkContext = { distinctId: 'user-123', @@ -64,317 +57,6 @@ describe('logs-utils', () => { }) }) - describe('toOtlpAnyValue', () => { - it('converts strings', () => { - expect(toOtlpAnyValue('hello')).toEqual({ stringValue: 'hello' }) - }) - - it('converts integers to decimal strings', () => { - expect(toOtlpAnyValue(42)).toEqual({ intValue: '42' }) - expect(toOtlpAnyValue(0)).toEqual({ intValue: '0' }) - expect(toOtlpAnyValue(-7)).toEqual({ intValue: '-7' }) - }) - - // Spec: outside int64 it is a stringValue, never an intValue. - it('converts integers outside int64 to stringValue', () => { - expect(toOtlpAnyValue(2 ** 63)).toEqual({ stringValue: '9223372036854775808' }) - expect(toOtlpAnyValue(-(2 ** 64))).toEqual({ stringValue: '-18446744073709551616' }) - expect(toOtlpAnyValue(1e21)).toEqual({ stringValue: '1000000000000000000000' }) - }) - - it('logs a debug line when an integer falls outside int64', () => { - const logger = { debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn(), critical: jest.fn() } - toOtlpAnyValue(2 ** 63, logger as any) - expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('outside the int64 range')) - }) - - it('keeps int64 min as intValue', () => { - // In range, but `String` renders it 192 below int64 min, so the decimal - // has to come from BigInt. - expect(toOtlpAnyValue(-(2 ** 63))).toEqual({ intValue: '-9223372036854775808' }) - }) - - it('keeps large in-range integers exact', () => { - expect(toOtlpAnyValue(Number.MAX_SAFE_INTEGER)).toEqual({ intValue: '9007199254740991' }) - // The largest double below 2^63 — no double exists between the two. - expect(toOtlpAnyValue(9223372036854774784)).toEqual({ intValue: '9223372036854774784' }) - expect(toOtlpAnyValue(2 ** 62)).toEqual({ intValue: '4611686018427387904' }) - }) - - it('converts floats to doubleValue', () => { - expect(toOtlpAnyValue(3.14)).toEqual({ doubleValue: 3.14 }) - }) - - it('converts booleans', () => { - expect(toOtlpAnyValue(true)).toEqual({ boolValue: true }) - expect(toOtlpAnyValue(false)).toEqual({ boolValue: false }) - }) - - // JSON has no representation for non-finite floats; without explicit - // handling, JSON.stringify silently turns them into `null` and the value - // is lost server-side. - it('converts NaN to stringValue', () => { - expect(toOtlpAnyValue(NaN)).toEqual({ stringValue: 'NaN' }) - }) - - it('converts +Infinity to stringValue', () => { - expect(toOtlpAnyValue(Infinity)).toEqual({ stringValue: 'Infinity' }) - }) - - it('converts -Infinity to stringValue', () => { - expect(toOtlpAnyValue(-Infinity)).toEqual({ stringValue: '-Infinity' }) - }) - - it('converts arrays of strings to arrayValue', () => { - expect(toOtlpAnyValue(['a', 'b'])).toEqual({ - arrayValue: { values: [{ stringValue: 'a' }, { stringValue: 'b' }] }, - }) - }) - - it('converts mixed primitive arrays recursively', () => { - expect(toOtlpAnyValue([1, 'x', true])).toEqual({ - arrayValue: { - values: [{ intValue: '1' }, { stringValue: 'x' }, { boolValue: true }], - }, - }) - }) - - it('converts plain objects to kvlistValue', () => { - expect(toOtlpAnyValue({ a: 1, b: 'two' })).toEqual({ - kvlistValue: { - values: [ - { key: 'a', value: { intValue: '1' } }, - { key: 'b', value: { stringValue: 'two' } }, - ], - }, - }) - }) - - it('converts nested objects recursively', () => { - expect(toOtlpAnyValue({ outer: { inner: 1 } })).toEqual({ - kvlistValue: { - values: [ - { - key: 'outer', - value: { kvlistValue: { values: [{ key: 'inner', value: { intValue: '1' } }] } }, - }, - ], - }, - }) - }) - - it('drops null and undefined keys inside objects', () => { - expect(toOtlpAnyValue({ kept: 1, gone: null, alsoGone: undefined })).toEqual({ - kvlistValue: { values: [{ key: 'kept', value: { intValue: '1' } }] }, - }) - }) - - // Not in LogAttributeValue, but reachable at runtime from untyped callers. - it('encodes Dates as ISO strings', () => { - expect(toOtlpAnyValue(new Date('2026-08-20T10:00:00.000Z') as unknown as LogAttributeValue)).toEqual({ - stringValue: '2026-08-20T10:00:00.000Z', - }) - }) - - it('marks circular references instead of recursing', () => { - const cyclic: Record = { name: 'root' } - cyclic.self = cyclic - expect(toOtlpAnyValue(cyclic)).toEqual({ - kvlistValue: { - values: [ - { key: 'name', value: { stringValue: 'root' } }, - { key: 'self', value: { stringValue: '[Circular]' } }, - ], - }, - }) - }) - - // An escaping error would surface in the caller's application code. - it('does not throw on an object nested past the depth cap', () => { - let deep: Record = { end: true } - for (let i = 0; i < 25000; i++) { - deep = { next: deep } - } - expect(() => toOtlpAnyValue(deep)).not.toThrow() - }) - - it('truncates at exactly 20 levels instead of recursing', () => { - let deep: Record = { end: true } - for (let i = 0; i < 25; i++) { - deep = { next: deep } - } - const encoded = JSON.stringify(toOtlpAnyValue(deep)) - expect(encoded).toContain('[Truncated]') - expect(encoded.split('"next"').length - 1).toBe(20) - }) - - it('marks a throwing getter without losing the rest of the object', () => { - const attrs = { - ok: 1, - get bad(): number { - throw new Error('getter blew up') - }, - } - expect(() => toOtlpKeyValueList(attrs)).not.toThrow() - expect(toOtlpKeyValueList(attrs)).toEqual([ - { key: 'ok', value: { intValue: '1' } }, - { key: 'bad', value: { stringValue: '[Unserializable]' } }, - ]) - }) - - // for...in walks the prototype chain once own keys are exhausted. - it('ignores inherited enumerable properties', () => { - const inherited: Record = Object.create({ fromPrototype: 'leaked' }) - inherited.own = 1 - expect(toOtlpAnyValue(inherited)).toEqual({ - kvlistValue: { values: [{ key: 'own', value: { intValue: '1' } }] }, - }) - }) - - // `String(fn)` would put the function's source text on the wire. - it('marks function and symbol values instead of stringifying them', () => { - expect(toOtlpAnyValue({ handler: () => 1, retries: 2 } as unknown as LogAttributeValue)).toEqual({ - kvlistValue: { - values: [ - { key: 'handler', value: { stringValue: '[Function]' } }, - { key: 'retries', value: { intValue: '2' } }, - ], - }, - }) - expect(toOtlpAnyValue({ sym: Symbol('x') } as unknown as LogAttributeValue)).toEqual({ - kvlistValue: { values: [{ key: 'sym', value: { stringValue: 'Symbol(x)' } }] }, - }) - }) - - // dayjs, Decimal, ORM documents. - it('honours toJSON', () => { - const wrapped = { toJSON: () => ({ amount: 5 }) } - expect(toOtlpAnyValue(wrapped as unknown as LogAttributeValue)).toEqual({ - kvlistValue: { values: [{ key: 'amount', value: { intValue: '5' } }] }, - }) - }) - - it('falls back to the plain walk when toJSON throws', () => { - const wrapped = { - kept: 1, - toJSON: () => { - throw new Error('nope') - }, - } - expect(toOtlpAnyValue(wrapped as unknown as LogAttributeValue)).toEqual({ - kvlistValue: { - values: [ - { key: 'kept', value: { intValue: '1' } }, - { key: 'toJSON', value: { stringValue: '[Function]' } }, - ], - }, - }) - }) - - // A toJSON returning its own object is a cycle like any other. - it('marks a cycle that runs through toJSON', () => { - const cyclic: Record = {} - cyclic.toJSON = () => ({ inner: cyclic }) - expect(toOtlpAnyValue(cyclic)).toEqual({ - kvlistValue: { values: [{ key: 'inner', value: { stringValue: '[Circular]' } }] }, - }) - }) - - // Both `null` and `{}` here are rejected for the whole request; iOS and - // Android drop them too. - it('drops holes and nullish elements from arrays', () => { - // eslint-disable-next-line no-sparse-arrays - expect(toOtlpAnyValue([1, , 3])).toEqual({ - arrayValue: { values: [{ intValue: '1' }, { intValue: '3' }] }, - }) - expect(toOtlpAnyValue([1, null, undefined, 3])).toEqual({ - arrayValue: { values: [{ intValue: '1' }, { intValue: '3' }] }, - }) - }) - - it('stops encoding array items once the node budget is spent', () => { - const row: Record = {} - for (let i = 0; i < 20; i++) { - row[`k${i}`] = i - } - const wide = Array.from({ length: 1000 }, () => ({ ...row })) - const values = toOtlpAnyValue(wide).arrayValue!.values - expect(values[values.length - 1]).toEqual({ stringValue: '[Truncated]' }) - // One marker, not one per unencodable item. - expect(values.filter((v) => v.stringValue === '[Truncated]')).toHaveLength(1) - }) - - it('caps a shared object graph instead of expanding it', () => { - let graph: Record = { leaf: true } - for (let i = 0; i < 20; i++) { - graph = { a: graph, b: graph } - } - const encoded = JSON.stringify(toOtlpAnyValue(graph)) - expect(encoded).toContain('[Truncated]') - expect(encoded.length).toBeLessThan(1_000_000) - }) - - it('caps a very wide object without inventing an attribute key', () => { - const wide: Record = {} - for (let i = 0; i < 5000; i++) { - wide[`k${i}`] = i - } - const logger = { debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn(), critical: jest.fn() } - const values = toOtlpAnyValue(wide, logger as any).kvlistValue!.values - expect(values).toHaveLength(1000) - expect(values.every((v) => v.key.startsWith('k'))).toBe(true) - expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('truncated')) - }) - - // Why the encoder does not delegate to toJsonSafeValue: that maps them to null. - it('keeps non-finite floats as strings inside nested objects', () => { - expect(toOtlpAnyValue({ nested: { ratio: NaN } })).toEqual({ - kvlistValue: { - values: [ - { - key: 'nested', - value: { kvlistValue: { values: [{ key: 'ratio', value: { stringValue: 'NaN' } }] } }, - }, - ], - }, - }) - }) - - // A lone surrogate survives JSON.stringify as a \uD800 escape, which the - // server rejects for the whole request. - it('replaces unpaired surrogates in values and keys', () => { - expect(toOtlpAnyValue('ok\ud83d')).toEqual({ stringValue: 'ok\ufffd' }) - expect(toOtlpAnyValue({ nested: 'ok\ud83d' })).toEqual({ - kvlistValue: { values: [{ key: 'nested', value: { stringValue: 'ok\ufffd' } }] }, - }) - expect(toOtlpKeyValueList({ 'key\ud83d': 1 })).toEqual([{ key: 'key\ufffd', value: { intValue: '1' } }]) - }) - - it('encodes empty containers with an explicit values array', () => { - expect(toOtlpAnyValue({})).toEqual({ kvlistValue: { values: [] } }) - expect(toOtlpAnyValue([])).toEqual({ arrayValue: { values: [] } }) - }) - - it('keeps a Date whose toISOString is overridden out of the wire format', () => { - const broken = new Date('2026-08-20T10:00:00.000Z') - - ;(broken as any).toISOString = () => ({}) - expect(typeof toOtlpAnyValue(broken as unknown as LogAttributeValue).stringValue).toBe('string') - }) - - it('encodes sibling references to one object twice, not as circular', () => { - const shared = { id: 1 } - expect(toOtlpAnyValue({ a: shared, b: shared })).toEqual({ - kvlistValue: { - values: [ - { key: 'a', value: { kvlistValue: { values: [{ key: 'id', value: { intValue: '1' } }] } } }, - { key: 'b', value: { kvlistValue: { values: [{ key: 'id', value: { intValue: '1' } }] } } }, - ], - }, - }) - }) - }) - describe('buildOtlpLogRecord attribute reads', () => { // Reading `options.attributes` happens before the encoder's per-key guard. it('marks an attribute whose getter throws without dropping the record', () => { @@ -425,36 +107,6 @@ describe('logs-utils', () => { }) }) - describe('toOtlpKeyValueList', () => { - it('converts a record to key-value list', () => { - expect( - toOtlpKeyValueList({ - name: 'test', - count: 5, - active: true, - }) - ).toEqual([ - { key: 'name', value: { stringValue: 'test' } }, - { key: 'count', value: { intValue: '5' } }, - { key: 'active', value: { boolValue: true } }, - ]) - }) - - it('handles empty record', () => { - expect(toOtlpKeyValueList({})).toEqual([]) - }) - - it('skips null and undefined values', () => { - expect( - toOtlpKeyValueList({ - kept: 'yes', - nullish: null, - missing: undefined, - }) - ).toEqual([{ key: 'kept', value: { stringValue: 'yes' } }]) - }) - }) - describe('buildOtlpLogRecord', () => { it('builds a minimal log record', () => { const record = buildOtlpLogRecord({ body: 'hello world' }, minimalSdkContext) diff --git a/packages/core/src/logs/logs-utils.ts b/packages/core/src/logs/logs-utils.ts index 589b9bc993..9bbac89505 100644 --- a/packages/core/src/logs/logs-utils.ts +++ b/packages/core/src/logs/logs-utils.ts @@ -2,8 +2,6 @@ import type { CaptureLogOptions, LogAttributeValue, LogSeverityLevel, - OtlpAnyValue, - OtlpKeyValue, OtlpLogRecord, OtlpLogsPayload, OtlpSeverityEntry, @@ -11,17 +9,9 @@ import type { } from '@posthog/types' import type { Logger } from '../types' import type { LogSdkContext, ResolvedPostHogLogsConfig } from './types' -import { isArray, isBoolean, isNull, isNullish, isNumber, isUndefined } from '../utils' -import { - CIRCULAR_VALUE, - FUNCTION_VALUE, - MAX_JSON_SAFE_VALUE_DEPTH, - MAX_JSON_SAFE_VALUE_ITEMS, - MAX_JSON_SAFE_VALUE_NODES, - sanitizeString, - TRUNCATED_VALUE, - UNSERIALIZABLE_VALUE, -} from '../utils/json-utils' +import { isNullish, isNumber, isUndefined } from '../utils' +import { sanitizeString, UNSERIALIZABLE_VALUE } from '../utils/json-utils' +import { toOtlpKeyValueList } from '../utils/otlp-any-value' // ============================================================================ // Severity mapping @@ -46,200 +36,6 @@ export function getOtlpSeverityNumber(level: LogSeverityLevel): number { return (OTLP_SEVERITY_MAP[level] || DEFAULT_OTLP_SEVERITY).number } -// ============================================================================ -// OTLP AnyValue conversion -// ============================================================================ - -// 2^63 — one past int64 max. -const INT64_RANGE_LIMIT = 9223372036854775808 - -const propertyIsEnumerable = Object.prototype.propertyIsEnumerable - -interface EncodeState { - /** Containers on the current path, so a back-reference becomes a marker. */ - ancestors: WeakSet - remainingNodes: number -} - -function newState(): EncodeState { - return { ancestors: new WeakSet(), remainingNodes: MAX_JSON_SAFE_VALUE_NODES } -} - -export function toOtlpAnyValue(value: LogAttributeValue, logger?: Logger): OtlpAnyValue { - try { - return encodeAnyValue(value, logger, newState(), 0) - } catch { - // Runs inside `captureLog` and the metrics flush: an error escaping here - // surfaces in the caller's own code. - return { stringValue: UNSERIALIZABLE_VALUE } - } -} - -export function toOtlpKeyValueList(attrs: Record, logger?: Logger): OtlpKeyValue[] { - try { - return encodeKeyValueList(attrs, logger, newState(), 0) - } catch { - return [] - } -} - -function encodeAnyValue( - value: LogAttributeValue, - logger: Logger | undefined, - state: EncodeState, - depth: number -): OtlpAnyValue { - if (state.remainingNodes <= 0) { - return { stringValue: TRUNCATED_VALUE } - } - state.remainingNodes-- - - if (isBoolean(value)) { - return { boolValue: value } - } - // typeof, not core's isNumber, which excludes NaN — proto3 JSON distinguishes - // a non-finite float from an ordinary string. - if (typeof value === 'number') { - if (!Number.isFinite(value)) { - return { stringValue: String(value) } - } - if (Number.isInteger(value)) { - if (Number.isSafeInteger(value)) { - return { intValue: String(value) } - } - // Past MAX_SAFE_INTEGER only BigInt gives the double's exact decimal: - // `String(-(2**63))` lands 192 below int64 min, outside the field it is - // about to be parsed into. Without BigInt the value rides as a string, - // which is never range-checked. - if (typeof BigInt === 'undefined') { - return { stringValue: String(value) } - } - const decimal = BigInt(value).toString() - if (value >= INT64_RANGE_LIMIT || value < -INT64_RANGE_LIMIT) { - // An out-of-range intValue 400s the whole logs request; on the metrics - // path it is swallowed server-side and the metric just disappears. - logger?.debug(`Attribute ${decimal} is outside the int64 range; encoding it as a string`) - return { stringValue: decimal } - } - return { intValue: decimal } - } - return { doubleValue: value } - } - if (typeof value === 'string') { - return { stringValue: sanitizeString(value) } - } - // `String(value)` would put a function's source text on the wire. - if (typeof value === 'function') { - return { stringValue: FUNCTION_VALUE } - } - if (typeof value === 'symbol') { - return { stringValue: String(value) } - } - if (typeof value === 'object' && value !== null) { - if (state.ancestors.has(value)) { - return { stringValue: CIRCULAR_VALUE } - } - if (depth >= MAX_JSON_SAFE_VALUE_DEPTH) { - return { stringValue: TRUNCATED_VALUE } - } - if (value instanceof Date) { - const time = value.getTime() - const iso = Number.isFinite(time) ? value.toISOString() : String(value) - // An overridden toISOString can return a non-string, which the server - // refuses for the whole request. - return { stringValue: typeof iso === 'string' ? sanitizeString(iso) : String(iso) } - } - // Registered before the toJSON probe: a toJSON returning a structure that - // references its own object is a cycle like any other. - state.ancestors.add(value) - try { - // The representation a value defines for itself — dayjs, Decimal, an ORM - // document, and a cross-realm Date that fails the `instanceof` above. - try { - const toJSON = (value as { toJSON?: unknown }).toJSON - if (typeof toJSON === 'function') { - return encodeAnyValue(toJSON.call(value) as LogAttributeValue, logger, state, depth + 1) - } - } catch { - // A throwing toJSON falls through to the plain walk. - } - if (isArray(value)) { - return { arrayValue: { values: encodeArrayValues(value, logger, state, depth + 1) } } - } - return { - kvlistValue: { - values: encodeKeyValueList(value as Record, logger, state, depth + 1), - }, - } - } finally { - // Siblings that reference the same object are duplication, not a cycle. - state.ancestors.delete(value) - } - } - return { stringValue: sanitizeString(String(value)) } -} - -function encodeArrayValues( - values: unknown[], - logger: Logger | undefined, - state: EncodeState, - depth: number -): OtlpAnyValue[] { - const result: OtlpAnyValue[] = [] - const itemCount = Math.min(values.length, MAX_JSON_SAFE_VALUE_ITEMS) - let index = 0 - for (; index < itemCount && state.remainingNodes > 0; index++) { - try { - const element = index in values ? values[index] : undefined - // Dropped, as iOS and Android do: proto3 JSON has no null AnyValue, and - // both `null` and `{}` here are rejected for the whole request. - if (isNullish(element)) { - continue - } - result.push(encodeAnyValue(element as LogAttributeValue, logger, state, depth)) - } catch { - result.push({ stringValue: UNSERIALIZABLE_VALUE }) - } - } - if (values.length > index) { - result.push({ stringValue: TRUNCATED_VALUE }) - } - return result -} - -function encodeKeyValueList( - attrs: Record, - logger: Logger | undefined, - state: EncodeState, - depth: number -): OtlpKeyValue[] { - const result: OtlpKeyValue[] = [] - for (const key in attrs) { - // for...in walks the prototype chain once own keys are exhausted. Skipped - // rather than broken out of: a proxy can yield keys in any order. - if (!propertyIsEnumerable.call(attrs, key)) { - continue - } - if (result.length >= MAX_JSON_SAFE_VALUE_ITEMS || state.remainingNodes <= 0) { - // Reported rather than written into the attributes: a synthetic key would - // land in the user's own namespace and could collide with a real one. - logger?.debug('Attributes truncated: the value exceeds the OTLP encoder budget') - break - } - try { - const value = attrs[key] - if (isNull(value) || isUndefined(value)) { - continue - } - result.push({ key: sanitizeString(key), value: encodeAnyValue(value, logger, state, depth) }) - } catch { - // A getter that throws costs its own key, not the whole record. - result.push({ key: sanitizeString(key), value: { stringValue: UNSERIALIZABLE_VALUE } }) - } - } - return result -} - // ============================================================================ // OTLP LogRecord construction // ============================================================================ diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index d88c595dc2..cb86a0336e 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -10,7 +10,7 @@ import type { } from '@posthog/types' import type { Logger } from '../types' import { isArray, safeSetTimeout } from '../utils' -import { toOtlpKeyValueList } from '../logs/logs-utils' +import { toOtlpKeyValueList } from '../utils/otlp-any-value' import { DEFAULT_HISTOGRAM_BOUNDS, bucketIndexFor, diff --git a/packages/core/src/metrics/metrics-utils.ts b/packages/core/src/metrics/metrics-utils.ts index 197455e7c0..4fb534bbba 100644 --- a/packages/core/src/metrics/metrics-utils.ts +++ b/packages/core/src/metrics/metrics-utils.ts @@ -1,5 +1,5 @@ import type { MetricAttributeValue, OtlpMetric, OtlpMetricsPayload } from '@posthog/types' -import { toOtlpKeyValueList } from '../logs/logs-utils' +import { toOtlpKeyValueList } from '../utils/otlp-any-value' import type { ResolvedPostHogMetricsConfig } from './types' /** diff --git a/packages/core/src/utils/otlp-any-value.spec.ts b/packages/core/src/utils/otlp-any-value.spec.ts new file mode 100644 index 0000000000..319ddc3be4 --- /dev/null +++ b/packages/core/src/utils/otlp-any-value.spec.ts @@ -0,0 +1,361 @@ +import type { LogAttributeValue } from '@posthog/types' +import { toOtlpAnyValue, toOtlpKeyValueList } from './otlp-any-value' + +describe('otlp-any-value', () => { + describe('toOtlpAnyValue', () => { + it('converts strings', () => { + expect(toOtlpAnyValue('hello')).toEqual({ stringValue: 'hello' }) + }) + + it('converts integers to decimal strings', () => { + expect(toOtlpAnyValue(42)).toEqual({ intValue: '42' }) + expect(toOtlpAnyValue(0)).toEqual({ intValue: '0' }) + expect(toOtlpAnyValue(-7)).toEqual({ intValue: '-7' }) + }) + + // Spec: outside int64 it is a stringValue, never an intValue. + it('converts integers outside int64 to stringValue', () => { + expect(toOtlpAnyValue(2 ** 63)).toEqual({ stringValue: '9223372036854775808' }) + expect(toOtlpAnyValue(-(2 ** 64))).toEqual({ stringValue: '-18446744073709551616' }) + expect(toOtlpAnyValue(1e21)).toEqual({ stringValue: '1000000000000000000000' }) + }) + + it('logs a debug line when an integer falls outside int64', () => { + const logger = { debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn(), critical: jest.fn() } + toOtlpAnyValue(2 ** 63, logger as any) + expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('outside the int64 range')) + }) + + it('keeps int64 min as intValue', () => { + // In range, but `String` renders it 192 below int64 min, so the decimal + // has to come from BigInt. + expect(toOtlpAnyValue(-(2 ** 63))).toEqual({ intValue: '-9223372036854775808' }) + }) + + it('keeps large in-range integers exact', () => { + expect(toOtlpAnyValue(Number.MAX_SAFE_INTEGER)).toEqual({ intValue: '9007199254740991' }) + // The largest double below 2^63 — no double exists between the two. + expect(toOtlpAnyValue(9223372036854774784)).toEqual({ intValue: '9223372036854774784' }) + expect(toOtlpAnyValue(2 ** 62)).toEqual({ intValue: '4611686018427387904' }) + }) + + it('converts a bigint inside int64 to a stringified intValue', () => { + // Span attributes accept bigint; a log attribute reaching here is typed + // out but still encodes correctly. + expect(toOtlpAnyValue(9007199254740993n as unknown as LogAttributeValue)).toEqual({ + intValue: '9007199254740993', + }) + }) + + it('converts a bigint beyond int64 to a string, with a warning', () => { + const logger = { debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn(), critical: jest.fn() } + expect(toOtlpAnyValue(18446744073709551616n as unknown as LogAttributeValue, logger as any)).toEqual({ + stringValue: '18446744073709551616', + }) + expect(logger.debug).toHaveBeenCalled() + }) + + it('converts floats to doubleValue', () => { + expect(toOtlpAnyValue(3.14)).toEqual({ doubleValue: 3.14 }) + }) + + it('converts booleans', () => { + expect(toOtlpAnyValue(true)).toEqual({ boolValue: true }) + expect(toOtlpAnyValue(false)).toEqual({ boolValue: false }) + }) + + // JSON has no representation for non-finite floats; without explicit + // handling, JSON.stringify silently turns them into `null` and the value + // is lost server-side. + it('converts NaN to stringValue', () => { + expect(toOtlpAnyValue(NaN)).toEqual({ stringValue: 'NaN' }) + }) + + it('converts +Infinity to stringValue', () => { + expect(toOtlpAnyValue(Infinity)).toEqual({ stringValue: 'Infinity' }) + }) + + it('converts -Infinity to stringValue', () => { + expect(toOtlpAnyValue(-Infinity)).toEqual({ stringValue: '-Infinity' }) + }) + + it('converts arrays of strings to arrayValue', () => { + expect(toOtlpAnyValue(['a', 'b'])).toEqual({ + arrayValue: { values: [{ stringValue: 'a' }, { stringValue: 'b' }] }, + }) + }) + + it('converts mixed primitive arrays recursively', () => { + expect(toOtlpAnyValue([1, 'x', true])).toEqual({ + arrayValue: { + values: [{ intValue: '1' }, { stringValue: 'x' }, { boolValue: true }], + }, + }) + }) + + it('converts plain objects to kvlistValue', () => { + expect(toOtlpAnyValue({ a: 1, b: 'two' })).toEqual({ + kvlistValue: { + values: [ + { key: 'a', value: { intValue: '1' } }, + { key: 'b', value: { stringValue: 'two' } }, + ], + }, + }) + }) + + it('converts nested objects recursively', () => { + expect(toOtlpAnyValue({ outer: { inner: 1 } })).toEqual({ + kvlistValue: { + values: [ + { + key: 'outer', + value: { kvlistValue: { values: [{ key: 'inner', value: { intValue: '1' } }] } }, + }, + ], + }, + }) + }) + + it('drops null and undefined keys inside objects', () => { + expect(toOtlpAnyValue({ kept: 1, gone: null, alsoGone: undefined })).toEqual({ + kvlistValue: { values: [{ key: 'kept', value: { intValue: '1' } }] }, + }) + }) + + // Not in LogAttributeValue, but reachable at runtime from untyped callers. + it('encodes Dates as ISO strings', () => { + expect(toOtlpAnyValue(new Date('2026-08-20T10:00:00.000Z') as unknown as LogAttributeValue)).toEqual({ + stringValue: '2026-08-20T10:00:00.000Z', + }) + }) + + it('marks circular references instead of recursing', () => { + const cyclic: Record = { name: 'root' } + cyclic.self = cyclic + expect(toOtlpAnyValue(cyclic)).toEqual({ + kvlistValue: { + values: [ + { key: 'name', value: { stringValue: 'root' } }, + { key: 'self', value: { stringValue: '[Circular]' } }, + ], + }, + }) + }) + + // An escaping error would surface in the caller's application code. + it('does not throw on an object nested past the depth cap', () => { + let deep: Record = { end: true } + for (let i = 0; i < 25000; i++) { + deep = { next: deep } + } + expect(() => toOtlpAnyValue(deep)).not.toThrow() + }) + + it('truncates at exactly 20 levels instead of recursing', () => { + let deep: Record = { end: true } + for (let i = 0; i < 25; i++) { + deep = { next: deep } + } + const encoded = JSON.stringify(toOtlpAnyValue(deep)) + expect(encoded).toContain('[Truncated]') + expect(encoded.split('"next"').length - 1).toBe(20) + }) + + it('marks a throwing getter without losing the rest of the object', () => { + const attrs = { + ok: 1, + get bad(): number { + throw new Error('getter blew up') + }, + } + expect(() => toOtlpKeyValueList(attrs)).not.toThrow() + expect(toOtlpKeyValueList(attrs)).toEqual([ + { key: 'ok', value: { intValue: '1' } }, + { key: 'bad', value: { stringValue: '[Unserializable]' } }, + ]) + }) + + // for...in walks the prototype chain once own keys are exhausted. + it('ignores inherited enumerable properties', () => { + const inherited: Record = Object.create({ fromPrototype: 'leaked' }) + inherited.own = 1 + expect(toOtlpAnyValue(inherited)).toEqual({ + kvlistValue: { values: [{ key: 'own', value: { intValue: '1' } }] }, + }) + }) + + // `String(fn)` would put the function's source text on the wire. + it('marks function and symbol values instead of stringifying them', () => { + expect(toOtlpAnyValue({ handler: () => 1, retries: 2 } as unknown as LogAttributeValue)).toEqual({ + kvlistValue: { + values: [ + { key: 'handler', value: { stringValue: '[Function]' } }, + { key: 'retries', value: { intValue: '2' } }, + ], + }, + }) + expect(toOtlpAnyValue({ sym: Symbol('x') } as unknown as LogAttributeValue)).toEqual({ + kvlistValue: { values: [{ key: 'sym', value: { stringValue: 'Symbol(x)' } }] }, + }) + }) + + // dayjs, Decimal, ORM documents. + it('honours toJSON', () => { + const wrapped = { toJSON: () => ({ amount: 5 }) } + expect(toOtlpAnyValue(wrapped as unknown as LogAttributeValue)).toEqual({ + kvlistValue: { values: [{ key: 'amount', value: { intValue: '5' } }] }, + }) + }) + + it('falls back to the plain walk when toJSON throws', () => { + const wrapped = { + kept: 1, + toJSON: () => { + throw new Error('nope') + }, + } + expect(toOtlpAnyValue(wrapped as unknown as LogAttributeValue)).toEqual({ + kvlistValue: { + values: [ + { key: 'kept', value: { intValue: '1' } }, + { key: 'toJSON', value: { stringValue: '[Function]' } }, + ], + }, + }) + }) + + // A toJSON returning its own object is a cycle like any other. + it('marks a cycle that runs through toJSON', () => { + const cyclic: Record = {} + cyclic.toJSON = () => ({ inner: cyclic }) + expect(toOtlpAnyValue(cyclic)).toEqual({ + kvlistValue: { values: [{ key: 'inner', value: { stringValue: '[Circular]' } }] }, + }) + }) + + // Both `null` and `{}` here are rejected for the whole request; iOS and + // Android drop them too. + it('drops holes and nullish elements from arrays', () => { + // eslint-disable-next-line no-sparse-arrays + expect(toOtlpAnyValue([1, , 3])).toEqual({ + arrayValue: { values: [{ intValue: '1' }, { intValue: '3' }] }, + }) + expect(toOtlpAnyValue([1, null, undefined, 3])).toEqual({ + arrayValue: { values: [{ intValue: '1' }, { intValue: '3' }] }, + }) + }) + + it('stops encoding array items once the node budget is spent', () => { + const row: Record = {} + for (let i = 0; i < 20; i++) { + row[`k${i}`] = i + } + const wide = Array.from({ length: 1000 }, () => ({ ...row })) + const values = toOtlpAnyValue(wide).arrayValue!.values + expect(values[values.length - 1]).toEqual({ stringValue: '[Truncated]' }) + // One marker, not one per unencodable item. + expect(values.filter((v) => v.stringValue === '[Truncated]')).toHaveLength(1) + }) + + it('caps a shared object graph instead of expanding it', () => { + let graph: Record = { leaf: true } + for (let i = 0; i < 20; i++) { + graph = { a: graph, b: graph } + } + const encoded = JSON.stringify(toOtlpAnyValue(graph)) + expect(encoded).toContain('[Truncated]') + expect(encoded.length).toBeLessThan(1_000_000) + }) + + it('caps a very wide object without inventing an attribute key', () => { + const wide: Record = {} + for (let i = 0; i < 5000; i++) { + wide[`k${i}`] = i + } + const logger = { debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn(), critical: jest.fn() } + const values = toOtlpAnyValue(wide, logger as any).kvlistValue!.values + expect(values).toHaveLength(1000) + expect(values.every((v) => v.key.startsWith('k'))).toBe(true) + expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining('truncated')) + }) + + // Why the encoder does not delegate to toJsonSafeValue: that maps them to null. + it('keeps non-finite floats as strings inside nested objects', () => { + expect(toOtlpAnyValue({ nested: { ratio: NaN } })).toEqual({ + kvlistValue: { + values: [ + { + key: 'nested', + value: { kvlistValue: { values: [{ key: 'ratio', value: { stringValue: 'NaN' } }] } }, + }, + ], + }, + }) + }) + + // A lone surrogate survives JSON.stringify as a \uD800 escape, which the + // server rejects for the whole request. + it('replaces unpaired surrogates in values and keys', () => { + expect(toOtlpAnyValue('ok\ud83d')).toEqual({ stringValue: 'ok\ufffd' }) + expect(toOtlpAnyValue({ nested: 'ok\ud83d' })).toEqual({ + kvlistValue: { values: [{ key: 'nested', value: { stringValue: 'ok\ufffd' } }] }, + }) + expect(toOtlpKeyValueList({ 'key\ud83d': 1 })).toEqual([{ key: 'key\ufffd', value: { intValue: '1' } }]) + }) + + it('encodes empty containers with an explicit values array', () => { + expect(toOtlpAnyValue({})).toEqual({ kvlistValue: { values: [] } }) + expect(toOtlpAnyValue([])).toEqual({ arrayValue: { values: [] } }) + }) + + it('keeps a Date whose toISOString is overridden out of the wire format', () => { + const broken = new Date('2026-08-20T10:00:00.000Z') + + ;(broken as any).toISOString = () => ({}) + expect(typeof toOtlpAnyValue(broken as unknown as LogAttributeValue).stringValue).toBe('string') + }) + + it('encodes sibling references to one object twice, not as circular', () => { + const shared = { id: 1 } + expect(toOtlpAnyValue({ a: shared, b: shared })).toEqual({ + kvlistValue: { + values: [ + { key: 'a', value: { kvlistValue: { values: [{ key: 'id', value: { intValue: '1' } }] } } }, + { key: 'b', value: { kvlistValue: { values: [{ key: 'id', value: { intValue: '1' } }] } } }, + ], + }, + }) + }) + }) + + describe('toOtlpKeyValueList', () => { + it('converts a record to key-value list', () => { + expect( + toOtlpKeyValueList({ + name: 'test', + count: 5, + active: true, + }) + ).toEqual([ + { key: 'name', value: { stringValue: 'test' } }, + { key: 'count', value: { intValue: '5' } }, + { key: 'active', value: { boolValue: true } }, + ]) + }) + + it('handles empty record', () => { + expect(toOtlpKeyValueList({})).toEqual([]) + }) + + it('skips null and undefined values', () => { + expect( + toOtlpKeyValueList({ + kept: 'yes', + nullish: null, + missing: undefined, + }) + ).toEqual([{ key: 'kept', value: { stringValue: 'yes' } }]) + }) + }) +}) diff --git a/packages/core/src/utils/otlp-any-value.ts b/packages/core/src/utils/otlp-any-value.ts new file mode 100644 index 0000000000..bed34139a8 --- /dev/null +++ b/packages/core/src/utils/otlp-any-value.ts @@ -0,0 +1,225 @@ +// The OTLP `AnyValue` encoder, shared by the logs, metrics and traces senders. +// +// Every value here comes from application code, so the encoder's job is to +// produce a payload the ingestion service accepts no matter what it is handed. +// A value the server refuses doesn't fail on its own — it 400s the whole +// request, taking every other record in the batch with it. + +import type { OtlpAnyValue, OtlpKeyValue } from '@posthog/types' +import type { Logger } from '../types' +import { isArray, isBoolean, isNull, isNullish, isUndefined } from './type-utils' +import { + CIRCULAR_VALUE, + FUNCTION_VALUE, + MAX_JSON_SAFE_VALUE_DEPTH, + MAX_JSON_SAFE_VALUE_ITEMS, + MAX_JSON_SAFE_VALUE_NODES, + sanitizeString, + TRUNCATED_VALUE, + UNSERIALIZABLE_VALUE, +} from './json-utils' + +// 2^63 — one past int64 max. +const INT64_RANGE_LIMIT = 9223372036854775808 + +// The same bound for the bigint branch. A decimal string rather than a `n` +// literal: this module reaches the browser bundle, which compiles to ES5, and +// a bigint literal there is a syntax error rather than a runtime fallback. +const INT64_RANGE_LIMIT_DECIMAL = '9223372036854775808' + +const propertyIsEnumerable = Object.prototype.propertyIsEnumerable + +interface EncodeState { + /** Containers on the current path, so a back-reference becomes a marker. */ + ancestors: WeakSet + remainingNodes: number +} + +function newState(): EncodeState { + return { ancestors: new WeakSet(), remainingNodes: MAX_JSON_SAFE_VALUE_NODES } +} + +export function toOtlpAnyValue(value: unknown, logger?: Logger): OtlpAnyValue { + try { + return encodeAnyValue(value, logger, newState(), 0) + } catch { + // Runs inside `captureLog`, the metrics flush and span encoding: an error + // escaping here surfaces in the caller's own code. + return { stringValue: UNSERIALIZABLE_VALUE } + } +} + +export function toOtlpKeyValueList(attrs: Record, logger?: Logger): OtlpKeyValue[] { + try { + return encodeKeyValueList(attrs, logger, newState(), 0) + } catch { + return [] + } +} + +function encodeBigInt(value: bigint, logger: Logger | undefined): OtlpAnyValue { + const decimal = value.toString() + const limit = BigInt(INT64_RANGE_LIMIT_DECIMAL) + if (value >= limit || value < -limit) { + logger?.debug(`Attribute ${decimal} is outside the int64 range; encoding it as a string`) + return { stringValue: decimal } + } + return { intValue: decimal } +} + +function encodeAnyValue(value: unknown, logger: Logger | undefined, state: EncodeState, depth: number): OtlpAnyValue { + if (state.remainingNodes <= 0) { + return { stringValue: TRUNCATED_VALUE } + } + state.remainingNodes-- + + if (isBoolean(value)) { + return { boolValue: value } + } + // Reaching this branch proves BigInt exists, so the limit can be built here + // rather than at module load. + if (typeof value === 'bigint') { + return encodeBigInt(value, logger) + } + // typeof, not core's isNumber, which excludes NaN — proto3 JSON distinguishes + // a non-finite float from an ordinary string. + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + return { stringValue: String(value) } + } + if (Number.isInteger(value)) { + if (Number.isSafeInteger(value)) { + return { intValue: String(value) } + } + // Past MAX_SAFE_INTEGER only BigInt gives the double's exact decimal: + // `String(-(2**63))` lands 192 below int64 min, outside the field it is + // about to be parsed into. Without BigInt the value rides as a string, + // which is never range-checked. + if (typeof BigInt === 'undefined') { + return { stringValue: String(value) } + } + const decimal = BigInt(value).toString() + if (value >= INT64_RANGE_LIMIT || value < -INT64_RANGE_LIMIT) { + // An out-of-range intValue 400s the whole logs request; on the metrics + // path it is swallowed server-side and the metric just disappears. + logger?.debug(`Attribute ${decimal} is outside the int64 range; encoding it as a string`) + return { stringValue: decimal } + } + return { intValue: decimal } + } + return { doubleValue: value } + } + if (typeof value === 'string') { + return { stringValue: sanitizeString(value) } + } + // `String(value)` would put a function's source text on the wire. + if (typeof value === 'function') { + return { stringValue: FUNCTION_VALUE } + } + if (typeof value === 'symbol') { + return { stringValue: String(value) } + } + if (typeof value === 'object' && value !== null) { + if (state.ancestors.has(value)) { + return { stringValue: CIRCULAR_VALUE } + } + if (depth >= MAX_JSON_SAFE_VALUE_DEPTH) { + return { stringValue: TRUNCATED_VALUE } + } + if (value instanceof Date) { + const time = value.getTime() + const iso = Number.isFinite(time) ? value.toISOString() : String(value) + // An overridden toISOString can return a non-string, which the server + // refuses for the whole request. + return { stringValue: typeof iso === 'string' ? sanitizeString(iso) : String(iso) } + } + // Registered before the toJSON probe: a toJSON returning a structure that + // references its own object is a cycle like any other. + state.ancestors.add(value) + try { + // The representation a value defines for itself — dayjs, Decimal, an ORM + // document, and a cross-realm Date that fails the `instanceof` above. + try { + const toJSON = (value as { toJSON?: unknown }).toJSON + if (typeof toJSON === 'function') { + return encodeAnyValue(toJSON.call(value), logger, state, depth + 1) + } + } catch { + // A throwing toJSON falls through to the plain walk. + } + if (isArray(value)) { + return { arrayValue: { values: encodeArrayValues(value, logger, state, depth + 1) } } + } + return { + kvlistValue: { + values: encodeKeyValueList(value as Record, logger, state, depth + 1), + }, + } + } finally { + // Siblings that reference the same object are duplication, not a cycle. + state.ancestors.delete(value) + } + } + return { stringValue: sanitizeString(String(value)) } +} + +function encodeArrayValues( + values: unknown[], + logger: Logger | undefined, + state: EncodeState, + depth: number +): OtlpAnyValue[] { + const result: OtlpAnyValue[] = [] + const itemCount = Math.min(values.length, MAX_JSON_SAFE_VALUE_ITEMS) + let index = 0 + for (; index < itemCount && state.remainingNodes > 0; index++) { + try { + const element = index in values ? values[index] : undefined + // Dropped: proto3 JSON has no null AnyValue, and both `null` and `{}` here + // are rejected for the whole request. + if (isNullish(element)) { + continue + } + result.push(encodeAnyValue(element, logger, state, depth)) + } catch { + result.push({ stringValue: UNSERIALIZABLE_VALUE }) + } + } + if (values.length > index) { + result.push({ stringValue: TRUNCATED_VALUE }) + } + return result +} + +function encodeKeyValueList( + attrs: Record, + logger: Logger | undefined, + state: EncodeState, + depth: number +): OtlpKeyValue[] { + const result: OtlpKeyValue[] = [] + for (const key in attrs) { + // for...in walks the prototype chain once own keys are exhausted. Skipped + // rather than broken out of: a proxy can yield keys in any order. + if (!propertyIsEnumerable.call(attrs, key)) { + continue + } + if (result.length >= MAX_JSON_SAFE_VALUE_ITEMS || state.remainingNodes <= 0) { + // Reported rather than written into the attributes: a synthetic key would + // land in the user's own namespace and could collide with a real one. + logger?.debug('Attributes truncated: the value exceeds the OTLP encoder budget') + break + } + try { + const value = attrs[key] + if (isNull(value) || isUndefined(value)) { + continue + } + result.push({ key: sanitizeString(key), value: encodeAnyValue(value, logger, state, depth) }) + } catch { + // A getter that throws costs its own key, not the whole record. + result.push({ key: sanitizeString(key), value: { stringValue: UNSERIALIZABLE_VALUE } }) + } + } + return result +}