From 3db04a2099588fcc3c4a2ffa0d819c6b86c09fde Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 1 Sep 2026 17:40:45 -0400 Subject: [PATCH 01/33] fix(core): honor Retry-After and skip oversized bodies on the OTLP queues Parses `Retry-After` once in the shared OTLP sender and applies it as a floor on each export queue's own backoff, and refuses a batch over the endpoint's 2 MB body limit without spending a request on it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Qp8JHHcGSf7mocdZtHDR29 --- .changeset/otlp-honor-retry-after.md | 5 + .changeset/otlp-skip-oversized-bodies.md | 5 + packages/browser/terser-mangled-names.json | 2 + .../posthog.otlp-retry-after.spec.ts | 108 +++++++++++++++ .../__tests__/posthog.otlp-too-large.spec.ts | 73 +++++++++++ .../core/src/__tests__/retry-after.spec.ts | 43 ++++++ packages/core/src/logs/index.spec.ts | 116 ++++++++++++++++ packages/core/src/logs/index.ts | 57 +++++++- packages/core/src/metrics/index.spec.ts | 72 ++++++++++ packages/core/src/metrics/index.ts | 37 +++++- packages/core/src/metrics/types.ts | 2 +- packages/core/src/posthog-core-stateless.ts | 124 ++++++++++++++---- packages/core/src/traces/index.spec.ts | 109 +++++++++++++++ packages/core/src/traces/index.ts | 46 ++++++- packages/core/src/traces/types.ts | 2 +- .../posthog-node-references-latest.json | 6 +- ...osthog-react-native-references-latest.json | 6 +- 17 files changed, 768 insertions(+), 45 deletions(-) create mode 100644 .changeset/otlp-honor-retry-after.md create mode 100644 .changeset/otlp-skip-oversized-bodies.md create mode 100644 packages/core/src/__tests__/posthog.otlp-retry-after.spec.ts create mode 100644 packages/core/src/__tests__/posthog.otlp-too-large.spec.ts create mode 100644 packages/core/src/__tests__/retry-after.spec.ts diff --git a/.changeset/otlp-honor-retry-after.md b/.changeset/otlp-honor-retry-after.md new file mode 100644 index 0000000000..47ceec3b97 --- /dev/null +++ b/.changeset/otlp-honor-retry-after.md @@ -0,0 +1,5 @@ +--- +'@posthog/core': patch +--- + +Honor `Retry-After` on the logs, metrics and traces export queues when a batch is refused. diff --git a/.changeset/otlp-skip-oversized-bodies.md b/.changeset/otlp-skip-oversized-bodies.md new file mode 100644 index 0000000000..9b332d1cd8 --- /dev/null +++ b/.changeset/otlp-skip-oversized-bodies.md @@ -0,0 +1,5 @@ +--- +'@posthog/core': patch +--- + +Stop sending logs, metrics and traces batches larger than the ingestion endpoint's 2 MB body limit. diff --git a/packages/browser/terser-mangled-names.json b/packages/browser/terser-mangled-names.json index da8592fcc8..d2c4bd9303 100644 --- a/packages/browser/terser-mangled-names.json +++ b/packages/browser/terser-mangled-names.json @@ -244,6 +244,7 @@ "_flushTimeout", "_flushTimeoutMs", "_flushTimer", + "_flushTimerFiresAt", "_flushToCapture", "_flushViaTransport", "_flushedSizeTracker", @@ -715,6 +716,7 @@ "_resumeSavedTour", "_resyncIntervalMs", "_resyncTimer", + "_retryAfterMs", "_retryQueue", "_rrwebError", "_rrwebStartAttempted", diff --git a/packages/core/src/__tests__/posthog.otlp-retry-after.spec.ts b/packages/core/src/__tests__/posthog.otlp-retry-after.spec.ts new file mode 100644 index 0000000000..8345841f46 --- /dev/null +++ b/packages/core/src/__tests__/posthog.otlp-retry-after.spec.ts @@ -0,0 +1,108 @@ +import { createTestClient, PostHogCoreTestClient, PostHogCoreTestClientMocks } from '@/testing' + +// The header has to survive the whole path: response → PostHogFetchHttpError → +// the retry-later outcome each queue reads. A unit test of the parser alone +// would still pass if the plumbing were missing. +describe('OTLP Retry-After', () => { + let posthog: PostHogCoreTestClient + let mocks: PostHogCoreTestClientMocks + + const respondWith = (status: number, retryAfter?: string): void => { + mocks.fetch.mockResolvedValue({ + status, + text: () => Promise.resolve(''), + json: () => Promise.resolve({}), + headers: { get: (name: string) => (name.toLowerCase() === 'retry-after' && retryAfter ? retryAfter : null) }, + }) + } + + beforeEach(() => { + ;[posthog, mocks] = createTestClient('TEST_API_KEY', { + host: 'http://example.com', + preloadFeatureFlags: false, + disableCompression: true, + fetchRetryCount: 0, + }) + }) + + it.each([ + ['logs', () => posthog._sendLogsBatch({ resourceLogs: [] } as any)], + ['metrics', () => posthog._sendMetricsBatch({ resourceMetrics: [] } as any)], + ['traces', () => posthog._sendTracesBatch({ resourceSpans: [] } as any)], + ])("surfaces the endpoint's Retry-After to the %s queue", async (_signal, send) => { + respondWith(429, '120') + + const outcome = await send() + + expect(outcome).toMatchObject({ kind: 'retry-later', retryAfterMs: 120_000 }) + }) + + it('sends once, not once per inner retry, when the endpoint names a wait', async () => { + // The inner retriable loop retries on a short fixed delay. Spending its + // attempts here would put three extra requests inside the very window the + // queue is about to back off for. + const [client, clientMocks] = createTestClient('TEST_API_KEY', { + host: 'http://example.com', + preloadFeatureFlags: false, + disableCompression: true, + }) + clientMocks.fetch.mockResolvedValue({ + status: 429, + text: () => Promise.resolve(''), + json: () => Promise.resolve({}), + headers: { get: (name: string) => (name.toLowerCase() === 'retry-after' ? '120' : null) }, + }) + + const pending = client._sendTracesBatch({ resourceSpans: [] } as any) + await jest.advanceTimersByTimeAsync(60_000) + + expect(await pending).toMatchObject({ kind: 'retry-later', retryAfterMs: 120_000 }) + expect(clientMocks.fetch).toHaveBeenCalledTimes(1) + }) + + it('still retries internally when the response names no wait', async () => { + const [client, clientMocks] = createTestClient('TEST_API_KEY', { + host: 'http://example.com', + preloadFeatureFlags: false, + disableCompression: true, + }) + clientMocks.fetch.mockResolvedValue({ + status: 503, + text: () => Promise.resolve(''), + json: () => Promise.resolve({}), + headers: { get: () => null }, + }) + + const pending = client._sendTracesBatch({ resourceSpans: [] } as any) + await jest.advanceTimersByTimeAsync(60_000) + + expect((await pending).kind).toBe('retry-later') + expect(clientMocks.fetch.mock.calls.length).toBeGreaterThan(1) + }) + + it('leaves retryAfterMs unset when the response sends no header', async () => { + respondWith(503) + + const outcome = await posthog._sendTracesBatch({ resourceSpans: [] } as any) + + expect(outcome.kind).toBe('retry-later') + expect((outcome as { retryAfterMs?: number }).retryAfterMs).toBeUndefined() + }) + + it('survives a transport whose headers accessor throws', async () => { + mocks.fetch.mockResolvedValue({ + status: 503, + text: () => Promise.resolve(''), + json: () => Promise.resolve({}), + headers: { + get: () => { + throw new Error('hostile transport') + }, + }, + }) + + const outcome = await posthog._sendTracesBatch({ resourceSpans: [] } as any) + + expect(outcome.kind).toBe('retry-later') + }) +}) diff --git a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts new file mode 100644 index 0000000000..f3b671f858 --- /dev/null +++ b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts @@ -0,0 +1,73 @@ +import { createTestClient, PostHogCoreTestClient, PostHogCoreTestClientMocks } from '@/testing' + +// The endpoint caps the request body at 2 MB. A batch over that can only come +// back 413, so it is reported as too-large without being sent — the caller's +// halving loop then isolates and drops the oversized record without spending a +// request on every attempt. +describe('OTLP bodies over the endpoint limit', () => { + let posthog: PostHogCoreTestClient + let mocks: PostHogCoreTestClientMocks + + const spansOf = (attributeBytes: number): any => ({ + resourceSpans: [ + { + resource: { attributes: [] }, + scopeSpans: [ + { + scope: { name: 'test' }, + spans: [ + { name: 'checkout', attributes: [{ key: 'blob', value: { stringValue: 'x'.repeat(attributeBytes) } }] }, + ], + }, + ], + }, + ], + }) + + beforeEach(() => { + ;[posthog, mocks] = createTestClient('TEST_API_KEY', { + host: 'http://example.com', + preloadFeatureFlags: false, + disableCompression: true, + fetchRetryCount: 0, + }) + mocks.fetch.mockResolvedValue({ + status: 200, + text: () => Promise.resolve(''), + json: () => Promise.resolve({}), + headers: { get: () => null }, + }) + }) + + it('sends a batch inside the limit', async () => { + await expect(posthog._sendTracesBatch(spansOf(1024))).resolves.toEqual({ kind: 'ok' }) + expect(mocks.fetch).toHaveBeenCalledTimes(1) + }) + + it.each([ + ['traces', (payload: any) => posthog._sendTracesBatch(payload)], + ['logs', (payload: any) => posthog._sendLogsBatch(payload)], + ['metrics', (payload: any) => posthog._sendMetricsBatch(payload)], + ])('reports a %s batch over the limit as too-large without a request', async (_signal, send) => { + await expect(send(spansOf(3 * 1024 * 1024))).resolves.toEqual({ kind: 'too-large' }) + expect(mocks.fetch).not.toHaveBeenCalled() + }) + + it('measures the payload, not the compressed body it is sent as', async () => { + // The endpoint decompresses before it measures, so a payload that gzips + // down to nothing is still refused on its decompressed size. + jest.spyOn(posthog as any, 'compressPayload').mockResolvedValue(new Uint8Array(1024)) + ;(posthog as any).disableCompression = false + + await expect(posthog._sendTracesBatch(spansOf(3 * 1024 * 1024))).resolves.toEqual({ kind: 'too-large' }) + expect(mocks.fetch).not.toHaveBeenCalled() + }) + + it('sends a compressible payload that is inside the limit before compression', async () => { + jest.spyOn(posthog as any, 'compressPayload').mockResolvedValue(new Uint8Array(1024)) + ;(posthog as any).disableCompression = false + + await expect(posthog._sendTracesBatch(spansOf(1024))).resolves.toEqual({ kind: 'ok' }) + expect(mocks.fetch).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/core/src/__tests__/retry-after.spec.ts b/packages/core/src/__tests__/retry-after.spec.ts new file mode 100644 index 0000000000..2c1157cbe3 --- /dev/null +++ b/packages/core/src/__tests__/retry-after.spec.ts @@ -0,0 +1,43 @@ +import { parseRetryAfterMs } from '../posthog-core-stateless' + +describe('parseRetryAfterMs', () => { + const now = Date.parse('2026-09-01T12:00:00Z') + + it('reads delta-seconds', () => { + expect(parseRetryAfterMs('60', now)).toBe(60_000) + expect(parseRetryAfterMs(' 7 ', now)).toBe(7_000) + }) + + it('reads an HTTP-date as a delay from now', () => { + expect(parseRetryAfterMs('Tue, 01 Sep 2026 12:00:30 GMT', now)).toBe(30_000) + }) + + it('ignores a header that does not name a future time', () => { + // A past date and a zero delta both mean "retry now", which is the caller's own backoff. + expect(parseRetryAfterMs('Tue, 01 Sep 2026 11:59:00 GMT', now)).toBeUndefined() + expect(parseRetryAfterMs('0', now)).toBeUndefined() + }) + + it('ignores numeric junk rather than letting Date.parse read it as a year', () => { + // `Date.parse` reads all three as dates in 2001, so a device clock earlier + // than that would otherwise turn them into a real wait. + for (const clock of [now, Date.parse('1999-01-01T00:00:00Z')]) { + expect(parseRetryAfterMs('-5', clock)).toBeUndefined() + expect(parseRetryAfterMs('+5', clock)).toBeUndefined() + expect(parseRetryAfterMs('5.5', clock)).toBeUndefined() + } + }) + + it('ignores a header it cannot parse rather than guessing', () => { + // "10 minutes" must not read as 10 seconds. + expect(parseRetryAfterMs('10 minutes', now)).toBeUndefined() + expect(parseRetryAfterMs('soon', now)).toBeUndefined() + expect(parseRetryAfterMs('', now)).toBeUndefined() + expect(parseRetryAfterMs(null, now)).toBeUndefined() + expect(parseRetryAfterMs(undefined, now)).toBeUndefined() + }) + + it('caps an unbounded value so a bogus header cannot strand a queue', () => { + expect(parseRetryAfterMs('86400', now)).toBe(5 * 60_000) + }) +}) diff --git a/packages/core/src/logs/index.spec.ts b/packages/core/src/logs/index.spec.ts index 28a424ece0..83447b358b 100644 --- a/packages/core/src/logs/index.spec.ts +++ b/packages/core/src/logs/index.spec.ts @@ -1166,6 +1166,122 @@ describe('PostHogLogs', () => { expect(readQueue(mockInstance)).toHaveLength(0) }) + it('waits out Retry-After even when a capture re-arms the timer mid-flush', async () => { + // The capture that lands while the send is in flight arms a timer at the + // plain interval; the 429 then asks for far longer. The earlier timer must + // not fire first, or the SDK sends inside the window it was told to skip. + mockInstance._sendLogsBatch = jest.fn(async () => { + logs.captureLog({ body: 'arrived mid-flush' }) + return { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 } + }) + + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 5000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + + await jest.advanceTimersByTimeAsync(5000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(6000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(300_000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) + }) + + it('does not let the size trigger send inside a Retry-After window', async () => { + mockInstance._sendLogsBatch = jest.fn(() => + Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) + ) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 1000, maxBufferSize: 2 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + // Enough records to trip the size trigger, well inside the window. + logs.captureLog({ body: 'second' }) + logs.captureLog({ body: 'third' }) + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(300_000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) + }) + + it('does not let onReconnect send inside a Retry-After window', async () => { + // `online` fires on every network handover; it says nothing about the + // rate-limit window the endpoint set. + mockInstance._sendLogsBatch = jest.fn(() => + Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) + ) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 1000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + logs.onReconnect() + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + }) + + it('flushes on reconnect once the Retry-After window has passed', async () => { + const outcomes: any[] = [{ kind: 'retry-later', error: new Error('429'), retryAfterMs: 5000 }] + mockInstance._sendLogsBatch = jest.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 1000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(5000) + logs.onReconnect() + await jest.advanceTimersByTimeAsync(1) + expect(mockInstance._sendLogsBatch.mock.calls.length).toBeGreaterThan(1) + }) + + it('keeps flushing on the interval while captures keep arriving', async () => { + // Every capture arms the timer. Re-arming a pending one would push the + // flush out for as long as logs keep coming, stranding a steady stream + // that never reaches the size trigger. + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 10_000, maxBufferSize: 100 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + + for (let i = 0; i < 30; i++) { + logs.captureLog({ body: `line ${i}` }) + await jest.advanceTimersByTimeAsync(2000) + } + + expect(mockInstance._sendLogsBatch).toHaveBeenCalled() + expect(readQueue(mockInstance)).toHaveLength(0) + }) + it('stops re-arming once the queue is empty', async () => { const logs = new PostHogLogs( mockInstance, diff --git a/packages/core/src/logs/index.ts b/packages/core/src/logs/index.ts index 06a7ca9b89..284e82dd65 100644 --- a/packages/core/src/logs/index.ts +++ b/packages/core/src/logs/index.ts @@ -27,6 +27,12 @@ export class PostHogLogs { // A batch captures this when it is assembled, so it can tell that the records it is // holding no longer correspond to anything queued. private _queueGeneration = 0 + // Absolute deadline from a `Retry-After` the endpoint sent, so every path that + // can start a send checks it — not just the retry timer. Cleared on the next + // success so one throttled response cannot pin the delay for the rest of the + // process. + private _retryAfterUntil = 0 + private _flushTimerFiresAt = 0 // Consecutive failed flushes; drives exponential backoff on the retry timer. // A successful flush resets it to 0. private _consecutiveFlushFailures = 0 @@ -91,14 +97,21 @@ export class PostHogLogs { this._intervalLogCount = 0 this._droppedWarned = false this._consecutiveFlushFailures = 0 + this._retryAfterUntil = 0 this._maxBatchRecordsPerPost = this._config.maxBatchRecordsPerPost } // Call when connectivity is restored: clear the failure backoff and flush now, // so records don't wait out a (possibly minutes-long) backoff delay after the // network returns. The host owns connectivity detection (web: `online` event). + // A `Retry-After` window survives this: the network coming back says nothing + // about the rate limit the endpoint set, and browsers fire `online` on every + // network handover. onReconnect(): void { this._consecutiveFlushFailures = 0 + if (this._isWaitingOutRetryAfter()) { + return + } this._flushInBackground() } @@ -292,7 +305,7 @@ export class PostHogLogs { if (outcome.kind === 'too-large' && batch.length > 1) { this._maxBatchRecordsPerPost = Math.max(1, Math.floor(batch.length / 2)) this._logger.warn( - `Received 413 when sending logs batch of size ${batch.length}, reducing batch size to ${this._maxBatchRecordsPerPost}` + `Logs batch of size ${batch.length} was too large for the ingestion endpoint, reducing batch size to ${this._maxBatchRecordsPerPost}` ) // Don't advance the queue — retry the same records with the smaller cap. continue @@ -301,6 +314,7 @@ export class PostHogLogs { if (outcome.kind === 'retry-later') { // Transient failure: keep records in the queue for the next flush cycle // and surface the error so the caller can log/react. + this._retryAfterUntil = outcome.retryAfterMs ? Date.now() + outcome.retryAfterMs : 0 throw outcome.error } @@ -364,22 +378,40 @@ export class PostHogLogs { this._instance.setPersistedProperty(PostHogPersistedProperty.LogsQueue, queue) // Flush trigger: drain now rather than waiting for the timer. The queue may - // grow past this up to the eviction cap while the flush is in flight. - if (queue.length >= this._maxBufferSize) { + // grow past this up to the eviction cap while the flush is in flight. Not + // while the endpoint has asked us to wait: the size trigger is the dominant + // one on a busy host, so sending here would ignore the window entirely. + if (queue.length >= this._maxBufferSize && !this._isWaitingOutRetryAfter()) { this._flushInBackground() return } - // Arm one timer at a time; re-arming within the window would push the flush out. + // Arm one timer at a time; re-arming on every enqueue would push the flush out. this._armFlushTimer() } // Arms the flush timer if none is pending. One-shot: the callback clears the // handle so the next enqueue (or a flush that left records) schedules again. - private _armFlushTimer(delayMs: number = this._flushIntervalMs): void { + private _armFlushTimer(): void { if (this._flushTimer) { return } + this._setFlushTimer(this._flushIntervalMs) + } + + // Backoff and `Retry-After` are floors, so a timer already armed at the plain + // interval has to give way to a longer one — otherwise a capture landing + // mid-flush would send inside the window the server asked us to skip. + private _armFlushTimerNoEarlierThan(delayMs: number): void { + if (this._flushTimer && Date.now() + delayMs <= this._flushTimerFiresAt) { + return + } + this._clearFlushTimer() + this._setFlushTimer(delayMs) + } + + private _setFlushTimer(delayMs: number): void { + this._flushTimerFiresAt = Date.now() + delayMs this._flushTimer = safeSetTimeout(() => { this._flushTimer = undefined this._flushInBackground() @@ -391,7 +423,17 @@ export class PostHogLogs { // retried every interval. private _nextFlushDelay(): number { const exponent = Math.min(Math.max(0, this._consecutiveFlushFailures - 1), MAX_FLUSH_BACKOFF_EXPONENT) - return this._flushIntervalMs * 2 ** exponent + // `Retry-After` is a floor, not a replacement: never retry before the server + // asked, and never more often than our own backoff would have. + return Math.max(this._flushIntervalMs * 2 ** exponent, this._retryAfterRemainingMs()) + } + + private _retryAfterRemainingMs(): number { + return Math.max(0, this._retryAfterUntil - Date.now()) + } + + private _isWaitingOutRetryAfter(): boolean { + return this._retryAfterRemainingMs() > 0 } private _hasQueuedRecords(): boolean { @@ -449,6 +491,7 @@ export class PostHogLogs { .then( () => { this._consecutiveFlushFailures = 0 + this._retryAfterUntil = 0 }, (err) => { this._consecutiveFlushFailures++ @@ -460,7 +503,7 @@ export class PostHogLogs { // sit undelivered on a quiet page; re-arm so the timer retries them, backing // off on consecutive failures. if (!this._instance.isDisabled && this._hasQueuedRecords()) { - this._armFlushTimer(this._nextFlushDelay()) + this._armFlushTimerNoEarlierThan(this._nextFlushDelay()) } }) } diff --git a/packages/core/src/metrics/index.spec.ts b/packages/core/src/metrics/index.spec.ts index 10e2ac574a..dc94ee7f51 100644 --- a/packages/core/src/metrics/index.spec.ts +++ b/packages/core/src/metrics/index.spec.ts @@ -243,6 +243,78 @@ describe('PostHogMetrics', () => { expect(mockInstance._sendMetricsBatch).toHaveBeenCalledTimes(1) }) + it('keeps flushing on the interval while captures keep arriving', async () => { + // Every capture arms the timer, and metrics have no size trigger to fall + // back on: re-arming a pending one would stop the window ever being sent. + const metrics = createMetrics({ flushIntervalMs: 10_000 }) + + for (let i = 0; i < 60; i++) { + metrics.count('orders_created', 1) + await jest.advanceTimersByTimeAsync(1000) + } + + expect(mockInstance._sendMetricsBatch).toHaveBeenCalled() + }) + + it('waits out Retry-After even when a flush timer is already pending', async () => { + // The capture arms a timer at the flush interval; the failed flush then asks + // for far longer. The pending timer must not fire first. + const instance = createMockInstance({ + _sendMetricsBatch: jest.fn((): Promise => + Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) + ), + }) + const metrics = createMetrics({ flushIntervalMs: 10_000 }, instance) + metrics.count('orders_created', 1) + await metrics.flush() + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(11_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(300_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) + }) + + it('clears Retry-After on an outcome that is not a retry', async () => { + // Only a retriable outcome carries a wait. A stale one left set here would + // pin every later flush at the server's old window forever. + const outcomes: SendMetricsBatchOutcome[] = [ + { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }, + { kind: 'fatal', error: new Error('400') }, + ] + const instance = createMockInstance({ + _sendMetricsBatch: jest.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })), + }) + const metrics = createMetrics({ flushIntervalMs: 10_000 }, instance) + metrics.count('orders_created', 1) + await metrics.flush() + + // The wait elapses, the retry lands a 400, and that ends the wait. + await jest.advanceTimersByTimeAsync(300_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) + + metrics.count('orders_created', 1) + await jest.advanceTimersByTimeAsync(10_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(3) + }) + + it('drops a Retry-After wait on reset', async () => { + const instance = createMockInstance({ + _sendMetricsBatch: jest.fn((): Promise => + Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) + ), + }) + const metrics = createMetrics({ flushIntervalMs: 10_000 }, instance) + metrics.count('orders_created', 1) + await metrics.flush() + + metrics.reset() + metrics.count('orders_created', 1) + await jest.advanceTimersByTimeAsync(10_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) + }) + it('does not send when the window is empty', async () => { createMetrics({ flushIntervalMs: 5000 }) await jest.advanceTimersByTimeAsync(15000) diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index cb86a0336e..3ec8b24b3b 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -73,6 +73,10 @@ export class PostHogMetrics { // types under one name produces charts that blend both series. private _typeByName = new Map() private _typeCollisionWarned = new Set() + // Set from a `Retry-After` the endpoint sent; cleared on any other outcome so + // one throttled response cannot pin the delay for the rest of the process. + private _retryAfterMs = 0 + private _flushTimerFiresAt = 0 // Bumped by reset(). A flush that was in flight when reset() ran (e.g. it // lost a shutdown race) sees a stale generation when its send settles and // discards its window instead of merging it back and re-arming the timer. @@ -136,6 +140,7 @@ export class PostHogMetrics { /** Clears the flush timer, drops the current window, and invalidates in-flight flushes. */ reset(): void { this._generation++ + this._retryAfterMs = 0 this._clearFlushTimer() this._series = new Map() this._flushPromise = null @@ -284,16 +289,41 @@ export class PostHogMetrics { return result } + // Arms the flush timer if none is pending. Every capture calls this, so it + // must leave a pending timer alone: re-arming on each one would push the + // flush out for as long as metrics keep arriving. private _armFlushTimer(): void { if (this._flushTimer) { return } + this._setFlushTimer(this._nextFlushDelay()) + } + + // `Retry-After` is a floor, so a timer already armed at the flush interval has + // to give way to a longer one rather than firing inside the window the server + // asked us to skip. + private _armFlushTimerNoEarlierThan(delayMs: number): void { + if (this._flushTimer && Date.now() + delayMs <= this._flushTimerFiresAt) { + return + } + this._clearFlushTimer() + this._setFlushTimer(delayMs) + } + + // `Retry-After` is a floor, not a replacement: never retry before the server + // asked, and never more often than the flush interval. + private _nextFlushDelay(): number { + return Math.max(this._config.flushIntervalMs, this._retryAfterMs) + } + + private _setFlushTimer(delayMs: number): void { + this._flushTimerFiresAt = Date.now() + delayMs this._flushTimer = safeSetTimeout(() => { this._flushTimer = undefined this.flush().catch((e) => { this._logger.error('Metrics flush failed:', e) }) - }, this._config.flushIntervalMs) + }, delayMs) } private _clearFlushTimer(): void { @@ -323,6 +353,9 @@ export class PostHogMetrics { // reconfigured, so this window is dropped whatever the outcome was. return } + // Only a retriable outcome carries a wait; anything else ends it, or a stale + // one would pin every later flush at the window the server has moved on from. + this._retryAfterMs = outcome.kind === 'retry-later' ? (outcome.retryAfterMs ?? 0) : 0 switch (outcome.kind) { case 'ok': return @@ -331,7 +364,7 @@ export class PostHogMetrics { // the next flush instead of being lost — and re-arm the timer, since // with no new captures nothing else would schedule that flush. this._mergeWindowBack(window) - this._armFlushTimer() + this._armFlushTimerNoEarlierThan(this._nextFlushDelay()) return case 'too-large': this._logger.warn('Metrics batch exceeded the server size limit and was dropped') diff --git a/packages/core/src/metrics/types.ts b/packages/core/src/metrics/types.ts index 7df8be3b1f..fb2685f421 100644 --- a/packages/core/src/metrics/types.ts +++ b/packages/core/src/metrics/types.ts @@ -20,7 +20,7 @@ import type { BeforeSendMetricFn, MetricAttributeValue, OtlpMetricsPayload } fro /** Same tagged outcome shape as `SendLogsBatchOutcome` — one policy for both signals. */ export type SendMetricsBatchOutcome = | { kind: 'ok' } - | { kind: 'retry-later'; error: unknown } + | { kind: 'retry-later'; error: unknown; retryAfterMs?: number } | { kind: 'too-large' } | { kind: 'fatal'; error: unknown } diff --git a/packages/core/src/posthog-core-stateless.ts b/packages/core/src/posthog-core-stateless.ts index 73565120ec..8ecbc09e86 100644 --- a/packages/core/src/posthog-core-stateless.ts +++ b/packages/core/src/posthog-core-stateless.ts @@ -51,6 +51,41 @@ import { createDefaultStackParser, } from './error-tracking' +/** + * Longest `Retry-After` the SDK will wait. A rate-limit window can legitimately + * be long, but nothing upstream of the SDK bounds this header — it is as likely + * to come from a proxy or CDN as from PostHog — and an unbounded value would + * strand a queue for hours, so the wait is capped and the retry happens early. + */ +const MAX_RETRY_AFTER_MS = 5 * 60_000 + +/** + * `Retry-After` as milliseconds from now. The header is either delta-seconds or + * an HTTP-date; anything else, a date already in the past, or a negative delta + * yields `undefined` so the caller keeps its own backoff. + * + * @internal Exposed for cross-package use within this SDK; not part of the stable public API. + */ +export function parseRetryAfterMs(value: string | null | undefined, now: number = Date.now()): number | undefined { + if (!value) { + return undefined + } + const trimmed = value.trim() + // Integer seconds. Not parseFloat: "10 minutes" must not read as 10 seconds. + const seconds = /^\d+$/.test(trimmed) ? Number(trimmed) : Number.NaN + if (!Number.isFinite(seconds) && /^[+-]?[\d.]+$/.test(trimmed)) { + // Numeric but not delta-seconds, so it is malformed. `Date.parse` reads + // "-5", "+5" and "5.5" as dates in 2001 rather than rejecting them, which + // on a device whose clock predates that would surface as a real wait. + return undefined + } + const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(trimmed) - now + if (!Number.isFinite(ms) || ms <= 0) { + return undefined + } + return Math.min(ms, MAX_RETRY_AFTER_MS) +} + class PostHogFetchHttpError extends Error { name = 'PostHogFetchHttpError' private responseBodyTextPromise?: Promise @@ -70,6 +105,17 @@ class PostHogFetchHttpError extends Error { return this.response.status } + /** The response's `Retry-After` as milliseconds from now, when it sent a usable one. */ + get retryAfterMs(): number | undefined { + try { + return parseRetryAfterMs(this.response.headers?.get('retry-after')) + } catch { + // `headers.get` is injected transport code; a throwing one must not turn a + // retriable failure into an unhandled rejection. + return undefined + } + } + get bodyReadTimedOut(): boolean { return this._bodyReadTimedOut } @@ -206,6 +252,32 @@ function isRetryableFlagsFetchError( return code !== 'ECONNREFUSED' } +/** + * The ingestion service's request body limit — `MAX_REQUEST_BODY_SIZE_BYTES`, + * which defaults to 2 MB and is applied to the body after the endpoint + * decompresses it, not to the compressed bytes on the wire. + * + * Used as a floor, never as a promise: the deployment can raise it, and a proxy + * in front can lower it. A body over this size cannot be accepted, so sending it + * only buys a 413; one under it is sent and may still be refused. + */ +const OTLP_MAX_BODY_BYTES = 2 * 1024 * 1024 + +/** A request body's size on the wire. `Buffer` where it exists, `TextEncoder` elsewhere. */ +function byteLengthOf(body: string | Blob | Uint8Array): number { + if (body instanceof Blob) { + return body.size + } + if (body instanceof Uint8Array) { + return body.byteLength + } + try { + return Buffer.byteLength(body, STRING_FORMAT) + } catch { + return new TextEncoder().encode(body).length + } +} + export function isPostHogFetchContentTooLargeError(err: unknown): err is PostHogFetchHttpError & { status: 413 } { return typeof err === 'object' && err instanceof PostHogFetchHttpError && err.status === 413 } @@ -234,7 +306,7 @@ function isPostHogEventProperties(value: JsonType | undefined): value is PostHog export type SendLogsBatchOutcome = | { kind: 'ok' } | { kind: 'too-large' } - | { kind: 'retry-later'; error: unknown } + | { kind: 'retry-later'; error: unknown; retryAfterMs?: number } | { kind: 'fatal'; error: unknown } /** @@ -245,7 +317,7 @@ export type SendLogsBatchOutcome = type SendOtlpBatchOutcome = | { kind: 'ok' } | { kind: 'too-large' } - | { kind: 'retry-later'; error: unknown } + | { kind: 'retry-later'; error: unknown; retryAfterMs?: number } | { kind: 'fatal'; error: unknown } export enum QuotaLimitedFeature { @@ -1671,7 +1743,24 @@ export abstract class PostHogCoreStateless { ? `${this.host}/i/v1/${path}` : `${this.host}/i/v1/${path}?token=${encodeURIComponent(this.apiKey)}` + // Measured before compression: the endpoint decompresses the body and applies + // its limit to what comes out, so a payload that gzips small is still refused + // on its decompressed size. A batch the endpoint cannot accept is reported + // without being sent, so the caller halves it — and ultimately isolates and + // drops the one oversized record — without spending a request on each attempt. + const payloadBytes = byteLengthOf(serialized) + if (payloadBytes > OTLP_MAX_BODY_BYTES) { + this.logMsgIfDebug(() => + console.warn( + `[PostHog] Not sending a ${path} batch of ${payloadBytes} bytes: the endpoint accepts at most ${OTLP_MAX_BODY_BYTES}` + ) + ) + return { kind: 'too-large' } + } + const gzippedPayload = !this.disableCompression ? await this.compressPayload(serialized) : null + const body = gzippedPayload || serialized + const fetchOptions: PostHogFetchOptions = { method: 'POST', headers: { @@ -1680,7 +1769,7 @@ export abstract class PostHogCoreStateless { ...(auth === 'bearer' && { Authorization: `Bearer ${this.apiKey}` }), ...(gzippedPayload !== null && { 'Content-Encoding': 'gzip' }), }, - body: gzippedPayload || serialized, + body, } try { @@ -1693,6 +1782,12 @@ export abstract class PostHogCoreStateless { if (isPostHogFetchContentTooLargeError(err)) { return false } + if (err instanceof PostHogFetchHttpError && err.retryAfterMs !== undefined) { + // The endpoint named a wait. This loop retries on a fixed short + // delay, so retrying here would spend every attempt inside the + // window; hand it to the queue's backoff instead. + return false + } return isPostHogFetchRetryableError(err) }, } @@ -1703,7 +1798,8 @@ export abstract class PostHogCoreStateless { return { kind: 'too-large' } } if (isPostHogFetchRetryableError(err)) { - return { kind: 'retry-later', error: err } + const retryAfterMs = err instanceof PostHogFetchHttpError ? err.retryAfterMs : undefined + return { kind: 'retry-later', error: err, ...(retryAfterMs !== undefined && { retryAfterMs }) } } return { kind: 'fatal', error: err } } @@ -1751,25 +1847,7 @@ export abstract class PostHogCoreStateless { requestTimeout?: number ): Promise { const body = options.body ? options.body : '' - let reqByteLength = -1 - try { - if (body instanceof Blob) { - reqByteLength = body.size - } else if (body instanceof Uint8Array) { - reqByteLength = body.byteLength - } else { - reqByteLength = Buffer.byteLength(body, STRING_FORMAT) - } - } catch { - if (body instanceof Blob) { - reqByteLength = body.size - } else if (body instanceof Uint8Array) { - reqByteLength = body.byteLength - } else { - const encoded = new TextEncoder().encode(body) - reqByteLength = encoded.length - } - } + const reqByteLength = byteLengthOf(body) const retriableOptions = { ...this._retryOptions, ...retryOptions } let attempt = 0 diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 1af123fe3b..022c45d4a0 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -1261,6 +1261,115 @@ describe('PostHogTraces', () => { }) }) + describe('Retry-After', () => { + it('waits at least as long as the endpoint asked', async () => { + mockInstance._sendTracesBatch.mockResolvedValue({ + kind: 'retry-later', + error: new Error('throttled'), + retryAfterMs: 90_000, + }) + const traces = createTraces({ flushIntervalMs: 1000, maxExportBatchSize: 1 }) + traces.startSpan('held').end() + + // One attempt, then a wait longer than the 30s exponential cap would give. + while (mockInstance._sendTracesBatch.mock.calls.length < 1) { + await jest.advanceTimersByTimeAsync(1000) + } + await jest.advanceTimersByTimeAsync(60_000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(31_000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(2) + }) + + it('keeps its own backoff when the endpoint asks for less', async () => { + mockInstance._sendTracesBatch.mockResolvedValue({ + kind: 'retry-later', + error: new Error('down'), + retryAfterMs: 10, + }) + const traces = createTraces({ flushIntervalMs: 5000, maxExportBatchSize: 1 }) + traces.startSpan('held').end() + + while (mockInstance._sendTracesBatch.mock.calls.length < 1) { + await jest.advanceTimersByTimeAsync(1000) + } + // A 10ms Retry-After must not turn the retry loop into a hot loop: the + // next attempt still waits out the queue's own backoff, not 10ms. + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(5000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(2) + }) + + it('waits out Retry-After even when a span ends mid-flush', async () => { + // The span that ends while the send is in flight arms a timer at the plain + // interval; the 429 then asks for far longer. The earlier timer must not + // fire first, or the SDK sends inside the window it was told to skip. + mockInstance._sendTracesBatch.mockImplementation(() => { + traces.startSpan('arrived mid-flush').end() + return Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) + }) + const traces = createTraces({ flushIntervalMs: 1000 }) + traces.startSpan('first').end() + + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(10_000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) + }) + + it('clears Retry-After on an outcome that is not a retry', async () => { + // Only a retriable outcome carries a wait. A stale one left set here would + // pin every later flush at the server's old window. + const outcomes: SendTracesBatchOutcome[] = [ + { kind: 'retry-later', error: new Error('throttled'), retryAfterMs: 300_000 }, + { kind: 'fatal', error: new Error('400') }, + ] + mockInstance._sendTracesBatch.mockImplementation(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) + const traces = createTraces({ flushIntervalMs: 1000, maxExportBatchSize: 1 }) + traces.startSpan('first').end() + + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) + + // The wait elapses, the retry lands a 400, and that ends the wait. + await jest.advanceTimersByTimeAsync(300_000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(2) + expect((traces as any)._retryAfterMs).toBe(0) + + traces.startSpan('second').end() + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(3) + }) + + it('stops honouring a stale Retry-After after a success', async () => { + let attempt = 0 + mockInstance._sendTracesBatch.mockImplementation(() => { + attempt++ + return Promise.resolve( + attempt === 1 ? { kind: 'retry-later', error: new Error('throttled'), retryAfterMs: 120_000 } : { kind: 'ok' } + ) + }) + const traces = createTraces({ flushIntervalMs: 1000, maxExportBatchSize: 1 }) + traces.startSpan('first').end() + await jest.advanceTimersByTimeAsync(200_000) + + traces.startSpan('second').end() + await traces.flush() + expect(sentSpans().map((s) => s.name)).toContain('second') + + // The wait is gone, so a later failure falls back to the plain interval. + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('down') }) + traces.startSpan('third').end() + const before = mockInstance._sendTracesBatch.mock.calls.length + await jest.advanceTimersByTimeAsync(2000) + expect(mockInstance._sendTracesBatch.mock.calls.length).toBeGreaterThan(before) + }) + }) + describe('live span bounds', () => { it('returns an inert handle once maxLiveSpans spans are live', async () => { const traces = createTraces({ maxLiveSpans: 2 }) diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 0281556118..9f6a54f176 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -73,6 +73,10 @@ export class PostHogTraces { private _lastDropWarningAt = 0 private _dropReasons = new Set() private _consecutiveFlushFailures = 0 + // Set from a `Retry-After` the endpoint sent; cleared on any other outcome so + // one throttled response cannot pin the delay for the rest of the process. + private _retryAfterMs = 0 + private _flushTimerFiresAt = 0 // Separate from the backoff counter: this one belongs to whatever batch is at // the head, and resets whenever that batch is removed or shrunk. private _headBatchFailures = 0 @@ -241,7 +245,7 @@ export class PostHogTraces { if (this._flushPromise === promise) { this._flushPromise = null } - this._armFlushTimerIfQueued() + this._armFlushTimerIfQueuedNoEarlierThan() }) this._flushPromise = promise return promise @@ -260,6 +264,7 @@ export class PostHogTraces { this._dropReasons.clear() this._lastDropWarningAt = 0 this._consecutiveFlushFailures = 0 + this._retryAfterMs = 0 this._headBatchFailures = 0 } @@ -499,6 +504,11 @@ export class PostHogTraces { return removed } + // Only a retriable outcome carries a wait; anything else ends it, or a + // stale one would pin every later flush at the window the server has + // moved on from. + this._retryAfterMs = outcome.kind === 'retry-later' ? (outcome.retryAfterMs ?? 0) : 0 + if (outcome.kind === 'ok') { this._consecutiveFlushFailures = 0 this._headBatchFailures = 0 @@ -518,7 +528,7 @@ export class PostHogTraces { this._queue.splice(0, 1) remaining -= 1 removed += 1 - this._recordDrop(1, 'the ingestion endpoint rejected it as too large') + this._recordDrop(1, 'it is too large for the ingestion endpoint') this._headBatchFailures = 0 continue } @@ -587,18 +597,41 @@ export class PostHogTraces { this._backgroundFlush = undefined // A trigger that arrived while this drain was finishing found the guard // set and the queue empty, so neither path armed a timer. - this._armFlushTimerIfQueued() + this._armFlushTimerIfQueuedNoEarlierThan() }) } + // Arms the flush timer if none is pending. Every span end can reach this, so + // it must leave a pending timer alone rather than pushing the flush out. private _armFlushTimerIfQueued(): void { if (this._flushTimer || !this._queue.length) { return } + this._setFlushTimer(this._nextFlushDelay()) + } + + // Backoff and `Retry-After` are floors, so a timer a span end armed at the + // plain interval while the send was in flight has to give way to a longer + // one — otherwise the retry lands inside the window the server asked us to + // skip. + private _armFlushTimerIfQueuedNoEarlierThan(): void { + if (!this._queue.length) { + return + } + const delayMs = this._nextFlushDelay() + if (this._flushTimer && Date.now() + delayMs <= this._flushTimerFiresAt) { + return + } + this._clearFlushTimer() + this._setFlushTimer(delayMs) + } + + private _setFlushTimer(delayMs: number): void { + this._flushTimerFiresAt = Date.now() + delayMs this._flushTimer = safeSetTimeout(() => { this._flushTimer = undefined this._flushInBackground() - }, this._nextFlushDelay()) + }, delayMs) } // Retry delay: base interval, doubling, capped at 30s — never below an interval @@ -606,7 +639,10 @@ export class PostHogTraces { private _nextFlushDelay(): number { const exponent = Math.min(Math.max(0, this._consecutiveFlushFailures - 1), MAX_FLUSH_BACKOFF_EXPONENT) const delay = this._config.flushIntervalMs * 2 ** exponent - return Math.min(delay, Math.max(MAX_FLUSH_BACKOFF_MS, this._config.flushIntervalMs)) + const capped = Math.min(delay, Math.max(MAX_FLUSH_BACKOFF_MS, this._config.flushIntervalMs)) + // `Retry-After` is a floor, not a replacement: never retry before the server + // asked, and never more often than our own backoff would have. + return Math.max(capped, this._retryAfterMs) } private _clearFlushTimer(): void { diff --git a/packages/core/src/traces/types.ts b/packages/core/src/traces/types.ts index 6551d0bcac..8e6c646ea3 100644 --- a/packages/core/src/traces/types.ts +++ b/packages/core/src/traces/types.ts @@ -19,7 +19,7 @@ import type { OtlpTracesPayload, Span, SpanAttributes, SpanKind, SpanStatusCode, /** Same tagged outcome shape as `SendLogsBatchOutcome` — one policy for all three signals. */ export type SendTracesBatchOutcome = | { kind: 'ok' } - | { kind: 'retry-later'; error: unknown } + | { kind: 'retry-later'; error: unknown; retryAfterMs?: number } | { kind: 'too-large' } | { kind: 'fatal'; error: unknown } diff --git a/packages/node/references/posthog-node-references-latest.json b/packages/node/references/posthog-node-references-latest.json index bf317fcc87..fd87cff466 100644 --- a/packages/node/references/posthog-node-references-latest.json +++ b/packages/node/references/posthog-node-references-latest.json @@ -4091,21 +4091,21 @@ "name": "SendLogsBatchOutcome", "properties": [], "path": "../core/src/posthog-core-stateless.ts", - "example": "{\n kind: 'ok';\n} | {\n kind: 'too-large';\n} | {\n kind: 'retry-later';\n error: unknown;\n} | {\n kind: 'fatal';\n error: unknown;\n}" + "example": "{\n kind: 'ok';\n} | {\n kind: 'too-large';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, { "id": "SendMetricsBatchOutcome", "name": "SendMetricsBatchOutcome", "properties": [], "path": "../core/src/metrics/types.ts", - "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" + "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, { "id": "SendTracesBatchOutcome", "name": "SendTracesBatchOutcome", "properties": [], "path": "../core/src/traces/types.ts", - "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" + "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, { "id": "SentryIntegrationOptions", diff --git a/packages/react-native/references/posthog-react-native-references-latest.json b/packages/react-native/references/posthog-react-native-references-latest.json index 326f31634e..6ba91d2ee5 100644 --- a/packages/react-native/references/posthog-react-native-references-latest.json +++ b/packages/react-native/references/posthog-react-native-references-latest.json @@ -4444,21 +4444,21 @@ "name": "SendLogsBatchOutcome", "properties": [], "path": "../core/src/posthog-core-stateless.ts", - "example": "{\n kind: 'ok';\n} | {\n kind: 'too-large';\n} | {\n kind: 'retry-later';\n error: unknown;\n} | {\n kind: 'fatal';\n error: unknown;\n}" + "example": "{\n kind: 'ok';\n} | {\n kind: 'too-large';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, { "id": "SendMetricsBatchOutcome", "name": "SendMetricsBatchOutcome", "properties": [], "path": "../core/src/metrics/types.ts", - "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" + "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, { "id": "SendTracesBatchOutcome", "name": "SendTracesBatchOutcome", "properties": [], "path": "../core/src/traces/types.ts", - "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" + "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, { "id": "SeverityLevel", From 098b59763b13ca84190ee0bcd9b824d8460cd02a Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Wed, 2 Sep 2026 11:35:39 -0400 Subject: [PATCH 02/33] fix(core): honor Retry-After on every send path, not just the retry timer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The window was only consulted where a timer was armed, so an explicit flush() — the lifecycle and per-request path — sent inside it. Traces spent its whole per-batch retry budget there and dropped the spans before the wait elapsed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019vkNv2jdvWeE8hoAuiwgCr --- .changeset/otlp-honor-retry-after.md | 6 +- .changeset/otlp-skip-oversized-bodies.md | 4 +- .../__tests__/posthog.otlp-too-large.spec.ts | 21 +++ .../core/src/__tests__/retry-after.spec.ts | 8 ++ packages/core/src/logs/index.spec.ts | 132 +++++++++++++++++- packages/core/src/logs/index.ts | 13 +- packages/core/src/metrics/index.spec.ts | 78 +++++++++++ packages/core/src/metrics/index.ts | 20 ++- packages/core/src/posthog-core-stateless.ts | 7 +- packages/core/src/traces/index.spec.ts | 74 +++++++++- packages/core/src/traces/index.ts | 36 ++++- packages/node/src/client.ts | 4 +- 12 files changed, 381 insertions(+), 22 deletions(-) diff --git a/.changeset/otlp-honor-retry-after.md b/.changeset/otlp-honor-retry-after.md index 47ceec3b97..0905c33b02 100644 --- a/.changeset/otlp-honor-retry-after.md +++ b/.changeset/otlp-honor-retry-after.md @@ -1,5 +1,9 @@ --- +'posthog-node': patch +'posthog-react-native': patch '@posthog/core': patch --- -Honor `Retry-After` on the logs, metrics and traces export queues when a batch is refused. +Honor `Retry-After` when the ingestion endpoint refuses a logs, metrics or traces batch, rather than retrying on the SDK's own backoff alone. The header acts as a floor, so a shorter value never makes the SDK retry sooner than it would have, and a wait longer than five minutes is capped. + +In `posthog-node`, `flush()` now leaves spans queued while such a wait is open instead of exporting them, so a service that flushes on every request cannot exhaust a batch's retry budget inside a window where every attempt is refused. Shutdown still exports. diff --git a/.changeset/otlp-skip-oversized-bodies.md b/.changeset/otlp-skip-oversized-bodies.md index 9b332d1cd8..d1243c199a 100644 --- a/.changeset/otlp-skip-oversized-bodies.md +++ b/.changeset/otlp-skip-oversized-bodies.md @@ -1,5 +1,7 @@ --- +'posthog-node': patch +'posthog-react-native': patch '@posthog/core': patch --- -Stop sending logs, metrics and traces batches larger than the ingestion endpoint's 2 MB body limit. +Stop uploading logs, metrics and traces batches larger than the ingestion endpoint's 2 MB body limit. Such a batch can only come back `413`, so it is split — and, when a single record is itself oversized, dropped — without spending a request on each attempt. The size is measured before compression, matching how the endpoint applies its limit. diff --git a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts index f3b671f858..7a867c5127 100644 --- a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts +++ b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts @@ -63,6 +63,27 @@ describe('OTLP bodies over the endpoint limit', () => { expect(mocks.fetch).not.toHaveBeenCalled() }) + // The endpoint rejects a body *over* its limit, so one exactly at it is + // accepted on both sides. Pinning both directions keeps the comparison from + // drifting to `>=`, which would refuse an acceptable batch without a request + // and — in traces — halve it down and drop the span with no 413 to show for it. + const overheadBytes = (): number => JSON.stringify(spansOf(1024)).length - 1024 + + it('sends a batch of exactly the limit', async () => { + const exact = 2 * 1024 * 1024 - overheadBytes() + expect(JSON.stringify(spansOf(exact)).length).toBe(2 * 1024 * 1024) + + await expect(posthog._sendTracesBatch(spansOf(exact))).resolves.toEqual({ kind: 'ok' }) + expect(mocks.fetch).toHaveBeenCalledTimes(1) + }) + + it('reports a batch one byte over the limit as too-large', async () => { + await expect(posthog._sendTracesBatch(spansOf(2 * 1024 * 1024 - overheadBytes() + 1))).resolves.toEqual({ + kind: 'too-large', + }) + expect(mocks.fetch).not.toHaveBeenCalled() + }) + it('sends a compressible payload that is inside the limit before compression', async () => { jest.spyOn(posthog as any, 'compressPayload').mockResolvedValue(new Uint8Array(1024)) ;(posthog as any).disableCompression = false diff --git a/packages/core/src/__tests__/retry-after.spec.ts b/packages/core/src/__tests__/retry-after.spec.ts index 2c1157cbe3..6b6eb3555d 100644 --- a/packages/core/src/__tests__/retry-after.spec.ts +++ b/packages/core/src/__tests__/retry-after.spec.ts @@ -12,6 +12,14 @@ describe('parseRetryAfterMs', () => { expect(parseRetryAfterMs('Tue, 01 Sep 2026 12:00:30 GMT', now)).toBe(30_000) }) + it('reads the outermost hop when two proxies each append one', () => { + // `headers.get` joins repeated headers with ", ". The date form carries a + // comma of its own, so only the delta-seconds form is split. + expect(parseRetryAfterMs('60, 120', now)).toBe(60_000) + expect(parseRetryAfterMs('60,120', now)).toBe(60_000) + expect(parseRetryAfterMs('Tue, 01 Sep 2026 12:00:30 GMT', now)).toBe(30_000) + }) + it('ignores a header that does not name a future time', () => { // A past date and a zero delta both mean "retry now", which is the caller's own backoff. expect(parseRetryAfterMs('Tue, 01 Sep 2026 11:59:00 GMT', now)).toBeUndefined() diff --git a/packages/core/src/logs/index.spec.ts b/packages/core/src/logs/index.spec.ts index 83447b358b..db5eea7801 100644 --- a/packages/core/src/logs/index.spec.ts +++ b/packages/core/src/logs/index.spec.ts @@ -1255,10 +1255,140 @@ describe('PostHogLogs', () => { await jest.advanceTimersByTimeAsync(1000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + // Stop just short of the deadline, then cross it without letting the + // re-armed timer fire — otherwise the timer satisfies the assertion and + // the test says nothing about onReconnect. + await jest.advanceTimersByTimeAsync(4999) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + jest.setSystemTime(Date.now() + 2) + + logs.onReconnect() + await jest.advanceTimersByTimeAsync(1) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) + }) + + it('does not let a capture after an explicit flush send inside the window', async () => { + // `flush()` is the lifecycle path (RN foreground/background, shutdown). + // It leaves no timer behind, so the next capture is the one that arms + // one — at the plain interval unless the window floors it. + mockInstance._sendLogsBatch = jest.fn(() => + Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) + ) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 5000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + await logs.flush().catch(() => {}) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + logs.captureLog({ body: 'second' }) + await jest.advanceTimersByTimeAsync(5000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(295_000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) + }) + + it('ends the window when an explicit flush succeeds', async () => { + // The endpoint just accepted a batch, so the wait it asked for earlier is + // over — the gated paths must not stay blocked for the rest of it. + const outcomes: any[] = [{ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }] + mockInstance._sendLogsBatch = jest.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 5000, maxBufferSize: 2 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) await jest.advanceTimersByTimeAsync(5000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + await logs.flush() + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) + + logs.captureLog({ body: 'second' }) + logs.onReconnect() + await jest.advanceTimersByTimeAsync(1) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(3) + }) + + it('drops a Retry-After wait on reset', async () => { + const outcomes: any[] = [{ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }] + mockInstance._sendLogsBatch = jest.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 1000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + // Asserted through a gated path: a plain capture would flush either way. + logs.reset() + logs.captureLog({ body: 'second' }) logs.onReconnect() await jest.advanceTimersByTimeAsync(1) - expect(mockInstance._sendLogsBatch.mock.calls.length).toBeGreaterThan(1) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) + }) + + it('keeps its own backoff when the endpoint asks for less', async () => { + // A proxy answering `Retry-After: 1` must not turn the retry into a + // one-second hot loop against an endpoint already refusing traffic. + mockInstance._sendLogsBatch = jest.fn(() => + Promise.resolve({ kind: 'retry-later', error: new Error('503'), retryAfterMs: 10 }) + ) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 5000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + await jest.advanceTimersByTimeAsync(5000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(5000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) + }) + + it('ends the wait when a later failure names none', async () => { + // 429 with a window, then a plain 503: the queue drops back to its own + // backoff rather than waiting the old window out on every attempt. + const outcomes: any[] = [ + { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }, + { kind: 'retry-later', error: new Error('503') }, + ] + mockInstance._sendLogsBatch = jest.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 1000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(300_000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) + + // Back on the plain backoff, not another 300s. + await jest.advanceTimersByTimeAsync(4000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(3) }) it('keeps flushing on the interval while captures keep arriving', async () => { diff --git a/packages/core/src/logs/index.ts b/packages/core/src/logs/index.ts index 284e82dd65..9d1314801b 100644 --- a/packages/core/src/logs/index.ts +++ b/packages/core/src/logs/index.ts @@ -311,10 +311,15 @@ export class PostHogLogs { continue } + // Only a retriable outcome carries a wait, and it is recorded here rather + // than on the background wrapper so an explicit `flush()` — which every + // lifecycle hook takes — records and clears it too. + this._retryAfterUntil = + outcome.kind === 'retry-later' && outcome.retryAfterMs ? Date.now() + outcome.retryAfterMs : 0 + if (outcome.kind === 'retry-later') { // Transient failure: keep records in the queue for the next flush cycle // and surface the error so the caller can log/react. - this._retryAfterUntil = outcome.retryAfterMs ? Date.now() + outcome.retryAfterMs : 0 throw outcome.error } @@ -396,7 +401,10 @@ export class PostHogLogs { if (this._flushTimer) { return } - this._setFlushTimer(this._flushIntervalMs) + // Floored by any open window: an explicit `flush()` leaves no timer behind, + // so this is the path a capture takes after one, and the plain interval + // would land inside the wait the endpoint asked for. + this._setFlushTimer(Math.max(this._flushIntervalMs, this._retryAfterRemainingMs())) } // Backoff and `Retry-After` are floors, so a timer already armed at the plain @@ -491,7 +499,6 @@ export class PostHogLogs { .then( () => { this._consecutiveFlushFailures = 0 - this._retryAfterUntil = 0 }, (err) => { this._consecutiveFlushFailures++ diff --git a/packages/core/src/metrics/index.spec.ts b/packages/core/src/metrics/index.spec.ts index dc94ee7f51..fbed08b347 100644 --- a/packages/core/src/metrics/index.spec.ts +++ b/packages/core/src/metrics/index.spec.ts @@ -315,6 +315,84 @@ describe('PostHogMetrics', () => { expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) }) + it('does not restart a served-out wait for a sample captured mid-retry', async () => { + // A deadline, not a duration. The retry's timer has already fired, so the + // capture below is the one that arms the next timer — and it sees the + // wait still set, because the send it belongs to has not settled. Holding + // a duration here re-arms for the whole window a second time and leaves + // the sample 300s behind on an endpoint that has already recovered. + let settleRetry: ((outcome: SendMetricsBatchOutcome) => void) | undefined + let call = 0 + const instance = createMockInstance({ + _sendMetricsBatch: jest.fn((): Promise => { + call++ + if (call === 1) { + return Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) + } + return new Promise((resolve) => { + settleRetry = resolve + }) + }), + }) + const metrics = createMetrics({ flushIntervalMs: 10_000 }, instance) + metrics.count('orders_created', 1) + await jest.advanceTimersByTimeAsync(10_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) + + // The wait elapses and the retry goes out, but hangs. + await jest.advanceTimersByTimeAsync(300_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) + + metrics.count('orders_created', 1) + settleRetry?.({ kind: 'ok' }) + await jest.advanceTimersByTimeAsync(0) + + // One interval, not another window. + await jest.advanceTimersByTimeAsync(10_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(3) + }) + + it('keeps its own interval when the endpoint asks for less', async () => { + const instance = createMockInstance({ + _sendMetricsBatch: jest.fn((): Promise => + Promise.resolve({ kind: 'retry-later', error: new Error('503'), retryAfterMs: 10 }) + ), + }) + const metrics = createMetrics({ flushIntervalMs: 10_000 }, instance) + metrics.count('orders_created', 1) + await jest.advanceTimersByTimeAsync(10_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(1000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(10_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) + }) + + it('ends the wait when a later failure names none', async () => { + const outcomes: SendMetricsBatchOutcome[] = [ + { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }, + { kind: 'retry-later', error: new Error('503') }, + ] + const instance = createMockInstance({ + _sendMetricsBatch: jest.fn((): Promise => + Promise.resolve(outcomes.shift() ?? { kind: 'ok' }) + ), + }) + const metrics = createMetrics({ flushIntervalMs: 10_000 }, instance) + metrics.count('orders_created', 1) + await jest.advanceTimersByTimeAsync(10_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) + + await jest.advanceTimersByTimeAsync(300_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) + + // Back on the plain interval, not another 300s. + await jest.advanceTimersByTimeAsync(10_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(3) + }) + it('does not send when the window is empty', async () => { createMetrics({ flushIntervalMs: 5000 }) await jest.advanceTimersByTimeAsync(15000) diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index 3ec8b24b3b..e9a7b1dd34 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -73,9 +73,12 @@ export class PostHogMetrics { // types under one name produces charts that blend both series. private _typeByName = new Map() private _typeCollisionWarned = new Set() - // Set from a `Retry-After` the endpoint sent; cleared on any other outcome so - // one throttled response cannot pin the delay for the rest of the process. - private _retryAfterMs = 0 + // Absolute deadline from a `Retry-After` the endpoint sent; cleared on any + // other outcome so one throttled response cannot pin the delay for the rest + // of the process. A deadline rather than a duration, so a timer armed while + // the wait is already part-served counts down the remainder instead of + // restarting it. + private _retryAfterUntil = 0 private _flushTimerFiresAt = 0 // Bumped by reset(). A flush that was in flight when reset() ran (e.g. it // lost a shutdown race) sees a stale generation when its send settles and @@ -140,7 +143,7 @@ export class PostHogMetrics { /** Clears the flush timer, drops the current window, and invalidates in-flight flushes. */ reset(): void { this._generation++ - this._retryAfterMs = 0 + this._retryAfterUntil = 0 this._clearFlushTimer() this._series = new Map() this._flushPromise = null @@ -313,7 +316,11 @@ export class PostHogMetrics { // `Retry-After` is a floor, not a replacement: never retry before the server // asked, and never more often than the flush interval. private _nextFlushDelay(): number { - return Math.max(this._config.flushIntervalMs, this._retryAfterMs) + return Math.max(this._config.flushIntervalMs, this._retryAfterRemainingMs()) + } + + private _retryAfterRemainingMs(): number { + return Math.max(0, this._retryAfterUntil - Date.now()) } private _setFlushTimer(delayMs: number): void { @@ -355,7 +362,8 @@ export class PostHogMetrics { } // Only a retriable outcome carries a wait; anything else ends it, or a stale // one would pin every later flush at the window the server has moved on from. - this._retryAfterMs = outcome.kind === 'retry-later' ? (outcome.retryAfterMs ?? 0) : 0 + this._retryAfterUntil = + outcome.kind === 'retry-later' && outcome.retryAfterMs ? Date.now() + outcome.retryAfterMs : 0 switch (outcome.kind) { case 'ok': return diff --git a/packages/core/src/posthog-core-stateless.ts b/packages/core/src/posthog-core-stateless.ts index 8ecbc09e86..3a0319f7b7 100644 --- a/packages/core/src/posthog-core-stateless.ts +++ b/packages/core/src/posthog-core-stateless.ts @@ -70,7 +70,12 @@ export function parseRetryAfterMs(value: string | null | undefined, now: number if (!value) { return undefined } - const trimmed = value.trim() + const raw = value.trim() + // `headers.get` joins repeated headers with ", ", so a CDN and a load + // balancer that each append one yield "60, 120". Read the first, which the + // outermost hop set — but only for delta-seconds, since an HTTP-date carries + // a comma of its own ("Wed, 21 Oct 2015 07:28:00 GMT"). + const trimmed = /^\d+\s*,/.test(raw) ? raw.slice(0, raw.indexOf(',')).trim() : raw // Integer seconds. Not parseFloat: "10 minutes" must not read as 10 seconds. const seconds = /^\d+$/.test(trimmed) ? Number(trimmed) : Number.NaN if (!Number.isFinite(seconds) && /^[+-]?[\d.]+$/.test(trimmed)) { diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 022c45d4a0..00540c9981 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -1338,7 +1338,6 @@ describe('PostHogTraces', () => { // The wait elapses, the retry lands a 400, and that ends the wait. await jest.advanceTimersByTimeAsync(300_000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(2) - expect((traces as any)._retryAfterMs).toBe(0) traces.startSpan('second').end() await jest.advanceTimersByTimeAsync(1000) @@ -1368,6 +1367,79 @@ describe('PostHogTraces', () => { await jest.advanceTimersByTimeAsync(2000) expect(mockInstance._sendTracesBatch.mock.calls.length).toBeGreaterThan(before) }) + + it('sends nothing on a host flush inside the window', async () => { + // posthog-node arms an events timer at `flushInterval` and its `flush()` + // override drains spans alongside them. Without the gate the window is + // spent one host cycle at a time. + mockInstance._sendTracesBatch.mockResolvedValue({ + kind: 'retry-later', + error: new Error('throttled'), + retryAfterMs: 300_000, + }) + const traces = createTraces({ flushIntervalMs: 10_000 }) + traces.startSpan('first').end() + + await traces.flush() + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) + + for (let i = 0; i < 9; i++) { + await jest.advanceTimersByTimeAsync(10_000) + await traces.flush() + } + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) + }) + + it('does not spend the head batch retry budget inside the window', async () => { + // The batch is dropped after MAX_RETRIES_PER_BATCH consecutive failures. + // At the host's cadence that budget burns out long before the window the + // endpoint asked for, and the spans are lost with it. + mockInstance._sendTracesBatch.mockResolvedValue({ + kind: 'retry-later', + error: new Error('throttled'), + retryAfterMs: 300_000, + }) + const traces = createTraces({ flushIntervalMs: 10_000 }) + traces.startSpan('first').end() + + for (let i = 0; i < 12; i++) { + await traces.flush() + await jest.advanceTimersByTimeAsync(10_000) + } + + expect(logger.warn).not.toHaveBeenCalledWith(expect.stringContaining('Dropping')) + }) + + it('sends on a forced flush inside the window', async () => { + // Teardown has no later attempt to honour the wait with. + mockInstance._sendTracesBatch.mockResolvedValue({ + kind: 'retry-later', + error: new Error('throttled'), + retryAfterMs: 300_000, + }) + const traces = createTraces({ flushIntervalMs: 10_000 }) + traces.startSpan('first').end() + + await traces.flush() + await traces.flush({ force: true }) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(2) + }) + + it('drops a Retry-After wait on reset', async () => { + const outcomes: SendTracesBatchOutcome[] = [ + { kind: 'retry-later', error: new Error('throttled'), retryAfterMs: 300_000 }, + ] + mockInstance._sendTracesBatch.mockImplementation(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) + const traces = createTraces({ flushIntervalMs: 1000 }) + traces.startSpan('first').end() + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) + + traces.reset() + traces.startSpan('second').end() + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(2) + }) }) describe('live span bounds', () => { diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 9f6a54f176..59f21e635c 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -73,9 +73,12 @@ export class PostHogTraces { private _lastDropWarningAt = 0 private _dropReasons = new Set() private _consecutiveFlushFailures = 0 - // Set from a `Retry-After` the endpoint sent; cleared on any other outcome so - // one throttled response cannot pin the delay for the rest of the process. - private _retryAfterMs = 0 + // Absolute deadline from a `Retry-After` the endpoint sent; cleared on any + // other outcome so one throttled response cannot pin the delay for the rest + // of the process. A deadline rather than a duration, so a timer armed while + // the wait is already part-served counts down the remainder instead of + // restarting it. + private _retryAfterUntil = 0 private _flushTimerFiresAt = 0 // Separate from the backoff counter: this one belongs to whatever batch is at // the head, and resets whenever that batch is removed or shrunk. @@ -219,13 +222,23 @@ export class PostHogTraces { * * A pass reports spans removed — queue length can't stand in, since a send * concurrent with an arrival leaves it unchanged. + * + * While the endpoint has asked for a wait this sends nothing and leaves the + * armed timer to retry, so a host that flushes on its own cadence can't spend + * the head batch's retry budget inside a window where every attempt is + * refused. `force` is for teardown, which has no later attempt to save. */ - async flush(): Promise { + async flush({ force = false }: { force?: boolean } = {}): Promise { for (;;) { if (!this._queue.length) { return } + if (!force && this._isWaitingOutRetryAfter()) { + this._armFlushTimerIfQueuedNoEarlierThan() + return + } + const inFlight = this._flushPromise const removed = await (inFlight ?? this._startFlush()) @@ -264,7 +277,7 @@ export class PostHogTraces { this._dropReasons.clear() this._lastDropWarningAt = 0 this._consecutiveFlushFailures = 0 - this._retryAfterMs = 0 + this._retryAfterUntil = 0 this._headBatchFailures = 0 } @@ -507,7 +520,8 @@ export class PostHogTraces { // Only a retriable outcome carries a wait; anything else ends it, or a // stale one would pin every later flush at the window the server has // moved on from. - this._retryAfterMs = outcome.kind === 'retry-later' ? (outcome.retryAfterMs ?? 0) : 0 + this._retryAfterUntil = + outcome.kind === 'retry-later' && outcome.retryAfterMs ? Date.now() + outcome.retryAfterMs : 0 if (outcome.kind === 'ok') { this._consecutiveFlushFailures = 0 @@ -642,7 +656,15 @@ export class PostHogTraces { const capped = Math.min(delay, Math.max(MAX_FLUSH_BACKOFF_MS, this._config.flushIntervalMs)) // `Retry-After` is a floor, not a replacement: never retry before the server // asked, and never more often than our own backoff would have. - return Math.max(capped, this._retryAfterMs) + return Math.max(capped, this._retryAfterRemainingMs()) + } + + private _retryAfterRemainingMs(): number { + return Math.max(0, this._retryAfterUntil - Date.now()) + } + + private _isWaitingOutRetryAfter(): boolean { + return this._retryAfterRemainingMs() > 0 } private _clearFlushTimer(): void { diff --git a/packages/node/src/client.ts b/packages/node/src/client.ts index 08cb08b189..2b7ec2d53b 100644 --- a/packages/node/src/client.ts +++ b/packages/node/src/client.ts @@ -2796,8 +2796,10 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen if (this._traces) { // Same treatment as metrics: send what's queued, raced against the shared // shutdown budget, then reset so a losing flush can't re-arm a timer. + // Forced past any `Retry-After` wait, which teardown has no later attempt + // to honour. await raceWithTimeout( - this._traces.flush().catch(() => {}), + this._traces.flush({ force: true }).catch(() => {}), Math.max(0, shutdownDeadlineMs - Date.now()) ) this._traces.reset() From f766c2129be8a80f8c37cae67d2c0761adcab384 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Wed, 2 Sep 2026 11:35:39 -0400 Subject: [PATCH 03/33] chore: regenerate the terser mangled-names list Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019vkNv2jdvWeE8hoAuiwgCr --- packages/browser/terser-mangled-names.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/browser/terser-mangled-names.json b/packages/browser/terser-mangled-names.json index d2c4bd9303..3f80e52346 100644 --- a/packages/browser/terser-mangled-names.json +++ b/packages/browser/terser-mangled-names.json @@ -25,6 +25,7 @@ "_applyTransforms", "_areWeOnline", "_armFlushTimer", + "_armFlushTimerNoEarlierThan", "_asRequiredConfig", "_autoSubmitPrefilledResponses", "_automaticDisplayDispose", @@ -421,6 +422,7 @@ "_isSurveyRefreshBackingOff", "_isSurveysEnabled", "_isTourEligible", + "_isWaitingOutRetryAfter", "_isWidgetEnabled", "_isWidgetOpen", "_isWidgetRendered", @@ -716,7 +718,8 @@ "_resumeSavedTour", "_resyncIntervalMs", "_resyncTimer", - "_retryAfterMs", + "_retryAfterRemainingMs", + "_retryAfterUntil", "_retryQueue", "_rrwebError", "_rrwebStartAttempted", @@ -766,6 +769,7 @@ "_setActivatedSession", "_setCrossTabFeatureFlagChangesPending", "_setFlushTimeout", + "_setFlushTimer", "_setPersonPropertiesForFlags", "_setProp", "_setProperty", From 9a2df41fb61efc2e37ab8dc0cdae1142a44acfb6 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Wed, 2 Sep 2026 13:22:11 -0400 Subject: [PATCH 04/33] fix(core): drain on explicit flush, and clamp the Retry-After deadline Gating traces flush() on the window stranded spans: it is what the serverless waitUntil keep-alive awaits, and the recovery timer is unref'd, so a frozen isolate never sent them. Protect the head batch's retry budget instead of suppressing the send. The wall-clock deadline is now clamped, so a backward clock step can't stretch a wait past the cap. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019vkNv2jdvWeE8hoAuiwgCr --- .changeset/otlp-honor-retry-after.md | 2 +- packages/core/src/logs/index.spec.ts | 60 ++++++++++++++ packages/core/src/logs/index.ts | 7 +- packages/core/src/metrics/index.spec.ts | 63 +++++++++++++++ packages/core/src/metrics/index.ts | 7 +- packages/core/src/posthog-core-stateless.ts | 9 +-- packages/core/src/traces/index.spec.ts | 89 +++++++++++++++------ packages/core/src/traces/index.ts | 34 +++++--- packages/core/src/utils/index.ts | 8 ++ packages/node/src/client.ts | 4 +- 10 files changed, 230 insertions(+), 53 deletions(-) diff --git a/.changeset/otlp-honor-retry-after.md b/.changeset/otlp-honor-retry-after.md index 0905c33b02..db11939853 100644 --- a/.changeset/otlp-honor-retry-after.md +++ b/.changeset/otlp-honor-retry-after.md @@ -6,4 +6,4 @@ Honor `Retry-After` when the ingestion endpoint refuses a logs, metrics or traces batch, rather than retrying on the SDK's own backoff alone. The header acts as a floor, so a shorter value never makes the SDK retry sooner than it would have, and a wait longer than five minutes is capped. -In `posthog-node`, `flush()` now leaves spans queued while such a wait is open instead of exporting them, so a service that flushes on every request cannot exhaust a batch's retry budget inside a window where every attempt is refused. Shutdown still exports. +The periodic flush waits the window out. An explicit `flush()` still sends, since it is a lifecycle or teardown boundary with no later attempt — but in `posthog-node` a refusal during the wait no longer counts against a span batch's retry budget, so a service that flushes on every request can no longer exhaust that budget and drop spans before the window has even elapsed. diff --git a/packages/core/src/logs/index.spec.ts b/packages/core/src/logs/index.spec.ts index db5eea7801..5120aeac53 100644 --- a/packages/core/src/logs/index.spec.ts +++ b/packages/core/src/logs/index.spec.ts @@ -1364,6 +1364,66 @@ describe('PostHogLogs', () => { expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) }) + it('does not restart a served-out wait for a capture during the retry', async () => { + // A deadline, not a duration. The retry's timer has already fired, so the + // capture below arms the next one — and it sees the wait still set, + // because the send it belongs to has not settled. Holding a duration here + // re-arms for the whole window again and leaves the record 300s behind an + // endpoint that has already recovered. + let settle: ((outcome: any) => void) | undefined + let call = 0 + mockInstance._sendLogsBatch = jest.fn(() => { + call++ + if (call === 1) { + return Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) + } + return new Promise((resolve) => { + settle = resolve + }) + }) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 5000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + await jest.advanceTimersByTimeAsync(5000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + // The wait elapses and the retry goes out, but hangs. + await jest.advanceTimersByTimeAsync(300_000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) + + logs.captureLog({ body: 'second' }) + settle?.({ kind: 'ok' }) + await jest.advanceTimersByTimeAsync(0) + + // One interval, not another window. + await jest.advanceTimersByTimeAsync(5000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(3) + }) + + it('does not let a backward clock step stretch the wait past the cap', async () => { + mockInstance._sendLogsBatch = jest.fn(() => + Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 60_000 }) + ) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 1000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + await jest.advanceTimersByTimeAsync(1000) + + const real = Date.now() + jest.spyOn(Date, 'now').mockImplementation(() => real - 3_600_000) + expect((logs as any)._retryAfterRemainingMs()).toBeLessThanOrEqual(5 * 60_000) + }) + it('ends the wait when a later failure names none', async () => { // 429 with a window, then a plain 503: the queue drops back to its own // backoff rather than waiting the old window out on every attempt. diff --git a/packages/core/src/logs/index.ts b/packages/core/src/logs/index.ts index 9d1314801b..f2d21cc80e 100644 --- a/packages/core/src/logs/index.ts +++ b/packages/core/src/logs/index.ts @@ -1,7 +1,7 @@ import type { LogAttributeValue } from '@posthog/types' import { buildOtlpLogRecord, buildOtlpLogsPayload, buildResourceAttributes } from './logs-utils' import { Logger, PostHogPersistedProperty } from '../types' -import { isArray, raceWithTimeout, safeSetTimeout } from '../utils' +import { MAX_RETRY_AFTER_MS, isArray, raceWithTimeout, safeSetTimeout } from '../utils' import type { BufferedLogEntry, CaptureLogOptions, LogSdkContext, LogsHost, ResolvedPostHogLogsConfig } from './types' // Caps the retry backoff at 2^6 = 64× the flush interval. @@ -437,7 +437,10 @@ export class PostHogLogs { } private _retryAfterRemainingMs(): number { - return Math.max(0, this._retryAfterUntil - Date.now()) + // Clamped, not just floored: the deadline is wall clock, so a backward step + // (NTP, a resumed VM, a user changing the date) would otherwise strand the + // queue far past the cap `parseRetryAfterMs` applied. + return Math.min(MAX_RETRY_AFTER_MS, Math.max(0, this._retryAfterUntil - Date.now())) } private _isWaitingOutRetryAfter(): boolean { diff --git a/packages/core/src/metrics/index.spec.ts b/packages/core/src/metrics/index.spec.ts index fbed08b347..7f73987f64 100644 --- a/packages/core/src/metrics/index.spec.ts +++ b/packages/core/src/metrics/index.spec.ts @@ -299,6 +299,33 @@ describe('PostHogMetrics', () => { expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(3) }) + it('does not install a Retry-After that lands after reset', async () => { + let settle: ((outcome: SendMetricsBatchOutcome) => void) | undefined + const instance = createMockInstance({ + _sendMetricsBatch: jest.fn( + (): Promise => + new Promise((resolve) => { + settle = resolve + }) + ), + }) + const metrics = createMetrics({ flushIntervalMs: 1000 }, instance) + metrics.count('orders_created', 1) + await jest.advanceTimersByTimeAsync(1000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) + + metrics.reset() + + // Settle first: the capture that arms the next timer must not find a + // window belonging to the client that was just torn down. + settle?.({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) + await jest.advanceTimersByTimeAsync(0) + + metrics.count('orders_created', 1) + await jest.advanceTimersByTimeAsync(1000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) + }) + it('drops a Retry-After wait on reset', async () => { const instance = createMockInstance({ _sendMetricsBatch: jest.fn((): Promise => @@ -370,6 +397,42 @@ describe('PostHogMetrics', () => { expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) }) + it('sends on an explicit flush inside the window', async () => { + // Lifecycle drains (RN background, shutdown) must not become no-ops for + // the length of a window; metrics has no `force` escape hatch. + const outcomes: SendMetricsBatchOutcome[] = [ + { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }, + ] + const instance = createMockInstance({ + _sendMetricsBatch: jest.fn((): Promise => + Promise.resolve(outcomes.shift() ?? { kind: 'ok' }) + ), + }) + const metrics = createMetrics({ flushIntervalMs: 10_000 }, instance) + metrics.count('orders_created', 1) + await jest.advanceTimersByTimeAsync(10_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) + + metrics.count('orders_created', 1) + await metrics.flush() + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) + }) + + it('does not let a backward clock step stretch the wait past the cap', async () => { + const instance = createMockInstance({ + _sendMetricsBatch: jest.fn((): Promise => + Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 60_000 }) + ), + }) + const metrics = createMetrics({ flushIntervalMs: 1000 }, instance) + metrics.count('orders_created', 1) + await metrics.flush() + + const real = Date.now() + jest.spyOn(Date, 'now').mockImplementation(() => real - 3_600_000) + expect((metrics as any)._retryAfterRemainingMs()).toBeLessThanOrEqual(5 * 60_000) + }) + it('ends the wait when a later failure names none', async () => { const outcomes: SendMetricsBatchOutcome[] = [ { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }, diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index e9a7b1dd34..dd06664d52 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -9,7 +9,7 @@ import type { OtlpNumberDataPoint, } from '@posthog/types' import type { Logger } from '../types' -import { isArray, safeSetTimeout } from '../utils' +import { MAX_RETRY_AFTER_MS, isArray, safeSetTimeout } from '../utils' import { toOtlpKeyValueList } from '../utils/otlp-any-value' import { DEFAULT_HISTOGRAM_BOUNDS, @@ -320,7 +320,10 @@ export class PostHogMetrics { } private _retryAfterRemainingMs(): number { - return Math.max(0, this._retryAfterUntil - Date.now()) + // Clamped, not just floored: the deadline is wall clock, so a backward step + // (NTP, a resumed VM, a user changing the date) would otherwise strand the + // queue far past the cap `parseRetryAfterMs` applied. + return Math.min(MAX_RETRY_AFTER_MS, Math.max(0, this._retryAfterUntil - Date.now())) } private _setFlushTimer(delayMs: number): void { diff --git a/packages/core/src/posthog-core-stateless.ts b/packages/core/src/posthog-core-stateless.ts index 3a0319f7b7..ce8fd68edc 100644 --- a/packages/core/src/posthog-core-stateless.ts +++ b/packages/core/src/posthog-core-stateless.ts @@ -29,6 +29,7 @@ import { import { allSettled, createNamedError, + MAX_RETRY_AFTER_MS, currentISOTime, PromiseQueue, removeTrailingSlash, @@ -51,14 +52,6 @@ import { createDefaultStackParser, } from './error-tracking' -/** - * Longest `Retry-After` the SDK will wait. A rate-limit window can legitimately - * be long, but nothing upstream of the SDK bounds this header — it is as likely - * to come from a proxy or CDN as from PostHog — and an unbounded value would - * strand a queue for hours, so the wait is capped and the retry happens early. - */ -const MAX_RETRY_AFTER_MS = 5 * 60_000 - /** * `Retry-After` as milliseconds from now. The header is either delta-seconds or * an HTTP-date; anything else, a date already in the past, or a negative delta diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 00540c9981..9aceac315c 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -1368,32 +1368,30 @@ describe('PostHogTraces', () => { expect(mockInstance._sendTracesBatch.mock.calls.length).toBeGreaterThan(before) }) - it('sends nothing on a host flush inside the window', async () => { - // posthog-node arms an events timer at `flushInterval` and its `flush()` - // override drains spans alongside them. Without the gate the window is - // spent one host cycle at a time. - mockInstance._sendTracesBatch.mockResolvedValue({ - kind: 'retry-later', - error: new Error('throttled'), - retryAfterMs: 300_000, - }) + it('still drains on an explicit flush inside the window', async () => { + // An explicit flush is a lifecycle or teardown boundary with no later + // attempt — on a serverless host the retry timer is unref'd and dies with + // the isolate — so the window must not turn it into a no-op. + const outcomes: SendTracesBatchOutcome[] = [ + { kind: 'retry-later', error: new Error('throttled'), retryAfterMs: 300_000 }, + ] + mockInstance._sendTracesBatch.mockImplementation(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) const traces = createTraces({ flushIntervalMs: 10_000 }) traces.startSpan('first').end() await traces.flush() expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) - for (let i = 0; i < 9; i++) { - await jest.advanceTimersByTimeAsync(10_000) - await traces.flush() - } - expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) + // The endpoint has recovered, and the caller asked explicitly. + await traces.flush() + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(2) + expect(sentSpans().map((s) => s.name)).toContain('first') }) it('does not spend the head batch retry budget inside the window', async () => { // The batch is dropped after MAX_RETRIES_PER_BATCH consecutive failures. - // At the host's cadence that budget burns out long before the window the - // endpoint asked for, and the spans are lost with it. + // At a host's flush cadence that budget would burn out long before the + // window the endpoint asked for, losing the spans with it. mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('throttled'), @@ -1408,20 +1406,63 @@ describe('PostHogTraces', () => { } expect(logger.warn).not.toHaveBeenCalledWith(expect.stringContaining('Dropping')) + expect(mockInstance._sendTracesBatch.mock.calls.length).toBeGreaterThan(1) }) - it('sends on a forced flush inside the window', async () => { - // Teardown has no later attempt to honour the wait with. - mockInstance._sendTracesBatch.mockResolvedValue({ - kind: 'retry-later', - error: new Error('throttled'), - retryAfterMs: 300_000, - }) + it('still charges the retry budget for failures outside any window', async () => { + // The budget must keep working when the endpoint names no wait, or a + // permanently failing head batch pins the queue forever. + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('503') }) const traces = createTraces({ flushIntervalMs: 10_000 }) traces.startSpan('first').end() + for (let i = 0; i < 12; i++) { + await traces.flush() + await jest.advanceTimersByTimeAsync(10_000) + } + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Dropping')) + }) + + it('does not let a backward clock step stretch the wait past the cap', async () => { + // The deadline is wall clock. Without a clamp an NTP correction or a + // resumed VM turns a 60s wait into an hours-long one. + mockInstance._sendTracesBatch.mockResolvedValueOnce({ + kind: 'retry-later', + error: new Error('429'), + retryAfterMs: 60_000, + }) + const traces = createTraces({ flushIntervalMs: 1000 }) + traces.startSpan('first').end() await traces.flush() - await traces.flush({ force: true }) + + const real = Date.now() + jest.spyOn(Date, 'now').mockImplementation(() => real - 3_600_000) + expect((traces as any)._retryAfterRemainingMs()).toBeLessThanOrEqual(5 * 60_000) + }) + + it('does not install a Retry-After that lands after reset', async () => { + // The generation check must precede the window assignment, or a 429 that + // settles after teardown pins a wait on the fresh client. + let settle: ((outcome: SendTracesBatchOutcome) => void) | undefined + mockInstance._sendTracesBatch.mockImplementation( + () => + new Promise((resolve) => { + settle = resolve + }) + ) + const traces = createTraces({ flushIntervalMs: 1000 }) + traces.startSpan('before-reset').end() + await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) + + traces.reset() + traces.startSpan('after-reset').end() + + settle?.({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) + await jest.advanceTimersByTimeAsync(0) + + await jest.advanceTimersByTimeAsync(1000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(2) }) diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 59f21e635c..93601134ef 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -13,7 +13,7 @@ import { newSpanId, newTraceId } from './ids' import { parseTraceparent, sanitizeTracestate } from './traceparent' import { assignUserAttributes, resolveStartTime, sanitizeName } from './sanitize' import { buildOtlpSpan, buildOtlpTracesPayload, buildTracesResourceAttributes } from './otlp' -import { isPromise, safeSetTimeout } from '../utils' +import { MAX_RETRY_AFTER_MS, isPromise, safeSetTimeout } from '../utils' // Retriable failures on the same head batch before it is dropped, so a stuck // batch cannot pin the queue while fresher spans are refused at the cap. The @@ -223,22 +223,18 @@ export class PostHogTraces { * A pass reports spans removed — queue length can't stand in, since a send * concurrent with an arrival leaves it unchanged. * - * While the endpoint has asked for a wait this sends nothing and leaves the - * armed timer to retry, so a host that flushes on its own cadence can't spend - * the head batch's retry budget inside a window where every attempt is - * refused. `force` is for teardown, which has no later attempt to save. + * An open `Retry-After` window does not stop an explicit flush: the caller is + * a lifecycle or teardown boundary that has no later attempt, and on a + * serverless host the armed timer is unref'd and dies with the isolate. The + * window is honoured by not charging such an attempt against the head batch's + * retry budget, so the wait costs a request rather than the spans. */ - async flush({ force = false }: { force?: boolean } = {}): Promise { + async flush(): Promise { for (;;) { if (!this._queue.length) { return } - if (!force && this._isWaitingOutRetryAfter()) { - this._armFlushTimerIfQueuedNoEarlierThan() - return - } - const inFlight = this._flushPromise const removed = await (inFlight ?? this._startFlush()) @@ -508,6 +504,10 @@ export class PostHogTraces { continue } + // Read before the send: the outcome overwrites the window, and an attempt + // the endpoint had already told us to skip says nothing about this batch. + const insideRetryAfterWindow = this._isWaitingOutRetryAfter() + const outcome = await this._instance._sendTracesBatch( buildOtlpTracesPayload(spans, resourceAttributes, scopeName, scopeVersion, this._logger) ) @@ -557,7 +557,12 @@ export class PostHogTraces { if (outcome.kind === 'retry-later') { this._consecutiveFlushFailures++ - this._headBatchFailures++ + // A refusal inside a window the endpoint asked us to wait out is not + // evidence against this batch. Charging it spends the whole budget on + // the wait and drops the spans before the window even elapses. + if (!insideRetryAfterWindow) { + this._headBatchFailures++ + } this._headBatchSize = size if (this._headBatchFailures < MAX_RETRIES_PER_BATCH) { // Keep the spans queued; the flush timer picks them up again. @@ -660,7 +665,10 @@ export class PostHogTraces { } private _retryAfterRemainingMs(): number { - return Math.max(0, this._retryAfterUntil - Date.now()) + // Clamped, not just floored: the deadline is wall clock, so a backward step + // (NTP, a resumed VM, a user changing the date) would otherwise strand the + // queue far past the cap `parseRetryAfterMs` applied. + return Math.min(MAX_RETRY_AFTER_MS, Math.max(0, this._retryAfterUntil - Date.now())) } private _isWaitingOutRetryAfter(): boolean { diff --git a/packages/core/src/utils/index.ts b/packages/core/src/utils/index.ts index 3f21f7c561..77b993c6cd 100644 --- a/packages/core/src/utils/index.ts +++ b/packages/core/src/utils/index.ts @@ -15,6 +15,14 @@ export * from './user-agent-utils' export const STRING_FORMAT = 'utf8' +/** + * Longest `Retry-After` the SDK will wait. A rate-limit window can legitimately + * be long, but nothing upstream of the SDK bounds this header — it is as likely + * to come from a proxy or CDN as from PostHog — and an unbounded value would + * strand a queue for hours, so the wait is capped and the retry happens early. + */ +export const MAX_RETRY_AFTER_MS = 5 * 60_000 + export const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i export function isValidUUID(value: unknown): value is string { diff --git a/packages/node/src/client.ts b/packages/node/src/client.ts index 2b7ec2d53b..08cb08b189 100644 --- a/packages/node/src/client.ts +++ b/packages/node/src/client.ts @@ -2796,10 +2796,8 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen if (this._traces) { // Same treatment as metrics: send what's queued, raced against the shared // shutdown budget, then reset so a losing flush can't re-arm a timer. - // Forced past any `Retry-After` wait, which teardown has no later attempt - // to honour. await raceWithTimeout( - this._traces.flush({ force: true }).catch(() => {}), + this._traces.flush().catch(() => {}), Math.max(0, shutdownDeadlineMs - Date.now()) ) this._traces.reset() From cc72d100de85424b1f4beff50fc9b8bd3df14990 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Wed, 2 Sep 2026 13:57:09 -0400 Subject: [PATCH 05/33] fix(core): stop an in-window refusal from extending the window The deadline was re-installed by the very refusal it excused, so a host flushing faster than the window kept it open forever: the traces retry budget never advanced, the head batch never retired, and everything behind it was dropped at maxQueueSize. Measured 0 releases over 60 flushes; now releases at the intended 8 x window. A backward clock step no longer holds a window open either, and MAX_RETRY_AFTER_MS is out of the public barrel. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019vkNv2jdvWeE8hoAuiwgCr --- packages/browser/terser-mangled-names.json | 1 + packages/core/src/logs/index.spec.ts | 48 ++++++++++++-- packages/core/src/logs/index.ts | 33 +++++++--- packages/core/src/metrics/index.spec.ts | 44 ++++++++----- packages/core/src/metrics/index.ts | 34 ++++++++-- packages/core/src/posthog-core-stateless.ts | 2 +- packages/core/src/traces/index.spec.ts | 71 +++++++++++++++++++-- packages/core/src/traces/index.ts | 30 ++++++--- packages/core/src/utils/index.ts | 8 --- packages/core/src/utils/retry-after.ts | 10 +++ 10 files changed, 223 insertions(+), 58 deletions(-) create mode 100644 packages/core/src/utils/retry-after.ts diff --git a/packages/browser/terser-mangled-names.json b/packages/browser/terser-mangled-names.json index 3f80e52346..73fc314f60 100644 --- a/packages/browser/terser-mangled-names.json +++ b/packages/browser/terser-mangled-names.json @@ -718,6 +718,7 @@ "_resumeSavedTour", "_resyncIntervalMs", "_resyncTimer", + "_retryAfterInstalledAt", "_retryAfterRemainingMs", "_retryAfterUntil", "_retryQueue", diff --git a/packages/core/src/logs/index.spec.ts b/packages/core/src/logs/index.spec.ts index 5120aeac53..c21ac6dd44 100644 --- a/packages/core/src/logs/index.spec.ts +++ b/packages/core/src/logs/index.spec.ts @@ -1405,23 +1405,63 @@ describe('PostHogLogs', () => { expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(3) }) - it('does not let a backward clock step stretch the wait past the cap', async () => { + it('does not let a host out-pacing the window keep it open forever', async () => { + // RN takes flush() on every app-state transition. If each refusal slid the + // deadline forward, the window would never elapse and the gated paths — + // the size trigger and onReconnect — would stay suppressed indefinitely. mockInstance._sendLogsBatch = jest.fn(() => - Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 60_000 }) + Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 30_000 }) ) const logs = new PostHogLogs( mockInstance, - resolveForTest({ flushIntervalMs: 1000 }), + resolveForTest({ flushIntervalMs: 60_000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + await logs.flush().catch(() => {}) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + // Lifecycle flushes every 5s, well past the 30s window. Sampled rather + // than asserted through onReconnect: whether a given moment falls inside + // a window is timing-dependent, but it must fall outside one *sometimes*. + let sawWindowClosed = false + for (let i = 0; i < 12; i++) { + await jest.advanceTimersByTimeAsync(5000) + // Sampled before the flush: a flush that finds the window closed opens + // a fresh one, so sampling after it would always look open. + if ((logs as any)._retryAfterRemainingMs() === 0) { + sawWindowClosed = true + } + logs.captureLog({ body: `line ${i}` }) + await logs.flush().catch(() => {}) + } + expect(sawWindowClosed).toBe(true) + }) + + it('keeps flushing after a backward clock step', async () => { + const outcomes: any[] = [{ kind: 'retry-later', error: new Error('429'), retryAfterMs: 60_000 }] + mockInstance._sendLogsBatch = jest.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 1000, maxBufferSize: 2 }), logger, getContextFor(mockInstance), immediateOnReady ) logs.captureLog({ body: 'first' }) await jest.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) const real = Date.now() jest.spyOn(Date, 'now').mockImplementation(() => real - 3_600_000) - expect((logs as any)._retryAfterRemainingMs()).toBeLessThanOrEqual(5 * 60_000) + + // A gated path: suppressed for the size of the step without the guard. + logs.captureLog({ body: 'second' }) + logs.onReconnect() + await jest.advanceTimersByTimeAsync(1) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) }) it('ends the wait when a later failure names none', async () => { diff --git a/packages/core/src/logs/index.ts b/packages/core/src/logs/index.ts index f2d21cc80e..8b66f03055 100644 --- a/packages/core/src/logs/index.ts +++ b/packages/core/src/logs/index.ts @@ -1,7 +1,8 @@ import type { LogAttributeValue } from '@posthog/types' import { buildOtlpLogRecord, buildOtlpLogsPayload, buildResourceAttributes } from './logs-utils' import { Logger, PostHogPersistedProperty } from '../types' -import { MAX_RETRY_AFTER_MS, isArray, raceWithTimeout, safeSetTimeout } from '../utils' +import { isArray, raceWithTimeout, safeSetTimeout } from '../utils' +import { MAX_RETRY_AFTER_MS } from '../utils/retry-after' import type { BufferedLogEntry, CaptureLogOptions, LogSdkContext, LogsHost, ResolvedPostHogLogsConfig } from './types' // Caps the retry backoff at 2^6 = 64× the flush interval. @@ -32,6 +33,7 @@ export class PostHogLogs { // success so one throttled response cannot pin the delay for the rest of the // process. private _retryAfterUntil = 0 + private _retryAfterInstalledAt = 0 private _flushTimerFiresAt = 0 // Consecutive failed flushes; drives exponential backoff on the retry timer. // A successful flush resets it to 0. @@ -98,6 +100,7 @@ export class PostHogLogs { this._droppedWarned = false this._consecutiveFlushFailures = 0 this._retryAfterUntil = 0 + this._retryAfterInstalledAt = 0 this._maxBatchRecordsPerPost = this._config.maxBatchRecordsPerPost } @@ -293,6 +296,10 @@ export class PostHogLogs { this._instance.getLibraryVersion() ) + // Read before the send: the outcome overwrites the window, and a refusal + // the endpoint had already told us to expect must not extend it. + const insideRetryAfterWindow = this._isWaitingOutRetryAfter() + const outcome = await this._instance._sendLogsBatch(payload) if (this._queueGeneration !== generation) { @@ -313,9 +320,15 @@ export class PostHogLogs { // Only a retriable outcome carries a wait, and it is recorded here rather // than on the background wrapper so an explicit `flush()` — which every - // lifecycle hook takes — records and clears it too. - this._retryAfterUntil = - outcome.kind === 'retry-later' && outcome.retryAfterMs ? Date.now() + outcome.retryAfterMs : 0 + // lifecycle hook takes — records and clears it too. A refusal from inside + // an open window does not extend it, or a host that flushes on its own + // cadence keeps the window from ever elapsing. + if (outcome.kind !== 'retry-later' || !outcome.retryAfterMs) { + this._retryAfterUntil = 0 + } else if (!insideRetryAfterWindow) { + this._retryAfterInstalledAt = Date.now() + this._retryAfterUntil = this._retryAfterInstalledAt + outcome.retryAfterMs + } if (outcome.kind === 'retry-later') { // Transient failure: keep records in the queue for the next flush cycle @@ -437,10 +450,14 @@ export class PostHogLogs { } private _retryAfterRemainingMs(): number { - // Clamped, not just floored: the deadline is wall clock, so a backward step - // (NTP, a resumed VM, a user changing the date) would otherwise strand the - // queue far past the cap `parseRetryAfterMs` applied. - return Math.min(MAX_RETRY_AFTER_MS, Math.max(0, this._retryAfterUntil - Date.now())) + const now = Date.now() + // The deadline is wall clock. A clock that has moved behind the point the + // window was installed can no longer measure it, so end the window rather + // than hold the queue for the size of the step; the cap bounds the rest. + if (now < this._retryAfterInstalledAt) { + return 0 + } + return Math.min(MAX_RETRY_AFTER_MS, Math.max(0, this._retryAfterUntil - now)) } private _isWaitingOutRetryAfter(): boolean { diff --git a/packages/core/src/metrics/index.spec.ts b/packages/core/src/metrics/index.spec.ts index 7f73987f64..842d1c3f2e 100644 --- a/packages/core/src/metrics/index.spec.ts +++ b/packages/core/src/metrics/index.spec.ts @@ -299,6 +299,35 @@ describe('PostHogMetrics', () => { expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(3) }) + it('does not let a host out-pacing the window keep it open forever', async () => { + // Each refusal sliding the deadline would keep `_nextFlushDelay` pinned at + // the full window, so the flush cadence would never recover. + const instance = createMockInstance({ + _sendMetricsBatch: jest.fn((): Promise => + Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 30_000 }) + ), + }) + const metrics = createMetrics({ flushIntervalMs: 60_000 }, instance) + metrics.count('orders_created', 1) + await metrics.flush() + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) + + // Sampled: whether a given moment falls inside a window is timing + // dependent, but it must fall outside one sometimes. + let sawWindowClosed = false + for (let i = 0; i < 12; i++) { + await jest.advanceTimersByTimeAsync(5000) + // Sampled before the flush: a flush that finds the window closed opens + // a fresh one, so sampling after it would always look open. + if ((metrics as any)._retryAfterRemainingMs() === 0) { + sawWindowClosed = true + } + metrics.count('orders_created', 1) + await metrics.flush() + } + expect(sawWindowClosed).toBe(true) + }) + it('does not install a Retry-After that lands after reset', async () => { let settle: ((outcome: SendMetricsBatchOutcome) => void) | undefined const instance = createMockInstance({ @@ -418,21 +447,6 @@ describe('PostHogMetrics', () => { expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) }) - it('does not let a backward clock step stretch the wait past the cap', async () => { - const instance = createMockInstance({ - _sendMetricsBatch: jest.fn((): Promise => - Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 60_000 }) - ), - }) - const metrics = createMetrics({ flushIntervalMs: 1000 }, instance) - metrics.count('orders_created', 1) - await metrics.flush() - - const real = Date.now() - jest.spyOn(Date, 'now').mockImplementation(() => real - 3_600_000) - expect((metrics as any)._retryAfterRemainingMs()).toBeLessThanOrEqual(5 * 60_000) - }) - it('ends the wait when a later failure names none', async () => { const outcomes: SendMetricsBatchOutcome[] = [ { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }, diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index dd06664d52..8584e611b0 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -9,7 +9,8 @@ import type { OtlpNumberDataPoint, } from '@posthog/types' import type { Logger } from '../types' -import { MAX_RETRY_AFTER_MS, isArray, safeSetTimeout } from '../utils' +import { isArray, safeSetTimeout } from '../utils' +import { MAX_RETRY_AFTER_MS } from '../utils/retry-after' import { toOtlpKeyValueList } from '../utils/otlp-any-value' import { DEFAULT_HISTOGRAM_BOUNDS, @@ -79,6 +80,7 @@ export class PostHogMetrics { // the wait is already part-served counts down the remainder instead of // restarting it. private _retryAfterUntil = 0 + private _retryAfterInstalledAt = 0 private _flushTimerFiresAt = 0 // Bumped by reset(). A flush that was in flight when reset() ran (e.g. it // lost a shutdown race) sees a stale generation when its send settles and @@ -144,6 +146,7 @@ export class PostHogMetrics { reset(): void { this._generation++ this._retryAfterUntil = 0 + this._retryAfterInstalledAt = 0 this._clearFlushTimer() this._series = new Map() this._flushPromise = null @@ -319,11 +322,19 @@ export class PostHogMetrics { return Math.max(this._config.flushIntervalMs, this._retryAfterRemainingMs()) } + private _isWaitingOutRetryAfter(): boolean { + return this._retryAfterRemainingMs() > 0 + } + private _retryAfterRemainingMs(): number { - // Clamped, not just floored: the deadline is wall clock, so a backward step - // (NTP, a resumed VM, a user changing the date) would otherwise strand the - // queue far past the cap `parseRetryAfterMs` applied. - return Math.min(MAX_RETRY_AFTER_MS, Math.max(0, this._retryAfterUntil - Date.now())) + const now = Date.now() + // The deadline is wall clock. A clock that has moved behind the point the + // window was installed can no longer measure it, so end the window rather + // than hold the queue for the size of the step; the cap bounds the rest. + if (now < this._retryAfterInstalledAt) { + return 0 + } + return Math.min(MAX_RETRY_AFTER_MS, Math.max(0, this._retryAfterUntil - now)) } private _setFlushTimer(delayMs: number): void { @@ -357,6 +368,9 @@ export class PostHogMetrics { this._typeCollisionWarned = new Set() const generation = this._generation + // Read before the send: the outcome overwrites the window, and a refusal + // the endpoint had already told us to expect must not extend it. + const insideRetryAfterWindow = this._isWaitingOutRetryAfter() const outcome = await this._instance._sendMetricsBatch(this._buildPayload(window)) if (generation !== this._generation) { // reset() ran while the send was in flight — the client was torn down or @@ -365,8 +379,14 @@ export class PostHogMetrics { } // Only a retriable outcome carries a wait; anything else ends it, or a stale // one would pin every later flush at the window the server has moved on from. - this._retryAfterUntil = - outcome.kind === 'retry-later' && outcome.retryAfterMs ? Date.now() + outcome.retryAfterMs : 0 + // A refusal from inside an open window does not extend it, or a host that + // flushes on its own cadence keeps the window from ever elapsing. + if (outcome.kind !== 'retry-later' || !outcome.retryAfterMs) { + this._retryAfterUntil = 0 + } else if (!insideRetryAfterWindow) { + this._retryAfterInstalledAt = Date.now() + this._retryAfterUntil = this._retryAfterInstalledAt + outcome.retryAfterMs + } switch (outcome.kind) { case 'ok': return diff --git a/packages/core/src/posthog-core-stateless.ts b/packages/core/src/posthog-core-stateless.ts index ce8fd68edc..6e674c97e9 100644 --- a/packages/core/src/posthog-core-stateless.ts +++ b/packages/core/src/posthog-core-stateless.ts @@ -29,7 +29,6 @@ import { import { allSettled, createNamedError, - MAX_RETRY_AFTER_MS, currentISOTime, PromiseQueue, removeTrailingSlash, @@ -42,6 +41,7 @@ import { getEventUuid, safeJsonStringify, } from './utils' +import { MAX_RETRY_AFTER_MS } from './utils/retry-after' import { uuidv7 } from './vendor/uuidv7' import { ErrorPropertiesBuilder, diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 9aceac315c..9342ec2fab 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -1407,6 +1407,55 @@ describe('PostHogTraces', () => { expect(logger.warn).not.toHaveBeenCalledWith(expect.stringContaining('Dropping')) expect(mockInstance._sendTracesBatch.mock.calls.length).toBeGreaterThan(1) + + // The budget is deferred, not disabled. Once the window elapses the batch + // must retire — a deadline that slid forward on each refusal would keep + // the window open forever and strand everything behind the head batch. + for (let i = 0; i < 12; i++) { + await jest.advanceTimersByTimeAsync(310_000) + await traces.flush() + } + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('failed 8 times in a row')) + }) + + it('charges the budget for a timer-path failure after the wait was served out', async () => { + // The exemption is narrow: an attempt made *early*, inside a window that + // has not elapsed. A retry the timer fired at the deadline has served the + // wait, so it must count — otherwise the budget never advances and a + // permanently refused head batch pins everything behind it. + mockInstance._sendTracesBatch.mockResolvedValue({ + kind: 'retry-later', + error: new Error('429'), + retryAfterMs: 60_000, + }) + const traces = createTraces({ flushIntervalMs: 1000 }) + traces.startSpan('first').end() + + for (let i = 0; i < 12; i++) { + await jest.advanceTimersByTimeAsync(61_000) + } + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('failed 8 times in a row')) + }) + + it('does not let a host out-pacing the window stall the retry budget', async () => { + // flush() more often than the window is long. If each refusal slid the + // deadline forward, the window would never elapse, the head batch would + // never retire, and every span behind it would be dropped at the cap. + mockInstance._sendTracesBatch.mockResolvedValue({ + kind: 'retry-later', + error: new Error('429'), + retryAfterMs: 30_000, + }) + const traces = createTraces({ flushIntervalMs: 10_000, maxQueueSize: 5, maxExportBatchSize: 2 }) + + for (let i = 0; i < 60; i++) { + traces.startSpan(`span-${i}`).end() + await traces.flush() + await jest.advanceTimersByTimeAsync(5_000) + } + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('failed 8 times in a row')) }) it('still charges the retry budget for failures outside any window', async () => { @@ -1424,10 +1473,11 @@ describe('PostHogTraces', () => { expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Dropping')) }) - it('does not let a backward clock step stretch the wait past the cap', async () => { - // The deadline is wall clock. Without a clamp an NTP correction or a - // resumed VM turns a 60s wait into an hours-long one. - mockInstance._sendTracesBatch.mockResolvedValueOnce({ + it('does not let a backward clock step hold the retry budget open', async () => { + // The deadline is wall clock. An NTP correction or a resumed VM would + // otherwise keep the window "open" for the size of the step — and with + // in-window refusals uncharged, the head batch would never retire. + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 60_000, @@ -1436,9 +1486,16 @@ describe('PostHogTraces', () => { traces.startSpan('first').end() await traces.flush() - const real = Date.now() - jest.spyOn(Date, 'now').mockImplementation(() => real - 3_600_000) - expect((traces as any)._retryAfterRemainingMs()).toBeLessThanOrEqual(5 * 60_000) + // Step the clock back an hour and let simulated time run from there. + jest.setSystemTime(Date.now() - 3_600_000) + + // Asserted on the queue, not the drop warning: `_recordDrop` paces that + // warning off the same clock and would suppress it here. + for (let i = 0; i < 12; i++) { + await traces.flush() + await jest.advanceTimersByTimeAsync(61_000) + } + expect((traces as any)._queue).toHaveLength(0) }) it('does not install a Retry-After that lands after reset', async () => { diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 93601134ef..2c198005cb 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -13,7 +13,8 @@ import { newSpanId, newTraceId } from './ids' import { parseTraceparent, sanitizeTracestate } from './traceparent' import { assignUserAttributes, resolveStartTime, sanitizeName } from './sanitize' import { buildOtlpSpan, buildOtlpTracesPayload, buildTracesResourceAttributes } from './otlp' -import { MAX_RETRY_AFTER_MS, isPromise, safeSetTimeout } from '../utils' +import { isPromise, safeSetTimeout } from '../utils' +import { MAX_RETRY_AFTER_MS } from '../utils/retry-after' // Retriable failures on the same head batch before it is dropped, so a stuck // batch cannot pin the queue while fresher spans are refused at the cap. The @@ -79,6 +80,7 @@ export class PostHogTraces { // the wait is already part-served counts down the remainder instead of // restarting it. private _retryAfterUntil = 0 + private _retryAfterInstalledAt = 0 private _flushTimerFiresAt = 0 // Separate from the backoff counter: this one belongs to whatever batch is at // the head, and resets whenever that batch is removed or shrunk. @@ -274,6 +276,7 @@ export class PostHogTraces { this._lastDropWarningAt = 0 this._consecutiveFlushFailures = 0 this._retryAfterUntil = 0 + this._retryAfterInstalledAt = 0 this._headBatchFailures = 0 } @@ -519,9 +522,16 @@ export class PostHogTraces { // Only a retriable outcome carries a wait; anything else ends it, or a // stale one would pin every later flush at the window the server has - // moved on from. - this._retryAfterUntil = - outcome.kind === 'retry-later' && outcome.retryAfterMs ? Date.now() + outcome.retryAfterMs : 0 + // moved on from. A refusal from inside an open window does not extend + // it: the endpoint is repeating an answer it already gave, and sliding + // the deadline on each one means the window never elapses — which would + // hold the head batch's retry budget at zero forever. + if (outcome.kind !== 'retry-later' || !outcome.retryAfterMs) { + this._retryAfterUntil = 0 + } else if (!insideRetryAfterWindow) { + this._retryAfterInstalledAt = Date.now() + this._retryAfterUntil = this._retryAfterInstalledAt + outcome.retryAfterMs + } if (outcome.kind === 'ok') { this._consecutiveFlushFailures = 0 @@ -665,10 +675,14 @@ export class PostHogTraces { } private _retryAfterRemainingMs(): number { - // Clamped, not just floored: the deadline is wall clock, so a backward step - // (NTP, a resumed VM, a user changing the date) would otherwise strand the - // queue far past the cap `parseRetryAfterMs` applied. - return Math.min(MAX_RETRY_AFTER_MS, Math.max(0, this._retryAfterUntil - Date.now())) + const now = Date.now() + // The deadline is wall clock. A clock that has moved behind the point the + // window was installed can no longer measure it, so end the window rather + // than hold the queue for the size of the step; the cap bounds the rest. + if (now < this._retryAfterInstalledAt) { + return 0 + } + return Math.min(MAX_RETRY_AFTER_MS, Math.max(0, this._retryAfterUntil - now)) } private _isWaitingOutRetryAfter(): boolean { diff --git a/packages/core/src/utils/index.ts b/packages/core/src/utils/index.ts index 77b993c6cd..3f21f7c561 100644 --- a/packages/core/src/utils/index.ts +++ b/packages/core/src/utils/index.ts @@ -15,14 +15,6 @@ export * from './user-agent-utils' export const STRING_FORMAT = 'utf8' -/** - * Longest `Retry-After` the SDK will wait. A rate-limit window can legitimately - * be long, but nothing upstream of the SDK bounds this header — it is as likely - * to come from a proxy or CDN as from PostHog — and an unbounded value would - * strand a queue for hours, so the wait is capped and the retry happens early. - */ -export const MAX_RETRY_AFTER_MS = 5 * 60_000 - export const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i export function isValidUUID(value: unknown): value is string { diff --git a/packages/core/src/utils/retry-after.ts b/packages/core/src/utils/retry-after.ts new file mode 100644 index 0000000000..5ce5fc6c82 --- /dev/null +++ b/packages/core/src/utils/retry-after.ts @@ -0,0 +1,10 @@ +/** + * Longest `Retry-After` the SDK will wait. A rate-limit window can legitimately + * be long, but nothing upstream of the SDK bounds this header — it is as likely + * to come from a proxy or CDN as from PostHog — and an unbounded value would + * strand a queue for hours, so the wait is capped and the retry happens early. + * + * Deliberately outside the `utils` barrel: that barrel is re-exported wholesale + * from the package entry point, and this is internal policy, not public API. + */ +export const MAX_RETRY_AFTER_MS = 5 * 60_000 From af652926ebe598f364c4d260e91c50318a9009c0 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Thu, 3 Sep 2026 10:26:53 -0400 Subject: [PATCH 06/33] Merge feat/traces-node-mvp into fix/otlp-honor-retry-after Port this PR's suites to vitest (#4739). --- .../posthog.otlp-retry-after.spec.ts | 4 +- .../__tests__/posthog.otlp-too-large.spec.ts | 4 +- packages/core/src/logs/index.spec.ts | 90 +++++++++---------- packages/core/src/metrics/index.spec.ts | 60 ++++++------- packages/core/src/traces/index.spec.ts | 50 +++++------ 5 files changed, 104 insertions(+), 104 deletions(-) diff --git a/packages/core/src/__tests__/posthog.otlp-retry-after.spec.ts b/packages/core/src/__tests__/posthog.otlp-retry-after.spec.ts index 8345841f46..b2029831af 100644 --- a/packages/core/src/__tests__/posthog.otlp-retry-after.spec.ts +++ b/packages/core/src/__tests__/posthog.otlp-retry-after.spec.ts @@ -54,7 +54,7 @@ describe('OTLP Retry-After', () => { }) const pending = client._sendTracesBatch({ resourceSpans: [] } as any) - await jest.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(60_000) expect(await pending).toMatchObject({ kind: 'retry-later', retryAfterMs: 120_000 }) expect(clientMocks.fetch).toHaveBeenCalledTimes(1) @@ -74,7 +74,7 @@ describe('OTLP Retry-After', () => { }) const pending = client._sendTracesBatch({ resourceSpans: [] } as any) - await jest.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(60_000) expect((await pending).kind).toBe('retry-later') expect(clientMocks.fetch.mock.calls.length).toBeGreaterThan(1) diff --git a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts index 7a867c5127..cb93788e24 100644 --- a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts +++ b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts @@ -56,7 +56,7 @@ describe('OTLP bodies over the endpoint limit', () => { it('measures the payload, not the compressed body it is sent as', async () => { // The endpoint decompresses before it measures, so a payload that gzips // down to nothing is still refused on its decompressed size. - jest.spyOn(posthog as any, 'compressPayload').mockResolvedValue(new Uint8Array(1024)) + vi.spyOn(posthog as any, 'compressPayload').mockResolvedValue(new Uint8Array(1024)) ;(posthog as any).disableCompression = false await expect(posthog._sendTracesBatch(spansOf(3 * 1024 * 1024))).resolves.toEqual({ kind: 'too-large' }) @@ -85,7 +85,7 @@ describe('OTLP bodies over the endpoint limit', () => { }) it('sends a compressible payload that is inside the limit before compression', async () => { - jest.spyOn(posthog as any, 'compressPayload').mockResolvedValue(new Uint8Array(1024)) + vi.spyOn(posthog as any, 'compressPayload').mockResolvedValue(new Uint8Array(1024)) ;(posthog as any).disableCompression = false await expect(posthog._sendTracesBatch(spansOf(1024))).resolves.toEqual({ kind: 'ok' }) diff --git a/packages/core/src/logs/index.spec.ts b/packages/core/src/logs/index.spec.ts index 8d5e5a79f3..a61f77c6eb 100644 --- a/packages/core/src/logs/index.spec.ts +++ b/packages/core/src/logs/index.spec.ts @@ -1170,7 +1170,7 @@ describe('PostHogLogs', () => { // The capture that lands while the send is in flight arms a timer at the // plain interval; the 429 then asks for far longer. The earlier timer must // not fire first, or the SDK sends inside the window it was told to skip. - mockInstance._sendLogsBatch = jest.fn(async () => { + mockInstance._sendLogsBatch = vi.fn(async () => { logs.captureLog({ body: 'arrived mid-flush' }) return { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 } }) @@ -1184,18 +1184,18 @@ describe('PostHogLogs', () => { ) logs.captureLog({ body: 'first' }) - await jest.advanceTimersByTimeAsync(5000) + await vi.advanceTimersByTimeAsync(5000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(6000) + await vi.advanceTimersByTimeAsync(6000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(300_000) + await vi.advanceTimersByTimeAsync(300_000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) }) it('does not let the size trigger send inside a Retry-After window', async () => { - mockInstance._sendLogsBatch = jest.fn(() => + mockInstance._sendLogsBatch = vi.fn(() => Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) ) const logs = new PostHogLogs( @@ -1206,23 +1206,23 @@ describe('PostHogLogs', () => { immediateOnReady ) logs.captureLog({ body: 'first' }) - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) // Enough records to trip the size trigger, well inside the window. logs.captureLog({ body: 'second' }) logs.captureLog({ body: 'third' }) - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(300_000) + await vi.advanceTimersByTimeAsync(300_000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) }) it('does not let onReconnect send inside a Retry-After window', async () => { // `online` fires on every network handover; it says nothing about the // rate-limit window the endpoint set. - mockInstance._sendLogsBatch = jest.fn(() => + mockInstance._sendLogsBatch = vi.fn(() => Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) ) const logs = new PostHogLogs( @@ -1233,17 +1233,17 @@ describe('PostHogLogs', () => { immediateOnReady ) logs.captureLog({ body: 'first' }) - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) logs.onReconnect() - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) }) it('flushes on reconnect once the Retry-After window has passed', async () => { const outcomes: any[] = [{ kind: 'retry-later', error: new Error('429'), retryAfterMs: 5000 }] - mockInstance._sendLogsBatch = jest.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) + mockInstance._sendLogsBatch = vi.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) const logs = new PostHogLogs( mockInstance, resolveForTest({ flushIntervalMs: 1000 }), @@ -1252,18 +1252,18 @@ describe('PostHogLogs', () => { immediateOnReady ) logs.captureLog({ body: 'first' }) - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) // Stop just short of the deadline, then cross it without letting the // re-armed timer fire — otherwise the timer satisfies the assertion and // the test says nothing about onReconnect. - await jest.advanceTimersByTimeAsync(4999) + await vi.advanceTimersByTimeAsync(4999) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) - jest.setSystemTime(Date.now() + 2) + vi.setSystemTime(Date.now() + 2) logs.onReconnect() - await jest.advanceTimersByTimeAsync(1) + await vi.advanceTimersByTimeAsync(1) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) }) @@ -1271,7 +1271,7 @@ describe('PostHogLogs', () => { // `flush()` is the lifecycle path (RN foreground/background, shutdown). // It leaves no timer behind, so the next capture is the one that arms // one — at the plain interval unless the window floors it. - mockInstance._sendLogsBatch = jest.fn(() => + mockInstance._sendLogsBatch = vi.fn(() => Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) ) const logs = new PostHogLogs( @@ -1286,10 +1286,10 @@ describe('PostHogLogs', () => { expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) logs.captureLog({ body: 'second' }) - await jest.advanceTimersByTimeAsync(5000) + await vi.advanceTimersByTimeAsync(5000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(295_000) + await vi.advanceTimersByTimeAsync(295_000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) }) @@ -1297,7 +1297,7 @@ describe('PostHogLogs', () => { // The endpoint just accepted a batch, so the wait it asked for earlier is // over — the gated paths must not stay blocked for the rest of it. const outcomes: any[] = [{ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }] - mockInstance._sendLogsBatch = jest.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) + mockInstance._sendLogsBatch = vi.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) const logs = new PostHogLogs( mockInstance, resolveForTest({ flushIntervalMs: 5000, maxBufferSize: 2 }), @@ -1306,7 +1306,7 @@ describe('PostHogLogs', () => { immediateOnReady ) logs.captureLog({ body: 'first' }) - await jest.advanceTimersByTimeAsync(5000) + await vi.advanceTimersByTimeAsync(5000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) await logs.flush() @@ -1314,13 +1314,13 @@ describe('PostHogLogs', () => { logs.captureLog({ body: 'second' }) logs.onReconnect() - await jest.advanceTimersByTimeAsync(1) + await vi.advanceTimersByTimeAsync(1) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(3) }) it('drops a Retry-After wait on reset', async () => { const outcomes: any[] = [{ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }] - mockInstance._sendLogsBatch = jest.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) + mockInstance._sendLogsBatch = vi.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) const logs = new PostHogLogs( mockInstance, resolveForTest({ flushIntervalMs: 1000 }), @@ -1329,21 +1329,21 @@ describe('PostHogLogs', () => { immediateOnReady ) logs.captureLog({ body: 'first' }) - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) // Asserted through a gated path: a plain capture would flush either way. logs.reset() logs.captureLog({ body: 'second' }) logs.onReconnect() - await jest.advanceTimersByTimeAsync(1) + await vi.advanceTimersByTimeAsync(1) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) }) it('keeps its own backoff when the endpoint asks for less', async () => { // A proxy answering `Retry-After: 1` must not turn the retry into a // one-second hot loop against an endpoint already refusing traffic. - mockInstance._sendLogsBatch = jest.fn(() => + mockInstance._sendLogsBatch = vi.fn(() => Promise.resolve({ kind: 'retry-later', error: new Error('503'), retryAfterMs: 10 }) ) const logs = new PostHogLogs( @@ -1354,13 +1354,13 @@ describe('PostHogLogs', () => { immediateOnReady ) logs.captureLog({ body: 'first' }) - await jest.advanceTimersByTimeAsync(5000) + await vi.advanceTimersByTimeAsync(5000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(5000) + await vi.advanceTimersByTimeAsync(5000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) }) @@ -1372,7 +1372,7 @@ describe('PostHogLogs', () => { // endpoint that has already recovered. let settle: ((outcome: any) => void) | undefined let call = 0 - mockInstance._sendLogsBatch = jest.fn(() => { + mockInstance._sendLogsBatch = vi.fn(() => { call++ if (call === 1) { return Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) @@ -1389,19 +1389,19 @@ describe('PostHogLogs', () => { immediateOnReady ) logs.captureLog({ body: 'first' }) - await jest.advanceTimersByTimeAsync(5000) + await vi.advanceTimersByTimeAsync(5000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) // The wait elapses and the retry goes out, but hangs. - await jest.advanceTimersByTimeAsync(300_000) + await vi.advanceTimersByTimeAsync(300_000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) logs.captureLog({ body: 'second' }) settle?.({ kind: 'ok' }) - await jest.advanceTimersByTimeAsync(0) + await vi.advanceTimersByTimeAsync(0) // One interval, not another window. - await jest.advanceTimersByTimeAsync(5000) + await vi.advanceTimersByTimeAsync(5000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(3) }) @@ -1409,7 +1409,7 @@ describe('PostHogLogs', () => { // RN takes flush() on every app-state transition. If each refusal slid the // deadline forward, the window would never elapse and the gated paths — // the size trigger and onReconnect — would stay suppressed indefinitely. - mockInstance._sendLogsBatch = jest.fn(() => + mockInstance._sendLogsBatch = vi.fn(() => Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 30_000 }) ) const logs = new PostHogLogs( @@ -1428,7 +1428,7 @@ describe('PostHogLogs', () => { // a window is timing-dependent, but it must fall outside one *sometimes*. let sawWindowClosed = false for (let i = 0; i < 12; i++) { - await jest.advanceTimersByTimeAsync(5000) + await vi.advanceTimersByTimeAsync(5000) // Sampled before the flush: a flush that finds the window closed opens // a fresh one, so sampling after it would always look open. if ((logs as any)._retryAfterRemainingMs() === 0) { @@ -1442,7 +1442,7 @@ describe('PostHogLogs', () => { it('keeps flushing after a backward clock step', async () => { const outcomes: any[] = [{ kind: 'retry-later', error: new Error('429'), retryAfterMs: 60_000 }] - mockInstance._sendLogsBatch = jest.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) + mockInstance._sendLogsBatch = vi.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) const logs = new PostHogLogs( mockInstance, resolveForTest({ flushIntervalMs: 1000, maxBufferSize: 2 }), @@ -1451,16 +1451,16 @@ describe('PostHogLogs', () => { immediateOnReady ) logs.captureLog({ body: 'first' }) - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) const real = Date.now() - jest.spyOn(Date, 'now').mockImplementation(() => real - 3_600_000) + vi.spyOn(Date, 'now').mockImplementation(() => real - 3_600_000) // A gated path: suppressed for the size of the step without the guard. logs.captureLog({ body: 'second' }) logs.onReconnect() - await jest.advanceTimersByTimeAsync(1) + await vi.advanceTimersByTimeAsync(1) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) }) @@ -1471,7 +1471,7 @@ describe('PostHogLogs', () => { { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }, { kind: 'retry-later', error: new Error('503') }, ] - mockInstance._sendLogsBatch = jest.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) + mockInstance._sendLogsBatch = vi.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) const logs = new PostHogLogs( mockInstance, resolveForTest({ flushIntervalMs: 1000 }), @@ -1480,14 +1480,14 @@ describe('PostHogLogs', () => { immediateOnReady ) logs.captureLog({ body: 'first' }) - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(300_000) + await vi.advanceTimersByTimeAsync(300_000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) // Back on the plain backoff, not another 300s. - await jest.advanceTimersByTimeAsync(4000) + await vi.advanceTimersByTimeAsync(4000) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(3) }) @@ -1505,7 +1505,7 @@ describe('PostHogLogs', () => { for (let i = 0; i < 30; i++) { logs.captureLog({ body: `line ${i}` }) - await jest.advanceTimersByTimeAsync(2000) + await vi.advanceTimersByTimeAsync(2000) } expect(mockInstance._sendLogsBatch).toHaveBeenCalled() diff --git a/packages/core/src/metrics/index.spec.ts b/packages/core/src/metrics/index.spec.ts index 879ff991a5..efcad4c8b4 100644 --- a/packages/core/src/metrics/index.spec.ts +++ b/packages/core/src/metrics/index.spec.ts @@ -250,7 +250,7 @@ describe('PostHogMetrics', () => { for (let i = 0; i < 60; i++) { metrics.count('orders_created', 1) - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) } expect(mockInstance._sendMetricsBatch).toHaveBeenCalled() @@ -260,7 +260,7 @@ describe('PostHogMetrics', () => { // The capture arms a timer at the flush interval; the failed flush then asks // for far longer. The pending timer must not fire first. const instance = createMockInstance({ - _sendMetricsBatch: jest.fn((): Promise => + _sendMetricsBatch: vi.fn((): Promise => Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) ), }) @@ -269,10 +269,10 @@ describe('PostHogMetrics', () => { await metrics.flush() expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(11_000) + await vi.advanceTimersByTimeAsync(11_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(300_000) + await vi.advanceTimersByTimeAsync(300_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) }) @@ -284,18 +284,18 @@ describe('PostHogMetrics', () => { { kind: 'fatal', error: new Error('400') }, ] const instance = createMockInstance({ - _sendMetricsBatch: jest.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })), + _sendMetricsBatch: vi.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })), }) const metrics = createMetrics({ flushIntervalMs: 10_000 }, instance) metrics.count('orders_created', 1) await metrics.flush() // The wait elapses, the retry lands a 400, and that ends the wait. - await jest.advanceTimersByTimeAsync(300_000) + await vi.advanceTimersByTimeAsync(300_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) metrics.count('orders_created', 1) - await jest.advanceTimersByTimeAsync(10_000) + await vi.advanceTimersByTimeAsync(10_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(3) }) @@ -303,7 +303,7 @@ describe('PostHogMetrics', () => { // Each refusal sliding the deadline would keep `_nextFlushDelay` pinned at // the full window, so the flush cadence would never recover. const instance = createMockInstance({ - _sendMetricsBatch: jest.fn((): Promise => + _sendMetricsBatch: vi.fn((): Promise => Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 30_000 }) ), }) @@ -316,7 +316,7 @@ describe('PostHogMetrics', () => { // dependent, but it must fall outside one sometimes. let sawWindowClosed = false for (let i = 0; i < 12; i++) { - await jest.advanceTimersByTimeAsync(5000) + await vi.advanceTimersByTimeAsync(5000) // Sampled before the flush: a flush that finds the window closed opens // a fresh one, so sampling after it would always look open. if ((metrics as any)._retryAfterRemainingMs() === 0) { @@ -331,7 +331,7 @@ describe('PostHogMetrics', () => { it('does not install a Retry-After that lands after reset', async () => { let settle: ((outcome: SendMetricsBatchOutcome) => void) | undefined const instance = createMockInstance({ - _sendMetricsBatch: jest.fn( + _sendMetricsBatch: vi.fn( (): Promise => new Promise((resolve) => { settle = resolve @@ -340,7 +340,7 @@ describe('PostHogMetrics', () => { }) const metrics = createMetrics({ flushIntervalMs: 1000 }, instance) metrics.count('orders_created', 1) - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) metrics.reset() @@ -348,16 +348,16 @@ describe('PostHogMetrics', () => { // Settle first: the capture that arms the next timer must not find a // window belonging to the client that was just torn down. settle?.({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) - await jest.advanceTimersByTimeAsync(0) + await vi.advanceTimersByTimeAsync(0) metrics.count('orders_created', 1) - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) }) it('drops a Retry-After wait on reset', async () => { const instance = createMockInstance({ - _sendMetricsBatch: jest.fn((): Promise => + _sendMetricsBatch: vi.fn((): Promise => Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) ), }) @@ -367,7 +367,7 @@ describe('PostHogMetrics', () => { metrics.reset() metrics.count('orders_created', 1) - await jest.advanceTimersByTimeAsync(10_000) + await vi.advanceTimersByTimeAsync(10_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) }) @@ -380,7 +380,7 @@ describe('PostHogMetrics', () => { let settleRetry: ((outcome: SendMetricsBatchOutcome) => void) | undefined let call = 0 const instance = createMockInstance({ - _sendMetricsBatch: jest.fn((): Promise => { + _sendMetricsBatch: vi.fn((): Promise => { call++ if (call === 1) { return Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) @@ -392,37 +392,37 @@ describe('PostHogMetrics', () => { }) const metrics = createMetrics({ flushIntervalMs: 10_000 }, instance) metrics.count('orders_created', 1) - await jest.advanceTimersByTimeAsync(10_000) + await vi.advanceTimersByTimeAsync(10_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) // The wait elapses and the retry goes out, but hangs. - await jest.advanceTimersByTimeAsync(300_000) + await vi.advanceTimersByTimeAsync(300_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) metrics.count('orders_created', 1) settleRetry?.({ kind: 'ok' }) - await jest.advanceTimersByTimeAsync(0) + await vi.advanceTimersByTimeAsync(0) // One interval, not another window. - await jest.advanceTimersByTimeAsync(10_000) + await vi.advanceTimersByTimeAsync(10_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(3) }) it('keeps its own interval when the endpoint asks for less', async () => { const instance = createMockInstance({ - _sendMetricsBatch: jest.fn((): Promise => + _sendMetricsBatch: vi.fn((): Promise => Promise.resolve({ kind: 'retry-later', error: new Error('503'), retryAfterMs: 10 }) ), }) const metrics = createMetrics({ flushIntervalMs: 10_000 }, instance) metrics.count('orders_created', 1) - await jest.advanceTimersByTimeAsync(10_000) + await vi.advanceTimersByTimeAsync(10_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(10_000) + await vi.advanceTimersByTimeAsync(10_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) }) @@ -433,13 +433,13 @@ describe('PostHogMetrics', () => { { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }, ] const instance = createMockInstance({ - _sendMetricsBatch: jest.fn((): Promise => + _sendMetricsBatch: vi.fn((): Promise => Promise.resolve(outcomes.shift() ?? { kind: 'ok' }) ), }) const metrics = createMetrics({ flushIntervalMs: 10_000 }, instance) metrics.count('orders_created', 1) - await jest.advanceTimersByTimeAsync(10_000) + await vi.advanceTimersByTimeAsync(10_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) metrics.count('orders_created', 1) @@ -453,20 +453,20 @@ describe('PostHogMetrics', () => { { kind: 'retry-later', error: new Error('503') }, ] const instance = createMockInstance({ - _sendMetricsBatch: jest.fn((): Promise => + _sendMetricsBatch: vi.fn((): Promise => Promise.resolve(outcomes.shift() ?? { kind: 'ok' }) ), }) const metrics = createMetrics({ flushIntervalMs: 10_000 }, instance) metrics.count('orders_created', 1) - await jest.advanceTimersByTimeAsync(10_000) + await vi.advanceTimersByTimeAsync(10_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(300_000) + await vi.advanceTimersByTimeAsync(300_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) // Back on the plain interval, not another 300s. - await jest.advanceTimersByTimeAsync(10_000) + await vi.advanceTimersByTimeAsync(10_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(3) }) diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 5146488404..e1e7fc73fd 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -1307,12 +1307,12 @@ describe('PostHogTraces', () => { // One attempt, then a wait longer than the 30s exponential cap would give. while (mockInstance._sendTracesBatch.mock.calls.length < 1) { - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) } - await jest.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(60_000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(31_000) + await vi.advanceTimersByTimeAsync(31_000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(2) }) @@ -1326,14 +1326,14 @@ describe('PostHogTraces', () => { traces.startSpan('held').end() while (mockInstance._sendTracesBatch.mock.calls.length < 1) { - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) } // A 10ms Retry-After must not turn the retry loop into a hot loop: the // next attempt still waits out the queue's own backoff, not 10ms. - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(5000) + await vi.advanceTimersByTimeAsync(5000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(2) }) @@ -1348,10 +1348,10 @@ describe('PostHogTraces', () => { const traces = createTraces({ flushIntervalMs: 1000 }) traces.startSpan('first').end() - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) - await jest.advanceTimersByTimeAsync(10_000) + await vi.advanceTimersByTimeAsync(10_000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) }) @@ -1366,15 +1366,15 @@ describe('PostHogTraces', () => { const traces = createTraces({ flushIntervalMs: 1000, maxExportBatchSize: 1 }) traces.startSpan('first').end() - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) // The wait elapses, the retry lands a 400, and that ends the wait. - await jest.advanceTimersByTimeAsync(300_000) + await vi.advanceTimersByTimeAsync(300_000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(2) traces.startSpan('second').end() - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(3) }) @@ -1388,7 +1388,7 @@ describe('PostHogTraces', () => { }) const traces = createTraces({ flushIntervalMs: 1000, maxExportBatchSize: 1 }) traces.startSpan('first').end() - await jest.advanceTimersByTimeAsync(200_000) + await vi.advanceTimersByTimeAsync(200_000) traces.startSpan('second').end() await traces.flush() @@ -1398,7 +1398,7 @@ describe('PostHogTraces', () => { mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('down') }) traces.startSpan('third').end() const before = mockInstance._sendTracesBatch.mock.calls.length - await jest.advanceTimersByTimeAsync(2000) + await vi.advanceTimersByTimeAsync(2000) expect(mockInstance._sendTracesBatch.mock.calls.length).toBeGreaterThan(before) }) @@ -1436,7 +1436,7 @@ describe('PostHogTraces', () => { for (let i = 0; i < 12; i++) { await traces.flush() - await jest.advanceTimersByTimeAsync(10_000) + await vi.advanceTimersByTimeAsync(10_000) } expect(logger.warn).not.toHaveBeenCalledWith(expect.stringContaining('Dropping')) @@ -1446,7 +1446,7 @@ describe('PostHogTraces', () => { // must retire — a deadline that slid forward on each refusal would keep // the window open forever and strand everything behind the head batch. for (let i = 0; i < 12; i++) { - await jest.advanceTimersByTimeAsync(310_000) + await vi.advanceTimersByTimeAsync(310_000) await traces.flush() } expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('failed 8 times in a row')) @@ -1466,7 +1466,7 @@ describe('PostHogTraces', () => { traces.startSpan('first').end() for (let i = 0; i < 12; i++) { - await jest.advanceTimersByTimeAsync(61_000) + await vi.advanceTimersByTimeAsync(61_000) } expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('failed 8 times in a row')) @@ -1486,7 +1486,7 @@ describe('PostHogTraces', () => { for (let i = 0; i < 60; i++) { traces.startSpan(`span-${i}`).end() await traces.flush() - await jest.advanceTimersByTimeAsync(5_000) + await vi.advanceTimersByTimeAsync(5_000) } expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('failed 8 times in a row')) @@ -1501,7 +1501,7 @@ describe('PostHogTraces', () => { for (let i = 0; i < 12; i++) { await traces.flush() - await jest.advanceTimersByTimeAsync(10_000) + await vi.advanceTimersByTimeAsync(10_000) } expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Dropping')) @@ -1521,13 +1521,13 @@ describe('PostHogTraces', () => { await traces.flush() // Step the clock back an hour and let simulated time run from there. - jest.setSystemTime(Date.now() - 3_600_000) + vi.setSystemTime(Date.now() - 3_600_000) // Asserted on the queue, not the drop warning: `_recordDrop` paces that // warning off the same clock and would suppress it here. for (let i = 0; i < 12; i++) { await traces.flush() - await jest.advanceTimersByTimeAsync(61_000) + await vi.advanceTimersByTimeAsync(61_000) } expect((traces as any)._queue).toHaveLength(0) }) @@ -1544,16 +1544,16 @@ describe('PostHogTraces', () => { ) const traces = createTraces({ flushIntervalMs: 1000 }) traces.startSpan('before-reset').end() - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) traces.reset() traces.startSpan('after-reset').end() settle?.({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) - await jest.advanceTimersByTimeAsync(0) + await vi.advanceTimersByTimeAsync(0) - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(2) }) @@ -1564,12 +1564,12 @@ describe('PostHogTraces', () => { mockInstance._sendTracesBatch.mockImplementation(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })) const traces = createTraces({ flushIntervalMs: 1000 }) traces.startSpan('first').end() - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) traces.reset() traces.startSpan('second').end() - await jest.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(1000) expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(2) }) }) From aea2b1510875fdb16f512e004ce3da6d18acf5a5 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Thu, 3 Sep 2026 14:05:38 -0400 Subject: [PATCH 07/33] fix(core): keep the Retry-After window off the public API and out of three copies Share one RetryAfterWindow across the logs, metrics and traces queues, move parseRetryAfterMs off the package entry point, and stop a header-less refusal, a size verdict or a clock step from discarding an open window. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017SWWFUdrPWaHTpwFNr4DgC --- .changeset/otlp-honor-retry-after.md | 2 +- packages/browser/terser-mangled-names.json | 7 +- .../__tests__/posthog.otlp-too-large.spec.ts | 11 ++ .../core/src/__tests__/retry-after.spec.ts | 120 +++++++++++++++++- packages/core/src/logs/index.spec.ts | 29 ++++- packages/core/src/logs/index.ts | 55 ++------ packages/core/src/metrics/index.spec.ts | 31 ++++- packages/core/src/metrics/index.ts | 44 +------ packages/core/src/posthog-core-stateless.ts | 94 ++++++-------- packages/core/src/traces/index.spec.ts | 55 ++++++++ packages/core/src/traces/index.ts | 77 ++++------- packages/core/src/utils/retry-after.ts | 115 +++++++++++++++++ 12 files changed, 445 insertions(+), 195 deletions(-) diff --git a/.changeset/otlp-honor-retry-after.md b/.changeset/otlp-honor-retry-after.md index db11939853..3096652802 100644 --- a/.changeset/otlp-honor-retry-after.md +++ b/.changeset/otlp-honor-retry-after.md @@ -6,4 +6,4 @@ Honor `Retry-After` when the ingestion endpoint refuses a logs, metrics or traces batch, rather than retrying on the SDK's own backoff alone. The header acts as a floor, so a shorter value never makes the SDK retry sooner than it would have, and a wait longer than five minutes is capped. -The periodic flush waits the window out. An explicit `flush()` still sends, since it is a lifecycle or teardown boundary with no later attempt — but in `posthog-node` a refusal during the wait no longer counts against a span batch's retry budget, so a service that flushes on every request can no longer exhaust that budget and drop spans before the window has even elapsed. +The periodic flush waits the window out. An explicit `flush()`, and a flush a host runs to keep a request alive, still sends — a short-lived process is not left holding data it will never get another chance to deliver. Those attempts no longer count against a span batch's retry budget either, so waiting out a rate limit no longer drops spans. diff --git a/packages/browser/terser-mangled-names.json b/packages/browser/terser-mangled-names.json index f5c716c4ca..4f0233b6af 100644 --- a/packages/browser/terser-mangled-names.json +++ b/packages/browser/terser-mangled-names.json @@ -376,6 +376,7 @@ "_initializeWidgetPromise", "_initialized", "_initializingClient", + "_installedAt", "_internalEventEmitter", "_internalFlagCheckSatisfied", "_intervalLogCount", @@ -425,7 +426,6 @@ "_isSurveyRefreshBackingOff", "_isSurveysEnabled", "_isTourEligible", - "_isWaitingOutRetryAfter", "_isWidgetEnabled", "_isWidgetOpen", "_isWidgetRendered", @@ -722,9 +722,7 @@ "_resumeSavedTour", "_resyncIntervalMs", "_resyncTimer", - "_retryAfterInstalledAt", - "_retryAfterRemainingMs", - "_retryAfterUntil", + "_retryAfter", "_retryQueue", "_rrwebError", "_rrwebStartAttempted", @@ -889,6 +887,7 @@ "_unsubscribeFeatureFlags", "_unsubscribeIdentifyListener", "_unsubscribeSessionId", + "_until", "_unwrapConsoleError", "_unwrapOnError", "_unwrapUnhandledRejection", diff --git a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts index cb93788e24..8d1b0029cb 100644 --- a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts +++ b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts @@ -84,6 +84,17 @@ describe('OTLP bodies over the endpoint limit', () => { expect(mocks.fetch).not.toHaveBeenCalled() }) + it('does not compress a batch it has already ruled out', async () => { + // Gzipping a multi-megabyte body only to throw it away is the whole cost of + // the attempt the size check exists to avoid, and the halving loops pay it + // again on every step down. + const compressPayload = vi.spyOn(posthog as any, 'compressPayload') + ;(posthog as any).disableCompression = false + + await expect(posthog._sendTracesBatch(spansOf(3 * 1024 * 1024))).resolves.toEqual({ kind: 'too-large' }) + expect(compressPayload).not.toHaveBeenCalled() + }) + it('sends a compressible payload that is inside the limit before compression', async () => { vi.spyOn(posthog as any, 'compressPayload').mockResolvedValue(new Uint8Array(1024)) ;(posthog as any).disableCompression = false diff --git a/packages/core/src/__tests__/retry-after.spec.ts b/packages/core/src/__tests__/retry-after.spec.ts index 6b6eb3555d..81cb865160 100644 --- a/packages/core/src/__tests__/retry-after.spec.ts +++ b/packages/core/src/__tests__/retry-after.spec.ts @@ -1,4 +1,4 @@ -import { parseRetryAfterMs } from '../posthog-core-stateless' +import { MAX_RETRY_AFTER_MS, parseRetryAfterMs, RetryAfterWindow } from '../utils/retry-after' describe('parseRetryAfterMs', () => { const now = Date.parse('2026-09-01T12:00:00Z') @@ -48,4 +48,122 @@ describe('parseRetryAfterMs', () => { it('caps an unbounded value so a bogus header cannot strand a queue', () => { expect(parseRetryAfterMs('86400', now)).toBe(5 * 60_000) }) + + it('ignores a value that is not a string', () => { + // The header comes from injected transport code, so `headers.get` can hand + // back anything at all. + expect(parseRetryAfterMs(60, now)).toBeUndefined() + expect(parseRetryAfterMs(['60'], now)).toBeUndefined() + expect(parseRetryAfterMs({}, now)).toBeUndefined() + }) +}) + +describe('RetryAfterWindow', () => { + const open = (retryAfterMs: number): RetryAfterWindow => { + const window = new RetryAfterWindow() + window.record({ kind: 'retry-later', retryAfterMs }) + return window + } + + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(Date.parse('2026-09-01T12:00:00Z')) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('counts down to the deadline the endpoint named', () => { + const window = open(60_000) + expect(window.remainingMs()).toBe(60_000) + + vi.setSystemTime(Date.now() + 20_000) + expect(window.remainingMs()).toBe(40_000) + expect(window.isOpen()).toBe(true) + }) + + it('does not bring back a window that already ran out', () => { + // The deadline is wall clock, so a backward step after the wait was served + // would otherwise make the queue wait it out a second time. + const window = open(60_000) + const installedAt = Date.now() + + vi.setSystemTime(installedAt + 600_000) + expect(window.remainingMs()).toBe(0) + + vi.setSystemTime(installedAt + 30_000) + expect(window.remainingMs()).toBe(0) + expect(window.isOpen()).toBe(false) + }) + + it('survives a clock correction of a few milliseconds', () => { + // NTP nudges the clock backwards by milliseconds routinely; discarding a + // five-minute wait over one would send straight back into the rate limit. + const window = open(300_000) + + vi.setSystemTime(Date.now() - 1) + expect(window.isOpen()).toBe(true) + }) + + it('discards the window on a real backward clock step', () => { + const window = open(300_000) + + vi.setSystemTime(Date.now() - 3_600_000) + expect(window.remainingMs()).toBe(0) + }) + + it('does not extend an open window on a repeated refusal', () => { + const window = open(60_000) + vi.setSystemTime(Date.now() + 20_000) + + window.record({ kind: 'retry-later', retryAfterMs: 60_000 }) + expect(window.remainingMs()).toBe(40_000) + }) + + it('keeps the window when a batch is refused for size', () => { + // `too-large` is a verdict on the body's size — the SDK's own or a 413 — so + // it carries nothing about the endpoint's rate limit. + const window = open(60_000) + + window.record({ kind: 'too-large' }) + expect(window.remainingMs()).toBe(60_000) + }) + + it('keeps the window when a retriable failure carries no header', () => { + // A network error, a timeout and a header-less 503 are what the outage that + // named the wait keeps producing; none of them revokes it. + const window = open(300_000) + vi.setSystemTime(Date.now() + 10_000) + + window.record({ kind: 'retry-later' }) + expect(window.remainingMs()).toBe(290_000) + }) + + it('installs a wait that arrives after the previous one has elapsed', () => { + // A send can outlive the window it was made under, and the refusal it comes + // back with names a deadline the endpoint still expects to be honored. + const window = open(60_000) + vi.setSystemTime(Date.now() + 60_000) + + window.record({ kind: 'retry-later', retryAfterMs: 300_000 }) + expect(window.remainingMs()).toBe(300_000) + }) + + it('caps a wait longer than the maximum when it is installed', () => { + // `retryAfterMs` reaches the window from the exported host interfaces and + // export outcomes, so it is not always a value the SDK parsed itself. + const window = open(60 * 60_000) + + vi.setSystemTime(Date.now() + MAX_RETRY_AFTER_MS) + expect(window.isOpen()).toBe(false) + }) + + it('ends the window on an outcome that is not a retry', () => { + for (const outcome of [{ kind: 'ok' } as const, { kind: 'fatal' } as const]) { + const window = open(60_000) + window.record(outcome) + expect(window.isOpen()).toBe(false) + } + }) }) diff --git a/packages/core/src/logs/index.spec.ts b/packages/core/src/logs/index.spec.ts index a61f77c6eb..38e6086dd2 100644 --- a/packages/core/src/logs/index.spec.ts +++ b/packages/core/src/logs/index.spec.ts @@ -1293,6 +1293,33 @@ describe('PostHogLogs', () => { expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) }) + it('keeps the Retry-After window when a batch is refused for size', async () => { + // `too-large` is a verdict on the body's size — the SDK's own or a 413 — + // so it says nothing about the endpoint's rate limit and must not end the + // wait. + const outcomes: any[] = [{ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }] + mockInstance._sendLogsBatch = vi.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'too-large' })) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 1000, maxBatchRecordsPerPost: 1 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + await vi.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + // A batch of one the endpoint cannot accept: the record is dropped. + await logs.flush() + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) + + logs.captureLog({ body: 'second' }) + logs.onReconnect() + await vi.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) + }) + it('ends the window when an explicit flush succeeds', async () => { // The endpoint just accepted a batch, so the wait it asked for earlier is // over — the gated paths must not stay blocked for the rest of it. @@ -1431,7 +1458,7 @@ describe('PostHogLogs', () => { await vi.advanceTimersByTimeAsync(5000) // Sampled before the flush: a flush that finds the window closed opens // a fresh one, so sampling after it would always look open. - if ((logs as any)._retryAfterRemainingMs() === 0) { + if ((logs as any)._retryAfter.remainingMs() === 0) { sawWindowClosed = true } logs.captureLog({ body: `line ${i}` }) diff --git a/packages/core/src/logs/index.ts b/packages/core/src/logs/index.ts index 8b66f03055..44d748ff62 100644 --- a/packages/core/src/logs/index.ts +++ b/packages/core/src/logs/index.ts @@ -2,7 +2,7 @@ import type { LogAttributeValue } from '@posthog/types' import { buildOtlpLogRecord, buildOtlpLogsPayload, buildResourceAttributes } from './logs-utils' import { Logger, PostHogPersistedProperty } from '../types' import { isArray, raceWithTimeout, safeSetTimeout } from '../utils' -import { MAX_RETRY_AFTER_MS } from '../utils/retry-after' +import { RetryAfterWindow } from '../utils/retry-after' import type { BufferedLogEntry, CaptureLogOptions, LogSdkContext, LogsHost, ResolvedPostHogLogsConfig } from './types' // Caps the retry backoff at 2^6 = 64× the flush interval. @@ -28,12 +28,8 @@ export class PostHogLogs { // A batch captures this when it is assembled, so it can tell that the records it is // holding no longer correspond to anything queued. private _queueGeneration = 0 - // Absolute deadline from a `Retry-After` the endpoint sent, so every path that - // can start a send checks it — not just the retry timer. Cleared on the next - // success so one throttled response cannot pin the delay for the rest of the - // process. - private _retryAfterUntil = 0 - private _retryAfterInstalledAt = 0 + // Every path that can start a send checks this, not just the retry timer. + private _retryAfter = new RetryAfterWindow() private _flushTimerFiresAt = 0 // Consecutive failed flushes; drives exponential backoff on the retry timer. // A successful flush resets it to 0. @@ -99,8 +95,7 @@ export class PostHogLogs { this._intervalLogCount = 0 this._droppedWarned = false this._consecutiveFlushFailures = 0 - this._retryAfterUntil = 0 - this._retryAfterInstalledAt = 0 + this._retryAfter.reset() this._maxBatchRecordsPerPost = this._config.maxBatchRecordsPerPost } @@ -112,7 +107,7 @@ export class PostHogLogs { // network handover. onReconnect(): void { this._consecutiveFlushFailures = 0 - if (this._isWaitingOutRetryAfter()) { + if (this._retryAfter.isOpen()) { return } this._flushInBackground() @@ -296,10 +291,6 @@ export class PostHogLogs { this._instance.getLibraryVersion() ) - // Read before the send: the outcome overwrites the window, and a refusal - // the endpoint had already told us to expect must not extend it. - const insideRetryAfterWindow = this._isWaitingOutRetryAfter() - const outcome = await this._instance._sendLogsBatch(payload) if (this._queueGeneration !== generation) { @@ -318,17 +309,10 @@ export class PostHogLogs { continue } - // Only a retriable outcome carries a wait, and it is recorded here rather - // than on the background wrapper so an explicit `flush()` — which every - // lifecycle hook takes — records and clears it too. A refusal from inside - // an open window does not extend it, or a host that flushes on its own - // cadence keeps the window from ever elapsing. - if (outcome.kind !== 'retry-later' || !outcome.retryAfterMs) { - this._retryAfterUntil = 0 - } else if (!insideRetryAfterWindow) { - this._retryAfterInstalledAt = Date.now() - this._retryAfterUntil = this._retryAfterInstalledAt + outcome.retryAfterMs - } + // Recorded here rather than on the background wrapper, so an explicit + // `flush()` — which every lifecycle hook takes — records and clears the + // window too. + this._retryAfter.record(outcome) if (outcome.kind === 'retry-later') { // Transient failure: keep records in the queue for the next flush cycle @@ -399,7 +383,7 @@ export class PostHogLogs { // grow past this up to the eviction cap while the flush is in flight. Not // while the endpoint has asked us to wait: the size trigger is the dominant // one on a busy host, so sending here would ignore the window entirely. - if (queue.length >= this._maxBufferSize && !this._isWaitingOutRetryAfter()) { + if (queue.length >= this._maxBufferSize && !this._retryAfter.isOpen()) { this._flushInBackground() return } @@ -417,7 +401,7 @@ export class PostHogLogs { // Floored by any open window: an explicit `flush()` leaves no timer behind, // so this is the path a capture takes after one, and the plain interval // would land inside the wait the endpoint asked for. - this._setFlushTimer(Math.max(this._flushIntervalMs, this._retryAfterRemainingMs())) + this._setFlushTimer(Math.max(this._flushIntervalMs, this._retryAfter.remainingMs())) } // Backoff and `Retry-After` are floors, so a timer already armed at the plain @@ -446,22 +430,7 @@ export class PostHogLogs { const exponent = Math.min(Math.max(0, this._consecutiveFlushFailures - 1), MAX_FLUSH_BACKOFF_EXPONENT) // `Retry-After` is a floor, not a replacement: never retry before the server // asked, and never more often than our own backoff would have. - return Math.max(this._flushIntervalMs * 2 ** exponent, this._retryAfterRemainingMs()) - } - - private _retryAfterRemainingMs(): number { - const now = Date.now() - // The deadline is wall clock. A clock that has moved behind the point the - // window was installed can no longer measure it, so end the window rather - // than hold the queue for the size of the step; the cap bounds the rest. - if (now < this._retryAfterInstalledAt) { - return 0 - } - return Math.min(MAX_RETRY_AFTER_MS, Math.max(0, this._retryAfterUntil - now)) - } - - private _isWaitingOutRetryAfter(): boolean { - return this._retryAfterRemainingMs() > 0 + return Math.max(this._flushIntervalMs * 2 ** exponent, this._retryAfter.remainingMs()) } private _hasQueuedRecords(): boolean { diff --git a/packages/core/src/metrics/index.spec.ts b/packages/core/src/metrics/index.spec.ts index efcad4c8b4..36c579c1d4 100644 --- a/packages/core/src/metrics/index.spec.ts +++ b/packages/core/src/metrics/index.spec.ts @@ -299,6 +299,35 @@ describe('PostHogMetrics', () => { expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(3) }) + it('keeps the Retry-After window when a batch is refused for size', async () => { + // `too-large` is a verdict on the body's size — the SDK's own or a 413 — + // so it says nothing about the endpoint's rate limit. Ending the wait on + // it lets the next refusal install a fresh window, pushing the retry out + // past the deadline the endpoint actually named — observed here as the + // flush landing at 300s rather than at 310s. + const outcomes: SendMetricsBatchOutcome[] = [ + { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }, + { kind: 'too-large' }, + { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }, + ] + const instance = createMockInstance({ + _sendMetricsBatch: vi.fn(() => Promise.resolve(outcomes.shift() ?? { kind: 'ok' })), + }) + const metrics = createMetrics({ flushIntervalMs: 1000 }, instance) + metrics.count('orders_created', 1) + await metrics.flush() + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(1) + + await vi.advanceTimersByTimeAsync(10_000) + await metrics.flush() + metrics.count('orders_created', 1) + await metrics.flush() + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(3) + + await vi.advanceTimersByTimeAsync(295_000) + expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(4) + }) + it('does not let a host out-pacing the window keep it open forever', async () => { // Each refusal sliding the deadline would keep `_nextFlushDelay` pinned at // the full window, so the flush cadence would never recover. @@ -319,7 +348,7 @@ describe('PostHogMetrics', () => { await vi.advanceTimersByTimeAsync(5000) // Sampled before the flush: a flush that finds the window closed opens // a fresh one, so sampling after it would always look open. - if ((metrics as any)._retryAfterRemainingMs() === 0) { + if ((metrics as any)._retryAfter.remainingMs() === 0) { sawWindowClosed = true } metrics.count('orders_created', 1) diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index 8584e611b0..f101e0c011 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -10,7 +10,7 @@ import type { } from '@posthog/types' import type { Logger } from '../types' import { isArray, safeSetTimeout } from '../utils' -import { MAX_RETRY_AFTER_MS } from '../utils/retry-after' +import { RetryAfterWindow } from '../utils/retry-after' import { toOtlpKeyValueList } from '../utils/otlp-any-value' import { DEFAULT_HISTOGRAM_BOUNDS, @@ -74,13 +74,7 @@ export class PostHogMetrics { // types under one name produces charts that blend both series. private _typeByName = new Map() private _typeCollisionWarned = new Set() - // Absolute deadline from a `Retry-After` the endpoint sent; cleared on any - // other outcome so one throttled response cannot pin the delay for the rest - // of the process. A deadline rather than a duration, so a timer armed while - // the wait is already part-served counts down the remainder instead of - // restarting it. - private _retryAfterUntil = 0 - private _retryAfterInstalledAt = 0 + private _retryAfter = new RetryAfterWindow() private _flushTimerFiresAt = 0 // Bumped by reset(). A flush that was in flight when reset() ran (e.g. it // lost a shutdown race) sees a stale generation when its send settles and @@ -145,8 +139,7 @@ export class PostHogMetrics { /** Clears the flush timer, drops the current window, and invalidates in-flight flushes. */ reset(): void { this._generation++ - this._retryAfterUntil = 0 - this._retryAfterInstalledAt = 0 + this._retryAfter.reset() this._clearFlushTimer() this._series = new Map() this._flushPromise = null @@ -319,22 +312,7 @@ export class PostHogMetrics { // `Retry-After` is a floor, not a replacement: never retry before the server // asked, and never more often than the flush interval. private _nextFlushDelay(): number { - return Math.max(this._config.flushIntervalMs, this._retryAfterRemainingMs()) - } - - private _isWaitingOutRetryAfter(): boolean { - return this._retryAfterRemainingMs() > 0 - } - - private _retryAfterRemainingMs(): number { - const now = Date.now() - // The deadline is wall clock. A clock that has moved behind the point the - // window was installed can no longer measure it, so end the window rather - // than hold the queue for the size of the step; the cap bounds the rest. - if (now < this._retryAfterInstalledAt) { - return 0 - } - return Math.min(MAX_RETRY_AFTER_MS, Math.max(0, this._retryAfterUntil - now)) + return Math.max(this._config.flushIntervalMs, this._retryAfter.remainingMs()) } private _setFlushTimer(delayMs: number): void { @@ -368,25 +346,13 @@ export class PostHogMetrics { this._typeCollisionWarned = new Set() const generation = this._generation - // Read before the send: the outcome overwrites the window, and a refusal - // the endpoint had already told us to expect must not extend it. - const insideRetryAfterWindow = this._isWaitingOutRetryAfter() const outcome = await this._instance._sendMetricsBatch(this._buildPayload(window)) if (generation !== this._generation) { // reset() ran while the send was in flight — the client was torn down or // reconfigured, so this window is dropped whatever the outcome was. return } - // Only a retriable outcome carries a wait; anything else ends it, or a stale - // one would pin every later flush at the window the server has moved on from. - // A refusal from inside an open window does not extend it, or a host that - // flushes on its own cadence keeps the window from ever elapsing. - if (outcome.kind !== 'retry-later' || !outcome.retryAfterMs) { - this._retryAfterUntil = 0 - } else if (!insideRetryAfterWindow) { - this._retryAfterInstalledAt = Date.now() - this._retryAfterUntil = this._retryAfterInstalledAt + outcome.retryAfterMs - } + this._retryAfter.record(outcome) switch (outcome.kind) { case 'ok': return diff --git a/packages/core/src/posthog-core-stateless.ts b/packages/core/src/posthog-core-stateless.ts index 6e674c97e9..06d8c113e2 100644 --- a/packages/core/src/posthog-core-stateless.ts +++ b/packages/core/src/posthog-core-stateless.ts @@ -41,7 +41,7 @@ import { getEventUuid, safeJsonStringify, } from './utils' -import { MAX_RETRY_AFTER_MS } from './utils/retry-after' +import { parseRetryAfterMs } from './utils/retry-after' import { uuidv7 } from './vendor/uuidv7' import { ErrorPropertiesBuilder, @@ -52,38 +52,6 @@ import { createDefaultStackParser, } from './error-tracking' -/** - * `Retry-After` as milliseconds from now. The header is either delta-seconds or - * an HTTP-date; anything else, a date already in the past, or a negative delta - * yields `undefined` so the caller keeps its own backoff. - * - * @internal Exposed for cross-package use within this SDK; not part of the stable public API. - */ -export function parseRetryAfterMs(value: string | null | undefined, now: number = Date.now()): number | undefined { - if (!value) { - return undefined - } - const raw = value.trim() - // `headers.get` joins repeated headers with ", ", so a CDN and a load - // balancer that each append one yield "60, 120". Read the first, which the - // outermost hop set — but only for delta-seconds, since an HTTP-date carries - // a comma of its own ("Wed, 21 Oct 2015 07:28:00 GMT"). - const trimmed = /^\d+\s*,/.test(raw) ? raw.slice(0, raw.indexOf(',')).trim() : raw - // Integer seconds. Not parseFloat: "10 minutes" must not read as 10 seconds. - const seconds = /^\d+$/.test(trimmed) ? Number(trimmed) : Number.NaN - if (!Number.isFinite(seconds) && /^[+-]?[\d.]+$/.test(trimmed)) { - // Numeric but not delta-seconds, so it is malformed. `Date.parse` reads - // "-5", "+5" and "5.5" as dates in 2001 rather than rejecting them, which - // on a device whose clock predates that would surface as a real wait. - return undefined - } - const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(trimmed) - now - if (!Number.isFinite(ms) || ms <= 0) { - return undefined - } - return Math.min(ms, MAX_RETRY_AFTER_MS) -} - class PostHogFetchHttpError extends Error { name = 'PostHogFetchHttpError' private responseBodyTextPromise?: Promise @@ -103,7 +71,10 @@ class PostHogFetchHttpError extends Error { return this.response.status } - /** The response's `Retry-After` as milliseconds from now, when it sent a usable one. */ + /** + * The response's `Retry-After` as milliseconds from now, when it sent a usable + * one, clamped to `MAX_RETRY_AFTER_MS`. + */ get retryAfterMs(): number | undefined { try { return parseRetryAfterMs(this.response.headers?.get('retry-after')) @@ -255,24 +226,35 @@ function isRetryableFlagsFetchError( * which defaults to 2 MB and is applied to the body after the endpoint * decompresses it, not to the compressed bytes on the wire. * - * Used as a floor, never as a promise: the deployment can raise it, and a proxy - * in front can lower it. A body over this size cannot be accepted, so sending it - * only buys a 413; one under it is sent and may still be refused. + * Applied as a ceiling on what the SDK will put on the wire: a body over it is + * reported as too large without a request being made, and a batch of one that + * still exceeds it is dropped. The cost is a deployment that raised + * `MAX_REQUEST_BODY_SIZE_BYTES` above 2 MB, where such a batch would have been + * accepted. A body under the limit is sent and may still be refused, by the + * endpoint or by a proxy in front of it with a lower one. */ const OTLP_MAX_BODY_BYTES = 2 * 1024 * 1024 -/** A request body's size on the wire. `Buffer` where it exists, `TextEncoder` elsewhere. */ +/** + * A request body's size on the wire. `Buffer` where it exists, `TextEncoder` + * elsewhere. + * + * Total by construction: it runs on hosts that define only part of the web + * platform — `Blob` in particular is absent on some server runtimes — and a + * size that cannot be measured is reported as `0`, leaving the body to be sent + * rather than turning a missing global into a failed export. + */ function byteLengthOf(body: string | Blob | Uint8Array): number { - if (body instanceof Blob) { - return body.size - } - if (body instanceof Uint8Array) { - return body.byteLength - } try { - return Buffer.byteLength(body, STRING_FORMAT) - } catch { + if (typeof body !== 'string') { + return body instanceof Uint8Array ? body.byteLength : body.size + } + if (typeof Buffer !== 'undefined') { + return Buffer.byteLength(body, STRING_FORMAT) + } return new TextEncoder().encode(body).length + } catch { + return 0 } } @@ -1736,16 +1718,13 @@ export abstract class PostHogCoreStateless { } const serialized = JSON.stringify(payload) - const url = - auth === 'bearer' - ? `${this.host}/i/v1/${path}` - : `${this.host}/i/v1/${path}?token=${encodeURIComponent(this.apiKey)}` - // Measured before compression: the endpoint decompresses the body and applies - // its limit to what comes out, so a payload that gzips small is still refused - // on its decompressed size. A batch the endpoint cannot accept is reported - // without being sent, so the caller halves it — and ultimately isolates and - // drops the one oversized record — without spending a request on each attempt. + // Measured on the uncompressed payload: the endpoint decompresses the body + // and applies its limit to what comes out, so one that gzips small is still + // refused on its decompressed size. A batch the endpoint cannot accept is + // reported without being sent — and before it is compressed — so the caller + // halves it, and ultimately isolates and drops the one oversized record, + // without spending a request or a gzip pass on each attempt. const payloadBytes = byteLengthOf(serialized) if (payloadBytes > OTLP_MAX_BODY_BYTES) { this.logMsgIfDebug(() => @@ -1756,6 +1735,11 @@ export abstract class PostHogCoreStateless { return { kind: 'too-large' } } + const url = + auth === 'bearer' + ? `${this.host}/i/v1/${path}` + : `${this.host}/i/v1/${path}?token=${encodeURIComponent(this.apiKey)}` + const gzippedPayload = !this.disableCompression ? await this.compressPayload(serialized) : null const body = gzippedPayload || serialized diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index e1e7fc73fd..507b40bd18 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -1532,6 +1532,61 @@ describe('PostHogTraces', () => { expect((traces as any)._queue).toHaveLength(0) }) + it('keeps the Retry-After window when a batch is refused for size', async () => { + // `too-large` is a verdict on the body's size — the SDK's own or a 413 — + // so it says nothing about the endpoint's rate limit and must not end the + // wait. + let first = true + mockInstance._sendTracesBatch.mockImplementation(() => { + const outcome: SendTracesBatchOutcome = first + ? { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 } + : { kind: 'too-large' } + first = false + return Promise.resolve(outcome) + }) + const traces = createTraces({ flushIntervalMs: 1000, maxExportBatchSize: 1 }) + traces.startSpan('first').end() + await vi.advanceTimersByTimeAsync(1000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(1) + + // Batches of one the endpoint cannot accept: the spans are dropped. + traces.startSpan('second').end() + await traces.flush() + const sends = mockInstance._sendTracesBatch.mock.calls.length + + traces.startSpan('third').end() + await vi.advanceTimersByTimeAsync(10_000) + expect(mockInstance._sendTracesBatch).toHaveBeenCalledTimes(sends) + }) + + it('stops the drain when it retires a batch inside the window', async () => { + // Retiring the head batch clears the failure counters, so the drain would + // otherwise carry straight on to the next batch — inside the wait the very + // refusal that retired it had just installed. One extra send still lands at + // that instant: `flush()`'s outer loop is deliberately not window-aware. + const sentAt: number[] = [] + mockInstance._sendTracesBatch.mockImplementation(() => { + sentAt.push(Date.now()) + return Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 60_000 }) + }) + const traces = createTraces({ flushIntervalMs: 1000, maxExportBatchSize: 1 }) + for (let i = 0; i < 4; i++) { + traces.startSpan(`span-${i}`).end() + } + + // Every attempt lands after the previous window elapsed, so every refusal + // is charged and the eighth retires the head batch. + for (let i = 0; i < 12; i++) { + await vi.advanceTimersByTimeAsync(61_000) + } + + const perInstant = new Map() + for (const at of sentAt) { + perInstant.set(at, (perInstant.get(at) ?? 0) + 1) + } + expect(Math.max(...perInstant.values())).toBe(2) + }) + it('does not install a Retry-After that lands after reset', async () => { // The generation check must precede the window assignment, or a 429 that // settles after teardown pins a wait on the fresh client. diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 1bf6506270..8dd83e5d24 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -14,7 +14,7 @@ import { parseTraceparent, sanitizeTracestate } from './traceparent' import { assignUserAttributes, resolveStartTime, sanitizeName } from './sanitize' import { buildOtlpSpan, buildOtlpTracesPayload, buildTracesResourceAttributes } from './otlp' import { isPromise, safeSetTimeout } from '../utils' -import { MAX_RETRY_AFTER_MS } from '../utils/retry-after' +import { RetryAfterWindow } from '../utils/retry-after' // Retriable failures on the same head batch before it is dropped, so a stuck // batch cannot pin the queue while fresher spans are refused at the cap. The @@ -77,13 +77,7 @@ export class PostHogTraces { private _lastDropWarningAt = 0 private _dropReasons = new Set() private _consecutiveFlushFailures = 0 - // Absolute deadline from a `Retry-After` the endpoint sent; cleared on any - // other outcome so one throttled response cannot pin the delay for the rest - // of the process. A deadline rather than a duration, so a timer armed while - // the wait is already part-served counts down the remainder instead of - // restarting it. - private _retryAfterUntil = 0 - private _retryAfterInstalledAt = 0 + private _retryAfter = new RetryAfterWindow() private _flushTimerFiresAt = 0 // Separate from the backoff counter: this one belongs to whatever batch is at // the head, and resets whenever that batch is removed or shrunk. @@ -230,11 +224,12 @@ export class PostHogTraces { * A pass reports spans removed — queue length can't stand in, since a send * concurrent with an arrival leaves it unchanged. * - * An open `Retry-After` window does not stop an explicit flush: the caller is - * a lifecycle or teardown boundary that has no later attempt, and on a - * serverless host the armed timer is unref'd and dies with the isolate. The - * window is honoured by not charging such an attempt against the head batch's - * retry budget, so the wait costs a request rather than the spans. + * An open `Retry-After` window does not stop this: an explicit flush, or one a + * host runs to keep a request alive, sends whatever is queued — a serverless + * isolate may not be around for the armed timer to fire. The window is honoured + * by not charging such an attempt against the head batch's retry budget, so the + * wait costs a request rather than the spans. The periodic flush does wait it + * out. */ async flush(): Promise { for (;;) { @@ -280,8 +275,7 @@ export class PostHogTraces { this._dropReasons.clear() this._lastDropWarningAt = 0 this._consecutiveFlushFailures = 0 - this._retryAfterUntil = 0 - this._retryAfterInstalledAt = 0 + this._retryAfter.reset() this._headBatchFailures = 0 } @@ -415,10 +409,14 @@ export class PostHogTraces { this._logger.debug('Span queue notification failed', error) } - // Not while a flush is failing: the queue stays above the batch size for the - // whole outage, so every further span end would re-POST immediately and the - // retry backoff would never apply. - if (this._queue.length >= this._maxExportBatchSize && !this._consecutiveFlushFailures) { + // Not while a flush is failing or the endpoint has asked us to wait: the + // queue stays above the batch size for the whole outage, so every further + // span end would re-POST immediately and the retry backoff would never apply. + if ( + this._queue.length >= this._maxExportBatchSize && + !this._consecutiveFlushFailures && + !this._retryAfter.isOpen() + ) { this._flushInBackground() } else { this._armFlushTimerIfQueued() @@ -514,9 +512,9 @@ export class PostHogTraces { continue } - // Read before the send: the outcome overwrites the window, and an attempt - // the endpoint had already told us to skip says nothing about this batch. - const insideRetryAfterWindow = this._isWaitingOutRetryAfter() + // Read before the send, so the retry budget below charges this attempt + // against the window it was actually made under. + const insideRetryAfterWindow = this._retryAfter.isOpen() const outcome = await this._instance._sendTracesBatch( buildOtlpTracesPayload(spans, resourceAttributes, scopeName, scopeVersion, this._logger) @@ -527,18 +525,7 @@ export class PostHogTraces { return removed } - // Only a retriable outcome carries a wait; anything else ends it, or a - // stale one would pin every later flush at the window the server has - // moved on from. A refusal from inside an open window does not extend - // it: the endpoint is repeating an answer it already gave, and sliding - // the deadline on each one means the window never elapses — which would - // hold the head batch's retry budget at zero forever. - if (outcome.kind !== 'retry-later' || !outcome.retryAfterMs) { - this._retryAfterUntil = 0 - } else if (!insideRetryAfterWindow) { - this._retryAfterInstalledAt = Date.now() - this._retryAfterUntil = this._retryAfterInstalledAt + outcome.retryAfterMs - } + this._retryAfter.record(outcome) if (outcome.kind === 'ok') { this._consecutiveFlushFailures = 0 @@ -594,6 +581,11 @@ export class PostHogTraces { this._consecutiveFlushFailures = 0 this._headBatchFailures = 0 this._recordDrop(size, `the ingestion endpoint failed ${MAX_RETRIES_PER_BATCH} times in a row`) + if (this._retryAfter.isOpen()) { + // Retiring the batch cleared the failure counters, so carrying on + // would send the next one straight away, inside the endpoint's wait. + return removed + } continue } @@ -678,22 +670,7 @@ export class PostHogTraces { const capped = Math.min(delay, Math.max(MAX_FLUSH_BACKOFF_MS, this._config.flushIntervalMs)) // `Retry-After` is a floor, not a replacement: never retry before the server // asked, and never more often than our own backoff would have. - return Math.max(capped, this._retryAfterRemainingMs()) - } - - private _retryAfterRemainingMs(): number { - const now = Date.now() - // The deadline is wall clock. A clock that has moved behind the point the - // window was installed can no longer measure it, so end the window rather - // than hold the queue for the size of the step; the cap bounds the rest. - if (now < this._retryAfterInstalledAt) { - return 0 - } - return Math.min(MAX_RETRY_AFTER_MS, Math.max(0, this._retryAfterUntil - now)) - } - - private _isWaitingOutRetryAfter(): boolean { - return this._retryAfterRemainingMs() > 0 + return Math.max(capped, this._retryAfter.remainingMs()) } private _clearFlushTimer(): void { diff --git a/packages/core/src/utils/retry-after.ts b/packages/core/src/utils/retry-after.ts index 5ce5fc6c82..17fce6247e 100644 --- a/packages/core/src/utils/retry-after.ts +++ b/packages/core/src/utils/retry-after.ts @@ -8,3 +8,118 @@ * from the package entry point, and this is internal policy, not public API. */ export const MAX_RETRY_AFTER_MS = 5 * 60_000 + +/** + * How far the wall clock may run behind the moment a window was installed before + * the window is discarded. NTP corrections on a healthy host are milliseconds, + * so a step past this is a clock change the deadline can no longer measure. + */ +const CLOCK_STEP_TOLERANCE_MS = 5_000 + +/** + * `Retry-After` as milliseconds from now. The header is either delta-seconds or + * an HTTP-date; anything else, a date already in the past, or a negative delta + * yields `undefined` so the caller keeps its own backoff. The result is capped + * at `MAX_RETRY_AFTER_MS`. + */ +export function parseRetryAfterMs(value: unknown, now: number = Date.now()): number | undefined { + if (typeof value !== 'string' || !value) { + return undefined + } + const raw = value.trim() + // `headers.get` joins repeated headers with ", ", so a CDN and a load + // balancer that each append one yield "60, 120". Read the first, which the + // outermost hop set — but only for delta-seconds, since an HTTP-date carries + // a comma of its own ("Wed, 21 Oct 2015 07:28:00 GMT"). + const trimmed = /^\d+\s*,/.test(raw) ? raw.slice(0, raw.indexOf(',')).trim() : raw + // Integer seconds. Not parseFloat: "10 minutes" must not read as 10 seconds. + const seconds = /^\d+$/.test(trimmed) ? Number(trimmed) : Number.NaN + if (!Number.isFinite(seconds) && /^[+-]?[\d.]+$/.test(trimmed)) { + // Numeric but not delta-seconds, so it is malformed. `Date.parse` reads + // "-5", "+5" and "5.5" as dates in 2001 rather than rejecting them, which + // on a device whose clock predates that would surface as a real wait. + return undefined + } + const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(trimmed) - now + if (!Number.isFinite(ms) || ms <= 0) { + return undefined + } + return Math.min(ms, MAX_RETRY_AFTER_MS) +} + +/** The part of an export outcome the window reads. */ +export type RetryAfterOutcome = { + kind: 'ok' | 'retry-later' | 'too-large' | 'fatal' + retryAfterMs?: number +} + +/** + * The wait an ingestion endpoint asked for, shared by the logs, metrics and + * traces export queues. + * + * Held as an absolute deadline rather than a duration, so a timer armed while + * the wait is already part-served counts down the remainder instead of + * restarting it. + */ +export class RetryAfterWindow { + private _until = 0 + private _installedAt = 0 + + /** + * Folds one export outcome into the window. + * + * A window still open is left as it stands: the refusal repeats an answer the + * endpoint has already given, and sliding the deadline on each one means the + * window never elapses for a host that sends on its own cadence. Whether it + * is still open is read here rather than passed in, so a send that outlives + * the wait it was made under installs the fresh deadline it came back with. + */ + record(outcome: RetryAfterOutcome): void { + if (outcome.kind === 'too-large') { + // A verdict on the body's size, whether the SDK's own or the endpoint's + // 413, and either way silent on the endpoint's rate limit. + return + } + if (outcome.kind !== 'retry-later') { + // A stale deadline left set here would pin every later send at a window + // the endpoint has moved on from. + this.reset() + return + } + if (!outcome.retryAfterMs || this.isOpen()) { + // Nothing to install, and nothing that revokes what is installed: the + // refusal either named no wait — a network error, a timeout or a + // header-less 503, all of which the outage that named the original wait + // keeps producing — or arrived while that wait is still being served. + return + } + this._installedAt = Date.now() + this._until = this._installedAt + Math.min(outcome.retryAfterMs, MAX_RETRY_AFTER_MS) + } + + /** + * Milliseconds left of the wait, `0` when none is open. Closes a window it + * finds spent, so a later backward clock step cannot bring it back. + */ + remainingMs(): number { + const now = Date.now() + if (now < this._installedAt - CLOCK_STEP_TOLERANCE_MS) { + this.reset() + return 0 + } + const remaining = Math.min(MAX_RETRY_AFTER_MS, Math.max(0, this._until - now)) + if (remaining === 0) { + this.reset() + } + return remaining + } + + isOpen(): boolean { + return this.remainingMs() > 0 + } + + reset(): void { + this._until = 0 + this._installedAt = 0 + } +} From 74ac21457f9a2d7ad87d1f316c3807c949ae1589 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Thu, 3 Sep 2026 14:41:53 -0400 Subject: [PATCH 08/33] fix(core): stop a closed Retry-After window from stranding the metrics queue Metrics never retired its pending timer when a flush started, so a wait armed from a Retry-After outlived the window a later successful flush had already closed, holding every subsequent capture for up to five minutes. Logs had the mirror gap: a capture landing mid-flush armed a plain-interval timer before the window existed, and only the background wrapper re-armed it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017SWWFUdrPWaHTpwFNr4DgC --- packages/core/src/logs/index.spec.ts | 31 +++++++++++++++++++++++++ packages/core/src/logs/index.ts | 7 ++++++ packages/core/src/metrics/index.spec.ts | 20 ++++++++++++++++ packages/core/src/metrics/index.ts | 6 +++++ 4 files changed, 64 insertions(+) diff --git a/packages/core/src/logs/index.spec.ts b/packages/core/src/logs/index.spec.ts index 38e6086dd2..6f8f66b228 100644 --- a/packages/core/src/logs/index.spec.ts +++ b/packages/core/src/logs/index.spec.ts @@ -1293,6 +1293,37 @@ describe('PostHogLogs', () => { expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) }) + it('does not let a capture during an explicit flush send inside the window', async () => { + // The sibling case covers a capture *after* the flush settles. This one + // lands while the send is in flight, so it arms the timer at the plain + // interval before the window exists. + let release: (v: any) => void = () => {} + mockInstance._sendLogsBatch = vi.fn( + () => + new Promise((resolve) => { + release = resolve + }) + ) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 5000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + const flushed = logs.flush().catch(() => {}) + await vi.advanceTimersByTimeAsync(0) + + logs.captureLog({ body: 'second' }) + release({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) + await flushed + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + await vi.advanceTimersByTimeAsync(5000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + }) + it('keeps the Retry-After window when a batch is refused for size', async () => { // `too-large` is a verdict on the body's size — the SDK's own or a 413 — // so it says nothing about the endpoint's rate limit and must not end the diff --git a/packages/core/src/logs/index.ts b/packages/core/src/logs/index.ts index 44d748ff62..aa0aff8efe 100644 --- a/packages/core/src/logs/index.ts +++ b/packages/core/src/logs/index.ts @@ -314,6 +314,13 @@ export class PostHogLogs { // window too. this._retryAfter.record(outcome) + // A capture that landed while this send was in flight armed the timer at + // the plain interval, before the window existed. Only `_flushInBackground` + // re-arms on settle, and an explicit `flush()` does not go through it. + if (this._flushTimer) { + this._armFlushTimerNoEarlierThan(Math.max(this._flushIntervalMs, this._retryAfter.remainingMs())) + } + if (outcome.kind === 'retry-later') { // Transient failure: keep records in the queue for the next flush cycle // and surface the error so the caller can log/react. diff --git a/packages/core/src/metrics/index.spec.ts b/packages/core/src/metrics/index.spec.ts index 36c579c1d4..b9f3487cf9 100644 --- a/packages/core/src/metrics/index.spec.ts +++ b/packages/core/src/metrics/index.spec.ts @@ -775,4 +775,24 @@ describe('PostHogMetrics', () => { expect(optedOutInstance._sendMetricsBatch).not.toHaveBeenCalled() }) }) + + it('does not hold a later capture at a window a successful flush already closed', async () => { + let sends = 0 + mockInstance._sendMetricsBatch = vi.fn(async (): Promise => { + sends += 1 + return sends === 1 ? { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 } : { kind: 'ok' } + }) + const metrics = createMetrics({ flushIntervalMs: 5000 }) + + metrics.count('a', 1) + await vi.advanceTimersByTimeAsync(5000) + await metrics.flush() + const closedAt = Date.now() + + metrics.count('b', 1) + await vi.advanceTimersByTimeAsync(5000) + + expect(mockInstance._sendMetricsBatch).toHaveBeenCalledTimes(3) + expect(Date.now() - closedAt).toBeLessThanOrEqual(5000) + }) }) diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index f101e0c011..f319836a0c 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -333,6 +333,12 @@ export class PostHogMetrics { } private async _doFlush(): Promise { + // A flush retires the pending timer, the way the logs and traces queues do. + // Without this, `_armFlushTimerNoEarlierThan` — which only ever ratchets a + // timer later — leaves a `Retry-After` delay armed after the window it came + // from has already been closed by a successful flush, and every later + // capture finds a timer pending and declines to arm a sooner one. + this._clearFlushTimer() if (this._series.size === 0) { return } From b91c94c87f91d0406534c39f72200c6a08f4c173 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Thu, 3 Sep 2026 15:02:20 -0400 Subject: [PATCH 09/33] fix(core): release a queue once the Retry-After window that held it closes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The re-arm helpers only ever move a timer later, so a record or series captured while a send was in flight stayed pinned to the window that send then closed — up to five minutes for a wait already over. Re-arm outright instead, and let onReconnect arm a timer during a window rather than returning with nothing scheduled, which stranded logs after an explicit flush was refused. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017SWWFUdrPWaHTpwFNr4DgC --- packages/core/src/logs/index.spec.ts | 57 +++++++++++++++++++++++++ packages/core/src/logs/index.ts | 16 +++++-- packages/core/src/metrics/index.spec.ts | 24 +++++++++++ packages/core/src/metrics/index.ts | 8 ++++ 4 files changed, 101 insertions(+), 4 deletions(-) diff --git a/packages/core/src/logs/index.spec.ts b/packages/core/src/logs/index.spec.ts index 6f8f66b228..f6efa8c83b 100644 --- a/packages/core/src/logs/index.spec.ts +++ b/packages/core/src/logs/index.spec.ts @@ -1324,6 +1324,63 @@ describe('PostHogLogs', () => { expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) }) + it('arms a timer when onReconnect lands inside the window after an explicit flush', async () => { + // `flush()` leaves no timer behind, so returning early here without + // arming one leaves the records with nothing scheduled at all. + mockInstance._sendLogsBatch = vi.fn(() => + Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 }) + ) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 10_000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'first' }) + await logs.flush().catch(() => {}) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + + await vi.advanceTimersByTimeAsync(10_000) + logs.onReconnect() + + await vi.advanceTimersByTimeAsync(289_000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(2000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(2) + }) + + it('releases a record captured mid-flush once that flush closes the window', async () => { + // The capture arms against the window that was open when it landed; the + // outcome then closes that window, so the timer has to come back down. + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 10_000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + let sends = 0 + mockInstance._sendLogsBatch = vi.fn(async () => { + sends += 1 + if (sends === 1) { + return { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 } + } + await Promise.resolve() + if (sends === 2) { + logs.captureLog({ body: 'mid' }) + } + return { kind: 'ok' } + }) + + logs.captureLog({ body: 'first' }) + await vi.advanceTimersByTimeAsync(10_000) + await logs.flush().catch(() => {}) + + await vi.advanceTimersByTimeAsync(10_000) + expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(3) + }) + it('keeps the Retry-After window when a batch is refused for size', async () => { // `too-large` is a verdict on the body's size — the SDK's own or a 413 — // so it says nothing about the endpoint's rate limit and must not end the diff --git a/packages/core/src/logs/index.ts b/packages/core/src/logs/index.ts index aa0aff8efe..8951b85569 100644 --- a/packages/core/src/logs/index.ts +++ b/packages/core/src/logs/index.ts @@ -108,6 +108,12 @@ export class PostHogLogs { onReconnect(): void { this._consecutiveFlushFailures = 0 if (this._retryAfter.isOpen()) { + // Connectivity is back but the endpoint's wait is not over. An explicit + // `flush()` leaves no timer behind, so returning without arming one is + // what strands the queue for the rest of the process. + if (this._hasQueuedRecords()) { + this._armFlushTimer() + } return } this._flushInBackground() @@ -314,11 +320,13 @@ export class PostHogLogs { // window too. this._retryAfter.record(outcome) - // A capture that landed while this send was in flight armed the timer at - // the plain interval, before the window existed. Only `_flushInBackground` - // re-arms on settle, and an explicit `flush()` does not go through it. + // A capture that landed while this send was in flight armed the timer + // against whatever the window was at the time. Re-arm outright rather + // than through the ratchet, which only ever moves a timer later and so + // would hold that capture at a window this very outcome just closed. if (this._flushTimer) { - this._armFlushTimerNoEarlierThan(Math.max(this._flushIntervalMs, this._retryAfter.remainingMs())) + this._clearFlushTimer() + this._setFlushTimer(Math.max(this._flushIntervalMs, this._retryAfter.remainingMs())) } if (outcome.kind === 'retry-later') { diff --git a/packages/core/src/metrics/index.spec.ts b/packages/core/src/metrics/index.spec.ts index b9f3487cf9..6577d72c07 100644 --- a/packages/core/src/metrics/index.spec.ts +++ b/packages/core/src/metrics/index.spec.ts @@ -795,4 +795,28 @@ describe('PostHogMetrics', () => { expect(mockInstance._sendMetricsBatch).toHaveBeenCalledTimes(3) expect(Date.now() - closedAt).toBeLessThanOrEqual(5000) }) + + it('releases a series captured mid-flush once that flush closes the window', async () => { + const metrics = createMetrics({ flushIntervalMs: 10_000 }) + let sends = 0 + mockInstance._sendMetricsBatch = vi.fn(async (): Promise => { + sends += 1 + if (sends === 1) { + return { kind: 'retry-later', error: new Error('429'), retryAfterMs: 300_000 } + } + await Promise.resolve() + if (sends === 2) { + metrics.count('mid', 1) + } + return { kind: 'ok' } + }) + + metrics.count('a', 1) + await vi.advanceTimersByTimeAsync(10_000) + await vi.advanceTimersByTimeAsync(10_000) + await metrics.flush() + + await vi.advanceTimersByTimeAsync(10_000) + expect(mockInstance._sendMetricsBatch).toHaveBeenCalledTimes(3) + }) }) diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index f319836a0c..5764231597 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -359,6 +359,14 @@ export class PostHogMetrics { return } this._retryAfter.record(outcome) + // A capture that landed while this send was in flight armed the timer + // against whatever the window was at the time. Re-arm outright rather than + // through the ratchet, which only ever moves a timer later and so would + // hold that capture at a window this very outcome just closed. + if (this._flushTimer) { + this._clearFlushTimer() + this._setFlushTimer(this._nextFlushDelay()) + } switch (outcome.kind) { case 'ok': return From 055e0b8405fc89676afbd0bfd606361d909e5903 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Thu, 3 Sep 2026 15:37:33 -0400 Subject: [PATCH 10/33] fix(traces): charge the retry budget once per backoff window An explicit flush() skips the wait but was charging the refusal, so a host draining per request retired a batch in eight calls and no elapsed time. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01L8eFZB35NExUZyq5hGgh43 --- packages/core/src/traces/index.spec.ts | 54 +++++++++++++++++++++++++- packages/core/src/traces/index.ts | 49 +++++++++++++++-------- 2 files changed, 84 insertions(+), 19 deletions(-) diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index 507b40bd18..ac8c00367b 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -1494,12 +1494,14 @@ describe('PostHogTraces', () => { it('still charges the retry budget for failures outside any window', async () => { // The budget must keep working when the endpoint names no wait, or a - // permanently failing head batch pins the queue forever. + // permanently failing head batch pins the queue forever. It takes the + // eight backoff windows the timer would have waited — roughly three + // minutes at this interval — however often the caller drains in between. mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('503') }) const traces = createTraces({ flushIntervalMs: 10_000 }) traces.startSpan('first').end() - for (let i = 0; i < 12; i++) { + for (let i = 0; i < 25; i++) { await traces.flush() await vi.advanceTimersByTimeAsync(10_000) } @@ -1507,6 +1509,54 @@ describe('PostHogTraces', () => { expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Dropping')) }) + it('does not spend the retry budget on a caller draining faster than the backoff', async () => { + // A serverless host calls `flush()` per invocation. Charging every refusal + // retired the batch in eight calls and no elapsed time at all, so a blip + // the timer would have ridden out cost the spans instead of a request. + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('503') }) + const traces = createTraces({ flushIntervalMs: 10_000 }) + traces.startSpan('first').end() + + for (let i = 0; i < 20; i++) { + await traces.flush() + } + + expect((traces as any)._queue).toHaveLength(1) + expect(logger.warn).not.toHaveBeenCalledWith(expect.stringContaining('Dropping')) + }) + + it('charges once per window however many attempts share it', async () => { + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('503') }) + const traces = createTraces({ flushIntervalMs: 10_000 }) + traces.startSpan('first').end() + + await traces.flush() + expect((traces as any)._headBatchFailures).toBe(1) + + await traces.flush() + await traces.flush() + expect((traces as any)._headBatchFailures).toBe(1) + + await vi.advanceTimersByTimeAsync(30_000) + await traces.flush() + expect((traces as any)._headBatchFailures).toBe(2) + }) + + it('gives a fresh batch a fresh window after the head is retired', async () => { + // The deadline belongs to the batch, not the queue: a new head must be + // chargeable straight away or it inherits the last one's served wait. + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'fatal', error: new Error('400') }) + const traces = createTraces({ flushIntervalMs: 10_000, maxExportBatchSize: 1 }) + traces.startSpan('first').end() + await traces.flush() + + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('503') }) + traces.startSpan('second').end() + await traces.flush() + + expect((traces as any)._headBatchFailures).toBe(1) + }) + it('does not let a backward clock step hold the retry budget open', async () => { // The deadline is wall clock. An NTP correction or a resumed VM would // otherwise keep the window "open" for the size of the step — and with diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 8dd83e5d24..23e87663de 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -18,9 +18,9 @@ import { RetryAfterWindow } from '../utils/retry-after' // Retriable failures on the same head batch before it is dropped, so a stuck // batch cannot pin the queue while fresher spans are refused at the cap. The -// budget counts attempts, not elapsed time: on the timer path the backoff -// spreads them over minutes, while a host that calls `flush()` per request -// spends them as fast as the requests arrive. +// budget counts backoff windows rather than attempts: a host that drains on +// every request would otherwise retire a batch in milliseconds, spending on +// its own call rate what the timer path spends over minutes. const MAX_RETRIES_PER_BATCH = 8 const MAX_FLUSH_BACKOFF_EXPONENT = 6 @@ -85,6 +85,9 @@ export class PostHogTraces { // Read only while a budget is in flight, so the head cannot grow to sweep in // fresh spans and drop them on a budget they never spent. private _headBatchSize = 0 + // When the head batch may next be charged, on `clockNow`'s basis: one failure + // per backoff window, whoever drove the attempt. + private _headBatchChargeableAt = 0 // Bumped by reset(); a pass whose generation is stale abandons the queue. private _generation = 0 // Live-span accounting: span id -> monotonic start. Ids and numbers only, @@ -276,7 +279,13 @@ export class PostHogTraces { this._lastDropWarningAt = 0 this._consecutiveFlushFailures = 0 this._retryAfter.reset() + this._resetHeadBatchBudget() + } + + /** Called wherever the head batch leaves or changes shape, so its budget goes with it. */ + private _resetHeadBatchBudget(): void { this._headBatchFailures = 0 + this._headBatchChargeableAt = 0 } /** @@ -508,13 +517,13 @@ export class PostHogTraces { this._queue.splice(0, size) remaining -= size removed += size - this._headBatchFailures = 0 + this._resetHeadBatchBudget() continue } - // Read before the send, so the retry budget below charges this attempt - // against the window it was actually made under. - const insideRetryAfterWindow = this._retryAfter.isOpen() + // Read before the send, so the budget below charges this attempt against + // the window it was actually made under. + const chargeable = clockNow() >= this._headBatchChargeableAt const outcome = await this._instance._sendTracesBatch( buildOtlpTracesPayload(spans, resourceAttributes, scopeName, scopeVersion, this._logger) @@ -529,7 +538,7 @@ export class PostHogTraces { if (outcome.kind === 'ok') { this._consecutiveFlushFailures = 0 - this._headBatchFailures = 0 + this._resetHeadBatchBudget() this._queue.splice(0, size) remaining -= size removed += size @@ -547,27 +556,33 @@ export class PostHogTraces { remaining -= 1 removed += 1 this._recordDrop(1, 'it is too large for the ingestion endpoint') - this._headBatchFailures = 0 + this._resetHeadBatchBudget() continue } // Halve the batch the server rejected, not the configured maximum: when the // queue is shallower than the maximum, shrinking it resends an identical body. this._maxExportBatchSize = Math.max(1, Math.floor(size / 2)) // A different batch from here on, so its budget starts fresh. - this._headBatchFailures = 0 + this._resetHeadBatchBudget() this._logger.debug(`Batch too large; retrying the same spans in batches of ${this._maxExportBatchSize}`) continue } if (outcome.kind === 'retry-later') { this._consecutiveFlushFailures++ - // A refusal inside a window the endpoint asked us to wait out is not - // evidence against this batch. Charging it spends the whole budget on - // the wait and drops the spans before the window even elapses. - if (!insideRetryAfterWindow) { + // One charge per backoff window. A refusal that arrives before the + // window the last one bought has elapsed — an explicit `flush()`, a + // per-request serverless drain, or a wait the endpoint asked for with + // `Retry-After` — is the same refusal seen again, not new evidence + // against the batch, so honoring the endpoint costs a request rather + // than the spans. + this._headBatchSize = size + if (chargeable) { this._headBatchFailures++ + // The window this charge buys is the delay the timer will wait, which + // grows with the failure count and honors `Retry-After`. + this._headBatchChargeableAt = clockNow() + this._nextFlushDelay() } - this._headBatchSize = size if (this._headBatchFailures < MAX_RETRIES_PER_BATCH) { // Keep the spans queued; the flush timer picks them up again. this._logger.debug('Span export failed; retrying on the next flush', outcome.error) @@ -579,7 +594,7 @@ export class PostHogTraces { remaining -= size removed += size this._consecutiveFlushFailures = 0 - this._headBatchFailures = 0 + this._resetHeadBatchBudget() this._recordDrop(size, `the ingestion endpoint failed ${MAX_RETRIES_PER_BATCH} times in a row`) if (this._retryAfter.isOpen()) { // Retiring the batch cleared the failure counters, so carrying on @@ -594,7 +609,7 @@ export class PostHogTraces { this._queue.splice(0, size) remaining -= size removed += size - this._headBatchFailures = 0 + this._resetHeadBatchBudget() this._recordDrop(size, 'the ingestion endpoint rejected the batch') } From 829f41fe448ae870b45645d2206e182138410f5a Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Thu, 3 Sep 2026 15:47:00 -0400 Subject: [PATCH 11/33] chore(core): declare posthog-js and trim the retry-after comments The shared timer change reaches the browser even though the window never opens there, so it needs its own changeset line. Comments that narrated the bugs behind each fix are cut back to the constraint a reader needs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017SWWFUdrPWaHTpwFNr4DgC --- .changeset/otlp-honor-retry-after.md | 3 +++ packages/core/src/logs/index.ts | 30 +++++++++++--------------- packages/core/src/metrics/index.ts | 27 +++++++++-------------- packages/core/src/traces/index.ts | 25 ++++++++------------- packages/core/src/utils/retry-after.ts | 6 ++---- 5 files changed, 36 insertions(+), 55 deletions(-) diff --git a/.changeset/otlp-honor-retry-after.md b/.changeset/otlp-honor-retry-after.md index 3096652802..2630f9178d 100644 --- a/.changeset/otlp-honor-retry-after.md +++ b/.changeset/otlp-honor-retry-after.md @@ -1,9 +1,12 @@ --- 'posthog-node': patch 'posthog-react-native': patch +'posthog-js': patch '@posthog/core': patch --- Honor `Retry-After` when the ingestion endpoint refuses a logs, metrics or traces batch, rather than retrying on the SDK's own backoff alone. The header acts as a floor, so a shorter value never makes the SDK retry sooner than it would have, and a wait longer than five minutes is capped. The periodic flush waits the window out. An explicit `flush()`, and a flush a host runs to keep a request alive, still sends — a short-lived process is not left holding data it will never get another chance to deliver. Those attempts no longer count against a span batch's retry budget either, so waiting out a rate limit no longer drops spans. + +In `posthog-js` the ingestion endpoint sends no `Retry-After`, so none of the above applies — but the log flush now keeps backing off while records are still arriving, instead of a new record resetting the retry to the flush interval. A failing endpoint is retried a handful of times over an outage rather than once per interval. diff --git a/packages/core/src/logs/index.ts b/packages/core/src/logs/index.ts index 8951b85569..6c54c1805b 100644 --- a/packages/core/src/logs/index.ts +++ b/packages/core/src/logs/index.ts @@ -108,9 +108,8 @@ export class PostHogLogs { onReconnect(): void { this._consecutiveFlushFailures = 0 if (this._retryAfter.isOpen()) { - // Connectivity is back but the endpoint's wait is not over. An explicit - // `flush()` leaves no timer behind, so returning without arming one is - // what strands the queue for the rest of the process. + // The wait outlives the reconnect, but something still has to schedule + // the retry: an explicit `flush()` leaves no timer behind. if (this._hasQueuedRecords()) { this._armFlushTimer() } @@ -315,15 +314,12 @@ export class PostHogLogs { continue } - // Recorded here rather than on the background wrapper, so an explicit - // `flush()` — which every lifecycle hook takes — records and clears the - // window too. + // Not on the background wrapper: every lifecycle hook takes `flush()`, + // which does not go through it. this._retryAfter.record(outcome) - // A capture that landed while this send was in flight armed the timer - // against whatever the window was at the time. Re-arm outright rather - // than through the ratchet, which only ever moves a timer later and so - // would hold that capture at a window this very outcome just closed. + // Outright, not through the ratchet: a timer a mid-flight capture armed + // is measured against a window this outcome may just have closed. if (this._flushTimer) { this._clearFlushTimer() this._setFlushTimer(Math.max(this._flushIntervalMs, this._retryAfter.remainingMs())) @@ -413,15 +409,13 @@ export class PostHogLogs { if (this._flushTimer) { return } - // Floored by any open window: an explicit `flush()` leaves no timer behind, - // so this is the path a capture takes after one, and the plain interval - // would land inside the wait the endpoint asked for. + // Floored by any open window: `flush()` leaves no timer behind, so a + // capture after one arrives here. this._setFlushTimer(Math.max(this._flushIntervalMs, this._retryAfter.remainingMs())) } - // Backoff and `Retry-After` are floors, so a timer already armed at the plain - // interval has to give way to a longer one — otherwise a capture landing - // mid-flush would send inside the window the server asked us to skip. + // Both floors, so a timer already armed at the plain interval gives way to a + // longer one. private _armFlushTimerNoEarlierThan(delayMs: number): void { if (this._flushTimer && Date.now() + delayMs <= this._flushTimerFiresAt) { return @@ -443,8 +437,8 @@ export class PostHogLogs { // retried every interval. private _nextFlushDelay(): number { const exponent = Math.min(Math.max(0, this._consecutiveFlushFailures - 1), MAX_FLUSH_BACKOFF_EXPONENT) - // `Retry-After` is a floor, not a replacement: never retry before the server - // asked, and never more often than our own backoff would have. + // A floor, not a replacement: the header never retries us sooner than our + // own backoff would have. return Math.max(this._flushIntervalMs * 2 ** exponent, this._retryAfter.remainingMs()) } diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index 5764231597..0d8b881ce9 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -288,9 +288,8 @@ export class PostHogMetrics { return result } - // Arms the flush timer if none is pending. Every capture calls this, so it - // must leave a pending timer alone: re-arming on each one would push the - // flush out for as long as metrics keep arriving. + // Every capture calls this, so a pending timer is left alone — re-arming on + // each one would push the flush out for as long as metrics keep arriving. private _armFlushTimer(): void { if (this._flushTimer) { return @@ -298,9 +297,8 @@ export class PostHogMetrics { this._setFlushTimer(this._nextFlushDelay()) } - // `Retry-After` is a floor, so a timer already armed at the flush interval has - // to give way to a longer one rather than firing inside the window the server - // asked us to skip. + // A floor, so a timer already armed at the flush interval gives way to a + // longer one. private _armFlushTimerNoEarlierThan(delayMs: number): void { if (this._flushTimer && Date.now() + delayMs <= this._flushTimerFiresAt) { return @@ -309,8 +307,8 @@ export class PostHogMetrics { this._setFlushTimer(delayMs) } - // `Retry-After` is a floor, not a replacement: never retry before the server - // asked, and never more often than the flush interval. + // A floor, not a replacement: the header never retries us sooner than the + // flush interval would have. private _nextFlushDelay(): number { return Math.max(this._config.flushIntervalMs, this._retryAfter.remainingMs()) } @@ -333,11 +331,8 @@ export class PostHogMetrics { } private async _doFlush(): Promise { - // A flush retires the pending timer, the way the logs and traces queues do. - // Without this, `_armFlushTimerNoEarlierThan` — which only ever ratchets a - // timer later — leaves a `Retry-After` delay armed after the window it came - // from has already been closed by a successful flush, and every later - // capture finds a timer pending and declines to arm a sooner one. + // A flush retires the pending timer, so a delay armed for a window this + // flush may close cannot outlive it. this._clearFlushTimer() if (this._series.size === 0) { return @@ -359,10 +354,8 @@ export class PostHogMetrics { return } this._retryAfter.record(outcome) - // A capture that landed while this send was in flight armed the timer - // against whatever the window was at the time. Re-arm outright rather than - // through the ratchet, which only ever moves a timer later and so would - // hold that capture at a window this very outcome just closed. + // Outright, not through the ratchet: a timer a mid-flight capture armed is + // measured against a window this outcome may just have closed. if (this._flushTimer) { this._clearFlushTimer() this._setFlushTimer(this._nextFlushDelay()) diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 23e87663de..74306e4f24 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -570,17 +570,12 @@ export class PostHogTraces { if (outcome.kind === 'retry-later') { this._consecutiveFlushFailures++ - // One charge per backoff window. A refusal that arrives before the - // window the last one bought has elapsed — an explicit `flush()`, a - // per-request serverless drain, or a wait the endpoint asked for with - // `Retry-After` — is the same refusal seen again, not new evidence - // against the batch, so honoring the endpoint costs a request rather - // than the spans. + // One charge per backoff window: a refusal arriving before the window + // the last charge bought has elapsed is the same refusal seen again, + // not new evidence against the batch. this._headBatchSize = size if (chargeable) { this._headBatchFailures++ - // The window this charge buys is the delay the timer will wait, which - // grows with the failure count and honors `Retry-After`. this._headBatchChargeableAt = clockNow() + this._nextFlushDelay() } if (this._headBatchFailures < MAX_RETRIES_PER_BATCH) { @@ -644,8 +639,8 @@ export class PostHogTraces { }) } - // Arms the flush timer if none is pending. Every span end can reach this, so - // it must leave a pending timer alone rather than pushing the flush out. + // Every span end can reach this, so a pending timer is left alone rather than + // pushing the flush out. private _armFlushTimerIfQueued(): void { if (this._flushTimer || !this._queue.length) { return @@ -653,10 +648,8 @@ export class PostHogTraces { this._setFlushTimer(this._nextFlushDelay()) } - // Backoff and `Retry-After` are floors, so a timer a span end armed at the - // plain interval while the send was in flight has to give way to a longer - // one — otherwise the retry lands inside the window the server asked us to - // skip. + // Both floors, so a timer a span end armed at the plain interval gives way to + // a longer one. private _armFlushTimerIfQueuedNoEarlierThan(): void { if (!this._queue.length) { return @@ -683,8 +676,8 @@ export class PostHogTraces { const exponent = Math.min(Math.max(0, this._consecutiveFlushFailures - 1), MAX_FLUSH_BACKOFF_EXPONENT) const delay = this._config.flushIntervalMs * 2 ** exponent const capped = Math.min(delay, Math.max(MAX_FLUSH_BACKOFF_MS, this._config.flushIntervalMs)) - // `Retry-After` is a floor, not a replacement: never retry before the server - // asked, and never more often than our own backoff would have. + // A floor, not a replacement: the header never retries us sooner than our + // own backoff would have. return Math.max(capped, this._retryAfter.remainingMs()) } diff --git a/packages/core/src/utils/retry-after.ts b/packages/core/src/utils/retry-after.ts index 17fce6247e..561404ddda 100644 --- a/packages/core/src/utils/retry-after.ts +++ b/packages/core/src/utils/retry-after.ts @@ -87,10 +87,8 @@ export class RetryAfterWindow { return } if (!outcome.retryAfterMs || this.isOpen()) { - // Nothing to install, and nothing that revokes what is installed: the - // refusal either named no wait — a network error, a timeout or a - // header-less 503, all of which the outage that named the original wait - // keeps producing — or arrived while that wait is still being served. + // A refusal that names no wait — a network error, a timeout, a + // header-less 503 — does not revoke one the endpoint already named. return } this._installedAt = Date.now() From de06acd3077de9b7a7ddb71b5cc1808b1bc92904 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 12:00:11 -0400 Subject: [PATCH 12/33] fix(core): measure OTLP bodies against the configured limit, not the fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ceiling was the 2 MB `MAX_REQUEST_BODY_SIZE_BYTES` falls back to, but the ingestion service runs with 10 MiB, so bodies between the two were refused without a request and their records dropped — a whole window, in metrics, which has no shrink path. Raise it to the largest limit any known deployment configures; the 413 path still covers the ones configured lower. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE1TyDWGyLwUXEX3ffy83K --- .changeset/otlp-skip-oversized-bodies.md | 2 +- .../__tests__/posthog.otlp-too-large.spec.ts | 39 ++++++++++++++----- packages/core/src/logs/index.spec.ts | 4 +- packages/core/src/logs/index.ts | 8 ++-- packages/core/src/logs/types.ts | 4 +- packages/core/src/posthog-core-stateless.ts | 23 ++++++----- packages/node/src/traces-defaults.ts | 2 +- 7 files changed, 53 insertions(+), 29 deletions(-) diff --git a/.changeset/otlp-skip-oversized-bodies.md b/.changeset/otlp-skip-oversized-bodies.md index d1243c199a..7a1f098f80 100644 --- a/.changeset/otlp-skip-oversized-bodies.md +++ b/.changeset/otlp-skip-oversized-bodies.md @@ -4,4 +4,4 @@ '@posthog/core': patch --- -Stop uploading logs, metrics and traces batches larger than the ingestion endpoint's 2 MB body limit. Such a batch can only come back `413`, so it is split — and, when a single record is itself oversized, dropped — without spending a request on each attempt. The size is measured before compression, matching how the endpoint applies its limit. +Stop uploading logs, metrics and traces batches larger than the ingestion endpoint's request body limit — such a batch is split, and a single oversized record dropped, without spending a request on each attempt. diff --git a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts index 8d1b0029cb..3b86821bdd 100644 --- a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts +++ b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts @@ -1,9 +1,15 @@ import { createTestClient, PostHogCoreTestClient, PostHogCoreTestClientMocks } from '@/testing' -// The endpoint caps the request body at 2 MB. A batch over that can only come -// back 413, so it is reported as too-large without being sent — the caller's -// halving loop then isolates and drops the oversized record without spending a -// request on every attempt. +// The SDK will not put a body larger than the endpoint's configured limit on +// the wire. Such a batch can only come back 413, so it is reported as too-large +// without being sent — the caller's halving loop then isolates and drops the +// oversized record without spending a request on every attempt. +// +// Mirrors OTLP_MAX_BODY_BYTES. Kept as a local literal so a change to the +// shipped ceiling has to be made deliberately here too. +const LIMIT_BYTES = 10 * 1024 * 1024 +const OVER_LIMIT_BYTES = LIMIT_BYTES + 1024 + describe('OTLP bodies over the endpoint limit', () => { let posthog: PostHogCoreTestClient let mocks: PostHogCoreTestClientMocks @@ -49,7 +55,7 @@ describe('OTLP bodies over the endpoint limit', () => { ['logs', (payload: any) => posthog._sendLogsBatch(payload)], ['metrics', (payload: any) => posthog._sendMetricsBatch(payload)], ])('reports a %s batch over the limit as too-large without a request', async (_signal, send) => { - await expect(send(spansOf(3 * 1024 * 1024))).resolves.toEqual({ kind: 'too-large' }) + await expect(send(spansOf(OVER_LIMIT_BYTES))).resolves.toEqual({ kind: 'too-large' }) expect(mocks.fetch).not.toHaveBeenCalled() }) @@ -59,7 +65,7 @@ describe('OTLP bodies over the endpoint limit', () => { vi.spyOn(posthog as any, 'compressPayload').mockResolvedValue(new Uint8Array(1024)) ;(posthog as any).disableCompression = false - await expect(posthog._sendTracesBatch(spansOf(3 * 1024 * 1024))).resolves.toEqual({ kind: 'too-large' }) + await expect(posthog._sendTracesBatch(spansOf(OVER_LIMIT_BYTES))).resolves.toEqual({ kind: 'too-large' }) expect(mocks.fetch).not.toHaveBeenCalled() }) @@ -70,15 +76,15 @@ describe('OTLP bodies over the endpoint limit', () => { const overheadBytes = (): number => JSON.stringify(spansOf(1024)).length - 1024 it('sends a batch of exactly the limit', async () => { - const exact = 2 * 1024 * 1024 - overheadBytes() - expect(JSON.stringify(spansOf(exact)).length).toBe(2 * 1024 * 1024) + const exact = LIMIT_BYTES - overheadBytes() + expect(JSON.stringify(spansOf(exact)).length).toBe(LIMIT_BYTES) await expect(posthog._sendTracesBatch(spansOf(exact))).resolves.toEqual({ kind: 'ok' }) expect(mocks.fetch).toHaveBeenCalledTimes(1) }) it('reports a batch one byte over the limit as too-large', async () => { - await expect(posthog._sendTracesBatch(spansOf(2 * 1024 * 1024 - overheadBytes() + 1))).resolves.toEqual({ + await expect(posthog._sendTracesBatch(spansOf(LIMIT_BYTES - overheadBytes() + 1))).resolves.toEqual({ kind: 'too-large', }) expect(mocks.fetch).not.toHaveBeenCalled() @@ -91,10 +97,23 @@ describe('OTLP bodies over the endpoint limit', () => { const compressPayload = vi.spyOn(posthog as any, 'compressPayload') ;(posthog as any).disableCompression = false - await expect(posthog._sendTracesBatch(spansOf(3 * 1024 * 1024))).resolves.toEqual({ kind: 'too-large' }) + await expect(posthog._sendTracesBatch(spansOf(OVER_LIMIT_BYTES))).resolves.toEqual({ kind: 'too-large' }) expect(compressPayload).not.toHaveBeenCalled() }) + it.each([ + ['traces', (payload: any) => posthog._sendTracesBatch(payload)], + ['logs', (payload: any) => posthog._sendLogsBatch(payload)], + ['metrics', (payload: any) => posthog._sendMetricsBatch(payload)], + ])('sends a %s batch the service accepts but its fallback default would not', async (_signal, send) => { + // 3 MiB sits above the 2 MB the service falls back to and below the limit it + // is configured with, so it is accepted today. A ceiling set to the fallback + // would refuse it here — costing the records, since a batch of one that is + // refused for size is dropped, and metrics drops the whole window. + await expect(send(spansOf(3 * 1024 * 1024))).resolves.toEqual({ kind: 'ok' }) + expect(mocks.fetch).toHaveBeenCalledTimes(1) + }) + it('sends a compressible payload that is inside the limit before compression', async () => { vi.spyOn(posthog as any, 'compressPayload').mockResolvedValue(new Uint8Array(1024)) ;(posthog as any).disableCompression = false diff --git a/packages/core/src/logs/index.spec.ts b/packages/core/src/logs/index.spec.ts index f6efa8c83b..4ef8d7e159 100644 --- a/packages/core/src/logs/index.spec.ts +++ b/packages/core/src/logs/index.spec.ts @@ -927,7 +927,7 @@ describe('PostHogLogs', () => { await logs.flush() expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('Dropping a single log record after 413 with batch size 1') + expect.stringContaining('Dropping a single log record with batch size 1') ) }) @@ -985,7 +985,7 @@ describe('PostHogLogs', () => { expect(sendSizes).toEqual([1]) expect(readQueue(mockInstance)).toHaveLength(0) expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('Dropping a single log record after 413 with batch size 1') + expect.stringContaining('Dropping a single log record with batch size 1') ) }) diff --git a/packages/core/src/logs/index.ts b/packages/core/src/logs/index.ts index 6c54c1805b..d074e9e404 100644 --- a/packages/core/src/logs/index.ts +++ b/packages/core/src/logs/index.ts @@ -332,13 +332,15 @@ export class PostHogLogs { } // ok | fatal | too-large-with-batch-of-1 → records are leaving the - // queue. 'fatal' and size-1 413s are dropped so we don't spin on the - // same record forever. Surface the size-1 413 explicitly so a single + // queue. 'fatal' and size-1 refusals are dropped so we don't spin on the + // same record forever. Surface the size-1 refusal explicitly so a single // oversized record (e.g. a giant body field) is visible in logs // instead of silently disappearing. if (outcome.kind === 'too-large') { + // Reached either from a 413 or from the size the SDK measured before + // sending, so the message names neither. this._logger.warn( - 'Dropping a single log record after 413 with batch size 1 — the record is larger than the server cap and cannot be split further.' + 'Dropping a single log record with batch size 1 — the record is larger than the server cap and cannot be split further.' ) } else if (outcome.kind === 'ok' && this._maxBatchRecordsPerPost < this._config.maxBatchRecordsPerPost) { // Linear recovery: each healthy send pushes the cap back up by 1 diff --git a/packages/core/src/logs/types.ts b/packages/core/src/logs/types.ts index 19c60ce5c8..93885fdbd7 100644 --- a/packages/core/src/logs/types.ts +++ b/packages/core/src/logs/types.ts @@ -128,8 +128,8 @@ export interface PostHogLogsConfig { /** * Max records per outbound POST. Keeps each request under the server's - * 2 MB cap. On a 413 response, the SDK halves this value, retries the - * same records, then ramps back up by 1 per healthy send. A 413 on a + * request body cap. On a 413 response, the SDK halves this value, retries + * the same records, then ramps back up by 1 per healthy send. A 413 on a * single-record batch drops the record (it's larger than the server can * accept regardless of batch size). Default: 50 (RN) / 100 (browser). */ diff --git a/packages/core/src/posthog-core-stateless.ts b/packages/core/src/posthog-core-stateless.ts index 06d8c113e2..497af6a68d 100644 --- a/packages/core/src/posthog-core-stateless.ts +++ b/packages/core/src/posthog-core-stateless.ts @@ -222,18 +222,21 @@ function isRetryableFlagsFetchError( } /** - * The ingestion service's request body limit — `MAX_REQUEST_BODY_SIZE_BYTES`, - * which defaults to 2 MB and is applied to the body after the endpoint - * decompresses it, not to the compressed bytes on the wire. + * Ceiling on what the SDK will put on the wire: a body over it is reported as + * too large without a request being made, and a batch of one that still exceeds + * it is dropped. The ingestion service applies its own + * `MAX_REQUEST_BODY_SIZE_BYTES` to the body after it decompresses it, not to + * the compressed bytes on the wire, so this is measured the same way. * - * Applied as a ceiling on what the SDK will put on the wire: a body over it is - * reported as too large without a request being made, and a batch of one that - * still exceeds it is dropped. The cost is a deployment that raised - * `MAX_REQUEST_BODY_SIZE_BYTES` above 2 MB, where such a batch would have been - * accepted. A body under the limit is sent and may still be refused, by the - * endpoint or by a proxy in front of it with a lower one. + * Set to the largest limit any known deployment configures — 10 MiB, what the + * ingestion service runs with — rather than the 2 MB the service falls back to + * when nothing configures it. The ceiling only earns its place by refusing a + * body that no deployment would have accepted: at 2 MB it would instead refuse + * bodies the service takes today, dropping records with no `413` to show for + * them. Deployments configured lower, and proxies in front of them, are covered + * by the `413` path, which stays the primary mechanism. */ -const OTLP_MAX_BODY_BYTES = 2 * 1024 * 1024 +const OTLP_MAX_BODY_BYTES = 10 * 1024 * 1024 /** * A request body's size on the wire. `Buffer` where it exists, `TextEncoder` diff --git a/packages/node/src/traces-defaults.ts b/packages/node/src/traces-defaults.ts index f4982505bc..87cb1f7011 100644 --- a/packages/node/src/traces-defaults.ts +++ b/packages/node/src/traces-defaults.ts @@ -2,7 +2,7 @@ import { assignUserAttributes } from '@posthog/core' import type { ResolvedTracesConfig, TracesConfig } from '@posthog/core' // OpenTelemetry's BatchSpanProcessor defaults, which sit comfortably under the -// server's 2 MB body cap. +// server's request body cap. const DEFAULT_FLUSH_INTERVAL_MS = 5000 const DEFAULT_MAX_EXPORT_BATCH_SIZE = 512 const DEFAULT_MAX_QUEUE_SIZE = 2048 From ea4e9a262ff37a82a1aadb623cbab7530e22416d Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 12:00:19 -0400 Subject: [PATCH 13/33] chore(core): correct the Retry-After parser and window comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parser rejects a zero delta as well as a negative one, and the window's justification named a retry-budget effect that no longer applies to traces — the rule holds for the logs and metrics gates instead. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE1TyDWGyLwUXEX3ffy83K --- packages/core/src/utils/retry-after.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/core/src/utils/retry-after.ts b/packages/core/src/utils/retry-after.ts index 561404ddda..5f77439934 100644 --- a/packages/core/src/utils/retry-after.ts +++ b/packages/core/src/utils/retry-after.ts @@ -18,9 +18,9 @@ const CLOCK_STEP_TOLERANCE_MS = 5_000 /** * `Retry-After` as milliseconds from now. The header is either delta-seconds or - * an HTTP-date; anything else, a date already in the past, or a negative delta - * yields `undefined` so the caller keeps its own backoff. The result is capped - * at `MAX_RETRY_AFTER_MS`. + * an HTTP-date; anything else, a date already in the past, or a non-positive + * delta yields `undefined` so the caller keeps its own backoff. The result is + * capped at `MAX_RETRY_AFTER_MS`. */ export function parseRetryAfterMs(value: unknown, now: number = Date.now()): number | undefined { if (typeof value !== 'string' || !value) { @@ -68,11 +68,14 @@ export class RetryAfterWindow { /** * Folds one export outcome into the window. * - * A window still open is left as it stands: the refusal repeats an answer the - * endpoint has already given, and sliding the deadline on each one means the - * window never elapses for a host that sends on its own cadence. Whether it - * is still open is read here rather than passed in, so a send that outlives - * the wait it was made under installs the fresh deadline it came back with. + * A window still open is left as it stands, even when the refusal names a + * longer wait than the one being served. Sliding the deadline on each refusal + * means the window never elapses for a host that flushes faster than the + * window is long: logs gates its size trigger and `onReconnect` on the + * window, and metrics re-arms its timer from it, so neither would recover + * while such a host kept sending. Whether it is still open is read here + * rather than passed in, so a send that outlives the wait it was made under + * installs the fresh deadline it came back with. */ record(outcome: RetryAfterOutcome): void { if (outcome.kind === 'too-large') { From 5885f79731113abf32900b491f86d41df11b54a9 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 12:58:56 -0400 Subject: [PATCH 14/33] chore(react-native): regenerate references for the body-cap doc wording Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE1TyDWGyLwUXEX3ffy83K --- .../references/posthog-react-native-references-latest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native/references/posthog-react-native-references-latest.json b/packages/react-native/references/posthog-react-native-references-latest.json index 6ba91d2ee5..2deb916774 100644 --- a/packages/react-native/references/posthog-react-native-references-latest.json +++ b/packages/react-native/references/posthog-react-native-references-latest.json @@ -3724,7 +3724,7 @@ "name": "flushIntervalMs" }, { - "description": "Max records per outbound POST. Keeps each request under the server's 2 MB cap. On a 413 response, the SDK halves this value, retries the same records, then ramps back up by 1 per healthy send. A 413 on a single-record batch drops the record (it's larger than the server can accept regardless of batch size). Default: 50 (RN) / 100 (browser).", + "description": "Max records per outbound POST. Keeps each request under the server's request body cap. On a 413 response, the SDK halves this value, retries the same records, then ramps back up by 1 per healthy send. A 413 on a single-record batch drops the record (it's larger than the server can accept regardless of batch size). Default: 50 (RN) / 100 (browser).", "type": "number", "name": "maxBatchRecordsPerPost" }, From 5fa7acad5fa07d1b8fef35cbc672a0f773696cba Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 14:51:05 -0400 Subject: [PATCH 15/33] refactor(core): hold the flush timer in one place for all three queues The logs, metrics and traces queues each carried their own handle, deadline and arm/clear pair, differing only in what fires. A `FlushTimer` holds the deadline next to the handle, so the ratchet is written and tested once. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE1TyDWGyLwUXEX3ffy83K --- .../core/src/__tests__/flush-timer.spec.ts | 100 ++++++++++++++++++ packages/core/src/logs/index.ts | 46 ++------ packages/core/src/metrics/index.ts | 50 +++------ packages/core/src/traces/index.ts | 36 ++----- packages/core/src/utils/flush-timer.ts | 53 ++++++++++ 5 files changed, 186 insertions(+), 99 deletions(-) create mode 100644 packages/core/src/__tests__/flush-timer.spec.ts create mode 100644 packages/core/src/utils/flush-timer.ts diff --git a/packages/core/src/__tests__/flush-timer.spec.ts b/packages/core/src/__tests__/flush-timer.spec.ts new file mode 100644 index 0000000000..c21862b6ab --- /dev/null +++ b/packages/core/src/__tests__/flush-timer.spec.ts @@ -0,0 +1,100 @@ +import { FlushTimer } from '../utils/flush-timer' + +describe('FlushTimer', () => { + it('fires once, after the delay', async () => { + const onFire = vi.fn() + const timer = new FlushTimer(onFire) + timer.arm(1000) + + await vi.advanceTimersByTimeAsync(999) + expect(onFire).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + expect(onFire).toHaveBeenCalledTimes(1) + + await vi.advanceTimersByTimeAsync(10_000) + expect(onFire).toHaveBeenCalledTimes(1) + }) + + it('releases the handle before firing, so the callback can arm again', async () => { + const timer: FlushTimer = new FlushTimer(() => { + pendingInsideCallback = timer.pending + }) + let pendingInsideCallback: boolean | undefined + timer.arm(1000) + expect(timer.pending).toBe(true) + + await vi.advanceTimersByTimeAsync(1000) + expect(pendingInsideCallback).toBe(false) + expect(timer.pending).toBe(false) + }) + + it('arm replaces a pending timer outright, in either direction', async () => { + const onFire = vi.fn() + const timer = new FlushTimer(onFire) + timer.arm(10_000) + timer.arm(1000) + + await vi.advanceTimersByTimeAsync(1000) + expect(onFire).toHaveBeenCalledTimes(1) + }) + + // The reason the deadline lives next to the handle: every capture reaches the + // arming path, and none of them may pull a flush in front of a longer wait. + it('armNoEarlierThan does not shorten a pending timer', async () => { + const onFire = vi.fn() + const timer = new FlushTimer(onFire) + timer.arm(10_000) + timer.armNoEarlierThan(1000) + + await vi.advanceTimersByTimeAsync(9999) + expect(onFire).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + expect(onFire).toHaveBeenCalledTimes(1) + }) + + it('armNoEarlierThan lengthens a pending timer', async () => { + const onFire = vi.fn() + const timer = new FlushTimer(onFire) + timer.arm(1000) + timer.armNoEarlierThan(10_000) + + await vi.advanceTimersByTimeAsync(1000) + expect(onFire).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(9000) + expect(onFire).toHaveBeenCalledTimes(1) + }) + + it('armNoEarlierThan counts down the remainder, not the whole delay', async () => { + const onFire = vi.fn() + const timer = new FlushTimer(onFire) + timer.arm(10_000) + await vi.advanceTimersByTimeAsync(6000) + // 4s left, so a 4s request is not longer and must not restart the wait. + timer.armNoEarlierThan(4000) + + await vi.advanceTimersByTimeAsync(4000) + expect(onFire).toHaveBeenCalledTimes(1) + }) + + it('armNoEarlierThan arms when nothing is pending', async () => { + const onFire = vi.fn() + const timer = new FlushTimer(onFire) + timer.armNoEarlierThan(1000) + expect(timer.pending).toBe(true) + + await vi.advanceTimersByTimeAsync(1000) + expect(onFire).toHaveBeenCalledTimes(1) + }) + + it('clear stops a pending timer and is safe to repeat', async () => { + const onFire = vi.fn() + const timer = new FlushTimer(onFire) + timer.arm(1000) + timer.clear() + timer.clear() + expect(timer.pending).toBe(false) + + await vi.advanceTimersByTimeAsync(10_000) + expect(onFire).not.toHaveBeenCalled() + }) +}) diff --git a/packages/core/src/logs/index.ts b/packages/core/src/logs/index.ts index d074e9e404..96ad4f66d3 100644 --- a/packages/core/src/logs/index.ts +++ b/packages/core/src/logs/index.ts @@ -1,7 +1,8 @@ import type { LogAttributeValue } from '@posthog/types' import { buildOtlpLogRecord, buildOtlpLogsPayload, buildResourceAttributes } from './logs-utils' import { Logger, PostHogPersistedProperty } from '../types' -import { isArray, raceWithTimeout, safeSetTimeout } from '../utils' +import { isArray, raceWithTimeout } from '../utils' +import { FlushTimer } from '../utils/flush-timer' import { RetryAfterWindow } from '../utils/retry-after' import type { BufferedLogEntry, CaptureLogOptions, LogSdkContext, LogsHost, ResolvedPostHogLogsConfig } from './types' @@ -18,7 +19,7 @@ export class PostHogLogs { // one record after each successful send so a one-off oversized payload // (e.g. a giant stack trace) doesn't permanently degrade throughput. private _maxBatchRecordsPerPost: number - private _flushTimer?: ReturnType + private readonly _flushTimer = new FlushTimer(() => this._flushInBackground()) // Serializes concurrent flushes — the second caller awaits the first rather // than racing it and double-sending the same head-of-queue records. private _flushPromise: Promise | null = null @@ -30,7 +31,6 @@ export class PostHogLogs { private _queueGeneration = 0 // Every path that can start a send checks this, not just the retry timer. private _retryAfter = new RetryAfterWindow() - private _flushTimerFiresAt = 0 // Consecutive failed flushes; drives exponential backoff on the retry timer. // A successful flush resets it to 0. private _consecutiveFlushFailures = 0 @@ -87,7 +87,7 @@ export class PostHogLogs { * and clears it separately (the browser empties its in-memory store). */ reset(): void { - this._clearFlushTimer() + this._flushTimer.clear() // `_flushPromise` is deliberately left alone: clearing it would let a second flush // run alongside the in-flight one. Retiring that flush is `clearQueue`'s job, // because the queue it holds belongs to the host, not to this state. @@ -264,7 +264,7 @@ export class PostHogLogs { } private async _flushInner(): Promise { - this._clearFlushTimer() + this._flushTimer.clear() let queue = this._instance.getPersistedProperty(PostHogPersistedProperty.LogsQueue) ?? [] if (queue.length === 0) { @@ -320,9 +320,8 @@ export class PostHogLogs { // Outright, not through the ratchet: a timer a mid-flight capture armed // is measured against a window this outcome may just have closed. - if (this._flushTimer) { - this._clearFlushTimer() - this._setFlushTimer(Math.max(this._flushIntervalMs, this._retryAfter.remainingMs())) + if (this._flushTimer.pending) { + this._flushTimer.arm(Math.max(this._flushIntervalMs, this._retryAfter.remainingMs())) } if (outcome.kind === 'retry-later') { @@ -408,32 +407,16 @@ export class PostHogLogs { // Arms the flush timer if none is pending. One-shot: the callback clears the // handle so the next enqueue (or a flush that left records) schedules again. private _armFlushTimer(): void { - if (this._flushTimer) { + if (this._flushTimer.pending) { return } // Floored by any open window: `flush()` leaves no timer behind, so a // capture after one arrives here. - this._setFlushTimer(Math.max(this._flushIntervalMs, this._retryAfter.remainingMs())) + this._flushTimer.arm(Math.max(this._flushIntervalMs, this._retryAfter.remainingMs())) } // Both floors, so a timer already armed at the plain interval gives way to a // longer one. - private _armFlushTimerNoEarlierThan(delayMs: number): void { - if (this._flushTimer && Date.now() + delayMs <= this._flushTimerFiresAt) { - return - } - this._clearFlushTimer() - this._setFlushTimer(delayMs) - } - - private _setFlushTimer(delayMs: number): void { - this._flushTimerFiresAt = Date.now() + delayMs - this._flushTimer = safeSetTimeout(() => { - this._flushTimer = undefined - this._flushInBackground() - }, delayMs) - } - // Retry delay after a flush that left records: the first retry is at the base // interval, then exponential backoff (capped) so a sustained outage isn't // retried every interval. @@ -460,7 +443,7 @@ export class PostHogLogs { * fetchRetryDelay)`, which can exceed the caller's shutdown SLA. */ async shutdown(timeoutMs?: number): Promise { - this._clearFlushTimer() + this._flushTimer.clear() const flushPromise = this.flush().catch(() => { // Best-effort: a logs-flush failure during shutdown is not actionable // and must not prevent the rest of shutdown from running. Errors are @@ -510,15 +493,8 @@ export class PostHogLogs { // sit undelivered on a quiet page; re-arm so the timer retries them, backing // off on consecutive failures. if (!this._instance.isDisabled && this._hasQueuedRecords()) { - this._armFlushTimerNoEarlierThan(this._nextFlushDelay()) + this._flushTimer.armNoEarlierThan(this._nextFlushDelay()) } }) } - - private _clearFlushTimer(): void { - if (this._flushTimer) { - clearTimeout(this._flushTimer) - this._flushTimer = undefined - } - } } diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index 0d8b881ce9..c9a8471e29 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -9,7 +9,8 @@ import type { OtlpNumberDataPoint, } from '@posthog/types' import type { Logger } from '../types' -import { isArray, safeSetTimeout } from '../utils' +import { isArray } from '../utils' +import { FlushTimer } from '../utils/flush-timer' import { RetryAfterWindow } from '../utils/retry-after' import { toOtlpKeyValueList } from '../utils/otlp-any-value' import { @@ -63,7 +64,11 @@ interface SeriesState { */ export class PostHogMetrics { private _series = new Map() - private _flushTimer?: ReturnType + private readonly _flushTimer = new FlushTimer(() => + this.flush().catch((e) => { + this._logger.error('Metrics flush failed:', e) + }) + ) // Serializes flushes — a manual flush() during an in-flight timer flush // queues behind it instead of racing it for the same window. private _flushPromise: Promise | null = null @@ -75,7 +80,6 @@ export class PostHogMetrics { private _typeByName = new Map() private _typeCollisionWarned = new Set() private _retryAfter = new RetryAfterWindow() - private _flushTimerFiresAt = 0 // Bumped by reset(). A flush that was in flight when reset() ran (e.g. it // lost a shutdown race) sees a stale generation when its send settles and // discards its window instead of merging it back and re-arming the timer. @@ -140,7 +144,7 @@ export class PostHogMetrics { reset(): void { this._generation++ this._retryAfter.reset() - this._clearFlushTimer() + this._flushTimer.clear() this._series = new Map() this._flushPromise = null this._seriesCapWarned = false @@ -291,49 +295,24 @@ export class PostHogMetrics { // Every capture calls this, so a pending timer is left alone — re-arming on // each one would push the flush out for as long as metrics keep arriving. private _armFlushTimer(): void { - if (this._flushTimer) { + if (this._flushTimer.pending) { return } - this._setFlushTimer(this._nextFlushDelay()) + this._flushTimer.arm(this._nextFlushDelay()) } // A floor, so a timer already armed at the flush interval gives way to a // longer one. - private _armFlushTimerNoEarlierThan(delayMs: number): void { - if (this._flushTimer && Date.now() + delayMs <= this._flushTimerFiresAt) { - return - } - this._clearFlushTimer() - this._setFlushTimer(delayMs) - } - // A floor, not a replacement: the header never retries us sooner than the // flush interval would have. private _nextFlushDelay(): number { return Math.max(this._config.flushIntervalMs, this._retryAfter.remainingMs()) } - private _setFlushTimer(delayMs: number): void { - this._flushTimerFiresAt = Date.now() + delayMs - this._flushTimer = safeSetTimeout(() => { - this._flushTimer = undefined - this.flush().catch((e) => { - this._logger.error('Metrics flush failed:', e) - }) - }, delayMs) - } - - private _clearFlushTimer(): void { - if (this._flushTimer) { - clearTimeout(this._flushTimer) - this._flushTimer = undefined - } - } - private async _doFlush(): Promise { // A flush retires the pending timer, so a delay armed for a window this // flush may close cannot outlive it. - this._clearFlushTimer() + this._flushTimer.clear() if (this._series.size === 0) { return } @@ -356,9 +335,8 @@ export class PostHogMetrics { this._retryAfter.record(outcome) // Outright, not through the ratchet: a timer a mid-flight capture armed is // measured against a window this outcome may just have closed. - if (this._flushTimer) { - this._clearFlushTimer() - this._setFlushTimer(this._nextFlushDelay()) + if (this._flushTimer.pending) { + this._flushTimer.arm(this._nextFlushDelay()) } switch (outcome.kind) { case 'ok': @@ -368,7 +346,7 @@ export class PostHogMetrics { // the next flush instead of being lost — and re-arm the timer, since // with no new captures nothing else would schedule that flush. this._mergeWindowBack(window) - this._armFlushTimerNoEarlierThan(this._nextFlushDelay()) + this._flushTimer.armNoEarlierThan(this._nextFlushDelay()) return case 'too-large': this._logger.warn('Metrics batch exceeded the server size limit and was dropped') diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 6c626f4623..fa30186284 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -23,7 +23,8 @@ import { parseTraceparent, sanitizeTracestate } from './traceparent' import { clampEndTime, resolveStartTime, resolveSuppliedTime, sanitizeName, toEpochMs } from './sanitize' import { assignUserAttributes } from '../utils/json-utils' import { buildOtlpSpan, buildOtlpTracesPayload, buildTracesResourceAttributes } from './otlp' -import { isPromise, safeSetTimeout } from '../utils' +import { isPromise } from '../utils' +import { FlushTimer } from '../utils/flush-timer' import { RetryAfterWindow } from '../utils/retry-after' // Retriable failures on the same head batch before it is dropped, so a stuck @@ -149,7 +150,7 @@ interface ParentContext { */ export class PostHogTraces { private _queue: SpanRecord[] = [] - private _flushTimer?: ReturnType + private readonly _flushTimer = new FlushTimer(() => this._flushInBackground()) // Serializes flushes: a second caller joins the first instead of double-sending the head. private _flushPromise: Promise | null = null // A trigger no-ops while a background drain is already pending. @@ -161,7 +162,6 @@ export class PostHogTraces { private _dropReasons = new Set() private _consecutiveFlushFailures = 0 private _retryAfter = new RetryAfterWindow() - private _flushTimerFiresAt = 0 // Separate from the backoff counter: this one belongs to whatever batch is at // the head, and resets whenever that batch is removed or shrunk. private _headBatchFailures = 0 @@ -339,7 +339,7 @@ export class PostHogTraces { } private _startFlush(): Promise { - this._clearFlushTimer() + this._flushTimer.clear() // Deferred by a microtask so the slot below is installed before the pass // reads anything: `_flushInner` runs synchronously as far as its first // await, and a resource-attribute getter or `toJSON` that ends a span in @@ -365,7 +365,7 @@ export class PostHogTraces { /** Clears the queue and timer. Used on shutdown and between tests. */ reset(): void { - this._clearFlushTimer() + this._flushTimer.clear() if (this._queue.length) { // Critical, and said here rather than counted: this is the last chance to // say anything about these spans, the drop warning is gated behind `debug` @@ -956,10 +956,10 @@ export class PostHogTraces { // Every span end can reach this, so a pending timer is left alone rather than // pushing the flush out. private _armFlushTimerIfQueued(): void { - if (this._flushTimer || !this._queue.length) { + if (this._flushTimer.pending || !this._queue.length) { return } - this._setFlushTimer(this._nextFlushDelay()) + this._flushTimer.arm(this._nextFlushDelay()) } // Both floors, so a timer a span end armed at the plain interval gives way to @@ -968,20 +968,7 @@ export class PostHogTraces { if (!this._queue.length) { return } - const delayMs = this._nextFlushDelay() - if (this._flushTimer && Date.now() + delayMs <= this._flushTimerFiresAt) { - return - } - this._clearFlushTimer() - this._setFlushTimer(delayMs) - } - - private _setFlushTimer(delayMs: number): void { - this._flushTimerFiresAt = Date.now() + delayMs - this._flushTimer = safeSetTimeout(() => { - this._flushTimer = undefined - this._flushInBackground() - }, delayMs) + this._flushTimer.armNoEarlierThan(this._nextFlushDelay()) } // Retry delay: base interval, doubling, capped at 30s — never below an interval @@ -994,11 +981,4 @@ export class PostHogTraces { // own backoff would have. return Math.max(capped, this._retryAfter.remainingMs()) } - - private _clearFlushTimer(): void { - if (this._flushTimer) { - clearTimeout(this._flushTimer) - this._flushTimer = undefined - } - } } diff --git a/packages/core/src/utils/flush-timer.ts b/packages/core/src/utils/flush-timer.ts new file mode 100644 index 0000000000..a50e8da648 --- /dev/null +++ b/packages/core/src/utils/flush-timer.ts @@ -0,0 +1,53 @@ +import { safeSetTimeout } from './index' + +/** + * The pending flush timer of an export queue, and when it is due. + * + * The queues arm this from two kinds of place: one that must not push a pending + * flush further out (every capture reaches it), and one that must not pull a + * flush back in front of a wait the endpoint asked for. Holding the deadline + * next to the handle is what lets the second kind compare against the first. + * + * Deliberately outside the `utils` barrel, like `RetryAfterWindow`: that barrel + * is re-exported wholesale from the package entry point, and this is internal. + */ +export class FlushTimer { + private _timer?: ReturnType + private _firesAt = 0 + + /** @param _onFire runs when the timer elapses, after the handle is released. */ + constructor(private readonly _onFire: () => void) {} + + /** Whether a flush is already scheduled. */ + get pending(): boolean { + return !!this._timer + } + + /** Schedules a flush in `delayMs`, replacing any timer already pending. */ + arm(delayMs: number): void { + this.clear() + this._firesAt = Date.now() + delayMs + this._timer = safeSetTimeout(() => { + this._timer = undefined + this._onFire() + }, delayMs) + } + + /** + * Arms only if it moves the flush later, so a timer armed for a longer wait + * survives a caller asking for a shorter one. + */ + armNoEarlierThan(delayMs: number): void { + if (this._timer && Date.now() + delayMs <= this._firesAt) { + return + } + this.arm(delayMs) + } + + clear(): void { + if (this._timer) { + clearTimeout(this._timer) + this._timer = undefined + } + } +} From dd5e9be7b72cec4359eac5bc5b5c3f7c485a4b07 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 15:10:57 -0400 Subject: [PATCH 16/33] chore(browser): regenerate mangled property names after the FlushTimer extraction Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE1TyDWGyLwUXEX3ffy83K --- packages/browser/terser-mangled-names.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/browser/terser-mangled-names.json b/packages/browser/terser-mangled-names.json index 4f0233b6af..fe12dcae46 100644 --- a/packages/browser/terser-mangled-names.json +++ b/packages/browser/terser-mangled-names.json @@ -25,7 +25,6 @@ "_applyTransforms", "_areWeOnline", "_armFlushTimer", - "_armFlushTimerNoEarlierThan", "_asRequiredConfig", "_autoSubmitPrefilledResponses", "_automaticDisplayDispose", @@ -111,7 +110,6 @@ "_clearDebouncer", "_clearFlushBufferTimer", "_clearFlushTimeout", - "_clearFlushTimer", "_clearSessionRegisteredProps", "_clearSessionState", "_clearSurveyTimeout", @@ -230,6 +228,7 @@ "_finishQueuedCompressionEvent", "_finishSetup", "_fireFeatureFlagsCallbacks", + "_firesAt", "_flagListenerCleanup", "_flagToExperiments", "_flagsLoadedFromRemote", @@ -246,7 +245,6 @@ "_flushTimeout", "_flushTimeoutMs", "_flushTimer", - "_flushTimerFiresAt", "_flushToCapture", "_flushViaTransport", "_flushedSizeTracker", @@ -537,6 +535,7 @@ "_onClick", "_onClickHandler", "_onDeadClick", + "_onFire", "_onFocusChange", "_onIdentityChanged", "_onIdentityCleared", @@ -772,7 +771,6 @@ "_setActivatedSession", "_setCrossTabFeatureFlagChangesPending", "_setFlushTimeout", - "_setFlushTimer", "_setPersonPropertiesForFlags", "_setProp", "_setProperty", @@ -860,6 +858,7 @@ "_teardown", "_throttledMutationsDropped", "_tickets", + "_timer", "_timestamp", "_totalBytes", "_touchStart", From 659f4e995b82ed00d8c3f847e6885989421759c5 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 15:12:53 -0400 Subject: [PATCH 17/33] fix(traces): reset the head batch budget when consent withdrawal clears the queue The discard emptied the queue but left the head batch's retry count and charge deadline behind, so a queue rebuilt after consent returned inherited a spent budget and an uncharged first refusal. Also corrects two comments the FlushTimer extraction left stacked, and one that claimed the in-window guard prevents the next send rather than deferring it a pass. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE1TyDWGyLwUXEX3ffy83K --- packages/core/src/logs/index.ts | 2 -- packages/core/src/metrics/index.ts | 2 -- packages/core/src/traces/index.spec.ts | 20 ++++++++++++++++++++ packages/core/src/traces/index.ts | 10 ++++++++-- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/packages/core/src/logs/index.ts b/packages/core/src/logs/index.ts index 96ad4f66d3..cc40f52d4b 100644 --- a/packages/core/src/logs/index.ts +++ b/packages/core/src/logs/index.ts @@ -415,8 +415,6 @@ export class PostHogLogs { this._flushTimer.arm(Math.max(this._flushIntervalMs, this._retryAfter.remainingMs())) } - // Both floors, so a timer already armed at the plain interval gives way to a - // longer one. // Retry delay after a flush that left records: the first retry is at the base // interval, then exponential backoff (capped) so a sustained outage isn't // retried every interval. diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index c9a8471e29..bdd690d009 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -301,8 +301,6 @@ export class PostHogMetrics { this._flushTimer.arm(this._nextFlushDelay()) } - // A floor, so a timer already armed at the flush interval gives way to a - // longer one. // A floor, not a replacement: the header never retries us sooner than the // flush interval would have. private _nextFlushDelay(): number { diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index e6bb45a871..e56bdd6c61 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -2748,6 +2748,26 @@ describe('PostHogTraces', () => { expect((traces as any)._headBatchFailures).toBe(2) }) + it('gives a fresh batch a fresh budget after an opt-out cleared the queue', async () => { + // The head batch leaves with the queue, so its budget must leave too: + // otherwise the next batch inherits a spent retry count and a charge + // deadline still in the future, and its first refusal goes uncharged. + mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('503') }) + const traces = createTraces({ flushIntervalMs: 10_000, maxExportBatchSize: 1 }) + traces.startSpan('before').end() + await traces.flush() + expect((traces as any)._headBatchFailures).toBe(1) + + mockInstance.optedOut = true + await traces.flush() + + // Asserted here rather than through a later refusal: the charge deadline + // the first failure installed is still in the future either way, so the + // next refusal is uncharged and the count reads 1 with or without the reset. + expect((traces as any)._headBatchFailures).toBe(0) + expect((traces as any)._headBatchChargeableAt).toBe(0) + }) + it('gives a fresh batch a fresh window after the head is retired', async () => { // The deadline belongs to the batch, not the queue: a new head must be // chargeable straight away or it inherits the last one's served wait. diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index fa30186284..4b6cc38a70 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -776,6 +776,9 @@ export class PostHogTraces { } const discarded = this._queue.length this._queue = [] + // The head batch left with the queue, so its budget goes too — anything + // queued after consent returns is a different batch. + this._resetHeadBatchBudget() this._recordDrop(discarded, 'the user has opted out') this._warnAboutDrops() return discarded @@ -905,8 +908,11 @@ export class PostHogTraces { this._resetHeadBatchBudget() this._recordDrop(size, `the ingestion endpoint failed ${MAX_RETRIES_PER_BATCH} times in a row`) if (this._retryAfter.isOpen()) { - // Retiring the batch cleared the failure counters, so carrying on - // would send the next one straight away, inside the endpoint's wait. + // Retiring the batch cleared the failure counters, so this drain + // would carry straight on to the next batch inside the endpoint's + // wait. Ending the pass here costs one send rather than the rest of + // the queue: `flush()` sees a non-zero count and loops, so the next + // batch still goes out, one pass later. return removed } continue From d06a6b2ececce581633893471e1843bf96c7dabb Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 17:01:20 -0400 Subject: [PATCH 18/33] docs: one-line the OTLP changesets and split the browser-visible half The Retry-After entry carried a paragraph telling posthog-js readers it did not apply to them; that half is now its own changeset scoped to the SDKs that see it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XBUa53f3QqECUyxnkSiWGA --- .changeset/logs-backoff-survives-captures.md | 7 +++++++ .changeset/otlp-honor-retry-after.md | 7 +------ .changeset/otlp-skip-oversized-bodies.md | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 .changeset/logs-backoff-survives-captures.md diff --git a/.changeset/logs-backoff-survives-captures.md b/.changeset/logs-backoff-survives-captures.md new file mode 100644 index 0000000000..7f68236c5b --- /dev/null +++ b/.changeset/logs-backoff-survives-captures.md @@ -0,0 +1,7 @@ +--- +'posthog-js': patch +'posthog-react-native': patch +'@posthog/core': patch +--- + +Keep backing off a failing log flush while new records arrive, instead of the next record resetting the retry to the flush interval. diff --git a/.changeset/otlp-honor-retry-after.md b/.changeset/otlp-honor-retry-after.md index 2630f9178d..af81a490f9 100644 --- a/.changeset/otlp-honor-retry-after.md +++ b/.changeset/otlp-honor-retry-after.md @@ -1,12 +1,7 @@ --- 'posthog-node': patch 'posthog-react-native': patch -'posthog-js': patch '@posthog/core': patch --- -Honor `Retry-After` when the ingestion endpoint refuses a logs, metrics or traces batch, rather than retrying on the SDK's own backoff alone. The header acts as a floor, so a shorter value never makes the SDK retry sooner than it would have, and a wait longer than five minutes is capped. - -The periodic flush waits the window out. An explicit `flush()`, and a flush a host runs to keep a request alive, still sends — a short-lived process is not left holding data it will never get another chance to deliver. Those attempts no longer count against a span batch's retry budget either, so waiting out a rate limit no longer drops spans. - -In `posthog-js` the ingestion endpoint sends no `Retry-After`, so none of the above applies — but the log flush now keeps backing off while records are still arriving, instead of a new record resetting the retry to the flush interval. A failing endpoint is retried a handful of times over an outage rather than once per interval. +Honor `Retry-After` when the ingestion endpoint refuses a logs, metrics or traces batch, instead of retrying on the SDK's own backoff alone. diff --git a/.changeset/otlp-skip-oversized-bodies.md b/.changeset/otlp-skip-oversized-bodies.md index 7a1f098f80..6e2801ec56 100644 --- a/.changeset/otlp-skip-oversized-bodies.md +++ b/.changeset/otlp-skip-oversized-bodies.md @@ -4,4 +4,4 @@ '@posthog/core': patch --- -Stop uploading logs, metrics and traces batches larger than the ingestion endpoint's request body limit — such a batch is split, and a single oversized record dropped, without spending a request on each attempt. +Stop uploading logs, metrics and traces batches over 10 MiB — the batch is split, and a single oversized record dropped, without spending a request on each attempt. From fa42a176cd163a31ab7cce267cd41579d8fad73f Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 17:01:25 -0400 Subject: [PATCH 19/33] fix(core): report an unserializable OTLP batch as too-large JSON.stringify sat outside the guard, so a payload past the runtime's max string length threw out of _sendOtlpBatch instead of returning an outcome. The traces queue only handles tagged outcomes, so it retried the batch forever without recording a failure. It now takes the halve-and-isolate path an oversized batch already takes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018AAPCyRnC5HbukEZV9hbai --- .../src/__tests__/posthog.otlp-too-large.spec.ts | 15 +++++++++++++++ packages/core/src/posthog-core-stateless.ts | 14 +++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts index 3b86821bdd..77a50bfa10 100644 --- a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts +++ b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts @@ -45,6 +45,21 @@ describe('OTLP bodies over the endpoint limit', () => { }) }) + it.each([ + ['traces', (payload: any) => posthog._sendTracesBatch(payload)], + ['logs', (payload: any) => posthog._sendLogsBatch(payload)], + ['metrics', (payload: any) => posthog._sendMetricsBatch(payload)], + ])('reports a %s batch it cannot serialize as too-large rather than throwing', async (_signal, send) => { + // A payload past the runtime's max string length throws out of + // `JSON.stringify`. Unguarded that escapes the tagged-outcome contract, and + // the caller retries a batch it can never send instead of halving it away. + const unserializable: any = spansOf(8) + unserializable.resourceSpans[0].scopeSpans[0].spans[0].self = unserializable + + await expect(send(unserializable)).resolves.toEqual({ kind: 'too-large' }) + expect(mocks.fetch).not.toHaveBeenCalled() + }) + it('sends a batch inside the limit', async () => { await expect(posthog._sendTracesBatch(spansOf(1024))).resolves.toEqual({ kind: 'ok' }) expect(mocks.fetch).toHaveBeenCalledTimes(1) diff --git a/packages/core/src/posthog-core-stateless.ts b/packages/core/src/posthog-core-stateless.ts index 497af6a68d..d5759ea030 100644 --- a/packages/core/src/posthog-core-stateless.ts +++ b/packages/core/src/posthog-core-stateless.ts @@ -1720,7 +1720,19 @@ export abstract class PostHogCoreStateless { return { kind: 'fatal', error: new Error('The client is disabled') } } - const serialized = JSON.stringify(payload) + // Serialised behind a guard: a payload too big to hold as one string throws + // `RangeError` here, which escapes the tagged-outcome contract and leaves the + // caller retrying a batch it can never send. Reported as too-large so it takes + // the same halve-and-isolate path as a batch that serialises but is oversized. + let serialized: string + try { + serialized = JSON.stringify(payload) + } catch (error) { + this.logMsgIfDebug(() => + console.warn(`[PostHog] Could not serialize a ${path} batch; reporting it as too large`, error) + ) + return { kind: 'too-large' } + } // Measured on the uncompressed payload: the endpoint decompresses the body // and applies its limit to what comes out, so one that gzips small is still From d1007d0dc917176c1811cf8963185dae0de72b22 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 17:05:55 -0400 Subject: [PATCH 20/33] docs(core): correct the unserializable-batch test comment, drop a duplicated aside The test builds a circular payload, not one past the max string length, and the barrel note was the same paragraph in two files. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XBUa53f3QqECUyxnkSiWGA --- .changeset/otlp-skip-oversized-bodies.md | 2 +- packages/core/src/__tests__/posthog.otlp-too-large.spec.ts | 7 ++++--- packages/core/src/utils/flush-timer.ts | 3 --- packages/core/src/utils/retry-after.ts | 3 --- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.changeset/otlp-skip-oversized-bodies.md b/.changeset/otlp-skip-oversized-bodies.md index 6e2801ec56..20ed94ae2a 100644 --- a/.changeset/otlp-skip-oversized-bodies.md +++ b/.changeset/otlp-skip-oversized-bodies.md @@ -4,4 +4,4 @@ '@posthog/core': patch --- -Stop uploading logs, metrics and traces batches over 10 MiB — the batch is split, and a single oversized record dropped, without spending a request on each attempt. +Stop uploading logs, metrics and traces batches over 10 MiB, or too large to serialize at all — the batch is split, and a single oversized record dropped, without spending a request on each attempt. diff --git a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts index 77a50bfa10..0ba60ae884 100644 --- a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts +++ b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts @@ -50,9 +50,10 @@ describe('OTLP bodies over the endpoint limit', () => { ['logs', (payload: any) => posthog._sendLogsBatch(payload)], ['metrics', (payload: any) => posthog._sendMetricsBatch(payload)], ])('reports a %s batch it cannot serialize as too-large rather than throwing', async (_signal, send) => { - // A payload past the runtime's max string length throws out of - // `JSON.stringify`. Unguarded that escapes the tagged-outcome contract, and - // the caller retries a batch it can never send instead of halving it away. + // A circular payload stands in for the reachable case: a batch past the + // runtime's max string length, which is too large to build in a test. Both + // leave `JSON.stringify` throwing, which unguarded escapes the tagged-outcome + // contract and leaves the caller retrying a batch it can never send. const unserializable: any = spansOf(8) unserializable.resourceSpans[0].scopeSpans[0].spans[0].self = unserializable diff --git a/packages/core/src/utils/flush-timer.ts b/packages/core/src/utils/flush-timer.ts index a50e8da648..a0ac9ae338 100644 --- a/packages/core/src/utils/flush-timer.ts +++ b/packages/core/src/utils/flush-timer.ts @@ -7,9 +7,6 @@ import { safeSetTimeout } from './index' * flush further out (every capture reaches it), and one that must not pull a * flush back in front of a wait the endpoint asked for. Holding the deadline * next to the handle is what lets the second kind compare against the first. - * - * Deliberately outside the `utils` barrel, like `RetryAfterWindow`: that barrel - * is re-exported wholesale from the package entry point, and this is internal. */ export class FlushTimer { private _timer?: ReturnType diff --git a/packages/core/src/utils/retry-after.ts b/packages/core/src/utils/retry-after.ts index 5f77439934..0ceaa8ef1d 100644 --- a/packages/core/src/utils/retry-after.ts +++ b/packages/core/src/utils/retry-after.ts @@ -3,9 +3,6 @@ * be long, but nothing upstream of the SDK bounds this header — it is as likely * to come from a proxy or CDN as from PostHog — and an unbounded value would * strand a queue for hours, so the wait is capped and the retry happens early. - * - * Deliberately outside the `utils` barrel: that barrel is re-exported wholesale - * from the package entry point, and this is internal policy, not public API. */ export const MAX_RETRY_AFTER_MS = 5 * 60_000 From 6df30f4cfdcab61f3dd4871c215f77f6d5d5dee8 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 17:12:49 -0400 Subject: [PATCH 21/33] docs: scope the OTLP entries to logs and metrics Traces ships new in the same release through #4579, so naming it here reads as a fix to behavior that never existed. Also drops the split-and-isolate claim from the oversized entry: metrics drops the window rather than halving. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XBUa53f3QqECUyxnkSiWGA --- .changeset/otlp-honor-retry-after.md | 2 +- .changeset/otlp-skip-oversized-bodies.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/otlp-honor-retry-after.md b/.changeset/otlp-honor-retry-after.md index af81a490f9..17f4f3e027 100644 --- a/.changeset/otlp-honor-retry-after.md +++ b/.changeset/otlp-honor-retry-after.md @@ -4,4 +4,4 @@ '@posthog/core': patch --- -Honor `Retry-After` when the ingestion endpoint refuses a logs, metrics or traces batch, instead of retrying on the SDK's own backoff alone. +Honor `Retry-After` when the ingestion endpoint refuses a logs or metrics batch, instead of retrying on the SDK's own schedule alone. diff --git a/.changeset/otlp-skip-oversized-bodies.md b/.changeset/otlp-skip-oversized-bodies.md index 20ed94ae2a..85a0bd8a27 100644 --- a/.changeset/otlp-skip-oversized-bodies.md +++ b/.changeset/otlp-skip-oversized-bodies.md @@ -4,4 +4,4 @@ '@posthog/core': patch --- -Stop uploading logs, metrics and traces batches over 10 MiB, or too large to serialize at all — the batch is split, and a single oversized record dropped, without spending a request on each attempt. +Stop sending logs and metrics batches over 10 MiB, or too large to serialize at all, instead of spending a request to discover the endpoint refuses them. From 030c294947679328cbc026a545188ac89efc3a78 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Fri, 4 Sep 2026 17:16:48 -0400 Subject: [PATCH 22/33] docs(core): correct where the ingestion body limit is applied capture-logs caps the raw request body (main.rs DefaultBodyLimit) and the gzip output (service.rs decompress_gzip_capped) at the same value, so the comment was wrong to say the wire bytes are not capped. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018AAPCyRnC5HbukEZV9hbai --- packages/core/src/posthog-core-stateless.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/core/src/posthog-core-stateless.ts b/packages/core/src/posthog-core-stateless.ts index d5759ea030..ab228bbd89 100644 --- a/packages/core/src/posthog-core-stateless.ts +++ b/packages/core/src/posthog-core-stateless.ts @@ -225,8 +225,9 @@ function isRetryableFlagsFetchError( * Ceiling on what the SDK will put on the wire: a body over it is reported as * too large without a request being made, and a batch of one that still exceeds * it is dropped. The ingestion service applies its own - * `MAX_REQUEST_BODY_SIZE_BYTES` to the body after it decompresses it, not to - * the compressed bytes on the wire, so this is measured the same way. + * `MAX_REQUEST_BODY_SIZE_BYTES` twice at the same value — once to the raw + * request body and again to the gzip output it decompresses — so measuring the + * uncompressed payload here matches the stricter of the two. * * Set to the largest limit any known deployment configures — 10 MiB, what the * ingestion service runs with — rather than the 2 MB the service falls back to From 7713aef7137ec111d799e3e2cfc2bac804ae8e31 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 09:53:06 -0400 Subject: [PATCH 23/33] docs(core): the ingestion limit sees the decompressed body Request decompression runs outside the body limit, so a gzip request is only measured once, after it is decoded. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjT26chjUzuPSyJZqh4AWQ --- packages/core/src/posthog-core-stateless.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/core/src/posthog-core-stateless.ts b/packages/core/src/posthog-core-stateless.ts index ab228bbd89..060c014eab 100644 --- a/packages/core/src/posthog-core-stateless.ts +++ b/packages/core/src/posthog-core-stateless.ts @@ -224,10 +224,9 @@ function isRetryableFlagsFetchError( /** * Ceiling on what the SDK will put on the wire: a body over it is reported as * too large without a request being made, and a batch of one that still exceeds - * it is dropped. The ingestion service applies its own - * `MAX_REQUEST_BODY_SIZE_BYTES` twice at the same value — once to the raw - * request body and again to the gzip output it decompresses — so measuring the - * uncompressed payload here matches the stricter of the two. + * it is dropped. The ingestion service decompresses a `Content-Encoding: gzip` + * request before it applies its own `MAX_REQUEST_BODY_SIZE_BYTES`, so the size + * that has to stay under the limit is the uncompressed one measured here. * * Set to the largest limit any known deployment configures — 10 MiB, what the * ingestion service runs with — rather than the 2 MB the service falls back to From 4db1e85a5a9627c8247c2faeb2bd0a984eee8130 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 10:39:32 -0400 Subject: [PATCH 24/33] chore: restore generated reference files to generator output The pre-commit JSON formatter added a trailing newline the api-extractor pipeline does not emit, so `Check public API references` saw a diff. --- packages/node/references/posthog-node-references-latest.json | 2 +- .../references/posthog-react-native-references-latest.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/node/references/posthog-node-references-latest.json b/packages/node/references/posthog-node-references-latest.json index 47b5b3a1ec..f694278b8c 100644 --- a/packages/node/references/posthog-node-references-latest.json +++ b/packages/node/references/posthog-node-references-latest.json @@ -4768,4 +4768,4 @@ "Traces", "Context" ] -} +} \ No newline at end of file diff --git a/packages/react-native/references/posthog-react-native-references-latest.json b/packages/react-native/references/posthog-react-native-references-latest.json index ac154cd424..aca00d6c1d 100644 --- a/packages/react-native/references/posthog-react-native-references-latest.json +++ b/packages/react-native/references/posthog-react-native-references-latest.json @@ -5211,4 +5211,4 @@ "Privacy", "LLM analytics" ] -} +} \ No newline at end of file From 6ee9467061fa7fab0d2d54f18661844b1145edc8 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 10:35:39 -0400 Subject: [PATCH 25/33] fix(traces): hold spans back during Retry-After and split locally The events flush timer no longer drains spans while the traces queue is honouring a Retry-After window. A batch the SDK measured as too large itself now splits that drain only, leaving the batch size the next drain starts from. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjT26chjUzuPSyJZqh4AWQ --- .../__tests__/posthog.otlp-too-large.spec.ts | 15 ++++-- packages/core/src/metrics/types.ts | 10 +++- packages/core/src/posthog-core-stateless.ts | 35 ++++++++++-- packages/core/src/traces/index.spec.ts | 27 ++++++++++ packages/core/src/traces/index.ts | 25 +++++++-- packages/core/src/traces/types.ts | 10 +++- packages/node/src/__tests__/traces.spec.ts | 53 +++++++++++++++++++ packages/node/src/client.ts | 16 ++++-- 8 files changed, 173 insertions(+), 18 deletions(-) diff --git a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts index 0ba60ae884..2126690ed3 100644 --- a/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts +++ b/packages/core/src/__tests__/posthog.otlp-too-large.spec.ts @@ -57,7 +57,7 @@ describe('OTLP bodies over the endpoint limit', () => { const unserializable: any = spansOf(8) unserializable.resourceSpans[0].scopeSpans[0].spans[0].self = unserializable - await expect(send(unserializable)).resolves.toEqual({ kind: 'too-large' }) + await expect(send(unserializable)).resolves.toEqual({ kind: 'too-large', measuredLocally: true }) expect(mocks.fetch).not.toHaveBeenCalled() }) @@ -71,7 +71,7 @@ describe('OTLP bodies over the endpoint limit', () => { ['logs', (payload: any) => posthog._sendLogsBatch(payload)], ['metrics', (payload: any) => posthog._sendMetricsBatch(payload)], ])('reports a %s batch over the limit as too-large without a request', async (_signal, send) => { - await expect(send(spansOf(OVER_LIMIT_BYTES))).resolves.toEqual({ kind: 'too-large' }) + await expect(send(spansOf(OVER_LIMIT_BYTES))).resolves.toEqual({ kind: 'too-large', measuredLocally: true }) expect(mocks.fetch).not.toHaveBeenCalled() }) @@ -81,7 +81,10 @@ describe('OTLP bodies over the endpoint limit', () => { vi.spyOn(posthog as any, 'compressPayload').mockResolvedValue(new Uint8Array(1024)) ;(posthog as any).disableCompression = false - await expect(posthog._sendTracesBatch(spansOf(OVER_LIMIT_BYTES))).resolves.toEqual({ kind: 'too-large' }) + await expect(posthog._sendTracesBatch(spansOf(OVER_LIMIT_BYTES))).resolves.toEqual({ + kind: 'too-large', + measuredLocally: true, + }) expect(mocks.fetch).not.toHaveBeenCalled() }) @@ -102,6 +105,7 @@ describe('OTLP bodies over the endpoint limit', () => { it('reports a batch one byte over the limit as too-large', async () => { await expect(posthog._sendTracesBatch(spansOf(LIMIT_BYTES - overheadBytes() + 1))).resolves.toEqual({ kind: 'too-large', + measuredLocally: true, }) expect(mocks.fetch).not.toHaveBeenCalled() }) @@ -113,7 +117,10 @@ describe('OTLP bodies over the endpoint limit', () => { const compressPayload = vi.spyOn(posthog as any, 'compressPayload') ;(posthog as any).disableCompression = false - await expect(posthog._sendTracesBatch(spansOf(OVER_LIMIT_BYTES))).resolves.toEqual({ kind: 'too-large' }) + await expect(posthog._sendTracesBatch(spansOf(OVER_LIMIT_BYTES))).resolves.toEqual({ + kind: 'too-large', + measuredLocally: true, + }) expect(compressPayload).not.toHaveBeenCalled() }) diff --git a/packages/core/src/metrics/types.ts b/packages/core/src/metrics/types.ts index fb2685f421..d5bbc28668 100644 --- a/packages/core/src/metrics/types.ts +++ b/packages/core/src/metrics/types.ts @@ -21,7 +21,15 @@ import type { BeforeSendMetricFn, MetricAttributeValue, OtlpMetricsPayload } fro export type SendMetricsBatchOutcome = | { kind: 'ok' } | { kind: 'retry-later'; error: unknown; retryAfterMs?: number } - | { kind: 'too-large' } + | { + kind: 'too-large' + /** + * True when the SDK measured the body itself rather than the endpoint + * refusing it, so the caller can split this drain without lowering the + * batch size it keeps between them. + */ + measuredLocally?: boolean + } | { kind: 'fatal'; error: unknown } /** diff --git a/packages/core/src/posthog-core-stateless.ts b/packages/core/src/posthog-core-stateless.ts index 060c014eab..f35fe5ca67 100644 --- a/packages/core/src/posthog-core-stateless.ts +++ b/packages/core/src/posthog-core-stateless.ts @@ -288,7 +288,15 @@ function isPostHogEventProperties(value: JsonType | undefined): value is PostHog */ export type SendLogsBatchOutcome = | { kind: 'ok' } - | { kind: 'too-large' } + | { + kind: 'too-large' + /** + * True when the SDK measured the body itself rather than the endpoint + * refusing it, so the caller can split this drain without lowering the + * batch size it keeps between them. + */ + measuredLocally?: boolean + } | { kind: 'retry-later'; error: unknown; retryAfterMs?: number } | { kind: 'fatal'; error: unknown } @@ -299,7 +307,15 @@ export type SendLogsBatchOutcome = */ type SendOtlpBatchOutcome = | { kind: 'ok' } - | { kind: 'too-large' } + | { + kind: 'too-large' + /** + * True when the SDK measured the body itself rather than the endpoint + * refusing it, so the caller can split this drain without lowering the + * batch size it keeps between them. + */ + measuredLocally?: boolean + } | { kind: 'retry-later'; error: unknown; retryAfterMs?: number } | { kind: 'fatal'; error: unknown } @@ -1382,11 +1398,20 @@ export abstract class PostHogCoreStateless { if (this.pendingFlushPromise) { return } - void this.flush().catch(async (err) => { + void this.flushAutomatic().catch(async (err) => { await logFlushError(err) }) } + /** + * The flush the SDK runs on its own, from the interval timer or the `flushAt` + * threshold. Separate from `flush()` so a host can hold back work that an + * endpoint has asked it to wait on, which an explicit flush overrides. + */ + protected flushAutomatic(): Promise { + return this.flush() + } + private async waitForPendingPromises( maxPromiseId: number, ignoredPromises: (Promise | null | undefined)[] = [] @@ -1731,7 +1756,7 @@ export abstract class PostHogCoreStateless { this.logMsgIfDebug(() => console.warn(`[PostHog] Could not serialize a ${path} batch; reporting it as too large`, error) ) - return { kind: 'too-large' } + return { kind: 'too-large', measuredLocally: true } } // Measured on the uncompressed payload: the endpoint decompresses the body @@ -1747,7 +1772,7 @@ export abstract class PostHogCoreStateless { `[PostHog] Not sending a ${path} batch of ${payloadBytes} bytes: the endpoint accepts at most ${OTLP_MAX_BODY_BYTES}` ) ) - return { kind: 'too-large' } + return { kind: 'too-large', measuredLocally: true } } const url = diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index c9aefb36af..55bd0cfce1 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -1844,6 +1844,33 @@ describe('PostHogTraces', () => { expect(batchSizes).toEqual([3, 1, 2]) }) + it('splits a locally measured batch without shrinking the next drain', async () => { + // The SDK measured this body itself, so it knows the oversized span is gone + // once the batch is isolated. The next drain starts at full size instead of + // ramping back one healthy batch at a time. + const instance = createMockInstance({ + _sendTracesBatch: vi + .fn() + .mockResolvedValueOnce({ kind: 'too-large', measuredLocally: true }) + .mockResolvedValue({ kind: 'ok' }), + }) + const traces = createTraces({ maxExportBatchSize: 512 }, instance) + for (let i = 0; i < 8; i++) { + traces.startSpan(`span-${i}`).end() + } + await traces.flush() + expect(sentPayloads(instance).map((p) => p.resourceSpans[0].scopeSpans[0].spans.length)).toEqual([8, 4, 4]) + + instance._sendTracesBatch.mockClear() + for (let i = 0; i < 8; i++) { + traces.startSpan(`later-${i}`).end() + } + await traces.flush() + + // One batch, not the 6-then-2 that a persistent shrink plus its +1 ramp gives. + expect(sentPayloads(instance).map((p) => p.resourceSpans[0].scopeSpans[0].spans.length)).toEqual([8]) + }) + it('ramps the batch size back up after a 413 shrink', async () => { // A one-off oversized payload shouldn't permanently halve throughput. const instance = createMockInstance({ diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 39d2928902..d82defaa7c 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -323,6 +323,14 @@ export class PostHogTraces { * wait costs a request rather than the spans. The periodic flush does wait it * out. */ + /** + * Whether the endpoint has asked this queue to wait. An automatic flush skips + * it; the traces flush timer already drains once the window closes. + */ + get throttled(): boolean { + return this._retryAfter.isOpen() + } + async flush(): Promise { for (;;) { if (!this._queue.length) { @@ -807,6 +815,10 @@ export class PostHogTraces { // Bounded by queue depth at flush start, so mid-drain arrivals ride the next flush. let remaining = this._queue.length let removed = 0 + // Splits this drain only. A batch the SDK measured as too large says nothing + // about the ones after the oversized span is gone, so the cap kept between + // drains stays where it is and the next one starts at full size. + let localCap = Number.POSITIVE_INFINITY const generation = this._generation try { @@ -823,7 +835,7 @@ export class PostHogTraces { this._headBatchFailures > 0 ? Math.min(this._maxExportBatchSize, this._headBatchSize) : this._maxExportBatchSize - const size = Math.max(1, Math.min(cap, remaining, this._queue.length)) + const size = Math.max(1, Math.min(cap, localCap, remaining, this._queue.length)) const batch = this._queue.slice(0, size) const spans = this._encodeBatch(batch) @@ -875,12 +887,17 @@ export class PostHogTraces { this._resetHeadBatchBudget() continue } - // Halve the batch the server rejected, not the configured maximum: when the + // Halve the batch that was refused, not the configured maximum: when the // queue is shallower than the maximum, shrinking it resends an identical body. - this._maxExportBatchSize = Math.max(1, Math.floor(size / 2)) + const halved = Math.max(1, Math.floor(size / 2)) + if (outcome.measuredLocally) { + localCap = halved + } else { + this._maxExportBatchSize = halved + } // A different batch from here on, so its budget starts fresh. this._resetHeadBatchBudget() - this._logger.debug(`Batch too large; retrying the same spans in batches of ${this._maxExportBatchSize}`) + this._logger.debug(`Batch too large; retrying the same spans in batches of ${halved}`) continue } diff --git a/packages/core/src/traces/types.ts b/packages/core/src/traces/types.ts index e992ae73c5..1f12eea7cf 100644 --- a/packages/core/src/traces/types.ts +++ b/packages/core/src/traces/types.ts @@ -30,7 +30,15 @@ import type { export type SendTracesBatchOutcome = | { kind: 'ok' } | { kind: 'retry-later'; error: unknown; retryAfterMs?: number } - | { kind: 'too-large' } + | { + kind: 'too-large' + /** + * True when the SDK measured the body itself rather than the endpoint + * refusing it, so the caller can split this drain without lowering the + * batch size it keeps between them. + */ + measuredLocally?: boolean + } | { kind: 'fatal'; error: unknown } /** The minimal host surface `PostHogTraces` depends on; `PostHogCoreStateless` satisfies it structurally. */ diff --git a/packages/node/src/__tests__/traces.spec.ts b/packages/node/src/__tests__/traces.spec.ts index 964bc075ab..1e1b5c80e2 100644 --- a/packages/node/src/__tests__/traces.spec.ts +++ b/packages/node/src/__tests__/traces.spec.ts @@ -537,6 +537,59 @@ describe('PostHog traces', () => { }) }) + describe('Retry-After', () => { + const refuseTraces = (retryAfterSeconds: number): void => { + mockedFetch.mockImplementation(async (url: any) => + String(url).includes('/i/v1/traces') + ? ({ + status: 429, + headers: { + get: (name: string) => (name.toLowerCase() === 'retry-after' ? `${retryAfterSeconds}` : null), + }, + text: async () => '', + json: async () => ({}), + } as any) + : ({ status: 200, headers: { get: () => null }, text: async () => '', json: async () => ({}) } as any) + ) + } + + it('does not drain spans on the events flush while the window is open', async () => { + const client = createClient({ flushAt: 1, flushInterval: 1000 }) + refuseTraces(300) + + client.startSpan('refused').end() + await flushTraces() + const afterRefusal = traceRequests().length + expect(afterRefusal).toBeGreaterThan(0) + + // Events keep arriving, so the events timer keeps firing. Spans must not + // ride along on it while the endpoint has asked the traces queue to wait. + client.startSpan('queued-during-window').end() + for (let i = 0; i < 5; i++) { + client.capture({ distinctId: 'user', event: 'tick' }) + await vi.advanceTimersByTimeAsync(1000) + await waitForPromises() + } + + expect(traceRequests()).toHaveLength(afterRefusal) + await client.shutdown() + }) + + it('still drains spans on an explicit flush', async () => { + const client = createClient({ flushAt: 100, flushInterval: 60_000 }) + refuseTraces(300) + + client.startSpan('refused').end() + await flushTraces() + const afterRefusal = traceRequests().length + + await client.flush() + + expect(traceRequests().length).toBeGreaterThan(afterRefusal) + await client.shutdown() + }) + }) + describe('beforeSpanSend', () => { it('scrubs attributes before they leave the process', async () => { const client = createClient({ diff --git a/packages/node/src/client.ts b/packages/node/src/client.ts index 5b83a9524f..df1c9da6c1 100644 --- a/packages/node/src/client.ts +++ b/packages/node/src/client.ts @@ -294,9 +294,9 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen * failed span export leaves the spans queued rather than rejecting, since * callers already treat `flush()` as safe to leave unwrapped. */ - private _flushEventsAndSpans(): Promise { + private _flushEventsAndSpans(skipThrottledSpans = false): Promise { const events = this.flushWithPendingPromises() - if (!this._traces) { + if (!this._traces || (skipThrottledSpans && this._traces.throttled)) { return events } // Settled, not `all`: `all` rejects the moment the event flush does, and a @@ -310,8 +310,18 @@ export abstract class PostHogBackendClient extends PostHogCoreStateless implemen }) } + protected override flushAutomatic(): Promise { + // The events timer must not drag spans through a `Retry-After` window the + // traces queue is honouring; its own timer picks them up when it closes. + return this._flushKeepingRuntimeAlive(true) + } + override async flush(): Promise { - const flushPromise = this._flushEventsAndSpans() + return this._flushKeepingRuntimeAlive(false) + } + + private _flushKeepingRuntimeAlive(skipThrottledSpans: boolean): Promise { + const flushPromise = this._flushEventsAndSpans(skipThrottledSpans) const waitUntil = this.options.waitUntil // Only register when no debounce promise is already keeping runtime alive if (waitUntil && !this._waitUntilCycle) { From d5c64e20b792be8f460f96f8f8ee612eb6b5edac Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 11:09:36 -0400 Subject: [PATCH 26/33] fix(otlp): extend an open Retry-After window and jitter our own backoff A refusal naming a longer wait now pushes the deadline out, bounded at five minutes from where the window was installed. Logs, metrics and traces jitter their own backoff, drawn once per failure, and metrics backs off exponentially rather than retrying on a fixed interval. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjT26chjUzuPSyJZqh4AWQ --- .../core/src/__tests__/retry-after.spec.ts | 26 ++++++++++++-- packages/core/src/logs/index.spec.ts | 20 ++++++----- packages/core/src/logs/index.ts | 18 +++++++--- packages/core/src/metrics/index.spec.ts | 12 ++++--- packages/core/src/metrics/index.ts | 25 +++++++++++-- packages/core/src/traces/index.spec.ts | 3 ++ packages/core/src/traces/index.ts | 25 +++++++++---- packages/core/src/utils/backoff.ts | 36 +++++++++++++++++++ packages/core/src/utils/retry-after.ts | 34 ++++++++++++------ 9 files changed, 160 insertions(+), 39 deletions(-) create mode 100644 packages/core/src/utils/backoff.ts diff --git a/packages/core/src/__tests__/retry-after.spec.ts b/packages/core/src/__tests__/retry-after.spec.ts index 81cb865160..8af1a12620 100644 --- a/packages/core/src/__tests__/retry-after.spec.ts +++ b/packages/core/src/__tests__/retry-after.spec.ts @@ -113,12 +113,34 @@ describe('RetryAfterWindow', () => { expect(window.remainingMs()).toBe(0) }) - it('does not extend an open window on a repeated refusal', () => { + it('extends an open window when a refusal names a longer wait', () => { const window = open(60_000) vi.setSystemTime(Date.now() + 20_000) window.record({ kind: 'retry-later', retryAfterMs: 60_000 }) - expect(window.remainingMs()).toBe(40_000) + expect(window.remainingMs()).toBe(60_000) + }) + + it('does not pull an open window in when a refusal names a shorter wait', () => { + // A shorter header must not cut a wait the endpoint has already asked for. + const window = open(120_000) + vi.setSystemTime(Date.now() + 10_000) + + window.record({ kind: 'retry-later', retryAfterMs: 5_000 }) + expect(window.remainingMs()).toBe(110_000) + }) + + it('caps an extension at five minutes from where the window was installed', () => { + // The SDK's own ceiling, not OTLP's: without it a host refused faster than + // the window is long would refresh the deadline indefinitely. + const window = open(240_000) + vi.setSystemTime(Date.now() + 200_000) + + window.record({ kind: 'retry-later', retryAfterMs: 240_000 }) + + // Asked for 440_000 from the install; capped at 300_000, of which 200_000 + // has been served. + expect(window.remainingMs()).toBe(100_000) }) it('keeps the window when a batch is refused for size', () => { diff --git a/packages/core/src/logs/index.spec.ts b/packages/core/src/logs/index.spec.ts index 4ef8d7e159..c7416b50d6 100644 --- a/packages/core/src/logs/index.spec.ts +++ b/packages/core/src/logs/index.spec.ts @@ -164,6 +164,9 @@ describe('PostHogLogs', () => { beforeEach(() => { mockInstance = createMockInstance() logger = createMockLogger() + // Retry delays carry jitter; pinned to its midpoint so every timing + // assertion here measures the backoff itself and cannot flake. + vi.spyOn(Math, 'random').mockReturnValue(0.5) }) it('constructs without throwing', () => { @@ -1520,10 +1523,10 @@ describe('PostHogLogs', () => { expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(3) }) - it('does not let a host out-pacing the window keep it open forever', async () => { - // RN takes flush() on every app-state transition. If each refusal slid the - // deadline forward, the window would never elapse and the gated paths — - // the size trigger and onReconnect — would stay suppressed indefinitely. + it('closes the window at the ceiling for a host out-pacing it', async () => { + // RN takes flush() on every app-state transition, and each refusal pushes + // the deadline out. The ceiling is what stops the gated paths — the size + // trigger and onReconnect — from being suppressed for good. mockInstance._sendLogsBatch = vi.fn(() => Promise.resolve({ kind: 'retry-later', error: new Error('429'), retryAfterMs: 30_000 }) ) @@ -1538,11 +1541,12 @@ describe('PostHogLogs', () => { await logs.flush().catch(() => {}) expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) - // Lifecycle flushes every 5s, well past the 30s window. Sampled rather - // than asserted through onReconnect: whether a given moment falls inside - // a window is timing-dependent, but it must fall outside one *sometimes*. + // Lifecycle flushes every 5s against a 30s window, for longer than the + // 5-minute ceiling. Sampled rather than asserted through onReconnect: + // whether a given moment falls inside a window is timing-dependent, but + // past the ceiling it must fall outside one. let sawWindowClosed = false - for (let i = 0; i < 12; i++) { + for (let i = 0; i < 70; i++) { await vi.advanceTimersByTimeAsync(5000) // Sampled before the flush: a flush that finds the window closed opens // a fresh one, so sampling after it would always look open. diff --git a/packages/core/src/logs/index.ts b/packages/core/src/logs/index.ts index cc40f52d4b..ca7b743628 100644 --- a/packages/core/src/logs/index.ts +++ b/packages/core/src/logs/index.ts @@ -4,10 +4,10 @@ import { Logger, PostHogPersistedProperty } from '../types' import { isArray, raceWithTimeout } from '../utils' import { FlushTimer } from '../utils/flush-timer' import { RetryAfterWindow } from '../utils/retry-after' +import { NO_JITTER, backoffDelayMs, drawJitter } from '../utils/backoff' import type { BufferedLogEntry, CaptureLogOptions, LogSdkContext, LogsHost, ResolvedPostHogLogsConfig } from './types' // Caps the retry backoff at 2^6 = 64× the flush interval. -const MAX_FLUSH_BACKOFF_EXPONENT = 6 export class PostHogLogs { private _maxBufferSize: number @@ -34,6 +34,7 @@ export class PostHogLogs { // Consecutive failed flushes; drives exponential backoff on the retry timer. // A successful flush resets it to 0. private _consecutiveFlushFailures = 0 + private _flushJitter = NO_JITTER // Fixed-window rate cap. Tumbling (not sliding) for cheap arithmetic on the // hot path. Window rolls the first time `captureLog` fires after the window @@ -95,6 +96,7 @@ export class PostHogLogs { this._intervalLogCount = 0 this._droppedWarned = false this._consecutiveFlushFailures = 0 + this._flushJitter = NO_JITTER this._retryAfter.reset() this._maxBatchRecordsPerPost = this._config.maxBatchRecordsPerPost } @@ -107,6 +109,7 @@ export class PostHogLogs { // network handover. onReconnect(): void { this._consecutiveFlushFailures = 0 + this._flushJitter = NO_JITTER if (this._retryAfter.isOpen()) { // The wait outlives the reconnect, but something still has to schedule // the retry: an explicit `flush()` leaves no timer behind. @@ -416,13 +419,16 @@ export class PostHogLogs { } // Retry delay after a flush that left records: the first retry is at the base - // interval, then exponential backoff (capped) so a sustained outage isn't - // retried every interval. + // interval, then exponential backoff so a sustained outage isn't retried every + // interval. Jitter is drawn once per failure and reused, so two delays taken + // for the same failure cannot disagree. private _nextFlushDelay(): number { - const exponent = Math.min(Math.max(0, this._consecutiveFlushFailures - 1), MAX_FLUSH_BACKOFF_EXPONENT) // A floor, not a replacement: the header never retries us sooner than our // own backoff would have. - return Math.max(this._flushIntervalMs * 2 ** exponent, this._retryAfter.remainingMs()) + return Math.max( + backoffDelayMs(this._flushIntervalMs, this._consecutiveFlushFailures, this._flushJitter), + this._retryAfter.remainingMs() + ) } private _hasQueuedRecords(): boolean { @@ -480,9 +486,11 @@ export class PostHogLogs { .then( () => { this._consecutiveFlushFailures = 0 + this._flushJitter = NO_JITTER }, (err) => { this._consecutiveFlushFailures++ + this._flushJitter = drawJitter() this._logger.error('PostHog logs flush failed:', err) } ) diff --git a/packages/core/src/metrics/index.spec.ts b/packages/core/src/metrics/index.spec.ts index 6577d72c07..feabff47b3 100644 --- a/packages/core/src/metrics/index.spec.ts +++ b/packages/core/src/metrics/index.spec.ts @@ -50,6 +50,9 @@ describe('PostHogMetrics', () => { vi.useFakeTimers() mockInstance = createMockInstance() logger = createMockLogger() + // Retry delays carry jitter; pinned to its midpoint so every timing + // assertion here measures the backoff itself and cannot flake. + vi.spyOn(Math, 'random').mockReturnValue(0.5) }) afterEach(() => { @@ -328,7 +331,7 @@ describe('PostHogMetrics', () => { expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(4) }) - it('does not let a host out-pacing the window keep it open forever', async () => { + it('closes the window at the ceiling for a host out-pacing it', async () => { // Each refusal sliding the deadline would keep `_nextFlushDelay` pinned at // the full window, so the flush cadence would never recover. const instance = createMockInstance({ @@ -344,7 +347,7 @@ describe('PostHogMetrics', () => { // Sampled: whether a given moment falls inside a window is timing // dependent, but it must fall outside one sometimes. let sawWindowClosed = false - for (let i = 0; i < 12; i++) { + for (let i = 0; i < 70; i++) { await vi.advanceTimersByTimeAsync(5000) // Sampled before the flush: a flush that finds the window closed opens // a fresh one, so sampling after it would always look open. @@ -494,8 +497,9 @@ describe('PostHogMetrics', () => { await vi.advanceTimersByTimeAsync(300_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(2) - // Back on the plain interval, not another 300s. - await vi.advanceTimersByTimeAsync(10_000) + // Off the 300s window and back on our own backoff: two consecutive + // failures, so one doubling of the interval rather than another 300s. + await vi.advanceTimersByTimeAsync(20_000) expect(instance._sendMetricsBatch).toHaveBeenCalledTimes(3) }) diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index bdd690d009..510c5612f2 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -12,6 +12,7 @@ import type { Logger } from '../types' import { isArray } from '../utils' import { FlushTimer } from '../utils/flush-timer' import { RetryAfterWindow } from '../utils/retry-after' +import { NO_JITTER, backoffDelayMs, drawJitter } from '../utils/backoff' import { toOtlpKeyValueList } from '../utils/otlp-any-value' import { DEFAULT_HISTOGRAM_BOUNDS, @@ -80,6 +81,8 @@ export class PostHogMetrics { private _typeByName = new Map() private _typeCollisionWarned = new Set() private _retryAfter = new RetryAfterWindow() + private _consecutiveFlushFailures = 0 + private _flushJitter = NO_JITTER // Bumped by reset(). A flush that was in flight when reset() ran (e.g. it // lost a shutdown race) sees a stale generation when its send settles and // discards its window instead of merging it back and re-arming the timer. @@ -144,6 +147,8 @@ export class PostHogMetrics { reset(): void { this._generation++ this._retryAfter.reset() + this._consecutiveFlushFailures = 0 + this._flushJitter = NO_JITTER this._flushTimer.clear() this._series = new Map() this._flushPromise = null @@ -301,10 +306,15 @@ export class PostHogMetrics { this._flushTimer.arm(this._nextFlushDelay()) } - // A floor, not a replacement: the header never retries us sooner than the - // flush interval would have. + // A floor, not a replacement: the header never retries us sooner than our own + // backoff would have. The backoff doubles per consecutive failure so a + // sustained outage is not retried every interval, and the jitter is drawn + // once per failure so a fleet refused together does not return together. private _nextFlushDelay(): number { - return Math.max(this._config.flushIntervalMs, this._retryAfter.remainingMs()) + return Math.max( + backoffDelayMs(this._config.flushIntervalMs, this._consecutiveFlushFailures, this._flushJitter), + this._retryAfter.remainingMs() + ) } private async _doFlush(): Promise { @@ -331,6 +341,15 @@ export class PostHogMetrics { return } this._retryAfter.record(outcome) + // Before any delay is taken from it, so the timer armed below is measured + // against this outcome rather than the one before it. + if (outcome.kind === 'retry-later') { + this._consecutiveFlushFailures++ + this._flushJitter = drawJitter() + } else { + this._consecutiveFlushFailures = 0 + this._flushJitter = NO_JITTER + } // Outright, not through the ratchet: a timer a mid-flight capture armed is // measured against a window this outcome may just have closed. if (this._flushTimer.pending) { diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index e1819fa3db..8fd8e53209 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -69,6 +69,9 @@ describe('PostHogTraces', () => { mockInstance = createMockInstance() logger = createMockLogger() context = {} + // Retry delays carry jitter; pinned to its midpoint so every timing + // assertion here measures the backoff itself and cannot flake. + vi.spyOn(Math, 'random').mockReturnValue(0.5) }) describe('startSpan', () => { diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index eb83504032..0aac1bf4b3 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -27,6 +27,7 @@ import { buildOtlpSpan, buildOtlpTracesPayload, buildTracesResourceAttributes } import { isPromise } from '../utils' import { FlushTimer } from '../utils/flush-timer' import { RetryAfterWindow } from '../utils/retry-after' +import { NO_JITTER, backoffDelayMs, drawJitter } from '../utils/backoff' // Retriable failures on the same head batch before it is dropped, so a stuck // batch cannot pin the queue while fresher spans are refused at the cap. The @@ -35,7 +36,6 @@ import { RetryAfterWindow } from '../utils/retry-after' // its own call rate what the timer path spends over minutes. const MAX_RETRIES_PER_BATCH = 8 -const MAX_FLUSH_BACKOFF_EXPONENT = 6 const MAX_FLUSH_BACKOFF_MS = 30_000 type SpanCallback = (span: Span) => T @@ -179,6 +179,7 @@ export class PostHogTraces { private _lastDropWarningAt = 0 private _dropReasons = new Set() private _consecutiveFlushFailures = 0 + private _flushJitter = NO_JITTER private _retryAfter = new RetryAfterWindow() // Separate from the backoff counter: this one belongs to whatever batch is at // the head, and resets whenever that batch is removed or shrunk. @@ -413,6 +414,7 @@ export class PostHogTraces { this._dropReasons.clear() this._lastDropWarningAt = 0 this._consecutiveFlushFailures = 0 + this._flushJitter = NO_JITTER this._retryAfter.reset() this._resetHeadBatchBudget() } @@ -882,6 +884,7 @@ export class PostHogTraces { if (outcome.kind === 'ok') { this._consecutiveFlushFailures = 0 + this._flushJitter = NO_JITTER this._resetHeadBatchBudget() this._queue.splice(0, size) remaining -= size @@ -901,6 +904,7 @@ export class PostHogTraces { removed += 1 this._recordDrop(1, 'it is too large for the ingestion endpoint') this._consecutiveFlushFailures = 0 + this._flushJitter = NO_JITTER this._resetHeadBatchBudget() continue } @@ -920,6 +924,7 @@ export class PostHogTraces { if (outcome.kind === 'retry-later') { this._consecutiveFlushFailures++ + this._flushJitter = drawJitter() // One charge per backoff window: a refusal arriving before the window // the last charge bought has elapsed is the same refusal seen again, // not new evidence against the batch. @@ -939,6 +944,7 @@ export class PostHogTraces { remaining -= size removed += size this._consecutiveFlushFailures = 0 + this._flushJitter = NO_JITTER this._resetHeadBatchBudget() this._recordDrop(size, `the ingestion endpoint failed ${MAX_RETRIES_PER_BATCH} times in a row`) if (this._retryAfter.isOpen()) { @@ -958,6 +964,7 @@ export class PostHogTraces { remaining -= size removed += size this._consecutiveFlushFailures = 0 + this._flushJitter = NO_JITTER this._resetHeadBatchBudget() this._recordDrop(size, 'the ingestion endpoint rejected the batch') } @@ -1012,13 +1019,19 @@ export class PostHogTraces { } // Retry delay: base interval, doubling, capped at 30s — never below an interval - // a host configured above the cap. + // a host configured above the cap. The jitter is drawn once per failure, so the + // timer and the retry-budget charge point are measured against the same delay. private _nextFlushDelay(): number { - const exponent = Math.min(Math.max(0, this._consecutiveFlushFailures - 1), MAX_FLUSH_BACKOFF_EXPONENT) - const delay = this._config.flushIntervalMs * 2 ** exponent - const capped = Math.min(delay, Math.max(MAX_FLUSH_BACKOFF_MS, this._config.flushIntervalMs)) // A floor, not a replacement: the header never retries us sooner than our // own backoff would have. - return Math.max(capped, this._retryAfter.remainingMs()) + return Math.max( + backoffDelayMs( + this._config.flushIntervalMs, + this._consecutiveFlushFailures, + this._flushJitter, + MAX_FLUSH_BACKOFF_MS + ), + this._retryAfter.remainingMs() + ) } } diff --git a/packages/core/src/utils/backoff.ts b/packages/core/src/utils/backoff.ts new file mode 100644 index 0000000000..029e10bd22 --- /dev/null +++ b/packages/core/src/utils/backoff.ts @@ -0,0 +1,36 @@ +/** Doublings the retry delay may grow by before it stops growing. */ +export const MAX_FLUSH_BACKOFF_EXPONENT = 6 + +/** + * How far a delay may be moved either side of its computed value. Clients that + * fail together otherwise retry together, and arrive at the endpoint as one + * burst each time it comes back — which is what OTel asks jitter to prevent. + * + * A quarter is enough to spread a fleet without a retry landing so early that + * it beats the interval the host configured, or so late that a recovered + * endpoint sits idle. + */ +const JITTER = 0.25 + +/** A multiplier in `[1 - JITTER, 1 + JITTER]`, drawn once per failure by the caller. */ +export function drawJitter(): number { + return 1 - JITTER + Math.random() * JITTER * 2 +} + +/** No jitter, for the delay a queue uses when nothing has failed. */ +export const NO_JITTER = 1 + +/** + * The delay before retrying an export, `baseMs` doubled once per failure past + * the first and capped at `maxMs`. + * + * `jitter` is applied to the SDK's own delay only. A `Retry-After` the endpoint + * sent is a floor underneath it, applied by the caller: spreading a fleet must + * never move a retry earlier than the endpoint asked for. + */ +export function backoffDelayMs(baseMs: number, failures: number, jitter: number, maxMs?: number): number { + const exponent = Math.min(Math.max(0, failures - 1), MAX_FLUSH_BACKOFF_EXPONENT) + const delay = baseMs * 2 ** exponent + const capped = maxMs === undefined ? delay : Math.min(delay, Math.max(maxMs, baseMs)) + return Math.round(capped * jitter) +} diff --git a/packages/core/src/utils/retry-after.ts b/packages/core/src/utils/retry-after.ts index 0ceaa8ef1d..3751af30f3 100644 --- a/packages/core/src/utils/retry-after.ts +++ b/packages/core/src/utils/retry-after.ts @@ -65,14 +65,17 @@ export class RetryAfterWindow { /** * Folds one export outcome into the window. * - * A window still open is left as it stands, even when the refusal names a - * longer wait than the one being served. Sliding the deadline on each refusal - * means the window never elapses for a host that flushes faster than the - * window is long: logs gates its size trigger and `onReconnect` on the - * window, and metrics re-arms its timer from it, so neither would recover - * while such a host kept sending. Whether it is still open is read here - * rather than passed in, so a send that outlives the wait it was made under - * installs the fresh deadline it came back with. + * A refusal naming a longer wait than the one being served pushes the + * deadline out, but never past `MAX_RETRY_AFTER_MS` from where the window was + * first installed. That ceiling is the SDK's own policy rather than OTLP's, + * which asks for the header to be honoured and names no cap: a host flushing + * faster than the window is long would otherwise refresh the deadline forever + * and never recover, since logs gates its size trigger and `onReconnect` on + * the window and metrics re-arms its timer from it. The cost is that a wait + * longer than the ceiling is served short. + * + * The deadline is never pulled in, so a shorter header cannot cut a wait the + * endpoint already asked for. */ record(outcome: RetryAfterOutcome): void { if (outcome.kind === 'too-large') { @@ -86,13 +89,22 @@ export class RetryAfterWindow { this.reset() return } - if (!outcome.retryAfterMs || this.isOpen()) { + if (!outcome.retryAfterMs) { // A refusal that names no wait — a network error, a timeout, a // header-less 503 — does not revoke one the endpoint already named. return } - this._installedAt = Date.now() - this._until = this._installedAt + Math.min(outcome.retryAfterMs, MAX_RETRY_AFTER_MS) + // Read before `_installedAt` is used: a spent window resets it, and a + // backward clock step is caught here rather than extending off a stale one. + const open = this.isOpen() + const now = Date.now() + const asked = Math.min(outcome.retryAfterMs, MAX_RETRY_AFTER_MS) + if (!open) { + this._installedAt = now + this._until = now + asked + return + } + this._until = Math.max(this._until, Math.min(now + asked, this._installedAt + MAX_RETRY_AFTER_MS)) } /** From b21fce35ccfbcff495f08f14255e708be0bb6fb1 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 11:09:52 -0400 Subject: [PATCH 27/33] docs(changeset): name the retry-window and backoff changes Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjT26chjUzuPSyJZqh4AWQ --- .changeset/otlp-honor-retry-after.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/otlp-honor-retry-after.md b/.changeset/otlp-honor-retry-after.md index 17f4f3e027..1ef2ab84a1 100644 --- a/.changeset/otlp-honor-retry-after.md +++ b/.changeset/otlp-honor-retry-after.md @@ -4,4 +4,4 @@ '@posthog/core': patch --- -Honor `Retry-After` when the ingestion endpoint refuses a logs or metrics batch, instead of retrying on the SDK's own schedule alone. +Honor `Retry-After` when the ingestion endpoint refuses a logs or metrics batch, instead of retrying on the SDK's own schedule alone. A refusal naming a longer wait extends the one being served, up to five minutes from when it started. Retry delays now carry jitter so clients refused together do not return together, and metrics backs off exponentially across consecutive failures rather than retrying on a fixed interval. From 3279b97cb568c0847ac3ed3de2bb4a111577d9d3 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 11:19:05 -0400 Subject: [PATCH 28/33] fix(otlp): cap the logs and metrics backoff at 30s, as the contracts state Both capabilities document the retry delay as exponential backoff capped at ~30s. Only traces applied it, so a 5s interval reached 320s after six doublings. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjT26chjUzuPSyJZqh4AWQ --- packages/core/src/logs/index.spec.ts | 23 +++++++++++++++++++++++ packages/core/src/logs/index.ts | 4 ++-- packages/core/src/metrics/index.ts | 9 +++++++-- packages/core/src/traces/index.ts | 4 +--- packages/core/src/utils/backoff.ts | 8 ++++++++ 5 files changed, 41 insertions(+), 7 deletions(-) diff --git a/packages/core/src/logs/index.spec.ts b/packages/core/src/logs/index.spec.ts index c7416b50d6..4ed6b5542b 100644 --- a/packages/core/src/logs/index.spec.ts +++ b/packages/core/src/logs/index.spec.ts @@ -1650,6 +1650,29 @@ describe('PostHogLogs', () => { expect(mockInstance._sendLogsBatch).toHaveBeenCalledTimes(1) }) + it('caps the retry delay at 30s however long the outage runs', async () => { + // The logs contract states the backoff as capped at ~30s. Without the cap + // a 5s interval reaches 320s after six doublings. + mockInstance._sendLogsBatch = vi.fn(() => Promise.resolve({ kind: 'retry-later', error: new Error('down') })) + const logs = new PostHogLogs( + mockInstance, + resolveForTest({ flushIntervalMs: 5000 }), + logger, + getContextFor(mockInstance), + immediateOnReady + ) + logs.captureLog({ body: 'retry-me' }) + + // Ten failures, well past the six doublings the exponent allows. + for (let i = 0; i < 10; i++) { + await vi.advanceTimersByTimeAsync(30_000) + } + // Capped, the delay settles at 30s and 300s of outage buys ten retries on + // top of the two the first doublings allow. Uncapped it reaches 320s and + // buys six attempts in total, so the difference is not a rounding one. + expect(mockInstance._sendLogsBatch.mock.calls.length).toBeGreaterThanOrEqual(11) + }) + it('backs off exponentially across consecutive failed flushes', async () => { // Every flush fails, so the record stays queued and the retry interval grows: // base (initial), base (1st retry), 2x, 4x, ... diff --git a/packages/core/src/logs/index.ts b/packages/core/src/logs/index.ts index ca7b743628..dd9f3e6cac 100644 --- a/packages/core/src/logs/index.ts +++ b/packages/core/src/logs/index.ts @@ -4,7 +4,7 @@ import { Logger, PostHogPersistedProperty } from '../types' import { isArray, raceWithTimeout } from '../utils' import { FlushTimer } from '../utils/flush-timer' import { RetryAfterWindow } from '../utils/retry-after' -import { NO_JITTER, backoffDelayMs, drawJitter } from '../utils/backoff' +import { MAX_FLUSH_BACKOFF_MS, NO_JITTER, backoffDelayMs, drawJitter } from '../utils/backoff' import type { BufferedLogEntry, CaptureLogOptions, LogSdkContext, LogsHost, ResolvedPostHogLogsConfig } from './types' // Caps the retry backoff at 2^6 = 64× the flush interval. @@ -426,7 +426,7 @@ export class PostHogLogs { // A floor, not a replacement: the header never retries us sooner than our // own backoff would have. return Math.max( - backoffDelayMs(this._flushIntervalMs, this._consecutiveFlushFailures, this._flushJitter), + backoffDelayMs(this._flushIntervalMs, this._consecutiveFlushFailures, this._flushJitter, MAX_FLUSH_BACKOFF_MS), this._retryAfter.remainingMs() ) } diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index 510c5612f2..8043ccf054 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -12,7 +12,7 @@ import type { Logger } from '../types' import { isArray } from '../utils' import { FlushTimer } from '../utils/flush-timer' import { RetryAfterWindow } from '../utils/retry-after' -import { NO_JITTER, backoffDelayMs, drawJitter } from '../utils/backoff' +import { MAX_FLUSH_BACKOFF_MS, NO_JITTER, backoffDelayMs, drawJitter } from '../utils/backoff' import { toOtlpKeyValueList } from '../utils/otlp-any-value' import { DEFAULT_HISTOGRAM_BOUNDS, @@ -312,7 +312,12 @@ export class PostHogMetrics { // once per failure so a fleet refused together does not return together. private _nextFlushDelay(): number { return Math.max( - backoffDelayMs(this._config.flushIntervalMs, this._consecutiveFlushFailures, this._flushJitter), + backoffDelayMs( + this._config.flushIntervalMs, + this._consecutiveFlushFailures, + this._flushJitter, + MAX_FLUSH_BACKOFF_MS + ), this._retryAfter.remainingMs() ) } diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index 0aac1bf4b3..9c8382af0c 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -27,7 +27,7 @@ import { buildOtlpSpan, buildOtlpTracesPayload, buildTracesResourceAttributes } import { isPromise } from '../utils' import { FlushTimer } from '../utils/flush-timer' import { RetryAfterWindow } from '../utils/retry-after' -import { NO_JITTER, backoffDelayMs, drawJitter } from '../utils/backoff' +import { MAX_FLUSH_BACKOFF_MS, NO_JITTER, backoffDelayMs, drawJitter } from '../utils/backoff' // Retriable failures on the same head batch before it is dropped, so a stuck // batch cannot pin the queue while fresher spans are refused at the cap. The @@ -36,8 +36,6 @@ import { NO_JITTER, backoffDelayMs, drawJitter } from '../utils/backoff' // its own call rate what the timer path spends over minutes. const MAX_RETRIES_PER_BATCH = 8 -const MAX_FLUSH_BACKOFF_MS = 30_000 - type SpanCallback = (span: Span) => T /** Monotonic where the platform has one, wall clock otherwise. Both are ms, and a platform never switches. */ diff --git a/packages/core/src/utils/backoff.ts b/packages/core/src/utils/backoff.ts index 029e10bd22..934291db81 100644 --- a/packages/core/src/utils/backoff.ts +++ b/packages/core/src/utils/backoff.ts @@ -1,6 +1,14 @@ /** Doublings the retry delay may grow by before it stops growing. */ export const MAX_FLUSH_BACKOFF_EXPONENT = 6 +/** + * Ceiling on the SDK's own retry delay, which the logs and traces contracts + * both state as "exponential backoff capped at ~30s". A host that configured a + * longer flush interval keeps it: the cap is there to stop the doubling running + * away, not to flush more often than asked. + */ +export const MAX_FLUSH_BACKOFF_MS = 30_000 + /** * How far a delay may be moved either side of its computed value. Clients that * fail together otherwise retry together, and arrive at the endpoint as one From 3a749d077d58e1971c71129b6540018bded3eb1f Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 11:26:14 -0400 Subject: [PATCH 29/33] chore(browser): record _flushJitter in the mangled property names Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjT26chjUzuPSyJZqh4AWQ --- packages/browser/terser-mangled-names.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/browser/terser-mangled-names.json b/packages/browser/terser-mangled-names.json index e354ee191f..b20405fad7 100644 --- a/packages/browser/terser-mangled-names.json +++ b/packages/browser/terser-mangled-names.json @@ -246,6 +246,7 @@ "_flushInner", "_flushInterval", "_flushIntervalMs", + "_flushJitter", "_flushPendingActivityTimestamp", "_flushPromise", "_flushTimeout", From 69d047588bed2e9bd426c59dd14edb1d271a5ecc Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 12:44:52 -0400 Subject: [PATCH 30/33] chore(references): regenerate for the measuredLocally too-large outcome Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjT26chjUzuPSyJZqh4AWQ --- .../node/references/posthog-node-references-latest.json | 8 ++++---- .../posthog-react-native-references-latest.json | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/node/references/posthog-node-references-latest.json b/packages/node/references/posthog-node-references-latest.json index f694278b8c..f45f81f272 100644 --- a/packages/node/references/posthog-node-references-latest.json +++ b/packages/node/references/posthog-node-references-latest.json @@ -4092,21 +4092,21 @@ "name": "SendLogsBatchOutcome", "properties": [], "path": "../core/src/posthog-core-stateless.ts", - "example": "{\n kind: 'ok';\n} | {\n kind: 'too-large';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'fatal';\n error: unknown;\n}" + "example": "{\n kind: 'ok';\n} | {\n kind: 'too-large';\n measuredLocally?: boolean;\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, { "id": "SendMetricsBatchOutcome", "name": "SendMetricsBatchOutcome", "properties": [], "path": "../core/src/metrics/types.ts", - "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" + "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'too-large';\n measuredLocally?: boolean;\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, { "id": "SendTracesBatchOutcome", "name": "SendTracesBatchOutcome", "properties": [], "path": "../core/src/traces/types.ts", - "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" + "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'too-large';\n measuredLocally?: boolean;\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, { "id": "SentryIntegrationOptions", @@ -4768,4 +4768,4 @@ "Traces", "Context" ] -} \ No newline at end of file +} diff --git a/packages/react-native/references/posthog-react-native-references-latest.json b/packages/react-native/references/posthog-react-native-references-latest.json index aca00d6c1d..06c403f3ff 100644 --- a/packages/react-native/references/posthog-react-native-references-latest.json +++ b/packages/react-native/references/posthog-react-native-references-latest.json @@ -4445,21 +4445,21 @@ "name": "SendLogsBatchOutcome", "properties": [], "path": "../core/src/posthog-core-stateless.ts", - "example": "{\n kind: 'ok';\n} | {\n kind: 'too-large';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'fatal';\n error: unknown;\n}" + "example": "{\n kind: 'ok';\n} | {\n kind: 'too-large';\n measuredLocally?: boolean;\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, { "id": "SendMetricsBatchOutcome", "name": "SendMetricsBatchOutcome", "properties": [], "path": "../core/src/metrics/types.ts", - "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" + "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'too-large';\n measuredLocally?: boolean;\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, { "id": "SendTracesBatchOutcome", "name": "SendTracesBatchOutcome", "properties": [], "path": "../core/src/traces/types.ts", - "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'too-large';\n} | {\n kind: 'fatal';\n error: unknown;\n}" + "example": "{\n kind: 'ok';\n} | {\n kind: 'retry-later';\n error: unknown;\n retryAfterMs?: number;\n} | {\n kind: 'too-large';\n measuredLocally?: boolean;\n} | {\n kind: 'fatal';\n error: unknown;\n}" }, { "id": "SeverityLevel", @@ -5211,4 +5211,4 @@ "Privacy", "LLM analytics" ] -} \ No newline at end of file +} From 8e95248e8372916ac2756d781771216f481d8b6f Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 13:41:47 -0400 Subject: [PATCH 31/33] docs(changeset): disclose the 30s cap on the logs retry delay The cap landed in 3279b97c with no changelog entry, and it changes retry cadence for posthog-js and posthog-react-native. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RWoFx4BctNmXnpnKS79Qgz --- .changeset/logs-backoff-cap.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/logs-backoff-cap.md diff --git a/.changeset/logs-backoff-cap.md b/.changeset/logs-backoff-cap.md new file mode 100644 index 0000000000..0f1248b8b8 --- /dev/null +++ b/.changeset/logs-backoff-cap.md @@ -0,0 +1,7 @@ +--- +'posthog-js': patch +'posthog-react-native': patch +'@posthog/core': patch +--- + +Cap the retry delay for log exports at 30 seconds, the ceiling the logs contract states. It previously doubled to 64 times the flush interval — 192s on web, 640s on React Native — so a log export now resumes within 30 seconds of a failing endpoint recovering, at the cost of more retry requests while that endpoint is down. From 4d02232f489a3ec594af42b84d1932196eb8e978 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 8 Sep 2026 13:50:39 -0400 Subject: [PATCH 32/33] chore: restore generated reference files to generator output The oxfmt-data pre-commit hook appends a trailing newline to any staged .json, which the api-extractor output does not carry, so committing these through the hook fails the Check public API references job. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RWoFx4BctNmXnpnKS79Qgz --- packages/node/references/posthog-node-references-latest.json | 2 +- .../references/posthog-react-native-references-latest.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/node/references/posthog-node-references-latest.json b/packages/node/references/posthog-node-references-latest.json index f45f81f272..7b452faf35 100644 --- a/packages/node/references/posthog-node-references-latest.json +++ b/packages/node/references/posthog-node-references-latest.json @@ -4768,4 +4768,4 @@ "Traces", "Context" ] -} +} \ No newline at end of file diff --git a/packages/react-native/references/posthog-react-native-references-latest.json b/packages/react-native/references/posthog-react-native-references-latest.json index 06c403f3ff..f3a8251800 100644 --- a/packages/react-native/references/posthog-react-native-references-latest.json +++ b/packages/react-native/references/posthog-react-native-references-latest.json @@ -5211,4 +5211,4 @@ "Privacy", "LLM analytics" ] -} +} \ No newline at end of file From c7e180293300cf6972b6f1de12606f058b73d93c Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Thu, 10 Sep 2026 10:51:04 -0400 Subject: [PATCH 33/33] fix(core): leave an exempt flush uncharged when a later refusal extends the window A send inside an open Retry-After window no longer spends the head batch's retry budget. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015x8Q3nnzBWKQau9tmrtDhj --- packages/core/src/traces/index.spec.ts | 34 +++++++++++++++++++++++--- packages/core/src/traces/index.ts | 6 +++-- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/packages/core/src/traces/index.spec.ts b/packages/core/src/traces/index.spec.ts index b0b63f1674..6d11c18c6b 100644 --- a/packages/core/src/traces/index.spec.ts +++ b/packages/core/src/traces/index.spec.ts @@ -2818,9 +2818,10 @@ describe('PostHogTraces', () => { }) it('does not let a host out-pacing the window stall the retry budget', async () => { - // flush() more often than the window is long. If each refusal slid the - // deadline forward, the window would never elapse, the head batch would - // never retire, and every span behind it would be dropped at the cap. + // flush() more often than the window is long. Each refusal slides the + // deadline, up to five minutes from where the window was installed; the + // in-window sends are uncharged, so each window costs one charge and eight + // of them retire the head batch rather than letting it hold the queue. mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('429'), @@ -2828,7 +2829,7 @@ describe('PostHogTraces', () => { }) const traces = createTraces({ flushIntervalMs: 10_000, maxQueueSize: 5, maxExportBatchSize: 2 }) - for (let i = 0; i < 60; i++) { + for (let i = 0; i < 500; i++) { traces.startSpan(`span-${i}`).end() await traces.flush() await vi.advanceTimersByTimeAsync(5_000) @@ -2870,6 +2871,31 @@ describe('PostHogTraces', () => { expect(logger.warn).not.toHaveBeenCalledWith(expect.stringContaining('Dropping')) }) + it('does not charge a caller-driven flush inside a window a later refusal extended', async () => { + // The charge point is set from the window as it stood at the first + // refusal. An in-window refusal then extends the window past it, and a + // flush between the two deadlines is still inside the endpoint's wait. + mockInstance._sendTracesBatch.mockResolvedValue({ + kind: 'retry-later', + error: new Error('429'), + retryAfterMs: 60_000, + }) + const traces = createTraces({ flushIntervalMs: 10_000 }) + traces.startSpan('first').end() + + await traces.flush() + expect((traces as any)._headBatchFailures).toBe(1) + + await vi.advanceTimersByTimeAsync(50_000) + await traces.flush() + await vi.advanceTimersByTimeAsync(10_000) + const sends = mockInstance._sendTracesBatch.mock.calls.length + await traces.flush() + + expect(mockInstance._sendTracesBatch.mock.calls.length).toBe(sends + 1) + expect((traces as any)._headBatchFailures).toBe(1) + }) + it('charges once per window however many attempts share it', async () => { mockInstance._sendTracesBatch.mockResolvedValue({ kind: 'retry-later', error: new Error('503') }) const traces = createTraces({ flushIntervalMs: 10_000 }) diff --git a/packages/core/src/traces/index.ts b/packages/core/src/traces/index.ts index f6a92693a9..6b2fac63ae 100644 --- a/packages/core/src/traces/index.ts +++ b/packages/core/src/traces/index.ts @@ -889,8 +889,10 @@ export class PostHogTraces { } // Read before the send, so the budget below charges this attempt against - // the window it was actually made under. - const chargeable = clockNow() >= this._headBatchChargeableAt + // the window it was actually made under. A send inside an open + // `Retry-After` window is caller-driven and exempt from the wait; a later + // refusal can extend that window past the charge point, so both are read. + const chargeable = clockNow() >= this._headBatchChargeableAt && !this._retryAfter.isOpen() const outcome = await this._instance._sendTracesBatch( buildOtlpTracesPayload(spans, resourceAttributes, scopeName, scopeVersion, this._logger)