From ec908caab147c04af83123f89831383a5aa8d433 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 05:25:52 +0000 Subject: [PATCH 01/17] fix(queue): keep buffered events through network outages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A transport failure (statusCode -1: lost connectivity, timeout, no route) was treated the same as a retriable server error, so a handful of failures backed off 1s/2s/3s would exceed maxRetries in ~6s and wipe the entire on-disk queue. Mobile networks flap constantly, so this silently discarded every buffered event for iOS customers. Transport failures now never drop the queue — events wait on disk and maxQueueSize bounds growth. A full drop is reserved for a responsive-but-unhealthy backend that keeps rejecting batches past the new maxRetryWindowSeconds window (default 24h). Every drop emits a $queue_records_dropped diagnostic event so the loss is measurable. maxRetries now governs only HTTP 413 batch halving. Generated-By: PostHog Desktop Task-Id: 4f8c390a-7251-4812-9ba9-57186baeb234 --- .changeset/quick-queues-keep.md | 5 ++ PostHog/PostHogConfig.swift | 19 ++++++-- PostHog/PostHogQueue.swift | 75 ++++++++++++++++++++++------- PostHog/PostHogSDK.swift | 10 ++++ PostHogTests/PostHogQueueTest.swift | 74 +++++++++++++++++++++++++++- 5 files changed, 162 insertions(+), 21 deletions(-) create mode 100644 .changeset/quick-queues-keep.md diff --git a/.changeset/quick-queues-keep.md b/.changeset/quick-queues-keep.md new file mode 100644 index 000000000..ff2d7a151 --- /dev/null +++ b/.changeset/quick-queues-keep.md @@ -0,0 +1,5 @@ +--- +"posthog-ios": patch +--- + +Stop dropping the whole on-disk event queue after a brief network outage. A transport failure (lost connectivity, timeout, no route) now keeps every buffered event and lets `maxQueueSize` bound the queue, instead of wiping it once a few retries exceeded `maxRetries`. A full queue drop is now reserved for a responsive-but-unhealthy backend that keeps rejecting batches past the new `maxRetryWindowSeconds` window (default 24 hours), and every drop emits a `$queue_records_dropped` diagnostic event so the loss is measurable. diff --git a/PostHog/PostHogConfig.swift b/PostHog/PostHogConfig.swift index c32745a8a..32553ed52 100644 --- a/PostHog/PostHogConfig.swift +++ b/PostHog/PostHogConfig.swift @@ -33,6 +33,7 @@ public typealias BeforeSendBlock = (PostHogEvent) -> PostHogEvent? static let maxBatchSize: Int = 50 static let flushIntervalSeconds: TimeInterval = 30 static let maxRetries: Int = 3 + static let maxRetryWindowSeconds: TimeInterval = 24 * 60 * 60 static let featureFlagRequestMaxRetries: Int = 1 } @@ -89,12 +90,24 @@ public typealias BeforeSendBlock = (PostHogEvent) -> PostHogEvent? /// Default: `30`. @objc public var flushIntervalSeconds: TimeInterval = Defaults.flushIntervalSeconds - /// Maximum number of consecutive flush attempts before the entire queue is - /// dropped to avoid infinite retries against a permanently-broken backend. - /// Increments on every retriable failure including HTTP 413 cap halving; + /// Maximum number of HTTP 413 batch-halving attempts before the oversized + /// batch is dropped. Increments on every 413 that halves the batch cap; /// resets on a successful 2xx response. Default 3. + /// + /// This no longer governs network or 5xx retries — those keep the queue + /// and are bounded by `maxRetryWindowSeconds` and `maxQueueSize` instead. @objc public var maxRetries: Int = Defaults.maxRetries + /// Sustained duration, in seconds, that flushes must keep failing against a + /// responsive-but-unhealthy backend (for example repeated HTTP 5xx) before + /// the on-disk queue is dropped to stop retrying forever. + /// + /// A network-level failure (lost connectivity, timeout, no route) never + /// triggers this drop: the queue is kept and bounded only by + /// `maxQueueSize`, so a brief connectivity blip cannot destroy buffered + /// events. The timer resets on any successful send. Default: 24 hours. + @objc public var maxRetryWindowSeconds: TimeInterval = Defaults.maxRetryWindowSeconds + /// Maximum number of retries for feature flag requests after transient network errors or retryable HTTP responses. /// Defaults to 1. Set to 0 to disable feature flag request retries. @objc public var featureFlagRequestMaxRetries: Int = Defaults.featureFlagRequestMaxRetries diff --git a/PostHog/PostHogQueue.swift b/PostHog/PostHogQueue.swift index edc82c71f..76e4bea21 100644 --- a/PostHog/PostHogQueue.swift +++ b/PostHog/PostHogQueue.swift @@ -65,6 +65,16 @@ class PostHogQueue { private var paused: Bool = false private var pausedUntil: Date? private var retryCount: Int = 0 + /// Wall clock of the first failure in the current unhealthy-backend streak, + /// or `nil` while healthy. Only server-side retriable failures (5xx, 429, + /// …) set it; transport failures never do. Cleared on any success. + private var failingSince: Date? + + /// Invoked from `dropAllQueuedRecords` with the number of records wiped and + /// the reason. The SDK wires this to emit a diagnostic event so a full + /// queue drop — the last remaining silent, total data-loss path — becomes + /// measurable. + var onRecordsDropped: ((Int, String) -> Void)? #if !os(watchOS) private let reachability: Reachability? private var reachableToken: RegistrationToken? @@ -150,22 +160,36 @@ class PostHogQueue { // -1 means its not anything related to the API but rather network or something else, so we try again let statusCode = result.statusCode ?? -1 - // Network error (-1) is universally retriable; everything else is - // up to the endpoint's retry policy. - let isRetriable = statusCode == -1 || endpoint.isRetriableStatusCode(statusCode) + // A transport failure (-1: lost connectivity, timeout, no route) means + // we could not reach the backend — not that it rejected our data. It is + // universally retriable and must never drop the queue. Everything else + // is up to the endpoint's retry policy. + let isTransportFailure = statusCode == -1 + let isRetriable = isTransportFailure || endpoint.isRetriableStatusCode(statusCode) if isRetriable { - let newCount = stateLock.withLock { () -> Int in + // `since` is the start of the current server-failure streak, or + // `nil` for a transport failure. Only server-side failures arm the + // clock, so an offline stretch never counts toward the drop window. + let (newCount, since): (Int, Date?) = stateLock.withLock { retryCount += 1 - return retryCount + if !isTransportFailure, failingSince == nil { + failingSince = now() + } + return (retryCount, isTransportFailure ? nil : failingSince) } - // `>` not `>=`: maxRetries is the count of retries allowed before - // dropping — default 3 retries attempts 1-3, drops on attempt 4. - if newCount > config.maxRetries { - dropAllQueuedRecords(reason: "max retries (\(config.maxRetries)) exceeded") + + // Drop the queue only when a responsive-but-unhealthy backend has + // kept rejecting our batches for a sustained window. Transport + // failures are excluded: the events wait on disk for connectivity + // to return and `maxQueueSize` bounds growth, so a flaky network + // can never wipe the buffer. + if let since, now().timeIntervalSince(since) >= config.maxRetryWindowSeconds { + dropAllQueuedRecords(reason: "backend failing for over \(config.maxRetryWindowSeconds)s") payload.completion(true) return } + let backoffDelay = min(TimeInterval(newCount) * retryDelay, maxRetryDelay) let delay = max(backoffDelay, result.retryAfter ?? 0) pauseFor(seconds: delay) @@ -207,29 +231,42 @@ class PostHogQueue { // Cap stays at 1 — the offender is gone but we keep being // cautious until a successful send. hedgeLog("Queue: dropping batch after HTTP 413 (cap == 1)") - stateLock.withLock { retryCount = 0 } + stateLock.withLock { + retryCount = 0 + failingSince = nil + } payload.completion(true) return } // 2xx success or non-retriable 4xx (auth, malformed, etc.): pop the // batch. Cap stays where it is — no ramp on success. - stateLock.withLock { retryCount = 0 } + stateLock.withLock { + retryCount = 0 + failingSince = nil + } payload.completion(true) } /// Drops every queued record from disk and resets the retry / pause state. - /// Called when `retryCount` exceeds `config.maxRetries` to avoid retrying - /// forever against a permanently-broken backend. Cap is left where it is - /// — new records starting against a known-bad backend benefit from the - /// conservative cap until proven otherwise. + /// Reserved for a backend that keeps rejecting our batches past + /// `maxRetryWindowSeconds`, or repeated HTTP 413 halving — never a network + /// blip. Reports the number of dropped records through `onRecordsDropped` + /// so the loss is measurable. Cap is left where it is — new records + /// starting against a known-bad backend benefit from the conservative cap + /// until proven otherwise. private func dropAllQueuedRecords(reason: String) { + let dropped = fileQueue.depth hedgeLog("Queue: dropping all queued records — \(reason)") fileQueue.clear() stateLock.withLock { retryCount = 0 + failingSince = nil pausedUntil = nil } + if dropped > 0 { + onRecordsDropped?(dropped, reason) + } } func start(disableReachabilityForTesting: Bool, @@ -481,7 +518,7 @@ class PostHogQueue { } private func pauseFor(seconds: TimeInterval) { - let until = Date().addingTimeInterval(seconds) + let until = now().addingTimeInterval(seconds) stateLock.withLock { pausedUntil = until } } @@ -491,7 +528,7 @@ class PostHogQueue { private func pauseReason() -> String? { let (isPaused, until) = stateLock.withLock { (paused, pausedUntil) } if isPaused { return "paused due to the reachability check" } - if let until, until > Date() { return "paused until `\(until)`" } + if let until, until > now() { return "paused until `\(until)`" } return nil } @@ -513,5 +550,9 @@ class PostHogQueue { var currentFlushAtForTesting: Int { batchLimitsLock.withLock { batchLimits.flushAt } } + + var currentRetryCountForTesting: Int { + stateLock.withLock { retryCount } + } } #endif diff --git a/PostHog/PostHogSDK.swift b/PostHog/PostHogSDK.swift index c91af0057..3bfca7bba 100644 --- a/PostHog/PostHogSDK.swift +++ b/PostHog/PostHogSDK.swift @@ -235,6 +235,16 @@ let maxRetryDelay = 30.0 logsQueue = PostHogQueue(config, theStorage, logsEndpoint) #endif + // Make a full queue drop measurable: emit a diagnostic event so + // the SDK stops discarding events silently. It queues like any + // other event and reaches PostHog once connectivity returns. + queue?.onRecordsDropped = { [weak self] count, reason in + self?.capture("$queue_records_dropped", properties: [ + "dropped_count": count, + "reason": reason, + ]) + } + queue?.start(disableReachabilityForTesting: config.disableReachabilityForTesting, disableQueueTimerForTesting: config.disableQueueTimerForTesting) diff --git a/PostHogTests/PostHogQueueTest.swift b/PostHogTests/PostHogQueueTest.swift index 09541661a..9c7bddea5 100644 --- a/PostHogTests/PostHogQueueTest.swift +++ b/PostHogTests/PostHogQueueTest.swift @@ -14,12 +14,13 @@ import Quick import XCTest class PostHogQueueTest: QuickSpec { - func getSut(flushAt: Int = 1, maxQueueSize: Int = 1000, maxBatchSize: Int = 50, maxRetries: Int = 3) -> PostHogQueue { + func getSut(flushAt: Int = 1, maxQueueSize: Int = 1000, maxBatchSize: Int = 50, maxRetries: Int = 3, maxRetryWindowSeconds: TimeInterval = 24 * 60 * 60) -> PostHogQueue { let config = PostHogConfig(projectToken: testProjectToken, host: "http://localhost:9001") config.flushAt = flushAt config.maxQueueSize = maxQueueSize config.maxBatchSize = maxBatchSize config.maxRetries = maxRetries + config.maxRetryWindowSeconds = maxRetryWindowSeconds config.sendFeatureFlagEvent = false let storage = PostHogStorage(config) let api = PostHogApi(config) @@ -409,6 +410,77 @@ class PostHogQueueTest: QuickSpec { sut.clear() } + it("never drops the queue on repeated network failures, even past the retry window") { + // The reported bug: ~6s of flaky connectivity used to wipe every + // buffered event. Transport failures (-1) must keep the queue no + // matter how many happen or how long they last — the window here + // is tiny to prove elapsed time alone can't trigger a drop. + let mockNow = MockDate() + now = { mockNow.date } + defer { now = { Date() } } + + let sut = self.getSut(flushAt: 100, maxBatchSize: 4, maxRetries: 3, maxRetryWindowSeconds: 1) + let networkError = NSError(domain: NSURLErrorDomain, code: NSURLErrorNotConnectedToInternet, userInfo: nil) + server.start(batchCount: 6) + server.batchResponseHandler = { _, _ in + HTTPStubsResponse(error: networkError) + } + + for i in 0 ..< 3 { + sut.add(PostHogEvent(event: "event\(i)", distinctId: "id\(i)")) + } + + for attempt in 1 ... 6 { + sut.flush() + // Wait for the failure to land, then step past the backoff pause + // and the retry window before the next attempt. + expect(sut.currentRetryCountForTesting).toEventually(equal(attempt)) + expect(sut.depth) == 3 + mockNow.date.addTimeInterval(60) + } + + expect(sut.depth) == 3 + + sut.clear() + } + + it("drops the queue after sustained server failures and reports the dropped count") { + let mockNow = MockDate() + now = { mockNow.date } + defer { now = { Date() } } + + let sut = self.getSut(flushAt: 100, maxBatchSize: 4, maxRetryWindowSeconds: 10) + var droppedCount = 0 + var droppedReason = "" + sut.onRecordsDropped = { count, reason in + droppedCount = count + droppedReason = reason + } + server.start(batchCount: 2) + server.batchResponseHandler = { _, _ in + HTTPStubsResponse(jsonObject: [], statusCode: 500, headers: nil) + } + + for i in 0 ..< 3 { + sut.add(PostHogEvent(event: "event\(i)", distinctId: "id\(i)")) + } + + // First failure starts the failure streak; the queue is retained. + sut.flush() + expect(sut.currentRetryCountForTesting).toEventually(equal(1)) + expect(sut.depth) == 3 + + // Past the window, the next server failure drops the whole queue and + // reports it so the loss is measurable. + mockNow.date.addTimeInterval(11) + sut.flush() + expect(sut.depth).toEventually(equal(0)) + expect(droppedCount) == 3 + expect(droppedReason).to(contain("backend failing")) + + sut.clear() + } + it("pops batch on non-retriable 4xx so a poison record cannot block the queue") { let sut = self.getSut(flushAt: 2, maxBatchSize: 4) server.batchResponseHandler = { _, _ in From 9ad59af188e01c3bac635e9b6918f742113c05d0 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 05:53:42 +0000 Subject: [PATCH 02/17] docs(queue): note maxRetryWindowSeconds is process-scoped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unhealthy-backend failure streak (`failingSince`) is tracked only in memory, so the `maxRetryWindowSeconds` drop window resets on every app launch and rarely fires on platforms that terminate processes frequently (iOS). Document this on both the private `failingSince` field and the public `maxRetryWindowSeconds` docstring so the documented "stop retrying forever" safeguard is honest about its per-process scope, and record why persisting the timestamp was declined (net-negative: adds a storage key and a per-first-failure disk write for a safeguard that never loses data or grows the buffer unbounded on its own — `maxQueueSize` already bounds it). Generated-By: PostHog Desktop Task-Id: 5e763b59-b175-4538-a60b-8aa86112df85 --- PostHog/PostHogConfig.swift | 6 ++++++ PostHog/PostHogQueue.swift | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/PostHog/PostHogConfig.swift b/PostHog/PostHogConfig.swift index 32553ed52..70efd652a 100644 --- a/PostHog/PostHogConfig.swift +++ b/PostHog/PostHogConfig.swift @@ -106,6 +106,12 @@ public typealias BeforeSendBlock = (PostHogEvent) -> PostHogEvent? /// triggers this drop: the queue is kept and bounded only by /// `maxQueueSize`, so a brief connectivity blip cannot destroy buffered /// events. The timer resets on any successful send. Default: 24 hours. + /// + /// The failure streak is tracked in memory and is not persisted across app + /// launches, so this window is measured within a single process lifetime + /// and restarts on every relaunch. On platforms that terminate processes + /// frequently (iOS) this drop rarely fires in practice, and `maxQueueSize` + /// stays the effective bound on buffered events. @objc public var maxRetryWindowSeconds: TimeInterval = Defaults.maxRetryWindowSeconds /// Maximum number of retries for feature flag requests after transient network errors or retryable HTTP responses. diff --git a/PostHog/PostHogQueue.swift b/PostHog/PostHogQueue.swift index 76e4bea21..b85bc1c43 100644 --- a/PostHog/PostHogQueue.swift +++ b/PostHog/PostHogQueue.swift @@ -68,6 +68,15 @@ class PostHogQueue { /// Wall clock of the first failure in the current unhealthy-backend streak, /// or `nil` while healthy. Only server-side retriable failures (5xx, 429, /// …) set it; transport failures never do. Cleared on any success. + /// + /// In-memory only — not persisted with the on-disk queue, so the streak and + /// therefore the `maxRetryWindowSeconds` drop window are measured within a + /// single process lifetime and restart from zero on every app launch. On + /// platforms that terminate processes frequently (iOS) the window-based + /// drop rarely fires; `maxQueueSize` remains the effective bound on the + /// buffer. Kept in memory deliberately: persisting it would add a storage + /// key and a disk write on each first failure for a safeguard that never + /// causes data loss or unbounded growth on its own. private var failingSince: Date? /// Invoked from `dropAllQueuedRecords` with the number of records wiped and From 9ee7767e893523ba7e54609cbc79aa2b0fcc20f2 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:00:40 +0000 Subject: [PATCH 03/17] fix(queue): scope maxRetries to HTTP 413, not transport/5xx failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HTTP 413 batch-halving branch compared `maxRetries` against the same `retryCount` that every retriable failure — transport `-1` and server 5xx/429 included — increments for backoff. So a burst of ordinary network flapping (the exact condition this PR exists to survive) could push retryCount past maxRetries, and then the very first 413 after connectivity returned would trip `newCount > maxRetries` and wipe the entire on-disk queue without a single halving attempt. That is total data loss on a reachable backend, and it contradicts the PR's own documented contract that maxRetries no longer governs network or 5xx retries. Introduce a dedicated `payloadTooLargeCount` that is incremented only when a 413 halves the cap and reset alongside `retryCount` on success, on the 413 poison drop, and on a full queue drop. `retryCount` now only ramps the backoff delay. Add a regression test: three transport failures followed by a 413 must halve the cap rather than drop the queue. Generated-By: PostHog Desktop Task-Id: 5e763b59-b175-4538-a60b-8aa86112df85 --- PostHog/PostHogQueue.swift | 35 +++++++++++----- PostHogTests/PostHogQueueTest.swift | 63 +++++++++++++++++++++++++---- 2 files changed, 81 insertions(+), 17 deletions(-) diff --git a/PostHog/PostHogQueue.swift b/PostHog/PostHogQueue.swift index b85bc1c43..ead2d117f 100644 --- a/PostHog/PostHogQueue.swift +++ b/PostHog/PostHogQueue.swift @@ -56,15 +56,27 @@ class PostHogQueue { private let configuredMaxQueueSize: Int private let timerInterval: TimeInterval - /// Guards `paused`, `pausedUntil`, and `retryCount`. These are touched from - /// the URLSession completion queue (handleResult), the timer's main-thread - /// callback (canFlush), and the reachability callbacks (onReachable / - /// onUnreachable) — all without coordination — so a single state lock keeps - /// the trio consistent against ThreadSanitizer. + /// Guards `paused`, `pausedUntil`, `retryCount`, `payloadTooLargeCount`, and + /// `failingSince`. These are touched from the URLSession completion queue + /// (handleResult), the timer's main-thread callback (canFlush), and the + /// reachability callbacks (onReachable / onUnreachable) — all without + /// coordination — so a single state lock keeps them consistent against + /// ThreadSanitizer. private let stateLock = NSLock() private var paused: Bool = false private var pausedUntil: Date? + /// Consecutive retriable failures (transport `-1` and server 5xx/429/…), + /// used only to ramp the backoff delay. Reset on any success or full drop. + /// It deliberately does *not* gate the HTTP 413 halving budget — see + /// `payloadTooLargeCount`. private var retryCount: Int = 0 + /// Consecutive HTTP 413 batch-halving attempts, compared against + /// `config.maxRetries`. Kept separate from `retryCount` so transport and + /// 5xx failures — which must never drop the queue — can't consume the 413 + /// halving budget and let the first 413 wipe every buffered record. + /// Incremented only when a 413 halves the cap; reset on success, on the + /// 413 poison drop, and on a full queue drop. + private var payloadTooLargeCount: Int = 0 /// Wall clock of the first failure in the current unhealthy-backend streak, /// or `nil` while healthy. Only server-side retriable failures (5xx, 429, /// …) set it; transport failures never do. Cleared on any success. @@ -208,9 +220,9 @@ class PostHogQueue { } // 413 Payload Too Large. Two paths: - // - cap > 1: this is a retry. Increment retryCount, drop all if - // `maxRetries` exceeded, otherwise halve cap and retry the same - // records. + // - cap > 1: this is a retry. Increment the 413 halving count, drop + // all if `maxRetries` exceeded, otherwise halve cap and retry the + // same records. // - cap == 1: poison drop. The offending record can't shrink any // further, so we drop the batch and apply the endpoint's poison // cap policy. Don't count it as a retry — the drop *is* the @@ -220,8 +232,8 @@ class PostHogQueue { if canHalve { let newCount = stateLock.withLock { () -> Int in - retryCount += 1 - return retryCount + payloadTooLargeCount += 1 + return payloadTooLargeCount } if newCount > config.maxRetries { dropAllQueuedRecords(reason: "max retries (\(config.maxRetries)) exceeded after repeated HTTP 413") @@ -242,6 +254,7 @@ class PostHogQueue { hedgeLog("Queue: dropping batch after HTTP 413 (cap == 1)") stateLock.withLock { retryCount = 0 + payloadTooLargeCount = 0 failingSince = nil } payload.completion(true) @@ -252,6 +265,7 @@ class PostHogQueue { // batch. Cap stays where it is — no ramp on success. stateLock.withLock { retryCount = 0 + payloadTooLargeCount = 0 failingSince = nil } payload.completion(true) @@ -270,6 +284,7 @@ class PostHogQueue { fileQueue.clear() stateLock.withLock { retryCount = 0 + payloadTooLargeCount = 0 failingSince = nil pausedUntil = nil } diff --git a/PostHogTests/PostHogQueueTest.swift b/PostHogTests/PostHogQueueTest.swift index 9c7bddea5..91fedfc56 100644 --- a/PostHogTests/PostHogQueueTest.swift +++ b/PostHogTests/PostHogQueueTest.swift @@ -186,13 +186,14 @@ class PostHogQueueTest: QuickSpec { sut.clear() } - it("drops the entire queue once retryCount exceeds maxRetries on repeated 413") { - // 413 with cap > 1 increments retryCount the same way 5xx / - // network errors do — both paths use the same - // `newCount > config.maxRetries` check — so this test covers - // both paths' drop logic. We use 413 here because it doesn't - // set `pausedUntil`, letting the test drive multiple retries - // without waiting out the exponential backoff. + it("drops the entire queue once the 413 halving count exceeds maxRetries") { + // A 413 with cap > 1 increments the dedicated 413 halving count + // and drops via the `newCount > config.maxRetries` check. Only the + // 413 path is bounded by `maxRetries`; transport and 5xx failures + // use a separate counter and the time window. We use 413 here + // because it doesn't set `pausedUntil`, letting the test drive + // multiple halving attempts without waiting out the exponential + // backoff. // // 20 events with maxBatchSize=20 so halving sequence is 10 → 5 // → drop — cap doesn't reach 1 before maxRetries=2 is exceeded @@ -444,6 +445,54 @@ class PostHogQueueTest: QuickSpec { sut.clear() } + it("a 413 after transport failures still halves instead of wiping the queue") { + // Regression: transport failures used to feed the same counter the + // 413 halving budget reads, so a burst of network flapping could + // push `retryCount` past `maxRetries` and make the very first 413 + // drop every buffered record without one halving attempt. Transport + // failures must not consume the 413 budget. + let mockNow = MockDate() + now = { mockNow.date } + defer { now = { Date() } } + + let sut = self.getSut(flushAt: 100, maxBatchSize: 20, maxRetries: 2) + let networkError = NSError(domain: NSURLErrorDomain, code: NSURLErrorNotConnectedToInternet, userInfo: nil) + var attempt = 0 + server.start(batchCount: 4) + server.batchResponseHandler = { _, _ in + attempt += 1 + // First three attempts are transport failures (flaky network); + // afterwards the backend responds 413. + return attempt <= 3 + ? HTTPStubsResponse(error: networkError) + : HTTPStubsResponse(jsonObject: [], statusCode: 413, headers: nil) + } + + for i in 0 ..< 20 { + sut.add(PostHogEvent(event: "evt\(i)", distinctId: "id")) + } + + // Three transport failures push retryCount to 3 (> maxRetries=2) + // without ever dropping the queue or touching the batch cap. + for attemptCount in 1 ... 3 { + sut.flush() + expect(sut.currentRetryCountForTesting).toEventually(equal(attemptCount)) + expect(sut.depth) == 20 + expect(sut.currentBatchCapForTesting) == 20 + // Step past the backoff pause before the next attempt. + mockNow.date.addTimeInterval(60) + } + + // The first 413 now arrives. Despite retryCount already exceeding + // maxRetries, the 413 budget is separate and starts fresh, so the + // cap halves and the batch is retained rather than wiped. + sut.flush() + expect(sut.currentBatchCapForTesting).toEventually(equal(10)) + expect(sut.depth) == 20 + + sut.clear() + } + it("drops the queue after sustained server failures and reports the dropped count") { let mockNow = MockDate() now = { mockNow.date } From d6a6cb644eeb5de4d6280c92210697f753684375 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:05:06 +0000 Subject: [PATCH 04/17] fix(queue): reset the failure window on transport failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `failingSince` arms the drop window on a server-side failure (5xx/429), but a subsequent transport failure (offline, timeout, no route) neither cleared nor paused it — the guard only skipped re-setting the field and the drop check was suppressed for that one call. The wall clock therefore kept spanning the whole offline period, so a single 500, hours offline, then one more 500 would see the entire gap as "sustained backend failure" and wipe the on-disk queue on just two error responses. That contradicts the window's own contract (it is scoped to a responsive-but-unhealthy backend) and is total data loss when reached. Clear `failingSince` on every transport failure so the window measures only a continuous run of server responses; a later server failure starts a fresh streak. This can only ever delay a drop, never cause an earlier one. Add a regression test: 500 arms the window, a transport failure past the window resets it, and the next 500 keeps the queue instead of wiping it. Generated-By: PostHog Desktop Task-Id: 5e763b59-b175-4538-a60b-8aa86112df85 --- PostHog/PostHogQueue.swift | 19 +++++++--- PostHogTests/PostHogQueueTest.swift | 57 +++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/PostHog/PostHogQueue.swift b/PostHog/PostHogQueue.swift index ead2d117f..c692b6a1d 100644 --- a/PostHog/PostHogQueue.swift +++ b/PostHog/PostHogQueue.swift @@ -79,7 +79,8 @@ class PostHogQueue { private var payloadTooLargeCount: Int = 0 /// Wall clock of the first failure in the current unhealthy-backend streak, /// or `nil` while healthy. Only server-side retriable failures (5xx, 429, - /// …) set it; transport failures never do. Cleared on any success. + /// …) arm it; a transport failure clears it instead, so an offline stretch + /// can't count toward the drop window. Also cleared on any success. /// /// In-memory only — not persisted with the on-disk queue, so the streak and /// therefore the `maxRetryWindowSeconds` drop window are measured within a @@ -190,14 +191,22 @@ class PostHogQueue { if isRetriable { // `since` is the start of the current server-failure streak, or - // `nil` for a transport failure. Only server-side failures arm the - // clock, so an offline stretch never counts toward the drop window. + // `nil` when the streak isn't armed. A transport failure *resets* + // the clock — the backend just went unreachable, so this is no + // longer a responsive-but-unhealthy streak — while a server-side + // failure arms it if it wasn't already. Clearing (not merely + // ignoring) `failingSince` on a transport failure is what keeps an + // offline stretch from counting toward the drop window: otherwise a + // 500, then hours offline, then one more 500 would see the whole + // gap as sustained backend failure and wipe the queue. let (newCount, since): (Int, Date?) = stateLock.withLock { retryCount += 1 - if !isTransportFailure, failingSince == nil { + if isTransportFailure { + failingSince = nil + } else if failingSince == nil { failingSince = now() } - return (retryCount, isTransportFailure ? nil : failingSince) + return (retryCount, failingSince) } // Drop the queue only when a responsive-but-unhealthy backend has diff --git a/PostHogTests/PostHogQueueTest.swift b/PostHogTests/PostHogQueueTest.swift index 91fedfc56..8cbb0af8b 100644 --- a/PostHogTests/PostHogQueueTest.swift +++ b/PostHogTests/PostHogQueueTest.swift @@ -530,6 +530,63 @@ class PostHogQueueTest: QuickSpec { sut.clear() } + it("a transport failure resets the backend failure window so an offline gap can't trigger a drop") { + // Regression: an initial 500 armed the failure clock, then a long + // offline stretch (transport failures) elapsed without clearing it, + // so the next 500 saw the whole offline gap as sustained backend + // failure and wiped the queue on only two error responses. A + // transport failure must reset the window; offline time must not + // count toward it. + let mockNow = MockDate() + now = { mockNow.date } + defer { now = { Date() } } + + let sut = self.getSut(flushAt: 100, maxBatchSize: 4, maxRetryWindowSeconds: 10) + var dropped = false + sut.onRecordsDropped = { _, _ in dropped = true } + + let networkError = NSError(domain: NSURLErrorDomain, code: NSURLErrorNotConnectedToInternet, userInfo: nil) + var attempt = 0 + server.start(batchCount: 3) + server.batchResponseHandler = { _, _ in + attempt += 1 + switch attempt { + case 1: return HTTPStubsResponse(jsonObject: [], statusCode: 500, headers: nil) // arms the window + case 2: return HTTPStubsResponse(error: networkError) // offline: must reset the window + default: return HTTPStubsResponse(jsonObject: [], statusCode: 500, headers: nil) // fresh window, no drop + } + } + + for i in 0 ..< 3 { + sut.add(PostHogEvent(event: "event\(i)", distinctId: "id\(i)")) + } + + // First 500 arms the failure streak. + sut.flush() + expect(sut.currentRetryCountForTesting).toEventually(equal(1)) + expect(sut.depth) == 3 + + // A long offline gap elapses, then a transport failure lands. Even + // though more than the window has passed since the first 500, the + // transport failure resets the clock and the queue is kept. + mockNow.date.addTimeInterval(11) + sut.flush() + expect(sut.currentRetryCountForTesting).toEventually(equal(2)) + expect(sut.depth) == 3 + + // A second 500 now, still past where the *original* window would + // have elapsed. Because the transport failure reset the clock, the + // offline time doesn't count: the queue survives instead of being + // wiped on two error responses. + mockNow.date.addTimeInterval(5) + sut.flush() + expect(sut.currentRetryCountForTesting).toEventually(equal(3)) + expect(sut.depth) == 3 + expect(dropped) == false + + sut.clear() + } + it("pops batch on non-retriable 4xx so a poison record cannot block the queue") { let sut = self.getSut(flushAt: 2, maxBatchSize: 4) server.batchResponseHandler = { _, _ in From 0f5ebc43411066f4da30339433481c9e131d10de Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:07:44 +0000 Subject: [PATCH 05/17] chore(api): add maxRetryWindowSeconds to public API snapshot This PR adds `@objc public var maxRetryWindowSeconds: TimeInterval` to the public `PostHogConfig`, but the generated snapshot at api/posthog-ios.public-api.txt was not regenerated, so the `public-api` CI job (`make apiCheck`) would fail its strict diff and block the merge. Add the missing entry in its sorted position, matching the exact format the generator emits for sibling `@objc` stored properties (the declaration column strips `public` and the default value; the USR follows the `c:@M@PostHog@objc(cs)PostHogConfig(py)` pattern). No Apple toolchain is available here to run `make apiUpdate`, so the line was reconstructed from the deterministic format of the neighbouring `maxRetries` / `flushIntervalSeconds` entries; `make apiCheck` should be re-run in CI to confirm. Generated-By: PostHog Desktop Task-Id: 5e763b59-b175-4538-a60b-8aa86112df85 --- api/posthog-ios.public-api.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/api/posthog-ios.public-api.txt b/api/posthog-ios.public-api.txt index f304d3e74..0cb397d4b 100644 --- a/api/posthog-ios.public-api.txt +++ b/api/posthog-ios.public-api.txt @@ -78,6 +78,7 @@ PostHog | PostHogConfig.logs | property | @objc let logs: PostHogLogsConfig | c: PostHog | PostHogConfig.maxBatchSize | property | @objc var maxBatchSize: Int | c:@M@PostHog@objc(cs)PostHogConfig(py)maxBatchSize PostHog | PostHogConfig.maxQueueSize | property | @objc var maxQueueSize: Int | c:@M@PostHog@objc(cs)PostHogConfig(py)maxQueueSize PostHog | PostHogConfig.maxRetries | property | @objc var maxRetries: Int | c:@M@PostHog@objc(cs)PostHogConfig(py)maxRetries +PostHog | PostHogConfig.maxRetryWindowSeconds | property | @objc var maxRetryWindowSeconds: TimeInterval | c:@M@PostHog@objc(cs)PostHogConfig(py)maxRetryWindowSeconds PostHog | PostHogConfig.optOut | property | @objc var optOut: Bool | c:@M@PostHog@objc(cs)PostHogConfig(py)optOut PostHog | PostHogConfig.personProfiles | property | @objc var personProfiles: PostHogPersonProfiles | c:@M@PostHog@objc(cs)PostHogConfig(py)personProfiles PostHog | PostHogConfig.preloadFeatureFlags | property | @objc var preloadFeatureFlags: Bool | c:@M@PostHog@objc(cs)PostHogConfig(py)preloadFeatureFlags From 98405816f437b1c18a4899916c43bbde7e967e6c Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:09:38 +0000 Subject: [PATCH 06/17] docs(config): note maxRetries still bounds push registration retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new maxRetries docstring claimed "This no longer governs network or 5xx retries" as an absolute statement, but PostHogPushSubscriptionHandler still reads config.maxRetries to bound retryable push-registration failures — and its classifier treats transport errors, 429, and 5xx as retryable, exactly the failure kinds the docstring said it no longer covers. A developer lowering maxRetries to tune 413 halving would unknowingly shorten push registration patience. Scope the "no longer governs network/5xx" claim to the event, replay, and log queues, and add a sentence stating that maxRetries still bounds push subscription registration retries. Documentation only; no behaviour or API change (the alternative of a separate pushSubscriptionMaxRetries option was deliberately not taken — it adds public API surface out of proportion to a one-sentence accuracy fix). Generated-By: PostHog Desktop Task-Id: 5e763b59-b175-4538-a60b-8aa86112df85 --- PostHog/PostHogConfig.swift | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/PostHog/PostHogConfig.swift b/PostHog/PostHogConfig.swift index 70efd652a..63f4403e8 100644 --- a/PostHog/PostHogConfig.swift +++ b/PostHog/PostHogConfig.swift @@ -94,8 +94,14 @@ public typealias BeforeSendBlock = (PostHogEvent) -> PostHogEvent? /// batch is dropped. Increments on every 413 that halves the batch cap; /// resets on a successful 2xx response. Default 3. /// - /// This no longer governs network or 5xx retries — those keep the queue - /// and are bounded by `maxRetryWindowSeconds` and `maxQueueSize` instead. + /// For the on-disk event, session-replay, and log queues this no longer + /// governs network or 5xx retries — those keep the queue and are bounded by + /// `maxRetryWindowSeconds` and `maxQueueSize` instead. + /// + /// It does still bound push-subscription registration retries: a transport + /// error, 429, or 5xx there is retried up to `maxRetries` times before the + /// device token is kept for the next app launch. Lowering this value to + /// tune 413 halving therefore also shortens push registration patience. @objc public var maxRetries: Int = Defaults.maxRetries /// Sustained duration, in seconds, that flushes must keep failing against a From adf3655595077bce9c47f0a0769804187995496a Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:14:08 +0000 Subject: [PATCH 07/17] fix(queue): don't pop the drop diagnostic after a full clear MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `dropAllQueuedRecords` clears the on-disk queue and then invokes `onRecordsDropped`, which the SDK wires to synchronously capture `$queue_records_dropped` back onto the same queue (the capture path has no dispatch hop, so the record is on disk before control returns). Both drop sites then called `payload.completion(true)`, whose closure runs `fileQueue.pop(items.count)` — a delete-by-position against the front of the queue. Because `clear()` already emptied the batch, that pop deletes the freshly written diagnostic (and any event an app thread captured in the same window), so the PR's headline observability event never survives to be sent. Complete the drop paths with `false` instead. `pop` is guarded by `if success`, so `false` skips the deletion (there is nothing legitimate to pop after a clear) while the closure still clears `isFlushing`. Add a queue-level regression test that reproduces the SDK's synchronous re-capture in `onRecordsDropped` and asserts the diagnostic remains on disk after the drop instead of being popped away. Generated-By: PostHog Desktop Task-Id: 5e763b59-b175-4538-a60b-8aa86112df85 --- PostHog/PostHogQueue.swift | 13 +++++++-- PostHogTests/PostHogQueueTest.swift | 44 +++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/PostHog/PostHogQueue.swift b/PostHog/PostHogQueue.swift index c692b6a1d..52cadba4b 100644 --- a/PostHog/PostHogQueue.swift +++ b/PostHog/PostHogQueue.swift @@ -216,7 +216,13 @@ class PostHogQueue { // can never wipe the buffer. if let since, now().timeIntervalSince(since) >= config.maxRetryWindowSeconds { dropAllQueuedRecords(reason: "backend failing for over \(config.maxRetryWindowSeconds)s") - payload.completion(true) + // Complete with `false`: `dropAllQueuedRecords` already cleared + // the batch from disk, and its `onRecordsDropped` callback may + // have synchronously enqueued the `$queue_records_dropped` + // diagnostic onto the now-empty queue. A `true` here would pop + // `items.count` records off the front and delete that fresh + // diagnostic (plus any event captured in the same window). + payload.completion(false) return } @@ -246,7 +252,10 @@ class PostHogQueue { } if newCount > config.maxRetries { dropAllQueuedRecords(reason: "max retries (\(config.maxRetries)) exceeded after repeated HTTP 413") - payload.completion(true) + // Complete with `false` — see the window-drop path above: + // the queue was just cleared, so popping would delete the + // diagnostic the drop callback enqueued onto it. + payload.completion(false) return } let actualBatchSize = payload.records.count diff --git a/PostHogTests/PostHogQueueTest.swift b/PostHogTests/PostHogQueueTest.swift index 8cbb0af8b..854a49b7c 100644 --- a/PostHogTests/PostHogQueueTest.swift +++ b/PostHogTests/PostHogQueueTest.swift @@ -587,6 +587,50 @@ class PostHogQueueTest: QuickSpec { sut.clear() } + it("keeps the diagnostic that onRecordsDropped enqueues during a full drop") { + // Regression: dropAllQueuedRecords fires onRecordsDropped, which the + // SDK wires to synchronously capture `$queue_records_dropped` back + // onto this same queue. The drop-path completion must NOT then pop: + // clear() already emptied the batch and `pop` deletes by position, + // so a pop would silently delete that fresh diagnostic (and any + // event captured in the same window), leaving the drop as invisible + // as it was before the feature existed. + let mockNow = MockDate() + now = { mockNow.date } + defer { now = { Date() } } + + let sut = self.getSut(flushAt: 100, maxBatchSize: 4, maxRetryWindowSeconds: 10) + + // Mimic the SDK's synchronous re-capture on drop: enqueue one + // diagnostic record onto the same queue from the callback. + sut.onRecordsDropped = { [weak sut] _, _ in + sut?.add(PostHogEvent(event: "$queue_records_dropped", distinctId: "id")) + } + + server.start(batchCount: 2) + server.batchResponseHandler = { _, _ in + HTTPStubsResponse(jsonObject: [], statusCode: 500, headers: nil) + } + + for i in 0 ..< 3 { + sut.add(PostHogEvent(event: "event\(i)", distinctId: "id\(i)")) + } + + // First 500 arms the failure window. + sut.flush() + expect(sut.currentRetryCountForTesting).toEventually(equal(1)) + expect(sut.depth) == 3 + + // Past the window: the next 500 drops all 3 and the callback + // enqueues the diagnostic. It must survive on disk (depth 1), not + // be popped away to 0. + mockNow.date.addTimeInterval(11) + sut.flush() + expect(sut.depth).toEventually(equal(1)) + + sut.clear() + } + it("pops batch on non-retriable 4xx so a poison record cannot block the queue") { let sut = self.getSut(flushAt: 2, maxBatchSize: 4) server.batchResponseHandler = { _, _ in From 0df92b275b7738ed386dafcd5eb2b5c3f2db6b81 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Mon, 31 Aug 2026 17:35:20 -0400 Subject: [PATCH 08/17] fix(queue): preserve durable records across retries --- .changeset/durable-queue-lifecycle.md | 5 + .changeset/quick-queues-keep.md | 5 - PostHog/PostHogApi.swift | 8 +- PostHog/PostHogConfig.swift | 32 +- PostHog/PostHogFileBackedQueue.swift | 76 ++- PostHog/PostHogQueue.swift | 222 +++------ PostHog/PostHogSDK.swift | 10 - PostHogTests/PostHogApiTest.swift | 19 + PostHogTests/PostHogFileBackedQueueTest.swift | 62 +++ PostHogTests/PostHogLogsQueueTest.swift | 66 +-- PostHogTests/PostHogQueueTest.swift | 458 +++++------------- api/posthog-ios.public-api.txt | 1 - 12 files changed, 368 insertions(+), 596 deletions(-) create mode 100644 .changeset/durable-queue-lifecycle.md delete mode 100644 .changeset/quick-queues-keep.md diff --git a/.changeset/durable-queue-lifecycle.md b/.changeset/durable-queue-lifecycle.md new file mode 100644 index 000000000..409088342 --- /dev/null +++ b/.changeset/durable-queue-lifecycle.md @@ -0,0 +1,5 @@ +--- +"posthog-ios": patch +--- + +Preserve bounded durable event, replay, and log queues across retryable upload failures, and acknowledge successful batches by their exact persisted entry identities. diff --git a/.changeset/quick-queues-keep.md b/.changeset/quick-queues-keep.md deleted file mode 100644 index ff2d7a151..000000000 --- a/.changeset/quick-queues-keep.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"posthog-ios": patch ---- - -Stop dropping the whole on-disk event queue after a brief network outage. A transport failure (lost connectivity, timeout, no route) now keeps every buffered event and lets `maxQueueSize` bound the queue, instead of wiping it once a few retries exceeded `maxRetries`. A full queue drop is now reserved for a responsive-but-unhealthy backend that keeps rejecting batches past the new `maxRetryWindowSeconds` window (default 24 hours), and every drop emits a `$queue_records_dropped` diagnostic event so the loss is measurable. diff --git a/PostHog/PostHogApi.swift b/PostHog/PostHogApi.swift index 3681d93ae..a4e3b3e60 100644 --- a/PostHog/PostHogApi.swift +++ b/PostHog/PostHogApi.swift @@ -10,19 +10,21 @@ import Foundation /// Common URLSession upload-response handler shared by `/batch`, `/snapshot`, /// and `/i/v1/logs`. Routes through `as?` so a missing HTTP response can't /// crash inside a customer process. -private func processUploadResponse( +func processUploadResponse( endpointName: String, data: Data?, response: URLResponse?, error: Error?, completion: @escaping (PostHogUploadInfo) -> Void ) { + let httpResponse = response as? HTTPURLResponse + if let error { hedgeLog("Error calling the \(endpointName) API: \(error).") - return completion(PostHogUploadInfo(statusCode: nil, error: error)) + return completion(PostHogUploadInfo(statusCode: httpResponse?.statusCode, error: error)) } - guard let httpResponse = response as? HTTPURLResponse else { + guard let httpResponse else { hedgeLog("\(endpointName) API returned no HTTP response") return completion(PostHogUploadInfo(statusCode: nil, error: nil)) } diff --git a/PostHog/PostHogConfig.swift b/PostHog/PostHogConfig.swift index 63f4403e8..92f463f73 100644 --- a/PostHog/PostHogConfig.swift +++ b/PostHog/PostHogConfig.swift @@ -33,7 +33,6 @@ public typealias BeforeSendBlock = (PostHogEvent) -> PostHogEvent? static let maxBatchSize: Int = 50 static let flushIntervalSeconds: TimeInterval = 30 static let maxRetries: Int = 3 - static let maxRetryWindowSeconds: TimeInterval = 24 * 60 * 60 static let featureFlagRequestMaxRetries: Int = 1 } @@ -90,36 +89,11 @@ public typealias BeforeSendBlock = (PostHogEvent) -> PostHogEvent? /// Default: `30`. @objc public var flushIntervalSeconds: TimeInterval = Defaults.flushIntervalSeconds - /// Maximum number of HTTP 413 batch-halving attempts before the oversized - /// batch is dropped. Increments on every 413 that halves the batch cap; - /// resets on a successful 2xx response. Default 3. - /// - /// For the on-disk event, session-replay, and log queues this no longer - /// governs network or 5xx retries — those keep the queue and are bounded by - /// `maxRetryWindowSeconds` and `maxQueueSize` instead. - /// - /// It does still bound push-subscription registration retries: a transport - /// error, 429, or 5xx there is retried up to `maxRetries` times before the - /// device token is kept for the next app launch. Lowering this value to - /// tune 413 halving therefore also shortens push registration patience. + /// Maximum number of retries for push-subscription registration failures. + /// Durable event, replay, and log queues retain retryable records and use + /// their configured capacity as the storage bound. Default: `3`. @objc public var maxRetries: Int = Defaults.maxRetries - /// Sustained duration, in seconds, that flushes must keep failing against a - /// responsive-but-unhealthy backend (for example repeated HTTP 5xx) before - /// the on-disk queue is dropped to stop retrying forever. - /// - /// A network-level failure (lost connectivity, timeout, no route) never - /// triggers this drop: the queue is kept and bounded only by - /// `maxQueueSize`, so a brief connectivity blip cannot destroy buffered - /// events. The timer resets on any successful send. Default: 24 hours. - /// - /// The failure streak is tracked in memory and is not persisted across app - /// launches, so this window is measured within a single process lifetime - /// and restarts on every relaunch. On platforms that terminate processes - /// frequently (iOS) this drop rarely fires in practice, and `maxQueueSize` - /// stays the effective bound on buffered events. - @objc public var maxRetryWindowSeconds: TimeInterval = Defaults.maxRetryWindowSeconds - /// Maximum number of retries for feature flag requests after transient network errors or retryable HTTP responses. /// Defaults to 1. Set to 0 to disable feature flag request retries. @objc public var featureFlagRequestMaxRetries: Int = Defaults.featureFlagRequestMaxRetries diff --git a/PostHog/PostHogFileBackedQueue.swift b/PostHog/PostHogFileBackedQueue.swift index bbad36801..5653b1c0a 100644 --- a/PostHog/PostHogFileBackedQueue.swift +++ b/PostHog/PostHogFileBackedQueue.swift @@ -8,7 +8,13 @@ import Foundation class PostHogFileBackedQueue { + struct Entry { + let id: String + let data: Data + } + let queue: URL + private let maxSize: Int? private var items = [String]() private let itemsLock = NSLock() @@ -16,8 +22,9 @@ class PostHogFileBackedQueue { itemsLock.withLock { items.count } } - init(queue: URL, oldQueues: [URL] = []) { + init(queue: URL, oldQueues: [URL] = [], maxSize: Int? = nil) { self.queue = queue + self.maxSize = maxSize.map { max(1, $0) } setup(oldQueues: oldQueues) } @@ -44,7 +51,7 @@ class PostHogFileBackedQueue { do { // when copying over buffered snapshots, content modification date will change, so we work off creation date instead. let sortedItems = try FileManager.default.contentsOfDirectory(at: queue, sortedBy: .creationDateKey) - itemsLock.withLock { items = sortedItems } + replaceItemsWithBounded(sortedItems) } catch { hedgeLog("Failed to load files for queue \(error)") // failed to read directory – bad permissions, perhaps? @@ -52,7 +59,11 @@ class PostHogFileBackedQueue { } func peek(_ count: Int) -> [Data] { - loadFiles(count) + peekEntries(count).map(\.data) + } + + func peekEntries(_ count: Int) -> [Entry] { + loadEntries(count) } func delete(index: Int) { @@ -70,13 +81,45 @@ class PostHogFileBackedQueue { deleteFiles(count) } - func add(_ contents: Data) { + func remove(ids: [String]) { + let ids = Set(ids) + let removed: [String] = itemsLock.withLock { + let removed = items.filter { ids.contains($0) } + items.removeAll { ids.contains($0) } + return removed + } + + for item in removed { + deleteSafely(queue.appendingPathComponent(item)) + } + } + + /// Persists one entry and optionally enforces a FIFO capacity in the same + /// critical section. Returning an evicted id lets the queue report + /// backpressure without racing a separate depth check against other adds. + @discardableResult + func add(_ contents: Data, maxSize: Int? = nil) -> String? { do { let filename = UUID.v7String() - try contents.write(to: queue.appendingPathComponent(filename)) - itemsLock.withLock { items.append(filename) } + let effectiveMaxSize = maxSize.map { max(1, $0) } ?? self.maxSize + var evicted: String? + + try itemsLock.withLock { + if let effectiveMaxSize, items.count >= effectiveMaxSize { + evicted = items.removeFirst() + if let evicted { + deleteSafely(queue.appendingPathComponent(evicted)) + } + } + + try contents.write(to: queue.appendingPathComponent(filename)) + items.append(filename) + } + + return evicted } catch { hedgeLog("Could not write file \(error)") + return nil } } @@ -91,14 +134,27 @@ class PostHogFileBackedQueue { func reloadFromDisk() { do { let sortedItems = try FileManager.default.contentsOfDirectory(at: queue, sortedBy: .creationDateKey) - itemsLock.withLock { items = sortedItems } + replaceItemsWithBounded(sortedItems) } catch { hedgeLog("Failed to reload files for queue \(error)") } } - private func loadFiles(_ count: Int) -> [Data] { - var results = [Data]() + private func replaceItemsWithBounded(_ sortedItems: [String]) { + let overflow = maxSize.map { max(0, sortedItems.count - $0) } ?? 0 + let dropped = sortedItems.prefix(overflow) + itemsLock.withLock { items = Array(sortedItems.dropFirst(overflow)) } + + for item in dropped { + deleteSafely(queue.appendingPathComponent(item)) + } + if overflow > 0 { + hedgeLog("Dropped \(overflow) oldest cached records to enforce queue capacity") + } + } + + private func loadEntries(_ count: Int) -> [Entry] { + var results = [Entry]() var skipped = Set() let itemsCopy = itemsLock.withLock { items } @@ -113,7 +169,7 @@ class PostHogFileBackedQueue { } let contents = try Data(contentsOf: itemURL) - results.append(contents) + results.append(Entry(id: item, data: contents)) } catch { if isTemporarilyUnavailable(error) { hedgeLog("File \(itemURL) is temporarily unavailable, will retry \(error)") diff --git a/PostHog/PostHogQueue.swift b/PostHog/PostHogQueue.swift index 52cadba4b..8d67e6240 100644 --- a/PostHog/PostHogQueue.swift +++ b/PostHog/PostHogQueue.swift @@ -56,47 +56,15 @@ class PostHogQueue { private let configuredMaxQueueSize: Int private let timerInterval: TimeInterval - /// Guards `paused`, `pausedUntil`, `retryCount`, `payloadTooLargeCount`, and - /// `failingSince`. These are touched from the URLSession completion queue - /// (handleResult), the timer's main-thread callback (canFlush), and the - /// reachability callbacks (onReachable / onUnreachable) — all without - /// coordination — so a single state lock keeps them consistent against - /// ThreadSanitizer. + /// Guards `paused`, `pausedUntil`, and `retryCount`. These are touched from + /// the URLSession completion queue (handleResult), the timer's main-thread + /// callback (canFlush), and the reachability callbacks (onReachable / + /// onUnreachable) — all without coordination — so a single state lock keeps + /// the trio consistent against ThreadSanitizer. private let stateLock = NSLock() private var paused: Bool = false private var pausedUntil: Date? - /// Consecutive retriable failures (transport `-1` and server 5xx/429/…), - /// used only to ramp the backoff delay. Reset on any success or full drop. - /// It deliberately does *not* gate the HTTP 413 halving budget — see - /// `payloadTooLargeCount`. private var retryCount: Int = 0 - /// Consecutive HTTP 413 batch-halving attempts, compared against - /// `config.maxRetries`. Kept separate from `retryCount` so transport and - /// 5xx failures — which must never drop the queue — can't consume the 413 - /// halving budget and let the first 413 wipe every buffered record. - /// Incremented only when a 413 halves the cap; reset on success, on the - /// 413 poison drop, and on a full queue drop. - private var payloadTooLargeCount: Int = 0 - /// Wall clock of the first failure in the current unhealthy-backend streak, - /// or `nil` while healthy. Only server-side retriable failures (5xx, 429, - /// …) arm it; a transport failure clears it instead, so an offline stretch - /// can't count toward the drop window. Also cleared on any success. - /// - /// In-memory only — not persisted with the on-disk queue, so the streak and - /// therefore the `maxRetryWindowSeconds` drop window are measured within a - /// single process lifetime and restart from zero on every app launch. On - /// platforms that terminate processes frequently (iOS) the window-based - /// drop rarely fires; `maxQueueSize` remains the effective bound on the - /// buffer. Kept in memory deliberately: persisting it would add a storage - /// key and a disk write on each first failure for a safeguard that never - /// causes data loss or unbounded growth on its own. - private var failingSince: Date? - - /// Invoked from `dropAllQueuedRecords` with the number of records wiped and - /// the reason. The SDK wires this to emit a diagnostic event so a full - /// queue drop — the last remaining silent, total data-loss path — becomes - /// measurable. - var onRecordsDropped: ((Int, String) -> Void)? #if !os(watchOS) private let reachability: Reachability? private var reachableToken: RegistrationToken? @@ -150,7 +118,8 @@ class PostHogQueue { rateCapWindowSeconds = endpoint.rateCapWindowSeconds(config) fileQueue = PostHogFileBackedQueue( queue: storage.url(forKey: endpoint.storageKey), - oldQueues: endpoint.oldStorageKeys.map { storage.url(forKey: $0) } + oldQueues: endpoint.oldStorageKeys.map { storage.url(forKey: $0) }, + maxSize: configuredMaxQueueSize ) dispatchQueue = DispatchQueue(label: endpoint.dispatchQueueLabel, target: .global(qos: .utility)) } @@ -165,7 +134,8 @@ class PostHogQueue { rateCapWindowSeconds = endpoint.rateCapWindowSeconds(config) fileQueue = PostHogFileBackedQueue( queue: storage.url(forKey: endpoint.storageKey), - oldQueues: endpoint.oldStorageKeys.map { storage.url(forKey: $0) } + oldQueues: endpoint.oldStorageKeys.map { storage.url(forKey: $0) }, + maxSize: configuredMaxQueueSize ) dispatchQueue = DispatchQueue(label: endpoint.dispatchQueueLabel, target: .global(qos: .utility)) } @@ -179,53 +149,13 @@ class PostHogQueue { } private func handleResult(_ result: PostHogUploadInfo, _ payload: PostHogConsumerPayload) { - // -1 means its not anything related to the API but rather network or something else, so we try again + // A missing response means no terminal delivery outcome is known, so + // the exact durable entries must remain eligible for a later flush. let statusCode = result.statusCode ?? -1 - - // A transport failure (-1: lost connectivity, timeout, no route) means - // we could not reach the backend — not that it rejected our data. It is - // universally retriable and must never drop the queue. Everything else - // is up to the endpoint's retry policy. - let isTransportFailure = statusCode == -1 - let isRetriable = isTransportFailure || endpoint.isRetriableStatusCode(statusCode) + let isRetriable = statusCode == -1 || endpoint.isRetriableStatusCode(statusCode) if isRetriable { - // `since` is the start of the current server-failure streak, or - // `nil` when the streak isn't armed. A transport failure *resets* - // the clock — the backend just went unreachable, so this is no - // longer a responsive-but-unhealthy streak — while a server-side - // failure arms it if it wasn't already. Clearing (not merely - // ignoring) `failingSince` on a transport failure is what keeps an - // offline stretch from counting toward the drop window: otherwise a - // 500, then hours offline, then one more 500 would see the whole - // gap as sustained backend failure and wipe the queue. - let (newCount, since): (Int, Date?) = stateLock.withLock { - retryCount += 1 - if isTransportFailure { - failingSince = nil - } else if failingSince == nil { - failingSince = now() - } - return (retryCount, failingSince) - } - - // Drop the queue only when a responsive-but-unhealthy backend has - // kept rejecting our batches for a sustained window. Transport - // failures are excluded: the events wait on disk for connectivity - // to return and `maxQueueSize` bounds growth, so a flaky network - // can never wipe the buffer. - if let since, now().timeIntervalSince(since) >= config.maxRetryWindowSeconds { - dropAllQueuedRecords(reason: "backend failing for over \(config.maxRetryWindowSeconds)s") - // Complete with `false`: `dropAllQueuedRecords` already cleared - // the batch from disk, and its `onRecordsDropped` callback may - // have synchronously enqueued the `$queue_records_dropped` - // diagnostic onto the now-empty queue. A `true` here would pop - // `items.count` records off the front and delete that fresh - // diagnostic (plus any event captured in the same window). - payload.completion(false) - return - } - + let newCount = nextRetryCount() let backoffDelay = min(TimeInterval(newCount) * retryDelay, maxRetryDelay) let delay = max(backoffDelay, result.retryAfter ?? 0) pauseFor(seconds: delay) @@ -234,30 +164,13 @@ class PostHogQueue { return } - // 413 Payload Too Large. Two paths: - // - cap > 1: this is a retry. Increment the 413 halving count, drop - // all if `maxRetries` exceeded, otherwise halve cap and retry the - // same records. - // - cap == 1: poison drop. The offending record can't shrink any - // further, so we drop the batch and apply the endpoint's poison - // cap policy. Don't count it as a retry — the drop *is* the - // resolution, not another attempt. + // A multi-record 413 is resolved by progressively shrinking the batch. + // Once a singleton still receives 413, only that poison entry is + // acknowledged for removal. Neither path clears unrelated records. if statusCode == 413 { let canHalve = batchLimitsLock.withLock { batchLimits.cap > 1 && payload.records.count > 1 } if canHalve { - let newCount = stateLock.withLock { () -> Int in - payloadTooLargeCount += 1 - return payloadTooLargeCount - } - if newCount > config.maxRetries { - dropAllQueuedRecords(reason: "max retries (\(config.maxRetries)) exceeded after repeated HTTP 413") - // Complete with `false` — see the window-drop path above: - // the queue was just cleared, so popping would delete the - // diagnostic the drop callback enqueued onto it. - payload.completion(false) - return - } let actualBatchSize = payload.records.count let halvedCap = batchLimitsLock.withLock { batchLimits.halve(actualBatchSize: actualBatchSize) @@ -267,48 +180,33 @@ class PostHogQueue { return } - // Cap stays at 1 — the offender is gone but we keep being - // cautious until a successful send. - hedgeLog("Queue: dropping batch after HTTP 413 (cap == 1)") - stateLock.withLock { - retryCount = 0 - payloadTooLargeCount = 0 - failingSince = nil - } + hedgeLog("Queue: dropping singleton batch after HTTP 413") + resetRetryState() payload.completion(true) return } - // 2xx success or non-retriable 4xx (auth, malformed, etc.): pop the - // batch. Cap stays where it is — no ramp on success. + // 2xx success or a terminal response removes the exact snapshotted + // entries. The adaptive cap stays where it is. + resetRetryState() + payload.completion(true) + } + + private func nextRetryCount() -> Int { stateLock.withLock { - retryCount = 0 - payloadTooLargeCount = 0 - failingSince = nil + let maximumBackoffStep = max(1, Int(maxRetryDelay / retryDelay)) + if retryCount < maximumBackoffStep { + retryCount += 1 + } + return retryCount } - payload.completion(true) } - /// Drops every queued record from disk and resets the retry / pause state. - /// Reserved for a backend that keeps rejecting our batches past - /// `maxRetryWindowSeconds`, or repeated HTTP 413 halving — never a network - /// blip. Reports the number of dropped records through `onRecordsDropped` - /// so the loss is measurable. Cap is left where it is — new records - /// starting against a known-bad backend benefit from the conservative cap - /// until proven otherwise. - private func dropAllQueuedRecords(reason: String) { - let dropped = fileQueue.depth - hedgeLog("Queue: dropping all queued records — \(reason)") - fileQueue.clear() + private func resetRetryState() { stateLock.withLock { retryCount = 0 - payloadTooLargeCount = 0 - failingSince = nil pausedUntil = nil } - if dropped > 0 { - onRecordsDropped?(dropped, reason) - } } func start(disableReachabilityForTesting: Bool, @@ -320,16 +218,7 @@ class PostHogQueue { // can all receive notifications without overwriting each other. reachableToken = reachability?.onReachable.subscribe { [weak self] reachability in guard let self else { return } - self.stateLock.withLock { - if self.config.dataMode == .wifi, reachability.connection != .wifi { - hedgeLog("Queue is paused because its not in WiFi mode") - self.paused = true - } else { - self.paused = false - } - } - - if reachability.connection == .wifi { + if self.updatePauseState(for: reachability) { self.flush() } } @@ -342,6 +231,13 @@ class PostHogQueue { } } + if let reachability { + // A shared notifier may already be running, so late queue + // subscribers must synchronously adopt its current state + // rather than waiting for a transition that may never come. + _ = updatePauseState(for: reachability) + } + do { try reachability?.startNotifier() } catch { @@ -451,18 +347,14 @@ class PostHogQueue { return } - if fileQueue.depth >= configuredMaxQueueSize { - hedgeLog("Queue is full, dropping oldest record") - // first is always oldest - fileQueue.delete(index: 0) - } - guard let data = endpoint.encode(record) else { hedgeLog("Tried to queue unserialisable record") return } - fileQueue.add(data) + if fileQueue.add(data, maxSize: configuredMaxQueueSize) != nil { + hedgeLog("Queue is full, dropping oldest record") + } hedgeLog("Queued \(endpoint.describe(record)). Depth: \(fileQueue.depth)") flushIfOverThreshold() } @@ -534,12 +426,12 @@ class PostHogQueue { return } - let items = self.fileQueue.peek(count) + let entries = self.fileQueue.peekEntries(count) var processing: [Record] = [] - for item in items { - guard let record = self.endpoint.decode(item) else { + for entry in entries { + guard let record = self.endpoint.decode(entry.data) else { continue } processing.append(record) @@ -547,8 +439,8 @@ class PostHogQueue { completion(PostHogConsumerPayload(records: processing) { [weak self] success in guard let self else { return } - if success, items.count > 0 { - self.fileQueue.pop(items.count) + if success, !entries.isEmpty { + self.fileQueue.remove(ids: entries.map(\.id)) hedgeLog("Completed!") } @@ -559,6 +451,26 @@ class PostHogQueue { } } + #if !os(watchOS) + /// Applies the latest shared reachability snapshot and returns whether + /// this queue may send under its configured data mode. + private func updatePauseState(for reachability: Reachability) -> Bool { + let connection = reachability.connection + return stateLock.withLock { + if connection == .unavailable { + hedgeLog("Queue is paused because network is unreachable") + paused = true + } else if config.dataMode == .wifi, connection != .wifi { + hedgeLog("Queue is paused because its not in WiFi mode") + paused = true + } else { + paused = false + } + return !paused + } + } + #endif + private func pauseFor(seconds: TimeInterval) { let until = now().addingTimeInterval(seconds) stateLock.withLock { pausedUntil = until } diff --git a/PostHog/PostHogSDK.swift b/PostHog/PostHogSDK.swift index 3bfca7bba..c91af0057 100644 --- a/PostHog/PostHogSDK.swift +++ b/PostHog/PostHogSDK.swift @@ -235,16 +235,6 @@ let maxRetryDelay = 30.0 logsQueue = PostHogQueue(config, theStorage, logsEndpoint) #endif - // Make a full queue drop measurable: emit a diagnostic event so - // the SDK stops discarding events silently. It queues like any - // other event and reaches PostHog once connectivity returns. - queue?.onRecordsDropped = { [weak self] count, reason in - self?.capture("$queue_records_dropped", properties: [ - "dropped_count": count, - "reason": reason, - ]) - } - queue?.start(disableReachabilityForTesting: config.disableReachabilityForTesting, disableQueueTimerForTesting: config.disableQueueTimerForTesting) diff --git a/PostHogTests/PostHogApiTest.swift b/PostHogTests/PostHogApiTest.swift index 891708b01..4d2f81c69 100644 --- a/PostHogTests/PostHogApiTest.swift +++ b/PostHogTests/PostHogApiTest.swift @@ -701,6 +701,25 @@ enum PostHogApiTests { } } + @Suite("Upload response handling", .serialized) + final class TestUploadResponseHandling { + @Test("preserves an HTTP status when URLSession also returns an error") + func preservesHTTPStatusAlongsideError() throws { + let url = try #require(URL(string: "http://localhost/batch")) + let httpResponse = try #require(HTTPURLResponse(url: url, statusCode: 503, httpVersion: nil, headerFields: nil)) + let error = URLError(.timedOut) + var uploadInfo: PostHogUploadInfo? + + processUploadResponse(endpointName: "batch", data: nil, response: httpResponse, error: error) { + uploadInfo = $0 + } + + let result = try #require(uploadInfo) + #expect(result.statusCode == 503) + #expect((result.error as? URLError)?.code == .timedOut) + } + } + @Suite("Non-HTTPURLResponse handling", .serialized) final class TestNonHTTPResponseHandling { private func getSut() -> PostHogApi { diff --git a/PostHogTests/PostHogFileBackedQueueTest.swift b/PostHogTests/PostHogFileBackedQueueTest.swift index 4f1ea6573..fc33ccfc4 100644 --- a/PostHogTests/PostHogFileBackedQueueTest.swift +++ b/PostHogTests/PostHogFileBackedQueueTest.swift @@ -97,6 +97,23 @@ class PostHogFileBackedQueueTest: QuickSpec { sut.clear() } + it("trims cached files to configured capacity on load") { + let baseUrl = applicationSupportDirectoryURL() + let newURL = baseUrl.appendingPathComponent("queue") + try FileManager.default.createDirectory(atPath: newURL.path, withIntermediateDirectories: true) + + for value in 0 ..< 3 { + try Data("cached-\(value)".utf8).write(to: newURL.appendingPathComponent("cached-\(value)")) + } + + let sut = PostHogFileBackedQueue(queue: newURL, maxSize: 2) + + expect(sut.depth) == 2 + expect(try? FileManager.default.contentsOfDirectory(atPath: newURL.path).count) == 2 + + sut.clear() + } + it("delete from queue and disk") { let baseUrl = applicationSupportDirectoryURL() let newURL = baseUrl.appendingPathComponent("queue") @@ -139,6 +156,51 @@ class PostHogFileBackedQueueTest: QuickSpec { sut.clear() } + it("removes exact stable entry identities") { + let sut = self.getSut() + let identicalData = self.eventJson.data(using: .utf8)! + + sut.add(identicalData) + sut.add(identicalData) + sut.add(identicalData) + + let entries = sut.peekEntries(3) + expect(Set(entries.map(\.id)).count) == 3 + expect(entries.map(\.data)) == [identicalData, identicalData, identicalData] + + sut.remove(ids: [entries[0].id, entries[2].id, "missing-entry"]) + + let remaining = sut.peekEntries(3) + expect(remaining.map(\.id)) == [entries[1].id] + expect(sut.depth) == 1 + expect(FileManager.default.fileExists(atPath: sut.queue.appendingPathComponent(entries[0].id).path)) == false + expect(FileManager.default.fileExists(atPath: sut.queue.appendingPathComponent(entries[1].id).path)) == true + expect(FileManager.default.fileExists(atPath: sut.queue.appendingPathComponent(entries[2].id).path)) == false + + sut.clear() + } + + it("enforces capacity atomically across concurrent adds") { + let sut = self.getSut() + let capacity = 3 + let group = DispatchGroup() + let producers = DispatchQueue(label: "com.posthog.file-queue-capacity-test", attributes: .concurrent) + + for value in 0 ..< 100 { + group.enter() + producers.async { + sut.add(Data("event-\(value)".utf8), maxSize: capacity) + group.leave() + } + } + + expect(group.wait(timeout: .now() + 5)) == .success + expect(sut.depth) == capacity + expect(try? FileManager.default.contentsOfDirectory(atPath: sut.queue.path).count) == capacity + + sut.clear() + } + it("add to queue and disk") { let baseUrl = applicationSupportDirectoryURL() let newURL = baseUrl.appendingPathComponent("queue") diff --git a/PostHogTests/PostHogLogsQueueTest.swift b/PostHogTests/PostHogLogsQueueTest.swift index 46bd055f2..ca8973ced 100644 --- a/PostHogTests/PostHogLogsQueueTest.swift +++ b/PostHogTests/PostHogLogsQueueTest.swift @@ -323,22 +323,11 @@ final class PostHogLogsQueueTests { #expect(queue.depth == 2) } - @Test("413 poison-drop does not consume the maxRetries budget — queue drains record-by-record") + @Test("413 halving reaches singleton poison drops and drains record-by-record") func handle413PoisonDropIsNotARetry() async throws { - // Pin down: cap=1 + 413 (poison drop) is a *resolution*, not a retry. - // It must not increment retryCount. - // - // Scenario: maxBatchSize=8 + 8 oversized records + default maxRetries=3. - // The cap halves 3 times (8→4→2→1, retryCount accumulating to 3) before - // reaching cap=1. If poison-drop counted as a retry, the next flush would - // push retryCount to 4 > 3 and fire dropAll — wiping ALL 8 records together. - // The correct behaviour treats poison-drop as a clean resolution: the - // offending record is popped, retryCount resets to 0, cap stays at 1, - // and the queue continues draining. - // - // Observable difference: the buggy path makes ~4 HTTP requests (3 - // halvings + 1 dropAll). The correct path makes far more — each record - // costs at least one halve cycle + one poison drop. + // Multi-record batches are retained while the cap halves to one. At + // cap one, each 413 removes only that singleton and leaves unrelated + // durable records for subsequent flushes. let (queue, _) = makeQueue(maxBufferSize: 100, maxBatchSize: 8, flushAt: 8) defer { queue.clear() queue.stop() @@ -365,43 +354,6 @@ final class PostHogLogsQueueTests { #expect(server.logsRequests.count > 8) } - @Test("drops the entire queue once retryCount exceeds maxRetries on repeated 413") - func handle413MaxRetriesDropsAll() async throws { - // Mirrors PostHogQueue's safeguard: a permanently-broken backend that - // keeps returning 413 should not leave the logs queue retrying forever. - // After config.maxRetries failed attempts, the queue drops everything - // and resets the retry / cap state. - let (queue, config) = makeQueue(maxBufferSize: 100, maxBatchSize: 64) - config.maxRetries = 1 - defer { queue.clear() - queue.stop() - } - - server.logsResponseHandler = { _, _ in - HTTPStubsResponse(jsonObject: ["error": "too large"], statusCode: 413, headers: nil) - } - - // Enough records that cap-halving alone won't drain the queue before - // retryCount exceeds maxRetries. - for i in 0 ..< 32 { - queue.add(makeRecord(body: "log-\(i)")) - } - await waitUntil { queue.depth == 32 } - - // Drive flushes until the queue drops everything via the maxRetries path. - // First flush: batchSize=32 → 413 → retryCount=1 (not > 1) → halve cap. - // Second flush: batchSize=16 → 413 → retryCount=2 (> 1) → drop ALL records. - queue.flush() - try? await Task.sleep(nanoseconds: 100_000_000) - queue.flush() - await waitUntil { queue.depth == 0 } - #expect(queue.depth == 0) - // Cap stays where it was when dropAll fired — no reset. Matches - // events / posthog-android behaviour: new records start at the - // halved cap until a successful send proves the backend is healthy. - #expect(queue.currentBatchCapForTesting == 16) - } - @Test("halves cap from min(cap, actualBatchSize) when queue depth was below cap") func handle413HalvesByActualBatchSize() async throws { // maxBatchSize=50 but only 4 records on disk. A 413 should halve from @@ -431,11 +383,6 @@ final class PostHogLogsQueueTests { #expect(queue.depth == 4) } - // Note: the 5xx path also goes through the maxRetries check → dropAllQueuedRecords, - // but testing it directly would require waiting out the 5+10+15s exponential backoff - // between attempts (`pausedUntil` blocks the next flush). The 413 maxRetries test - // above covers the shared drop logic without that latency since 413 doesn't pause. - @Test("non-413 4xx drops the batch (poison-pill)") func handleNon413_4xxDrops() async throws { let (queue, _) = makeQueue(maxBufferSize: 100, maxBatchSize: 100) @@ -743,10 +690,13 @@ final class PostHogLogsQueueTests { reachability.onUnreachable.invoke(reachability) queue.add(makeRecord(body: "while-offline")) - queue.flush() + for _ in 0 ..< 3 { + queue.flush() + } try? await Task.sleep(nanoseconds: 300_000_000) #expect(server.logsRequests.isEmpty) #expect(queue.depth == 1) + #expect(queue.currentRetryCountForTesting == 0) // Simulate WiFi back. The reachable callback unpauses and proactively // triggers a flush. diff --git a/PostHogTests/PostHogQueueTest.swift b/PostHogTests/PostHogQueueTest.swift index 854a49b7c..85ff3f7f6 100644 --- a/PostHogTests/PostHogQueueTest.swift +++ b/PostHogTests/PostHogQueueTest.swift @@ -13,14 +13,31 @@ import OHHTTPStubsSwift import Quick import XCTest +private final class ControlledBatchSender { + private let lock = NSLock() + private var completions = [(PostHogUploadInfo) -> Void]() + + var requestCount: Int { + lock.withLock { completions.count } + } + + func send(_: [PostHogEvent], completion: @escaping (PostHogUploadInfo) -> Void) { + lock.withLock { completions.append(completion) } + } + + func completeRequest(at index: Int, with result: PostHogUploadInfo) { + let completion = lock.withLock { completions[index] } + completion(result) + } +} + class PostHogQueueTest: QuickSpec { - func getSut(flushAt: Int = 1, maxQueueSize: Int = 1000, maxBatchSize: Int = 50, maxRetries: Int = 3, maxRetryWindowSeconds: TimeInterval = 24 * 60 * 60) -> PostHogQueue { + func getSut(flushAt: Int = 1, maxQueueSize: Int = 1000, maxBatchSize: Int = 50, maxRetries: Int = 3) -> PostHogQueue { let config = PostHogConfig(projectToken: testProjectToken, host: "http://localhost:9001") config.flushAt = flushAt config.maxQueueSize = maxQueueSize config.maxBatchSize = maxBatchSize config.maxRetries = maxRetries - config.maxRetryWindowSeconds = maxRetryWindowSeconds config.sendFeatureFlagEvent = false let storage = PostHogStorage(config) let api = PostHogApi(config) @@ -186,114 +203,44 @@ class PostHogQueueTest: QuickSpec { sut.clear() } - it("drops the entire queue once the 413 halving count exceeds maxRetries") { - // A 413 with cap > 1 increments the dedicated 413 halving count - // and drops via the `newCount > config.maxRetries` check. Only the - // 413 path is bounded by `maxRetries`; transport and 5xx failures - // use a separate counter and the time window. We use 413 here - // because it doesn't set `pausedUntil`, letting the test drive - // multiple halving attempts without waiting out the exponential - // backoff. - // - // 20 events with maxBatchSize=20 so halving sequence is 10 → 5 - // → drop — cap doesn't reach 1 before maxRetries=2 is exceeded - // on the third attempt; the maxRetries cap fires first instead - // of the poison-drop path. Each flush is awaited via - // `currentBatchCapForTesting` so the gate inside `take()` - // doesn't swallow back-to-back calls. - let sut = self.getSut(flushAt: 100, maxBatchSize: 20, maxRetries: 2) - server.start(batchCount: 3) - server.batchResponseHandler = { _, _ in - HTTPStubsResponse(jsonObject: [], statusCode: 413, headers: nil) - } - - for i in 0 ..< 20 { - sut.add(PostHogEvent(event: "evt\(i)", distinctId: "id")) - } - - sut.flush() - expect(sut.currentBatchCapForTesting).toEventually(equal(10)) - sut.flush() - expect(sut.currentBatchCapForTesting).toEventually(equal(5)) - sut.flush() - expect(sut.depth).toEventually(equal(0)) - - sut.clear() - } - - it("maxRetries drop wipes the entire queue, not just the current batch") { - // Multiple events queued. After enough 413s to trip maxRetries, - // the drop must clear ALL of them, not just whatever batch was - // in flight. - let sut = self.getSut(flushAt: 100, maxBatchSize: 50, maxRetries: 1) - server.start(batchCount: 2) + it("retains batch on retriable 5xx and does not change cap") { + let sut = self.getSut(flushAt: 2, maxBatchSize: 4) server.batchResponseHandler = { _, _ in - HTTPStubsResponse(jsonObject: [], statusCode: 413, headers: nil) + HTTPStubsResponse(jsonObject: [], statusCode: 500, headers: nil) } - for i in 0 ..< 5 { - sut.add(PostHogEvent(event: "event\(i)", distinctId: "id\(i)")) - } + sut.add(PostHogEvent(event: "event1", distinctId: "id1")) + sut.add(PostHogEvent(event: "event2", distinctId: "id2")) - // First flush: batch=5 → 413 → retryCount=1 (not > 1) → halve. - sut.flush() - expect(sut.currentBatchCapForTesting).toEventually(equal(2)) - expect(sut.depth) == 5 + _ = getBatchedEvents(server) - // Second flush: batch=2 → 413 → retryCount=2 (> 1) → drop ALL, - // not just the 2 in this batch. - sut.flush() - expect(sut.depth).toEventually(equal(0)) + expect(sut.depth).toEventually(equal(2)) + expect(sut.currentBatchCapForTesting).toEventually(equal(4)) sut.clear() } - it("queue keeps working after a maxRetries drop — retryCount is reset") { - // After events get dropped the queue must continue to accept and - // flush new ones; otherwise the SDK is permanently broken until - // the host app restarts. - let sut = self.getSut(flushAt: 100, maxBatchSize: 50, maxRetries: 1) - var attempt = 0 - server.start(batchCount: 3) + it("retains batch on HTTP 429 and does not change cap") { + let sut = self.getSut(flushAt: 2, maxBatchSize: 4) server.batchResponseHandler = { _, _ in - attempt += 1 - // First two attempts fail with 413 → triggers maxRetries drop. - // Third attempt (the post-drop add) succeeds. - return attempt <= 2 - ? HTTPStubsResponse(jsonObject: [], statusCode: 413, headers: nil) - : HTTPStubsResponse(jsonObject: ["status": "ok"], statusCode: 200, headers: nil) - } - - for i in 0 ..< 5 { - sut.add(PostHogEvent(event: "doomed\(i)", distinctId: "id\(i)")) + HTTPStubsResponse(jsonObject: [], statusCode: 429, headers: nil) } - sut.flush() - expect(sut.currentBatchCapForTesting).toEventually(equal(2)) - sut.flush() - expect(sut.depth).toEventually(equal(0)) + sut.add(PostHogEvent(event: "event1", distinctId: "id1")) + sut.add(PostHogEvent(event: "event2", distinctId: "id2")) - // dropAll never resets the adaptive cap — it stays where the - // last 413 left it, for both events and logs. New records start - // against the conservative cap until a successful send proves - // the backend is healthy. - expect(sut.currentBatchCapForTesting) == 2 + _ = getBatchedEvents(server) - // New event after the drop should flush successfully — retryCount - // and pausedUntil were reset by dropAllQueuedEvents. - sut.add(PostHogEvent(event: "after-drop", distinctId: "id")) - sut.flush() - let events = getBatchedEvents(server) - expect(events.contains(where: { $0.event == "after-drop" })) == true - expect(sut.depth).toEventually(equal(0)) + expect(sut.depth).toEventually(equal(2)) + expect(sut.currentBatchCapForTesting).toEventually(equal(4)) sut.clear() } - it("retains batch on retriable 5xx and does not change cap") { + it("retains batch on HTTP 408 (request timeout is retriable) and does not change cap") { let sut = self.getSut(flushAt: 2, maxBatchSize: 4) server.batchResponseHandler = { _, _ in - HTTPStubsResponse(jsonObject: [], statusCode: 500, headers: nil) + HTTPStubsResponse(jsonObject: [], statusCode: 408, headers: nil) } sut.add(PostHogEvent(event: "event1", distinctId: "id1")) @@ -307,44 +254,125 @@ class PostHogQueueTest: QuickSpec { sut.clear() } - it("retains batch on HTTP 429 and does not change cap") { - let sut = self.getSut(flushAt: 2, maxBatchSize: 4) - server.batchResponseHandler = { _, _ in - HTTPStubsResponse(jsonObject: [], statusCode: 429, headers: nil) + it("retains retryable HTTP failures past maxRetries and drains after recovery") { + let mockNow = MockDate() + now = { mockNow.date } + defer { now = { Date() } } + + let sut = self.getSut(flushAt: 100, maxBatchSize: 4, maxRetries: 1) + server.start(batchCount: 4) + server.batchResponseHandler = { _, requestNumber in + requestNumber <= 3 + ? HTTPStubsResponse(jsonObject: [], statusCode: 503, headers: nil) + : HTTPStubsResponse(jsonObject: ["status": "ok"], statusCode: 200, headers: nil) } sut.add(PostHogEvent(event: "event1", distinctId: "id1")) sut.add(PostHogEvent(event: "event2", distinctId: "id2")) - _ = getBatchedEvents(server) + for expectedAttempt in 1 ... 3 { + sut.flush() + expect(server.batchRequests.count).toEventually(equal(expectedAttempt)) + expect(sut.currentRetryCountForTesting).toEventually(equal(expectedAttempt)) + expect(sut.depth) == 2 + mockNow.date.addTimeInterval(60) + } - expect(sut.depth).toEventually(equal(2)) - expect(sut.currentBatchCapForTesting).toEventually(equal(4)) + sut.flush() + expect(server.batchRequests.count).toEventually(equal(4)) + expect(sut.depth).toEventually(equal(0)) + expect(sut.currentRetryCountForTesting).toEventually(equal(0)) sut.clear() } - it("retains batch on HTTP 408 (request timeout is retriable) and does not change cap") { - let sut = self.getSut(flushAt: 2, maxBatchSize: 4) - server.batchResponseHandler = { _, _ in - HTTPStubsResponse(jsonObject: [], statusCode: 408, headers: nil) + it("retains transport failures past maxRetries and drains after recovery") { + let mockNow = MockDate() + now = { mockNow.date } + defer { now = { Date() } } + + let sut = self.getSut(flushAt: 100, maxBatchSize: 4, maxRetries: 0) + let networkError = NSError(domain: NSURLErrorDomain, code: NSURLErrorNetworkConnectionLost, userInfo: nil) + server.start(batchCount: 4) + server.batchResponseHandler = { _, requestNumber in + requestNumber <= 3 + ? HTTPStubsResponse(error: networkError) + : HTTPStubsResponse(jsonObject: ["status": "ok"], statusCode: 200, headers: nil) } sut.add(PostHogEvent(event: "event1", distinctId: "id1")) sut.add(PostHogEvent(event: "event2", distinctId: "id2")) - _ = getBatchedEvents(server) + for expectedAttempt in 1 ... 3 { + sut.flush() + expect(server.batchRequests.count).toEventually(equal(expectedAttempt)) + expect(sut.currentRetryCountForTesting).toEventually(equal(expectedAttempt)) + expect(sut.depth) == 2 + mockNow.date.addTimeInterval(60) + } - expect(sut.depth).toEventually(equal(2)) - expect(sut.currentBatchCapForTesting).toEventually(equal(4)) + sut.flush() + expect(server.batchRequests.count).toEventually(equal(4)) + expect(sut.depth).toEventually(equal(0)) sut.clear() } + it("late success removes exact in-flight identities after full-capacity replacement") { + let config = PostHogConfig(projectToken: "queue_identity_\(UUID().uuidString)", host: "http://localhost:9001") + config.flushAt = 100 + config.maxQueueSize = 2 + config.maxBatchSize = 2 + let storage = PostHogStorage(config) + let sender = ControlledBatchSender() + let endpoint = QueueEndpoint( + storageKey: .queue, + oldStorageKeys: [], + dispatchQueueLabel: "com.posthog.Queue.IdentityTest", + initialCap: { $0.maxBatchSize }, + initialFlushAt: { $0.flushAt }, + maxQueueSize: { $0.maxQueueSize }, + flushIntervalSeconds: { $0.flushIntervalSeconds }, + rateCapMax: { _ in 0 }, + rateCapWindowSeconds: { _ in 0 }, + encode: { toJSONData($0.toJSON()) }, + decode: { PostHogEvent.fromJSON($0) }, + describe: { $0.event }, + send: sender.send, + isRetriableStatusCode: { _ in false } + ) + let sut = PostHogQueue(config, storage, endpoint, nil) + defer { sut.clear() } + let identicalEvent = PostHogEvent(event: "identical", distinctId: "same-id") + + sut.add(identicalEvent) + sut.add(identicalEvent) + let inFlightIds = sut.fileQueue.peekEntries(2).map(\.id) + + sut.flush() + expect(sender.requestCount).toEventually(equal(1)) + + // Adding while full evicts the first in-flight entry and appends a + // byte-identical replacement with a new durable identity. + sut.add(identicalEvent) + let replacementId = sut.fileQueue.peekEntries(2).last!.id + expect(inFlightIds).notTo(contain(replacementId)) + + sender.completeRequest(at: 0, with: PostHogUploadInfo(statusCode: 200, error: nil)) + + expect(sut.depth).toEventually(equal(1)) + expect(sut.fileQueue.peekEntries(2).map(\.id)) == [replacementId] + + sut.flush() + expect(sender.requestCount).toEventually(equal(2)) + sender.completeRequest(at: 1, with: PostHogUploadInfo(statusCode: 200, error: nil)) + expect(sut.depth).toEventually(equal(0)) + } + it("halves cap repeatedly across multiple 413s and drops once cap reaches 1") { // flushAt is high so add() doesn't trigger an auto-flush — we drive // each flush manually to observe the multi-step halving sequence. - let sut = self.getSut(flushAt: 100, maxBatchSize: 4) + let sut = self.getSut(flushAt: 100, maxBatchSize: 4, maxRetries: 0) server.start(batchCount: 3) server.batchResponseHandler = { _, _ in HTTPStubsResponse(jsonObject: [], statusCode: 413, headers: nil) @@ -411,226 +439,6 @@ class PostHogQueueTest: QuickSpec { sut.clear() } - it("never drops the queue on repeated network failures, even past the retry window") { - // The reported bug: ~6s of flaky connectivity used to wipe every - // buffered event. Transport failures (-1) must keep the queue no - // matter how many happen or how long they last — the window here - // is tiny to prove elapsed time alone can't trigger a drop. - let mockNow = MockDate() - now = { mockNow.date } - defer { now = { Date() } } - - let sut = self.getSut(flushAt: 100, maxBatchSize: 4, maxRetries: 3, maxRetryWindowSeconds: 1) - let networkError = NSError(domain: NSURLErrorDomain, code: NSURLErrorNotConnectedToInternet, userInfo: nil) - server.start(batchCount: 6) - server.batchResponseHandler = { _, _ in - HTTPStubsResponse(error: networkError) - } - - for i in 0 ..< 3 { - sut.add(PostHogEvent(event: "event\(i)", distinctId: "id\(i)")) - } - - for attempt in 1 ... 6 { - sut.flush() - // Wait for the failure to land, then step past the backoff pause - // and the retry window before the next attempt. - expect(sut.currentRetryCountForTesting).toEventually(equal(attempt)) - expect(sut.depth) == 3 - mockNow.date.addTimeInterval(60) - } - - expect(sut.depth) == 3 - - sut.clear() - } - - it("a 413 after transport failures still halves instead of wiping the queue") { - // Regression: transport failures used to feed the same counter the - // 413 halving budget reads, so a burst of network flapping could - // push `retryCount` past `maxRetries` and make the very first 413 - // drop every buffered record without one halving attempt. Transport - // failures must not consume the 413 budget. - let mockNow = MockDate() - now = { mockNow.date } - defer { now = { Date() } } - - let sut = self.getSut(flushAt: 100, maxBatchSize: 20, maxRetries: 2) - let networkError = NSError(domain: NSURLErrorDomain, code: NSURLErrorNotConnectedToInternet, userInfo: nil) - var attempt = 0 - server.start(batchCount: 4) - server.batchResponseHandler = { _, _ in - attempt += 1 - // First three attempts are transport failures (flaky network); - // afterwards the backend responds 413. - return attempt <= 3 - ? HTTPStubsResponse(error: networkError) - : HTTPStubsResponse(jsonObject: [], statusCode: 413, headers: nil) - } - - for i in 0 ..< 20 { - sut.add(PostHogEvent(event: "evt\(i)", distinctId: "id")) - } - - // Three transport failures push retryCount to 3 (> maxRetries=2) - // without ever dropping the queue or touching the batch cap. - for attemptCount in 1 ... 3 { - sut.flush() - expect(sut.currentRetryCountForTesting).toEventually(equal(attemptCount)) - expect(sut.depth) == 20 - expect(sut.currentBatchCapForTesting) == 20 - // Step past the backoff pause before the next attempt. - mockNow.date.addTimeInterval(60) - } - - // The first 413 now arrives. Despite retryCount already exceeding - // maxRetries, the 413 budget is separate and starts fresh, so the - // cap halves and the batch is retained rather than wiped. - sut.flush() - expect(sut.currentBatchCapForTesting).toEventually(equal(10)) - expect(sut.depth) == 20 - - sut.clear() - } - - it("drops the queue after sustained server failures and reports the dropped count") { - let mockNow = MockDate() - now = { mockNow.date } - defer { now = { Date() } } - - let sut = self.getSut(flushAt: 100, maxBatchSize: 4, maxRetryWindowSeconds: 10) - var droppedCount = 0 - var droppedReason = "" - sut.onRecordsDropped = { count, reason in - droppedCount = count - droppedReason = reason - } - server.start(batchCount: 2) - server.batchResponseHandler = { _, _ in - HTTPStubsResponse(jsonObject: [], statusCode: 500, headers: nil) - } - - for i in 0 ..< 3 { - sut.add(PostHogEvent(event: "event\(i)", distinctId: "id\(i)")) - } - - // First failure starts the failure streak; the queue is retained. - sut.flush() - expect(sut.currentRetryCountForTesting).toEventually(equal(1)) - expect(sut.depth) == 3 - - // Past the window, the next server failure drops the whole queue and - // reports it so the loss is measurable. - mockNow.date.addTimeInterval(11) - sut.flush() - expect(sut.depth).toEventually(equal(0)) - expect(droppedCount) == 3 - expect(droppedReason).to(contain("backend failing")) - - sut.clear() - } - - it("a transport failure resets the backend failure window so an offline gap can't trigger a drop") { - // Regression: an initial 500 armed the failure clock, then a long - // offline stretch (transport failures) elapsed without clearing it, - // so the next 500 saw the whole offline gap as sustained backend - // failure and wiped the queue on only two error responses. A - // transport failure must reset the window; offline time must not - // count toward it. - let mockNow = MockDate() - now = { mockNow.date } - defer { now = { Date() } } - - let sut = self.getSut(flushAt: 100, maxBatchSize: 4, maxRetryWindowSeconds: 10) - var dropped = false - sut.onRecordsDropped = { _, _ in dropped = true } - - let networkError = NSError(domain: NSURLErrorDomain, code: NSURLErrorNotConnectedToInternet, userInfo: nil) - var attempt = 0 - server.start(batchCount: 3) - server.batchResponseHandler = { _, _ in - attempt += 1 - switch attempt { - case 1: return HTTPStubsResponse(jsonObject: [], statusCode: 500, headers: nil) // arms the window - case 2: return HTTPStubsResponse(error: networkError) // offline: must reset the window - default: return HTTPStubsResponse(jsonObject: [], statusCode: 500, headers: nil) // fresh window, no drop - } - } - - for i in 0 ..< 3 { - sut.add(PostHogEvent(event: "event\(i)", distinctId: "id\(i)")) - } - - // First 500 arms the failure streak. - sut.flush() - expect(sut.currentRetryCountForTesting).toEventually(equal(1)) - expect(sut.depth) == 3 - - // A long offline gap elapses, then a transport failure lands. Even - // though more than the window has passed since the first 500, the - // transport failure resets the clock and the queue is kept. - mockNow.date.addTimeInterval(11) - sut.flush() - expect(sut.currentRetryCountForTesting).toEventually(equal(2)) - expect(sut.depth) == 3 - - // A second 500 now, still past where the *original* window would - // have elapsed. Because the transport failure reset the clock, the - // offline time doesn't count: the queue survives instead of being - // wiped on two error responses. - mockNow.date.addTimeInterval(5) - sut.flush() - expect(sut.currentRetryCountForTesting).toEventually(equal(3)) - expect(sut.depth) == 3 - expect(dropped) == false - - sut.clear() - } - - it("keeps the diagnostic that onRecordsDropped enqueues during a full drop") { - // Regression: dropAllQueuedRecords fires onRecordsDropped, which the - // SDK wires to synchronously capture `$queue_records_dropped` back - // onto this same queue. The drop-path completion must NOT then pop: - // clear() already emptied the batch and `pop` deletes by position, - // so a pop would silently delete that fresh diagnostic (and any - // event captured in the same window), leaving the drop as invisible - // as it was before the feature existed. - let mockNow = MockDate() - now = { mockNow.date } - defer { now = { Date() } } - - let sut = self.getSut(flushAt: 100, maxBatchSize: 4, maxRetryWindowSeconds: 10) - - // Mimic the SDK's synchronous re-capture on drop: enqueue one - // diagnostic record onto the same queue from the callback. - sut.onRecordsDropped = { [weak sut] _, _ in - sut?.add(PostHogEvent(event: "$queue_records_dropped", distinctId: "id")) - } - - server.start(batchCount: 2) - server.batchResponseHandler = { _, _ in - HTTPStubsResponse(jsonObject: [], statusCode: 500, headers: nil) - } - - for i in 0 ..< 3 { - sut.add(PostHogEvent(event: "event\(i)", distinctId: "id\(i)")) - } - - // First 500 arms the failure window. - sut.flush() - expect(sut.currentRetryCountForTesting).toEventually(equal(1)) - expect(sut.depth) == 3 - - // Past the window: the next 500 drops all 3 and the callback - // enqueues the diagnostic. It must survive on disk (depth 1), not - // be popped away to 0. - mockNow.date.addTimeInterval(11) - sut.flush() - expect(sut.depth).toEventually(equal(1)) - - sut.clear() - } - it("pops batch on non-retriable 4xx so a poison record cannot block the queue") { let sut = self.getSut(flushAt: 2, maxBatchSize: 4) server.batchResponseHandler = { _, _ in diff --git a/api/posthog-ios.public-api.txt b/api/posthog-ios.public-api.txt index 0cb397d4b..f304d3e74 100644 --- a/api/posthog-ios.public-api.txt +++ b/api/posthog-ios.public-api.txt @@ -78,7 +78,6 @@ PostHog | PostHogConfig.logs | property | @objc let logs: PostHogLogsConfig | c: PostHog | PostHogConfig.maxBatchSize | property | @objc var maxBatchSize: Int | c:@M@PostHog@objc(cs)PostHogConfig(py)maxBatchSize PostHog | PostHogConfig.maxQueueSize | property | @objc var maxQueueSize: Int | c:@M@PostHog@objc(cs)PostHogConfig(py)maxQueueSize PostHog | PostHogConfig.maxRetries | property | @objc var maxRetries: Int | c:@M@PostHog@objc(cs)PostHogConfig(py)maxRetries -PostHog | PostHogConfig.maxRetryWindowSeconds | property | @objc var maxRetryWindowSeconds: TimeInterval | c:@M@PostHog@objc(cs)PostHogConfig(py)maxRetryWindowSeconds PostHog | PostHogConfig.optOut | property | @objc var optOut: Bool | c:@M@PostHog@objc(cs)PostHogConfig(py)optOut PostHog | PostHogConfig.personProfiles | property | @objc var personProfiles: PostHogPersonProfiles | c:@M@PostHog@objc(cs)PostHogConfig(py)personProfiles PostHog | PostHogConfig.preloadFeatureFlags | property | @objc var preloadFeatureFlags: Bool | c:@M@PostHog@objc(cs)PostHogConfig(py)preloadFeatureFlags From 3233d8bd9f590cbe836a37b8f9d193bc3aaa2687 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Mon, 31 Aug 2026 18:55:32 -0400 Subject: [PATCH 09/17] test(queue): verify FIFO trimming on reload --- PostHogTests/PostHogFileBackedQueueTest.swift | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/PostHogTests/PostHogFileBackedQueueTest.swift b/PostHogTests/PostHogFileBackedQueueTest.swift index fc33ccfc4..f4f0fb2c5 100644 --- a/PostHogTests/PostHogFileBackedQueueTest.swift +++ b/PostHogTests/PostHogFileBackedQueueTest.swift @@ -102,14 +102,30 @@ class PostHogFileBackedQueueTest: QuickSpec { let newURL = baseUrl.appendingPathComponent("queue") try FileManager.default.createDirectory(atPath: newURL.path, withIntermediateDirectories: true) - for value in 0 ..< 3 { - try Data("cached-\(value)".utf8).write(to: newURL.appendingPathComponent("cached-\(value)")) + let oldestURL = newURL.appendingPathComponent("cached-oldest") + let middleURL = newURL.appendingPathComponent("cached-middle") + let newestURL = newURL.appendingPathComponent("cached-newest") + let oldestData = Data("cached-0".utf8) + let middleData = Data("cached-1".utf8) + let newestData = Data("cached-2".utf8) + + for (url, data, creationDate, modificationDate) in [ + (newestURL, newestData, Date(timeIntervalSince1970: 300), Date(timeIntervalSince1970: 100)), + (oldestURL, oldestData, Date(timeIntervalSince1970: 100), Date(timeIntervalSince1970: 300)), + (middleURL, middleData, Date(timeIntervalSince1970: 200), Date(timeIntervalSince1970: 200)), + ] { + try data.write(to: url) + try FileManager.default.setAttributes([.modificationDate: modificationDate], ofItemAtPath: url.path) + try FileManager.default.setAttributes([.creationDate: creationDate], ofItemAtPath: url.path) } let sut = PostHogFileBackedQueue(queue: newURL, maxSize: 2) expect(sut.depth) == 2 - expect(try? FileManager.default.contentsOfDirectory(atPath: newURL.path).count) == 2 + expect(sut.peek(2)) == [middleData, newestData] + expect(FileManager.default.fileExists(atPath: oldestURL.path)) == false + expect(FileManager.default.fileExists(atPath: middleURL.path)) == true + expect(FileManager.default.fileExists(atPath: newestURL.path)) == true sut.clear() } From 2e86090fda4a59af9a4fae1a308205b04d28b16f Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:50:25 +0000 Subject: [PATCH 10/17] fix(queue): clear reachability pause when the notifier fails to start The initial reachability state check can pause the queue before startNotifier() runs. Reachability.connection reports .unavailable both for a genuinely offline device and for a failed flags read, and the same flags read is what makes startNotifier() throw, so the two failures are correlated. With no notifier running, no onReachable callback can ever clear the pause, leaving the events, replay, and logs queues paused for the process lifetime and dropping records at the queue cap without ever attempting a send. Clear the pause in the catch so a dead notifier leaves the queue sending, matching the previous behaviour where URLSession errors and the retry backoff handled connectivity. Generated-By: PostHog Desktop Task-Id: b3234784-8ba6-44c2-9a45-4e81711a5320 --- PostHog/PostHogQueue.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/PostHog/PostHogQueue.swift b/PostHog/PostHogQueue.swift index 8d67e6240..06baf7a83 100644 --- a/PostHog/PostHogQueue.swift +++ b/PostHog/PostHogQueue.swift @@ -241,7 +241,11 @@ class PostHogQueue { do { try reachability?.startNotifier() } catch { + // Without a running notifier no callback can ever clear a + // reachability pause, so it must not outlive the failure. + // URLSession errors and the retry backoff cover connectivity. hedgeLog("Error: Unable to monitor network reachability: \(error)") + stateLock.withLock { paused = false } } #endif } From ebfbf4752e81fe2bdaefcf6759bfc1b5e8c68944 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:53:35 +0000 Subject: [PATCH 11/17] fix(queue): enumerate the queue directory under itemsLock on reload reloadFromDisk read the queue directory outside itemsLock and then replaced items from that snapshot under the lock. A filename appended by a concurrent add between the two steps was silently dropped from the in-memory index while its file stayed on disk, so the record could not be sent until the next reload or process start, and the stale count could let the physical queue exceed its configured capacity. Session replay opens exactly this window on purpose: it flips hasPassedMinimumDuration before migrateBufferToQueue so new snapshots route straight to the inner queue, and migrateAll finishes with target reloadFromDisk on the buffer IO queue while capture keeps adding. Fold the enumeration and the replacement into one critical section by moving both into reindexFromDisk, which replaces replaceItemsWithBounded and is now the single re-index path for setup and reload. Capacity trimming and the file deletes it triggers are unchanged and still run outside the lock. Generated-By: PostHog Desktop Task-Id: b3234784-8ba6-44c2-9a45-4e81711a5320 --- PostHog/PostHogFileBackedQueue.swift | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/PostHog/PostHogFileBackedQueue.swift b/PostHog/PostHogFileBackedQueue.swift index 5653b1c0a..813784e25 100644 --- a/PostHog/PostHogFileBackedQueue.swift +++ b/PostHog/PostHogFileBackedQueue.swift @@ -49,9 +49,7 @@ class PostHogFileBackedQueue { } do { - // when copying over buffered snapshots, content modification date will change, so we work off creation date instead. - let sortedItems = try FileManager.default.contentsOfDirectory(at: queue, sortedBy: .creationDateKey) - replaceItemsWithBounded(sortedItems) + try reindexFromDisk() } catch { hedgeLog("Failed to load files for queue \(error)") // failed to read directory – bad permissions, perhaps? @@ -133,23 +131,30 @@ class PostHogFileBackedQueue { /// Use after externally adding files to the queue directory. func reloadFromDisk() { do { - let sortedItems = try FileManager.default.contentsOfDirectory(at: queue, sortedBy: .creationDateKey) - replaceItemsWithBounded(sortedItems) + try reindexFromDisk() } catch { hedgeLog("Failed to reload files for queue \(error)") } } - private func replaceItemsWithBounded(_ sortedItems: [String]) { - let overflow = maxSize.map { max(0, sortedItems.count - $0) } ?? 0 - let dropped = sortedItems.prefix(overflow) - itemsLock.withLock { items = Array(sortedItems.dropFirst(overflow)) } + /// Re-reads the queue directory and replaces the in-memory index with it, + /// enforcing the FIFO capacity. The enumeration runs inside `itemsLock` so a + /// filename appended by a concurrent `add` can't be dropped from the index by + /// the replacement while its file stays on disk. + private func reindexFromDisk() throws { + let dropped: [String] = try itemsLock.withLock { + // when copying over buffered snapshots, content modification date will change, so we work off creation date instead. + let sortedItems = try FileManager.default.contentsOfDirectory(at: queue, sortedBy: .creationDateKey) + let overflow = maxSize.map { max(0, sortedItems.count - $0) } ?? 0 + items = Array(sortedItems.dropFirst(overflow)) + return Array(sortedItems.prefix(overflow)) + } for item in dropped { deleteSafely(queue.appendingPathComponent(item)) } - if overflow > 0 { - hedgeLog("Dropped \(overflow) oldest cached records to enforce queue capacity") + if !dropped.isEmpty { + hedgeLog("Dropped \(dropped.count) oldest cached records to enforce queue capacity") } } From 3db55302c0532f82e96f0de2f6ba4874608c8305 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 21:03:14 +0000 Subject: [PATCH 12/17] fix(api): keep Retry-After when a response arrives with an error URLSession can hand back both an HTTPURLResponse and an error when the transfer fails after the headers arrive. processUploadResponse parsed the Retry-After header below the error branch, so on that path the header was dropped and only the local backoff applied: the queue retried after its own delay and the push subscription handler fell back to its exponential backoff, both ignoring the delay a 429 or 503 asked for. Parse the header once, before the error branch, and pass it on both paths. The queue already takes max(backoff, retryAfter), so this can only lengthen its pause; the push handler prefers the server value, matching what it already does for a plain 429. The no-response path is unchanged - there are no headers to read there. Adds a response-plus-error test covering a 429 with Retry-After alongside the existing status-preservation test. Generated-By: PostHog Desktop Task-Id: 5cb2da82-834d-4210-b996-d20454cc7e22 --- PostHog/PostHogApi.swift | 6 ++++-- PostHogTests/PostHogApiTest.swift | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/PostHog/PostHogApi.swift b/PostHog/PostHogApi.swift index a4e3b3e60..25657f793 100644 --- a/PostHog/PostHogApi.swift +++ b/PostHog/PostHogApi.swift @@ -18,10 +18,13 @@ func processUploadResponse( completion: @escaping (PostHogUploadInfo) -> Void ) { let httpResponse = response as? HTTPURLResponse + // Parsed before the error branch: URLSession can deliver headers and then fail the + // transfer, and a rate-limited response still carries the delay the server asked for. + let retryAfter = httpResponse.flatMap { $0.value(forHTTPHeaderField: "Retry-After") }.flatMap(parseRetryAfter) if let error { hedgeLog("Error calling the \(endpointName) API: \(error).") - return completion(PostHogUploadInfo(statusCode: httpResponse?.statusCode, error: error)) + return completion(PostHogUploadInfo(statusCode: httpResponse?.statusCode, error: error, retryAfter: retryAfter)) } guard let httpResponse else { @@ -36,7 +39,6 @@ func processUploadResponse( hedgeLog("\(endpointName) sent successfully.") } - let retryAfter = httpResponse.value(forHTTPHeaderField: "Retry-After").flatMap(parseRetryAfter) completion(PostHogUploadInfo(statusCode: httpResponse.statusCode, error: nil, retryAfter: retryAfter)) } diff --git a/PostHogTests/PostHogApiTest.swift b/PostHogTests/PostHogApiTest.swift index 4d2f81c69..96caef95c 100644 --- a/PostHogTests/PostHogApiTest.swift +++ b/PostHogTests/PostHogApiTest.swift @@ -718,6 +718,22 @@ enum PostHogApiTests { #expect(result.statusCode == 503) #expect((result.error as? URLError)?.code == .timedOut) } + + @Test("preserves Retry-After when URLSession also returns an error") + func preservesRetryAfterAlongsideError() throws { + let url = try #require(URL(string: "http://localhost/batch")) + let httpResponse = try #require(HTTPURLResponse(url: url, statusCode: 429, httpVersion: nil, headerFields: ["Retry-After": "120"])) + let error = URLError(.networkConnectionLost) + var uploadInfo: PostHogUploadInfo? + + processUploadResponse(endpointName: "batch", data: nil, response: httpResponse, error: error) { + uploadInfo = $0 + } + + let result = try #require(uploadInfo) + #expect(result.statusCode == 429) + #expect(result.retryAfter == 120) + } } @Suite("Non-HTTPURLResponse handling", .serialized) From 1332b7acd6834f6c9b6b13fbd9041ec4ba831b31 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Mon, 7 Sep 2026 16:32:15 -0400 Subject: [PATCH 13/17] fix(queue): preserve existing records when enqueue fails --- .changeset/durable-queue-lifecycle.md | 7 +- PostHog/PostHogFileBackedQueue.swift | 3 +- PostHogTests/PostHogApiTest.swift | 8 +- .../PostHogFileBackedQueueAlignmentTest.swift | 31 +++++ PostHogTests/PostHogQueueTest.swift | 130 ++++++++++++++++-- 5 files changed, 165 insertions(+), 14 deletions(-) diff --git a/.changeset/durable-queue-lifecycle.md b/.changeset/durable-queue-lifecycle.md index 409088342..9534d1e89 100644 --- a/.changeset/durable-queue-lifecycle.md +++ b/.changeset/durable-queue-lifecycle.md @@ -2,4 +2,9 @@ "posthog-ios": patch --- -Preserve bounded durable event, replay, and log queues across retryable upload failures, and acknowledge successful batches by their exact persisted entry identities. +- Preserve bounded durable event, replay, and log queues across retryable upload failures instead of clearing them. +- Limit `maxRetries` to push-subscription registration; it no longer bounds event, replay, or log queue flush attempts. +- Acknowledge successful and terminal uploads by their exact persisted entry identities so full-queue replacements accepted during an upload are not deleted. +- Trim persisted queues to `maxQueueSize` (or `logs.maxBufferSize` for logs) in FIFO order when loading them from disk. +- Honor the received HTTP status and `Retry-After` when an upload also returns a transport error: successful and terminal responses remove the sent entries, while retryable responses retain them. +- Preserve existing queued records when writing a new record to a full queue fails. diff --git a/PostHog/PostHogFileBackedQueue.swift b/PostHog/PostHogFileBackedQueue.swift index 813784e25..b701a5526 100644 --- a/PostHog/PostHogFileBackedQueue.swift +++ b/PostHog/PostHogFileBackedQueue.swift @@ -103,6 +103,8 @@ class PostHogFileBackedQueue { var evicted: String? try itemsLock.withLock { + try contents.write(to: queue.appendingPathComponent(filename)) + if let effectiveMaxSize, items.count >= effectiveMaxSize { evicted = items.removeFirst() if let evicted { @@ -110,7 +112,6 @@ class PostHogFileBackedQueue { } } - try contents.write(to: queue.appendingPathComponent(filename)) items.append(filename) } diff --git a/PostHogTests/PostHogApiTest.swift b/PostHogTests/PostHogApiTest.swift index 96caef95c..cff78614f 100644 --- a/PostHogTests/PostHogApiTest.swift +++ b/PostHogTests/PostHogApiTest.swift @@ -703,10 +703,10 @@ enum PostHogApiTests { @Suite("Upload response handling", .serialized) final class TestUploadResponseHandling { - @Test("preserves an HTTP status when URLSession also returns an error") - func preservesHTTPStatusAlongsideError() throws { + @Test("preserves an HTTP status when URLSession also returns an error", arguments: [200, 400, 408, 429, 503]) + func preservesHTTPStatusAlongsideError(statusCode: Int) throws { let url = try #require(URL(string: "http://localhost/batch")) - let httpResponse = try #require(HTTPURLResponse(url: url, statusCode: 503, httpVersion: nil, headerFields: nil)) + let httpResponse = try #require(HTTPURLResponse(url: url, statusCode: statusCode, httpVersion: nil, headerFields: nil)) let error = URLError(.timedOut) var uploadInfo: PostHogUploadInfo? @@ -715,7 +715,7 @@ enum PostHogApiTests { } let result = try #require(uploadInfo) - #expect(result.statusCode == 503) + #expect(result.statusCode == statusCode) #expect((result.error as? URLError)?.code == .timedOut) } diff --git a/PostHogTests/PostHogFileBackedQueueAlignmentTest.swift b/PostHogTests/PostHogFileBackedQueueAlignmentTest.swift index 6774e02d0..fbb3ed0ed 100644 --- a/PostHogTests/PostHogFileBackedQueueAlignmentTest.swift +++ b/PostHogTests/PostHogFileBackedQueueAlignmentTest.swift @@ -31,6 +31,37 @@ struct PostHogFileBackedQueueAlignmentTest { data.map { String(data: $0, encoding: .utf8)! } } + @Test("a failed write to a full queue preserves existing entries", .enabled(if: geteuid() != 0)) + func failedWritePreservesFullQueue() throws { + let (queue, dir) = makeQueue() + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: dir.path) + try? FileManager.default.removeItem(at: dir) + } + + queue.add(Data("A".utf8), maxSize: 2) + queue.add(Data("B".utf8), maxSize: 2) + let originalIds = queue.peekEntries(2).map(\.id) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: dir.path) + #expect(throws: (any Error).self) { + try Data("probe".utf8).write(to: dir.appendingPathComponent("write-probe")) + } + + #expect(queue.add(Data("C".utf8), maxSize: 2) == nil) + #expect(queue.depth == 2) + #expect(queue.peekEntries(2).map(\.id) == originalIds) + #expect(decode(queue.peek(2)) == ["A", "B"]) + #expect(Set(try FileManager.default.contentsOfDirectory(atPath: dir.path)) == Set(originalIds)) + + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: dir.path) + #expect(queue.add(Data("C".utf8), maxSize: 2) == originalIds.first) + #expect(queue.depth == 2) + #expect(decode(queue.peek(2)) == ["B", "C"]) + let reloaded = PostHogFileBackedQueue(queue: dir, maxSize: 2) + #expect(Set(reloaded.peekEntries(2).map(\.id)) == Set(queue.peekEntries(2).map(\.id))) + #expect(reloaded.depth == 2) + } + @Test("delivers every record once in FIFO order on the happy path") func happyPath() throws { let (queue, dir) = makeQueue() diff --git a/PostHogTests/PostHogQueueTest.swift b/PostHogTests/PostHogQueueTest.swift index 85ff3f7f6..76f5d3dbc 100644 --- a/PostHogTests/PostHogQueueTest.swift +++ b/PostHogTests/PostHogQueueTest.swift @@ -11,6 +11,7 @@ import OHHTTPStubs import OHHTTPStubsSwift @testable import PostHog import Quick +import Testing import XCTest private final class ControlledBatchSender { @@ -260,9 +261,9 @@ class PostHogQueueTest: QuickSpec { defer { now = { Date() } } let sut = self.getSut(flushAt: 100, maxBatchSize: 4, maxRetries: 1) - server.start(batchCount: 4) + server.start(batchCount: 6) server.batchResponseHandler = { _, requestNumber in - requestNumber <= 3 + requestNumber <= 3 || requestNumber == 5 ? HTTPStubsResponse(jsonObject: [], statusCode: 503, headers: nil) : HTTPStubsResponse(jsonObject: ["status": "ok"], statusCode: 200, headers: nil) } @@ -283,6 +284,19 @@ class PostHogQueueTest: QuickSpec { expect(sut.depth).toEventually(equal(0)) expect(sut.currentRetryCountForTesting).toEventually(equal(0)) + sut.add(PostHogEvent(event: "fresh", distinctId: "id3")) + sut.flush() + expect(server.batchRequests.count).toEventually(equal(5)) + expect(sut.currentRetryCountForTesting).toEventually(equal(1)) + expect(sut.depth) == 1 + sut.flush() + expect(server.batchRequests.count) == 5 + mockNow.date.addTimeInterval(1) + sut.flush() + expect(server.batchRequests.count).toEventually(equal(6)) + expect(sut.depth).toEventually(equal(0)) + expect(sut.currentRetryCountForTesting).toEventually(equal(0)) + sut.clear() } @@ -352,16 +366,16 @@ class PostHogQueueTest: QuickSpec { sut.flush() expect(sender.requestCount).toEventually(equal(1)) - // Adding while full evicts the first in-flight entry and appends a - // byte-identical replacement with a new durable identity. + // Replace the entire in-flight batch with byte-identical payloads. sut.add(identicalEvent) - let replacementId = sut.fileQueue.peekEntries(2).last!.id - expect(inFlightIds).notTo(contain(replacementId)) + sut.add(identicalEvent) + let replacementIds = sut.fileQueue.peekEntries(2).map(\.id) + expect(Set(inFlightIds).isDisjoint(with: replacementIds)) == true sender.completeRequest(at: 0, with: PostHogUploadInfo(statusCode: 200, error: nil)) - expect(sut.depth).toEventually(equal(1)) - expect(sut.fileQueue.peekEntries(2).map(\.id)) == [replacementId] + expect(sut.depth).toEventually(equal(2)) + expect(sut.fileQueue.peekEntries(2).map(\.id)) == replacementIds sut.flush() expect(sender.requestCount).toEventually(equal(2)) @@ -472,3 +486,103 @@ class PostHogQueueTest: QuickSpec { } } } + +@Suite("PostHog queue upload disposition", .serialized, .resetsGlobalState) +struct PostHogQueueUploadDispositionTest { + private func makeQueue(snapshot: Bool, sender: ControlledBatchSender) -> PostHogQueue { + let config = PostHogConfig(projectToken: "queue_disposition_\(UUID().uuidString)", host: "http://localhost:9001") + config.flushAt = 100 + config.maxBatchSize = 4 + config.maxRetries = 0 + let api = PostHogApi(config) + let base: QueueEndpoint = snapshot ? .snapshot(api: api) : .batch(api: api) + let endpoint = QueueEndpoint( + storageKey: base.storageKey, + oldStorageKeys: [], + dispatchQueueLabel: base.dispatchQueueLabel, + initialCap: base.initialCap, + initialFlushAt: base.initialFlushAt, + maxQueueSize: base.maxQueueSize, + flushIntervalSeconds: base.flushIntervalSeconds, + rateCapMax: base.rateCapMax, + rateCapWindowSeconds: base.rateCapWindowSeconds, + encode: base.encode, + decode: base.decode, + describe: base.describe, + send: sender.send, + isRetriableStatusCode: base.isRetriableStatusCode + ) + return PostHogQueue(config, PostHogStorage(config), endpoint, nil) + } + + private func waitForRequest(_ count: Int, sender: ControlledBatchSender) async throws { + await waitUntil { sender.requestCount == count } + try #require(sender.requestCount == count) + } + + @Test("received HTTP disposition wins over an accompanying transport error", arguments: [-1, 200, 400, 408, 429, 503], [false, true]) + func receivedHTTPDisposition(statusCode: Int, snapshot: Bool) async throws { + let sender = ControlledBatchSender() + let queue = makeQueue(snapshot: snapshot, sender: sender) + defer { queue.clear() } + queue.add(PostHogEvent(event: "sent", distinctId: "id")) + let sentIds = queue.fileQueue.peekEntries(1).map(\.id) + queue.flush() + try await waitForRequest(1, sender: sender) + queue.add(PostHogEvent(event: "not-sent", distinctId: "id")) + let allIds = queue.fileQueue.peekEntries(2).map(\.id) + let response = statusCode == -1 ? nil : HTTPURLResponse( + url: try #require(URL(string: "http://localhost/batch")), + statusCode: statusCode, httpVersion: nil, headerFields: nil + ) + processUploadResponse(endpointName: "test", data: nil, response: response, error: URLError(.networkConnectionLost)) { + sender.completeRequest(at: 0, with: $0) + } + + let retryable = [-1, 408, 429, 503].contains(statusCode) + let expectedIds = retryable ? allIds : allIds.filter { !sentIds.contains($0) } + #expect(queue.fileQueue.peekEntries(2).map(\.id) == expectedIds) + #expect(queue.currentRetryCountForTesting == (retryable ? 1 : 0)) + let reloaded = PostHogFileBackedQueue(queue: queue.fileQueue.queue) + #expect(Set(reloaded.peekEntries(2).map(\.id)) == Set(expectedIds)) + } + + @Test("413 reaches singleton after retryable failures and preserves later records", arguments: [false, true]) + func shrinkingAfterRetryableFailures(snapshot: Bool) async throws { + let mockNow = MockDate() + now = { mockNow.date } + defer { now = { Date() } } + let sender = ControlledBatchSender() + let queue = makeQueue(snapshot: snapshot, sender: sender) + defer { queue.clear() } + for name in ["poison", "later-1", "later-2", "later-3"] { + queue.add(PostHogEvent(event: name, distinctId: "id")) + } + let originalIds = queue.fileQueue.peekEntries(4).map(\.id) + for attempt in 0 ..< 3 { + queue.flush() + try await waitForRequest(attempt + 1, sender: sender) + sender.completeRequest(at: attempt, with: PostHogUploadInfo(statusCode: 503, error: nil)) + #expect(queue.fileQueue.peekEntries(4).map(\.id) == originalIds) + mockNow.date.addTimeInterval(60) + } + for (attempt, cap) in [(3, 2), (4, 1)] { + queue.flush() + try await waitForRequest(attempt + 1, sender: sender) + sender.completeRequest(at: attempt, with: PostHogUploadInfo(statusCode: 413, error: nil)) + #expect(queue.currentBatchCapForTesting == cap) + #expect(queue.fileQueue.peekEntries(4).map(\.id) == originalIds) + } + queue.flush() + try await waitForRequest(6, sender: sender) + sender.completeRequest(at: 5, with: PostHogUploadInfo(statusCode: 413, error: nil)) + #expect(queue.fileQueue.peekEntries(4).map(\.id) == Array(originalIds.dropFirst())) + #expect(queue.currentRetryCountForTesting == 0) + for attempt in 6 ..< 9 { + queue.flush() + try await waitForRequest(attempt + 1, sender: sender) + sender.completeRequest(at: attempt, with: PostHogUploadInfo(statusCode: 200, error: nil)) + } + #expect(queue.depth == 0) + } +} From 3506456c0319ce3e7355fd0d79a47a6e0e6264cc Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Mon, 7 Sep 2026 16:46:27 -0400 Subject: [PATCH 14/17] chore(release): classify durable queue changes as minor --- .changeset/durable-queue-lifecycle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/durable-queue-lifecycle.md b/.changeset/durable-queue-lifecycle.md index 9534d1e89..ecbe5f1d4 100644 --- a/.changeset/durable-queue-lifecycle.md +++ b/.changeset/durable-queue-lifecycle.md @@ -1,5 +1,5 @@ --- -"posthog-ios": patch +"posthog-ios": minor --- - Preserve bounded durable event, replay, and log queues across retryable upload failures instead of clearing them. From 06e53683b61206b23f15eddd251ee56c116609be Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:04:18 +0000 Subject: [PATCH 15/17] fix(push): classify HTTP 408 as retryable for push subscriptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `isRetryable` excluded 408, so a request timeout was terminal. On the unregister path the terminal branch calls `clearPendingUnregister`, which removes the durable `pushPendingUnregister` record — a logout DELETE that met a 408 was never re-attempted, leaving the device subscribed under the logged-out distinct id. Forwarding the received status alongside a transport error widened the shapes that reach this gap: a 408 that also failed in URLSession previously arrived with no status and fell into the retryable default. 408 now joins 429 and 5xx, matching the policy `QueueEndpoint+Factories` already applies to the event, replay, and log queues. The registration path shares the classifier and also improves: a 408 backs off and retries instead of halting the handler. Generated-By: PostHog Desktop Task-Id: d785939f-f0b4-4db4-bbf9-c87f1c2364cb --- .changeset/durable-queue-lifecycle.md | 1 + .../PushNotifications/PostHogPushSubscriptionHandler.swift | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.changeset/durable-queue-lifecycle.md b/.changeset/durable-queue-lifecycle.md index ecbe5f1d4..6ab5cf868 100644 --- a/.changeset/durable-queue-lifecycle.md +++ b/.changeset/durable-queue-lifecycle.md @@ -8,3 +8,4 @@ - Trim persisted queues to `maxQueueSize` (or `logs.maxBufferSize` for logs) in FIFO order when loading them from disk. - Honor the received HTTP status and `Retry-After` when an upload also returns a transport error: successful and terminal responses remove the sent entries, while retryable responses retain them. - Preserve existing queued records when writing a new record to a full queue fails. +- Retry push-subscription requests that answer with HTTP 408 instead of treating the timeout as terminal, so a pending logout unregister survives until it succeeds. diff --git a/PostHog/PushNotifications/PostHogPushSubscriptionHandler.swift b/PostHog/PushNotifications/PostHogPushSubscriptionHandler.swift index 607e7b4c0..9a9fb3ff3 100644 --- a/PostHog/PushNotifications/PostHogPushSubscriptionHandler.swift +++ b/PostHog/PushNotifications/PostHogPushSubscriptionHandler.swift @@ -830,9 +830,11 @@ final class PostHogPushSubscriptionHandler { storage.remove(key: .pushPendingUnregister) } - /// Transport error (no status), 429, or 5xx is retryable; everything else (4xx) is terminal. + /// Transport error (no status), 408, 429, or 5xx is retryable; everything else (4xx) is terminal. + /// 408 matches the queues' policy in `QueueEndpoint+Factories`: a request timeout can arrive as a + /// status alongside a URLSession error, and treating it as terminal drops the unregister intent. private func isRetryable(_ info: PostHogUploadInfo) -> Bool { - info.statusCode.map { $0 == 429 || (500 ... 599 ~= $0) } ?? true + info.statusCode.map { $0 == 408 || $0 == 429 || (500 ... 599 ~= $0) } ?? true } private func statusString(_ info: PostHogUploadInfo) -> String { From ef27c0c72ecf60b83352cebee3d7a8ccc9c87b71 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:44:58 +0000 Subject: [PATCH 16/17] fix(queue): sort durable records in a total order before trimming Loading the queue from disk now trims to capacity, so the sort order decides which persisted records are deleted. The comparator was neither total nor stable: files with equal creation dates were left in unspecified order, and a file whose creation date could not be read fell back to `.distantPast`, sorting it ahead of every dated record and making it the first candidate for deletion. Fall back to `.distantFuture` so an unreadable record is retained rather than culled, and break ties on the filename so the order is total. UUID v7 filenames sort lexicographically by their embedded timestamp, so the tie-breaker keeps contemporaneous records in creation order. Generated-By: PostHog Desktop Task-Id: 8f1e9af4-1dec-48b8-a9e1-d834f9314cf1 --- PostHog/PostHogFileBackedQueue.swift | 14 +++++++---- .../PostHogFileBackedQueueAlignmentTest.swift | 24 +++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/PostHog/PostHogFileBackedQueue.swift b/PostHog/PostHogFileBackedQueue.swift index b701a5526..a19e9fbad 100644 --- a/PostHog/PostHogFileBackedQueue.swift +++ b/PostHog/PostHogFileBackedQueue.swift @@ -253,12 +253,18 @@ private func migrateOldQueueFolder(queue: URL, oldQueueFolder: URL) { } private extension FileManager { - /// Returns filenames sorted by resource key + /// Returns filenames in a total order: by resource key, then by filename. + /// `reindexFromDisk` deletes the head of this order to enforce capacity, so a file + /// whose date can't be read must not sort oldest, and equal dates need a tie-breaker + /// because `sorted` isn't stable. UUID v7 names sort by their embedded timestamp. func contentsOfDirectory(at url: URL, sortedBy key: URLResourceKey) throws -> [String] { let urls = try contentsOfDirectory(at: url, includingPropertiesForKeys: [key]) - return urls.sorted { - let date1 = (try? $0.resourceValues(forKeys: [key]).allValues[key] as? Date) ?? .distantPast - let date2 = (try? $1.resourceValues(forKeys: [key]).allValues[key] as? Date) ?? .distantPast + return urls.sorted { lhs, rhs in + let date1 = (try? lhs.resourceValues(forKeys: [key]).allValues[key] as? Date) ?? .distantFuture + let date2 = (try? rhs.resourceValues(forKeys: [key]).allValues[key] as? Date) ?? .distantFuture + if date1 == date2 { + return lhs.lastPathComponent < rhs.lastPathComponent + } return date1 < date2 }.map(\.lastPathComponent) } diff --git a/PostHogTests/PostHogFileBackedQueueAlignmentTest.swift b/PostHogTests/PostHogFileBackedQueueAlignmentTest.swift index fbb3ed0ed..211f87368 100644 --- a/PostHogTests/PostHogFileBackedQueueAlignmentTest.swift +++ b/PostHogTests/PostHogFileBackedQueueAlignmentTest.swift @@ -83,6 +83,30 @@ struct PostHogFileBackedQueueAlignmentTest { #expect(queue.depth == 0) } + @Test("breaks creation-date ties by filename when trimming to capacity on load") + func tiedCreationDatesTrimDeterministically() throws { + let dir = FileManager.default.temporaryDirectory + .appendingPathComponent("ph-queue-tie-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: dir) } + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + + // written out of filename order so an enumeration- or write-order-dependent + // sort would disagree with the tie-breaker + let tied = Date(timeIntervalSince1970: 100) + for name in ["record-4", "record-1", "record-3", "record-2"] { + let url = dir.appendingPathComponent(name) + try Data(name.utf8).write(to: url) + try FileManager.default.setAttributes([.creationDate: tied], ofItemAtPath: url.path) + } + + let queue = PostHogFileBackedQueue(queue: dir, maxSize: 2) + + #expect(queue.depth == 2) + #expect(decode(queue.peek(2)) == ["record-3", "record-4"]) + #expect(!FileManager.default.fileExists(atPath: dir.appendingPathComponent("record-1").path)) + #expect(!FileManager.default.fileExists(atPath: dir.appendingPathComponent("record-2").path)) + } + enum UnreadableHead { case missing // vanished from disk: !fileExists, pruned via `continue` case corrupt // present but unreadable (Data(contentsOf:) throws): deleted then pruned From 20259ef74224e077f20bfb782c11316d955a1d7a Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:49:22 +0000 Subject: [PATCH 17/17] fix(api): do not report a failed redirect as a delivery status The error branch of `processUploadResponse` now forwards the status recorded on the task, so a 3xx that URLSession was still following when the transfer failed was reported as the upload's outcome. Events and replay classify 3xx as retryable, but the logs policy and the push handler's `isRetryable` retry only 408, 429, and 5xx, so the same wire condition deleted a durable log batch there and cleared the `.pushPendingUnregister` intent, leaving the device subscribed under a signed-out distinct id. A redirect is not a delivery confirmation for the payload, so report no status when a 3xx is left on a failed task. `Retry-After` and the error are kept, and every other status arriving alongside an error is still honored. Generated-By: PostHog Desktop Task-Id: 8f1e9af4-1dec-48b8-a9e1-d834f9314cf1 --- .changeset/durable-queue-lifecycle.md | 2 +- PostHog/PostHogApi.swift | 8 +++++++- PostHogTests/PostHogApiTest.swift | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/.changeset/durable-queue-lifecycle.md b/.changeset/durable-queue-lifecycle.md index 6ab5cf868..aa338be5d 100644 --- a/.changeset/durable-queue-lifecycle.md +++ b/.changeset/durable-queue-lifecycle.md @@ -6,6 +6,6 @@ - Limit `maxRetries` to push-subscription registration; it no longer bounds event, replay, or log queue flush attempts. - Acknowledge successful and terminal uploads by their exact persisted entry identities so full-queue replacements accepted during an upload are not deleted. - Trim persisted queues to `maxQueueSize` (or `logs.maxBufferSize` for logs) in FIFO order when loading them from disk. -- Honor the received HTTP status and `Retry-After` when an upload also returns a transport error: successful and terminal responses remove the sent entries, while retryable responses retain them. +- Honor the received HTTP status and `Retry-After` when an upload also returns a transport error: successful and terminal responses remove the sent entries, while retryable responses retain them. A `3xx` left on a failed request is reported as no status, because a redirect does not confirm delivery to the final host. - Preserve existing queued records when writing a new record to a full queue fails. - Retry push-subscription requests that answer with HTTP 408 instead of treating the timeout as terminal, so a pending logout unregister survives until it succeeds. diff --git a/PostHog/PostHogApi.swift b/PostHog/PostHogApi.swift index 25657f793..7b3cd2ef2 100644 --- a/PostHog/PostHogApi.swift +++ b/PostHog/PostHogApi.swift @@ -24,7 +24,13 @@ func processUploadResponse( if let error { hedgeLog("Error calling the \(endpointName) API: \(error).") - return completion(PostHogUploadInfo(statusCode: httpResponse?.statusCode, error: error, retryAfter: retryAfter)) + // A 3xx left on a failed task is the redirect URLSession was still following, not an + // outcome for the payload, so it's reported as no status. Honoring it would let the + // policies that treat 3xx as terminal (logs, push unregister) delete durable records + // that never reached the final host. + let status = httpResponse?.statusCode + let delivered = status.flatMap { 300 ... 399 ~= $0 ? nil : $0 } + return completion(PostHogUploadInfo(statusCode: delivered, error: error, retryAfter: retryAfter)) } guard let httpResponse else { diff --git a/PostHogTests/PostHogApiTest.swift b/PostHogTests/PostHogApiTest.swift index cff78614f..603999684 100644 --- a/PostHogTests/PostHogApiTest.swift +++ b/PostHogTests/PostHogApiTest.swift @@ -719,6 +719,25 @@ enum PostHogApiTests { #expect((result.error as? URLError)?.code == .timedOut) } + @Test("reports no status for a redirect left on a failed task", arguments: [301, 302, 307, 308]) + func dropsRedirectStatusAlongsideError(statusCode: Int) throws { + let url = try #require(URL(string: "http://localhost/i/v1/logs")) + let httpResponse = try #require(HTTPURLResponse(url: url, statusCode: statusCode, httpVersion: nil, headerFields: ["Retry-After": "30"])) + let error = URLError(.httpTooManyRedirects) + var uploadInfo: PostHogUploadInfo? + + processUploadResponse(endpointName: "logs", data: nil, response: httpResponse, error: error) { + uploadInfo = $0 + } + + let result = try #require(uploadInfo) + // no status keeps the records retryable for the logs and push-unregister + // policies, which classify 3xx as terminal + #expect(result.statusCode == nil) + #expect(result.retryAfter == 30) + #expect((result.error as? URLError)?.code == .httpTooManyRedirects) + } + @Test("preserves Retry-After when URLSession also returns an error") func preservesRetryAfterAlongsideError() throws { let url = try #require(URL(string: "http://localhost/batch"))