diff --git a/.changeset/quiet-otters-listen.md b/.changeset/quiet-otters-listen.md new file mode 100644 index 000000000..a2a3b389a --- /dev/null +++ b/.changeset/quiet-otters-listen.md @@ -0,0 +1,5 @@ +--- +'posthog-ios': patch +--- + +Omit null-valued custom object properties recursively when serializing events for delivery and disk queues, while preserving null array entries. diff --git a/PostHog.xcodeproj/project.pbxproj b/PostHog.xcodeproj/project.pbxproj index 5a37ee449..f5a94cf15 100644 --- a/PostHog.xcodeproj/project.pbxproj +++ b/PostHog.xcodeproj/project.pbxproj @@ -13,6 +13,7 @@ BF5E00000000000000000002 /* PHObjCExceptionCatcher.h in Headers */ = {isa = PBXBuildFile; fileRef = BF5E00000000000000000006 /* PHObjCExceptionCatcher.h */; settings = {ATTRIBUTES = (Private, ); }; }; BF5E00000000000000000003 /* PHObjCExceptionCatcher.m in Sources */ = {isa = PBXBuildFile; fileRef = BF5E00000000000000000007 /* PHObjCExceptionCatcher.m */; }; D0E500012FA0000100000002 /* PostHogEventSnapshotTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E500012FA0000100000001 /* PostHogEventSnapshotTests.swift */; }; + A1B2C3D42FA0000100000002 /* PostHogEventNullSerializationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D42FA0000100000001 /* PostHogEventNullSerializationTests.swift */; }; D0E500012FA0000100000021 /* event-shapes-batch.json in Resources */ = {isa = PBXBuildFile; fileRef = D0E500012FA0000100000011 /* event-shapes-batch.json */; }; D0E500012FA0000100000022 /* feature-flags-request.json in Resources */ = {isa = PBXBuildFile; fileRef = D0E500012FA0000100000012 /* feature-flags-request.json */; }; D0E500012FA0000100000023 /* session-replay-request.json in Resources */ = {isa = PBXBuildFile; fileRef = D0E500012FA0000100000013 /* session-replay-request.json */; }; @@ -787,6 +788,7 @@ BF5E00000000000000000006 /* PHObjCExceptionCatcher.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PHObjCExceptionCatcher.h; sourceTree = ""; }; BF5E00000000000000000007 /* PHObjCExceptionCatcher.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = PHObjCExceptionCatcher.m; sourceTree = ""; }; D0E500012FA0000100000001 /* PostHogEventSnapshotTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PostHogEventSnapshotTests.swift; sourceTree = ""; }; + A1B2C3D42FA0000100000001 /* PostHogEventNullSerializationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PostHogEventNullSerializationTests.swift; sourceTree = ""; }; D0E500012FA0000100000011 /* event-shapes-batch.json */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.json; path = "event-shapes-batch.json"; sourceTree = ""; }; D0E500012FA0000100000012 /* feature-flags-request.json */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.json; path = "feature-flags-request.json"; sourceTree = ""; }; D0E500012FA0000100000013 /* session-replay-request.json */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.json; path = "session-replay-request.json"; sourceTree = ""; }; @@ -1709,6 +1711,7 @@ B74800000000000000000004 /* PostHogTests-Bridging-Header.h */, B74800000000000000000003 /* PostHogPushNotificationSwizzlingTest.swift */, D0E500012FA0000100000001 /* PostHogEventSnapshotTests.swift */, + A1B2C3D42FA0000100000001 /* PostHogEventNullSerializationTests.swift */, DA27AEC82F56B3F3002A59CE /* PostHogSamplingTest.swift */, DB7100052F80000100000001 /* PostHogTracingHeadersTest.swift */, DA27AEC02F56B3E3002A59CE /* PostHogSessionReplayEventTriggersTest.swift */, @@ -3489,6 +3492,7 @@ BF5E00000000000000000001 /* PHBeforeSendExceptionTestFixture.m in Sources */, B74800000000000000000002 /* PHNotificationDelegateTestFixture.m in Sources */, D0E500012FA0000100000002 /* PostHogEventSnapshotTests.swift in Sources */, + A1B2C3D42FA0000100000002 /* PostHogEventNullSerializationTests.swift in Sources */, DA979D7B2CD370B700F56BAE /* PostHogAutocaptureEventTrackerSpec.swift in Sources */, DA0C944B2D54CDD000BFD9FB /* PostHogStorageMigrationTest.swift in Sources */, 690FF0F52AF0F06100A0B06B /* PostHogSDKTest.swift in Sources */, diff --git a/PostHog/Models/PostHogEvent.swift b/PostHog/Models/PostHogEvent.swift index ccd248ab1..fd432bb45 100644 --- a/PostHog/Models/PostHogEvent.swift +++ b/PostHog/Models/PostHogEvent.swift @@ -90,7 +90,7 @@ import Foundation var json: [String: Any] = [ "event": event, "distinct_id": distinctId, - "properties": properties, + "properties": Self.serializedProperties(properties, event: event), "timestamp": toISO8601String(timestamp), "uuid": uuid.postHogUuidString, ] @@ -102,6 +102,47 @@ import Foundation return json } + + /// Event-only cleanup shared by wire, queue encoding and legacy event rewrites. + /// Typed SDK payloads retain their own null semantics; custom `$set`/`$group_set` + /// dictionaries are not exempt. Generic storage and JSON sanitization are unchanged. + static func serializedProperties(_ properties: [String: Any], event: String) -> [String: Any] { + let reserved: Set + switch event { + case "$snapshot": + reserved = ["$snapshot_data"] + case "$exception": + reserved = ["$exception_list", "$debug_images"] + case "$feature_flag_called": + reserved = ["$feature_flag_response", "$feature_flag_reason", "$feature_flag_id", "$feature_flag_version"] + default: + reserved = [] + } + return properties.reduce(into: [:]) { result, entry in + if reserved.contains(entry.key) { + result[entry.key] = entry.value + } else { + result[entry.key] = removingNullObjectMembers(entry.value) + } + } + } +} + +private func removingNullObjectMembers(_ value: Any) -> Any? { + // Foundation bridges boxed Optional.none to NSNull, as JSONSerialization does. + if value as AnyObject is NSNull { + return nil + } + if let dictionary = value as? [String: Any] { + return dictionary.reduce(into: [String: Any]()) { result, entry in + result[entry.key] = removingNullObjectMembers(entry.value) + } + } + if let array = value as? [Any] { + // Do not compact null slots or dictionaries emptied by recursive cleanup. + return array.map { removingNullObjectMembers($0) ?? NSNull() } + } + return value } enum PostHogKnownUnsafeEditableEvent: String { diff --git a/PostHog/PostHogLegacyQueue.swift b/PostHog/PostHogLegacyQueue.swift index 8b7285259..3d916cd65 100644 --- a/PostHog/PostHogLegacyQueue.swift +++ b/PostHog/PostHogLegacyQueue.swift @@ -25,9 +25,17 @@ func migrateOldQueue(queue: URL, oldQueue: URL) { } for item in array { - guard let event = item as? [String: Any] else { + guard var event = item as? [String: Any] else { continue } + let eventName = event["event"] as? String ?? "" + if let properties = event["properties"] as? [String: Any] { + event["properties"] = PostHogEvent.serializedProperties(properties, event: eventName) + } + // v2 stored person properties outside the properties container. + if let setProperties = event["$set"] as? [String: Any] { + event["$set"] = PostHogEvent.serializedProperties(setProperties, event: "") + } let timestamp = event["timestamp"] as? String ?? toISO8601String(Date()) let timestampDate = toISO8601Date(timestamp) ?? Date() diff --git a/PostHogTests/PostHogEventNullSerializationTests.swift b/PostHogTests/PostHogEventNullSerializationTests.swift new file mode 100644 index 000000000..6be93d2c3 --- /dev/null +++ b/PostHogTests/PostHogEventNullSerializationTests.swift @@ -0,0 +1,229 @@ +import Foundation +@testable import PostHog +import Testing + +@Suite(.serialized) +struct PostHogEventNullSerializationTests { + private func properties() -> [String: Any] { + let none: String? = nil + return [ + "test": NSNull(), + "optional": none as Any, + "nested": ["drop": NSNull()], + "items": ["1", NSNull(), 2, ["drop": NSNull()], [NSNull()]], + "empty": "", "zero": 0, "enabled": false, + "literal": "null", "literalUndefined": "undefined", + "emptyArray": [], "emptyObject": [:], + "$set": ["drop": NSNull()], + "$set_once": ["drop": NSNull()], + "$group_set": ["drop": NSNull()], + ] + } + + private var expected: [String: Any] { + [ + "nested": [:], "items": ["1", NSNull(), 2, [:], [NSNull()]], + "empty": "", "zero": 0, "enabled": false, + "literal": "null", "literalUndefined": "undefined", + "emptyArray": [], "emptyObject": [:], + "$set": [:], "$set_once": [:], "$group_set": [:], + ] + } + + private func expectProperties(_ json: [String: Any], _ expected: [String: Any]) throws { + let actual = try #require(json["properties"] as? [String: Any]) + #expect(NSDictionary(dictionary: actual).isEqual(to: expected)) + } + + @Test(arguments: ["Nullable Properties", "$screen", "$identify", "$groupidentify", "$exception", "$ai_generation", "$snapshot"]) + func eventSerialization(eventName: String) throws { + let input = properties() + let event = PostHogEvent(event: eventName, distinctId: "test-user", properties: input) + let json = try #require(fromJSONData(try JSONSerialization.data(withJSONObject: event.toJSON()))) + try expectProperties(json, expected) + #expect(event.properties["test"] is NSNull) + #expect(input["test"] is NSNull) + #expect(json["event"] as? String == eventName) + #expect(json["distinct_id"] as? String == "test-user") + #expect(json["uuid"] as? String == event.uuid.postHogUuidString) + } + + @Test + func foundationBridgingAndOptionalArraySlots() throws { + let none: Int? = nil + let nested = NSMutableDictionary(dictionary: ["drop": NSNull()]) + let items = NSMutableArray(array: [NSNull(), nested]) + let event = PostHogEvent(event: "bridged", distinctId: "test-user", properties: [ + "nested": nested, "items": items, "optionals": [none, 1] as [Int?], + ]) + try expectProperties(event.toJSON(), ["nested": [:], "items": [NSNull(), [:]], "optionals": [NSNull(), 1]]) + #expect(nested["drop"] is NSNull) + #expect(items.count == 2) + } + + @Test + func allNullPropertiesKeepEventAndGenericJSONIsUnchanged() throws { + let event = PostHogEvent(event: "only null", distinctId: "test-user", properties: ["test": NSNull()]) + try expectProperties(event.toJSON(), [:]) + let generic = try #require(toJSONData(["test": NSNull()])) + #expect(fromJSONData(generic)?["test"] is NSNull) + } + + @Test + func typedPayloadsKeepNullsOnlyOnTheirOwnEvents() throws { + let reserved: [(String, [String: Any])] = [ + ("$snapshot", ["$snapshot_data": [["data": ["parentId": NSNull()]]]]), + ("$exception", ["$exception_list": [["value": NSNull()]], "$debug_images": [["debug_id": NSNull()]]]), + ("$feature_flag_called", [ + "$feature_flag_response": NSNull(), "$feature_flag_reason": NSNull(), + "$feature_flag_id": NSNull(), "$feature_flag_version": NSNull(), + ]), + ] + for (name, payload) in reserved { + var input = payload + input["custom"] = ["drop": NSNull()] + var expected = payload + expected["custom"] = [String: Any]() + try expectProperties(PostHogEvent(event: name, distinctId: "test-user", properties: input).toJSON(), expected) + } + try expectProperties(PostHogEvent(event: "custom", distinctId: "test-user", properties: [ + "$snapshot_data": ["drop": NSNull()], "$feature_flag_response": NSNull(), + ]).toJSON(), ["$snapshot_data": [:]]) + } + + // No SDK setup or shared Application Support storage. Every file belongs to this test. + private func withDirectory(_ body: (URL) throws -> Void) throws { + let root = URL(fileURLWithPath: #filePath).deletingLastPathComponent() + .appendingPathComponent(".null-serialization-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + try body(root) + } + + @Test + func beforeSendAdditionsPrivacyFilteringAndDrops() throws { + let chain = BeforeSendChain() + chain.set([{ event in + event.properties.removeValue(forKey: "private") + event.properties["hookNull"] = NSNull() + event.properties["hookItems"] = [NSNull(), ["drop": NSNull()]] + return event + }]) + let event = PostHogEvent(event: "hook", distinctId: "test-user", properties: ["private": "secret"]) + let result = try #require(chain.run(event)) + try expectProperties(result.toJSON(), ["hookItems": [NSNull(), [:]]]) + chain.set([{ _ in nil }]) + #expect(chain.run(event) == nil) + } + + @Test + func diskWriteRestoreAndLateMutation() throws { + try withDirectory { root in + let event = PostHogEvent(event: "disk", distinctId: "test-user", properties: properties()) + // Models values introduced downstream of capture-time sanitization, by enrichment/hooks. + event.properties["hookNull"] = NSNull() + event.properties["hookItems"] = [NSNull(), ["drop": NSNull()]] + var expected = expected + expected["hookItems"] = [NSNull(), [:]] + let queue = PostHogFileBackedQueue(queue: root) + queue.add(try #require(toJSONData(event.toJSON()))) + let data = try #require(PostHogFileBackedQueue(queue: root).peek(1).first) + try expectProperties(try #require(fromJSONData(data)), expected) + let restored = try #require(PostHogEvent.fromJSON(data)) + restored.properties["lateNull"] = NSNull() + try expectProperties(restored.toJSON(), expected) + } + } + + @Test(arguments: [false, true]) + func apiWireSerialization(snapshot: Bool) async throws { + let config = PostHogConfig(projectToken: "test-null-serialization", host: "http://127.0.0.1:1") + let session = URLSessionConfiguration.ephemeral + session.protocolClasses = [NullSerializationURLProtocol.self] + config.urlSessionConfiguration = session + let api = PostHogApi(config) + // Old persisted events can still contain nulls; the final API encoder must clean them. + let oldData = try JSONSerialization.data(withJSONObject: [ + "event": snapshot ? "$snapshot" : "wire", "distinct_id": "test-user", "properties": properties(), + ]) + let event = try #require(PostHogEvent.fromJSON(oldData)) + event.properties["lateNull"] = NSNull() + let info: PostHogUploadInfo = await withCheckedContinuation { continuation in + if snapshot { + api.snapshot(events: [event]) { continuation.resume(returning: $0) } + } else { + api.batch(events: [event]) { continuation.resume(returning: $0) } + } + } + #expect(info.statusCode == 200) + let body = try #require(NullSerializationURLProtocol.body) + let json = try JSONSerialization.jsonObject(with: body) + let events = try #require(snapshot ? json as? [[String: Any]] : (json as? [String: Any])?["batch"] as? [[String: Any]]) + try expectProperties(try #require(events.first), expected) + } + + @Test + func legacyRewriteCleansPropertiesWithoutChangingEnvelope() throws { + try withDirectory { root in + let old = root.appendingPathComponent("v2.json") + let destination = root.appendingPathComponent("v3") + let legacy: [String: Any] = [ + "event": "legacy", "distinct_id": "test-user", "timestamp": "2024-01-01T00:00:00Z", + "message_id": "old-id", "envelopeNull": NSNull(), + "properties": properties().filter { $0.key != "optional" }, "$set": ["drop": NSNull()], + ] + try JSONSerialization.data(withJSONObject: [legacy]).write(to: old) + let queue = PostHogFileBackedQueue(queue: destination, oldQueues: [old]) + let data = try #require(queue.peek(1).first) + let json = try #require(fromJSONData(data)) + try expectProperties(json, expected) + #expect((json["$set"] as? [String: Any])?.isEmpty == true) + #expect(json["envelopeNull"] is NSNull) + #expect(json["message_id"] as? String == "old-id") + try expectProperties(try #require(PostHogEvent.fromJSON(data)).toJSON(), expected) + } + } +} + +// Intercepts every URL in the test session; unexpected requests cannot reach the network. +private final class NullSerializationURLProtocol: URLProtocol { + private static let lock = NSLock() + private static var capturedBody: Data? + static var body: Data? { lock.withLock { capturedBody } } + + override static func canInit(with _: URLRequest) -> Bool { + true + } + override static func canonicalRequest(for request: URLRequest) -> URLRequest { + request + } + + override func startLoading() { + do { + #expect(request.url?.host == "127.0.0.1") + var data = request.httpBody ?? Data() + if let stream = request.httpBodyStream { + stream.open() + defer { stream.close() } + var buffer = [UInt8](repeating: 0, count: 4096) + while stream.hasBytesAvailable { + let count = stream.read(&buffer, maxLength: buffer.count) + if count <= 0 { break } + data.append(contentsOf: buffer.prefix(count)) + } + } + if request.value(forHTTPHeaderField: "Content-Encoding") == "gzip" { + data = try data.gunzipped() + } + Self.lock.withLock { Self.capturedBody = data } + let response = HTTPURLResponse(url: request.url!, statusCode: 200, httpVersion: nil, headerFields: nil)! + client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) + client?.urlProtocol(self, didLoad: Data("{}".utf8)) + client?.urlProtocolDidFinishLoading(self) + } catch { + client?.urlProtocol(self, didFailWithError: error) + } + } + + override func stopLoading() {} +}