fix: omit null object properties when serializing events

.changeset/quiet-otters-listen.md

@@ -0,0 +1,5 @@
+---
+'posthog-ios': patch
+---
+
+Omit null-valued custom object properties recursively when serializing events for delivery and disk queues, while preserving null array entries.

PostHog.xcodeproj/project.pbxproj

@@ -13,6 +13,7 @@
 		BF5E00000000000000000002 /* PHObjCExceptionCatcher.h in Headers */ = {isa = PBXBuildFile; fileRef = BF5E00000000000000000006 /* PHObjCExceptionCatcher.h */; settings = {ATTRIBUTES = (Private, ); }; };
 		BF5E00000000000000000003 /* PHObjCExceptionCatcher.m in Sources */ = {isa = PBXBuildFile; fileRef = BF5E00000000000000000007 /* PHObjCExceptionCatcher.m */; };
 		D0E500012FA0000100000002 /* PostHogEventSnapshotTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E500012FA0000100000001 /* PostHogEventSnapshotTests.swift */; };
+		A1B2C3D42FA0000100000002 /* PostHogEventNullSerializationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D42FA0000100000001 /* PostHogEventNullSerializationTests.swift */; };
 		D0E500012FA0000100000021 /* event-shapes-batch.json in Resources */ = {isa = PBXBuildFile; fileRef = D0E500012FA0000100000011 /* event-shapes-batch.json */; };
 		D0E500012FA0000100000022 /* feature-flags-request.json in Resources */ = {isa = PBXBuildFile; fileRef = D0E500012FA0000100000012 /* feature-flags-request.json */; };
 		D0E500012FA0000100000023 /* session-replay-request.json in Resources */ = {isa = PBXBuildFile; fileRef = D0E500012FA0000100000013 /* session-replay-request.json */; };
@@ -787,6 +788,7 @@
 		BF5E00000000000000000006 /* PHObjCExceptionCatcher.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PHObjCExceptionCatcher.h; sourceTree = "<group>"; };
 		BF5E00000000000000000007 /* PHObjCExceptionCatcher.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = PHObjCExceptionCatcher.m; sourceTree = "<group>"; };
 		D0E500012FA0000100000001 /* PostHogEventSnapshotTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PostHogEventSnapshotTests.swift; sourceTree = "<group>"; };
+		A1B2C3D42FA0000100000001 /* PostHogEventNullSerializationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PostHogEventNullSerializationTests.swift; sourceTree = "<group>"; };
 		D0E500012FA0000100000011 /* event-shapes-batch.json */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.json; path = "event-shapes-batch.json"; sourceTree = "<group>"; };
 		D0E500012FA0000100000012 /* feature-flags-request.json */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.json; path = "feature-flags-request.json"; sourceTree = "<group>"; };
 		D0E500012FA0000100000013 /* session-replay-request.json */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.json; path = "session-replay-request.json"; sourceTree = "<group>"; };
@@ -1709,6 +1711,7 @@
 				B74800000000000000000004 /* PostHogTests-Bridging-Header.h */,
 				B74800000000000000000003 /* PostHogPushNotificationSwizzlingTest.swift */,
 				D0E500012FA0000100000001 /* PostHogEventSnapshotTests.swift */,
+				A1B2C3D42FA0000100000001 /* PostHogEventNullSerializationTests.swift */,
 				DA27AEC82F56B3F3002A59CE /* PostHogSamplingTest.swift */,
 				DB7100052F80000100000001 /* PostHogTracingHeadersTest.swift */,
 				DA27AEC02F56B3E3002A59CE /* PostHogSessionReplayEventTriggersTest.swift */,
@@ -3489,6 +3492,7 @@
 				BF5E00000000000000000001 /* PHBeforeSendExceptionTestFixture.m in Sources */,
 				B74800000000000000000002 /* PHNotificationDelegateTestFixture.m in Sources */,
 				D0E500012FA0000100000002 /* PostHogEventSnapshotTests.swift in Sources */,
+				A1B2C3D42FA0000100000002 /* PostHogEventNullSerializationTests.swift in Sources */,
 				DA979D7B2CD370B700F56BAE /* PostHogAutocaptureEventTrackerSpec.swift in Sources */,
 				DA0C944B2D54CDD000BFD9FB /* PostHogStorageMigrationTest.swift in Sources */,
 				690FF0F52AF0F06100A0B06B /* PostHogSDKTest.swift in Sources */,

PostHog/Models/PostHogEvent.swift

@@ -90,7 +90,7 @@ import Foundation
         var json: [String: Any] = [
             "event": event,
             "distinct_id": distinctId,
-            "properties": properties,
+            "properties": Self.serializedProperties(properties, event: event),
             "timestamp": toISO8601String(timestamp),
             "uuid": uuid.postHogUuidString,
         ]
@@ -102,6 +102,47 @@ import Foundation
 
         return json
     }
+
+    /// Event-only cleanup shared by wire, queue encoding and legacy event rewrites.
+    /// Typed SDK payloads retain their own null semantics; custom `$set`/`$group_set`
+    /// dictionaries are not exempt. Generic storage and JSON sanitization are unchanged.
+    static func serializedProperties(_ properties: [String: Any], event: String) -> [String: Any] {
+        let reserved: Set<String>
+        switch event {
+        case "$snapshot":
+            reserved = ["$snapshot_data"]
+        case "$exception":
+            reserved = ["$exception_list", "$debug_images"]
+        case "$feature_flag_called":
+            reserved = ["$feature_flag_response", "$feature_flag_reason", "$feature_flag_id", "$feature_flag_version"]
+        default:
+            reserved = []
+        }
+        return properties.reduce(into: [:]) { result, entry in
+            if reserved.contains(entry.key) {
+                result[entry.key] = entry.value
+            } else {
+                result[entry.key] = removingNullObjectMembers(entry.value)
+            }
+        }
+    }
+}
+
+private func removingNullObjectMembers(_ value: Any) -> Any? {
+    // Foundation bridges boxed Optional.none to NSNull, as JSONSerialization does.
+    if value as AnyObject is NSNull {
+        return nil
+    }
+    if let dictionary = value as? [String: Any] {
+        return dictionary.reduce(into: [String: Any]()) { result, entry in
+            result[entry.key] = removingNullObjectMembers(entry.value)
+        }
+    }
+    if let array = value as? [Any] {
+        // Do not compact null slots or dictionaries emptied by recursive cleanup.
+        return array.map { removingNullObjectMembers($0) ?? NSNull() }
+    }
+    return value
 }
 
 enum PostHogKnownUnsafeEditableEvent: String {

PostHog/PostHogLegacyQueue.swift

@@ -25,9 +25,17 @@ func migrateOldQueue(queue: URL, oldQueue: URL) {
         }
 
         for item in array {
-            guard let event = item as? [String: Any] else {
+            guard var event = item as? [String: Any] else {
                 continue
             }
+            let eventName = event["event"] as? String ?? ""
+            if let properties = event["properties"] as? [String: Any] {
+                event["properties"] = PostHogEvent.serializedProperties(properties, event: eventName)
+            }
+            // v2 stored person properties outside the properties container.
+            if let setProperties = event["$set"] as? [String: Any] {
+                event["$set"] = PostHogEvent.serializedProperties(setProperties, event: "")
+            }
             let timestamp = event["timestamp"] as? String ?? toISO8601String(Date())
 
             let timestampDate = toISO8601Date(timestamp) ?? Date()

PostHogTests/PostHogEventNullSerializationTests.swift

@@ -0,0 +1,229 @@
+import Foundation
+@testable import PostHog
+import Testing
+
+@Suite(.serialized)
+struct PostHogEventNullSerializationTests {
+    private func properties() -> [String: Any] {
+        let none: String? = nil
+        return [
+            "test": NSNull(),
+            "optional": none as Any,
+            "nested": ["drop": NSNull()],
+            "items": ["1", NSNull(), 2, ["drop": NSNull()], [NSNull()]],
+            "empty": "", "zero": 0, "enabled": false,
+            "literal": "null", "literalUndefined": "undefined",
+            "emptyArray": [], "emptyObject": [:],
+            "$set": ["drop": NSNull()],
+            "$set_once": ["drop": NSNull()],
+            "$group_set": ["drop": NSNull()],
+        ]
+    }
+
+    private var expected: [String: Any] {
+        [
+            "nested": [:], "items": ["1", NSNull(), 2, [:], [NSNull()]],
+            "empty": "", "zero": 0, "enabled": false,
+            "literal": "null", "literalUndefined": "undefined",
+            "emptyArray": [], "emptyObject": [:],
+            "$set": [:], "$set_once": [:], "$group_set": [:],
+        ]
+    }
+
+    private func expectProperties(_ json: [String: Any], _ expected: [String: Any]) throws {
+        let actual = try #require(json["properties"] as? [String: Any])
+        #expect(NSDictionary(dictionary: actual).isEqual(to: expected))
+    }
+
+    @Test(arguments: ["Nullable Properties", "$screen", "$identify", "$groupidentify", "$exception", "$ai_generation", "$snapshot"])
+    func eventSerialization(eventName: String) throws {
+        let input = properties()
+        let event = PostHogEvent(event: eventName, distinctId: "test-user", properties: input)
+        let json = try #require(fromJSONData(try JSONSerialization.data(withJSONObject: event.toJSON())))
+        try expectProperties(json, expected)
+        #expect(event.properties["test"] is NSNull)
+        #expect(input["test"] is NSNull)
+        #expect(json["event"] as? String == eventName)
+        #expect(json["distinct_id"] as? String == "test-user")
+        #expect(json["uuid"] as? String == event.uuid.postHogUuidString)
+    }
+
+    @Test
+    func foundationBridgingAndOptionalArraySlots() throws {
+        let none: Int? = nil
+        let nested = NSMutableDictionary(dictionary: ["drop": NSNull()])
+        let items = NSMutableArray(array: [NSNull(), nested])
+        let event = PostHogEvent(event: "bridged", distinctId: "test-user", properties: [
+            "nested": nested, "items": items, "optionals": [none, 1] as [Int?],
+        ])
+        try expectProperties(event.toJSON(), ["nested": [:], "items": [NSNull(), [:]], "optionals": [NSNull(), 1]])
+        #expect(nested["drop"] is NSNull)
+        #expect(items.count == 2)
+    }
+
+    @Test
+    func allNullPropertiesKeepEventAndGenericJSONIsUnchanged() throws {
+        let event = PostHogEvent(event: "only null", distinctId: "test-user", properties: ["test": NSNull()])
+        try expectProperties(event.toJSON(), [:])
+        let generic = try #require(toJSONData(["test": NSNull()]))
+        #expect(fromJSONData(generic)?["test"] is NSNull)
+    }
+
+    @Test
+    func typedPayloadsKeepNullsOnlyOnTheirOwnEvents() throws {
+        let reserved: [(String, [String: Any])] = [
+            ("$snapshot", ["$snapshot_data": [["data": ["parentId": NSNull()]]]]),
+            ("$exception", ["$exception_list": [["value": NSNull()]], "$debug_images": [["debug_id": NSNull()]]]),
+            ("$feature_flag_called", [
+                "$feature_flag_response": NSNull(), "$feature_flag_reason": NSNull(),
+                "$feature_flag_id": NSNull(), "$feature_flag_version": NSNull(),
+            ]),
+        ]
+        for (name, payload) in reserved {
+            var input = payload
+            input["custom"] = ["drop": NSNull()]
+            var expected = payload
+            expected["custom"] = [String: Any]()
+            try expectProperties(PostHogEvent(event: name, distinctId: "test-user", properties: input).toJSON(), expected)
+        }
+        try expectProperties(PostHogEvent(event: "custom", distinctId: "test-user", properties: [
+            "$snapshot_data": ["drop": NSNull()], "$feature_flag_response": NSNull(),
+        ]).toJSON(), ["$snapshot_data": [:]])
+    }
+
+    // No SDK setup or shared Application Support storage. Every file belongs to this test.
+    private func withDirectory(_ body: (URL) throws -> Void) throws {
+        let root = URL(fileURLWithPath: #filePath).deletingLastPathComponent()
+            .appendingPathComponent(".null-serialization-\(UUID().uuidString)", isDirectory: true)
+        try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
+        defer { try? FileManager.default.removeItem(at: root) }
+        try body(root)
+    }
+
+    @Test
+    func beforeSendAdditionsPrivacyFilteringAndDrops() throws {
+        let chain = BeforeSendChain<PostHogEvent>()
+        chain.set([{ event in
+            event.properties.removeValue(forKey: "private")
+            event.properties["hookNull"] = NSNull()
+            event.properties["hookItems"] = [NSNull(), ["drop": NSNull()]]
+            return event
+        }])
+        let event = PostHogEvent(event: "hook", distinctId: "test-user", properties: ["private": "secret"])
+        let result = try #require(chain.run(event))
+        try expectProperties(result.toJSON(), ["hookItems": [NSNull(), [:]]])
+        chain.set([{ _ in nil }])
+        #expect(chain.run(event) == nil)
+    }
+
+    @Test
+    func diskWriteRestoreAndLateMutation() throws {
+        try withDirectory { root in
+            let event = PostHogEvent(event: "disk", distinctId: "test-user", properties: properties())
+            // Models values introduced downstream of capture-time sanitization, by enrichment/hooks.
+            event.properties["hookNull"] = NSNull()
+            event.properties["hookItems"] = [NSNull(), ["drop": NSNull()]]
+            var expected = expected
+            expected["hookItems"] = [NSNull(), [:]]
+            let queue = PostHogFileBackedQueue(queue: root)
+            queue.add(try #require(toJSONData(event.toJSON())))
+            let data = try #require(PostHogFileBackedQueue(queue: root).peek(1).first)
+            try expectProperties(try #require(fromJSONData(data)), expected)
+            let restored = try #require(PostHogEvent.fromJSON(data))
+            restored.properties["lateNull"] = NSNull()
+            try expectProperties(restored.toJSON(), expected)
+        }
+    }
+
+    @Test(arguments: [false, true])
+    func apiWireSerialization(snapshot: Bool) async throws {
+        let config = PostHogConfig(projectToken: "test-null-serialization", host: "http://127.0.0.1:1")
+        let session = URLSessionConfiguration.ephemeral
+        session.protocolClasses = [NullSerializationURLProtocol.self]
+        config.urlSessionConfiguration = session
+        let api = PostHogApi(config)
+        // Old persisted events can still contain nulls; the final API encoder must clean them.
+        let oldData = try JSONSerialization.data(withJSONObject: [
+            "event": snapshot ? "$snapshot" : "wire", "distinct_id": "test-user", "properties": properties(),
+        ])
+        let event = try #require(PostHogEvent.fromJSON(oldData))
+        event.properties["lateNull"] = NSNull()
+        let info: PostHogUploadInfo = await withCheckedContinuation { continuation in
+            if snapshot {
+                api.snapshot(events: [event]) { continuation.resume(returning: $0) }
+            } else {
+                api.batch(events: [event]) { continuation.resume(returning: $0) }
+            }
+        }
+        #expect(info.statusCode == 200)
+        let body = try #require(NullSerializationURLProtocol.body)
+        let json = try JSONSerialization.jsonObject(with: body)
+        let events = try #require(snapshot ? json as? [[String: Any]] : (json as? [String: Any])?["batch"] as? [[String: Any]])
+        try expectProperties(try #require(events.first), expected)
+    }
+
+    @Test
+    func legacyRewriteCleansPropertiesWithoutChangingEnvelope() throws {
+        try withDirectory { root in
+            let old = root.appendingPathComponent("v2.json")
+            let destination = root.appendingPathComponent("v3")
+            let legacy: [String: Any] = [
+                "event": "legacy", "distinct_id": "test-user", "timestamp": "2024-01-01T00:00:00Z",
+                "message_id": "old-id", "envelopeNull": NSNull(),
+                "properties": properties().filter { $0.key != "optional" }, "$set": ["drop": NSNull()],
+            ]
+            try JSONSerialization.data(withJSONObject: [legacy]).write(to: old)
+            let queue = PostHogFileBackedQueue(queue: destination, oldQueues: [old])
+            let data = try #require(queue.peek(1).first)
+            let json = try #require(fromJSONData(data))
+            try expectProperties(json, expected)
+            #expect((json["$set"] as? [String: Any])?.isEmpty == true)
+            #expect(json["envelopeNull"] is NSNull)
+            #expect(json["message_id"] as? String == "old-id")
+            try expectProperties(try #require(PostHogEvent.fromJSON(data)).toJSON(), expected)
+        }
+    }
+}
+
+// Intercepts every URL in the test session; unexpected requests cannot reach the network.
+private final class NullSerializationURLProtocol: URLProtocol {
+    private static let lock = NSLock()
+    private static var capturedBody: Data?
+    static var body: Data? { lock.withLock { capturedBody } }
+
+    override static func canInit(with _: URLRequest) -> Bool {
+        true
+    }
+    override static func canonicalRequest(for request: URLRequest) -> URLRequest {
+        request
+    }
+
+    override func startLoading() {
+        do {
+            #expect(request.url?.host == "127.0.0.1")
+            var data = request.httpBody ?? Data()
+            if let stream = request.httpBodyStream {
+                stream.open()
+                defer { stream.close() }
+                var buffer = [UInt8](repeating: 0, count: 4096)
+                while stream.hasBytesAvailable {
+                    let count = stream.read(&buffer, maxLength: buffer.count)
+                    if count <= 0 { break }
+                    data.append(contentsOf: buffer.prefix(count))
+                }
+            }
+            if request.value(forHTTPHeaderField: "Content-Encoding") == "gzip" {
+                data = try data.gunzipped()
+            }
+            Self.lock.withLock { Self.capturedBody = data }
+            let response = HTTPURLResponse(url: request.url!, statusCode: 200, httpVersion: nil, headerFields: nil)!
+            client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed)
+            client?.urlProtocol(self, didLoad: Data("{}".utf8))
+            client?.urlProtocolDidFinishLoading(self)
+        } catch {
+            client?.urlProtocol(self, didFailWithError: error)
+        }
+    }
+
+    override func stopLoading() {}
+}