fix: preserve null array positions in event serialization

.changeset/preserve-null-array-positions.md

@@ -0,0 +1,7 @@
+---
+"posthog": patch
+"posthog-android": patch
+"posthog-server": patch
+---
+
+Preserve null array positions when serializing event properties, while continuing to omit null-valued object members recursively.

posthog-server/src/test/java/com/posthog/server/PostHogNullPropertyTest.kt

@@ -0,0 +1,199 @@
+package com.posthog.server
+
+import com.google.gson.JsonObject
+import com.google.gson.JsonParser
+import com.posthog.internal.PostHogApi
+import com.posthog.server.internal.PostHogFeatureFlags
+import okhttp3.OkHttpClient
+import okhttp3.mockwebserver.MockResponse
+import okhttp3.mockwebserver.MockWebServer
+import java.io.IOException
+import java.net.InetAddress
+import java.net.Proxy
+import java.util.concurrent.TimeUnit
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertNotNull
+import kotlin.test.assertTrue
+
+internal class PostHogNullPropertyTest {
+    @Test
+    fun `server capture exception and hook properties preserve null slots in memory queue batch`() {
+        withServer { sut, http ->
+            val properties = mapOf("nested" to mapOf("drop" to null), "items" to listOf("1", null, 2, mapOf("drop" to null), listOf(null)))
+            sut.capture("user", "Nullable", properties)
+            sut.captureException(IllegalStateException("synthetic"), "user", properties)
+            sut.capture("user", "Dropped", properties)
+            assertEquals(0, http.requestCount)
+            http.enqueue(MockResponse().setBody("{}"))
+            sut.flush()
+            val request = assertNotNull(http.takeRequest(5, TimeUnit.SECONDS))
+            assertEquals("/batch", request.path)
+            val events = JsonParser.parseString(request.body.readUtf8()).asJsonObject.getAsJsonArray("batch")
+            assertEquals(setOf("Nullable", "${'$'}exception"), events.map { it.asJsonObject["event"].asString }.toSet())
+            events.forEach {
+                val actual = it.asJsonObject.getAsJsonObject("properties")
+                assertEquals(JsonParser.parseString("""["1",null,2,{},[null]]"""), actual["items"])
+                assertEquals(JsonObject(), actual["nested"])
+                assertHook(actual)
+            }
+            assertTrue((properties["nested"] as Map<*, *>).containsKey("drop"))
+            assertEquals(5, (properties["items"] as List<*>).size)
+        }
+    }
+
+    @Test
+    fun `fatal server memory queue serializes hook properties on blocking path`() {
+        withServer { sut, http ->
+            http.enqueue(MockResponse().setBody("{}"))
+            // Exercise the fatal queue marker without installing a handler or crashing the process.
+            sut.capture("user", "${'$'}exception", mapOf("${'$'}exception_level" to "fatal"))
+            assertEquals(1, http.requestCount)
+            val request = assertNotNull(http.takeRequest(5, TimeUnit.SECONDS))
+            assertEquals("/batch", request.path)
+            val event = JsonParser.parseString(request.body.readUtf8()).asJsonObject.getAsJsonArray("batch").single().asJsonObject
+            assertEquals("fatal", event.getAsJsonObject("properties")["${'$'}exception_level"].asString)
+            assertHook(event.getAsJsonObject("properties"))
+        }
+    }
+
+    @Test
+    fun `reading flag definitions from cache retains baseline null array compaction`() {
+        withFlagCache(fetch = false)
+    }
+
+    @Test
+    fun `writing flag definitions to cache retains baseline null array compaction`() {
+        withFlagCache(fetch = true)
+    }
+
+    private fun withFlagCache(fetch: Boolean) {
+        val http = MockWebServer()
+        http.start(InetAddress.getByName("127.0.0.1"), 0)
+        val transport =
+            OkHttpClient.Builder()
+                .proxy(Proxy.NO_PROXY)
+                .followRedirects(false)
+                .followSslRedirects(false)
+                .addInterceptor { chain ->
+                    if (!fetch || chain.request().url.host != "127.0.0.1") {
+                        throw IOException("Unexpected SDK request: ${chain.request().url}")
+                    }
+                    chain.proceed(chain.request())
+                }
+                .build()
+        val config =
+            com.posthog.PostHogConfig(
+                "null-properties-cache-test",
+                http.url("/").newBuilder().host("127.0.0.1").build().toString(),
+            ).apply { httpClient = transport }
+        val definition =
+            """
+            {"flags":[{"id":1,"name":"cached","key":"cached","active":true,"version":1,
+            "filters":{"groups":[{"properties":[],"rollout_percentage":100}],"payloads":{"true":[null,"x"]}}}],
+            "group_type_mapping":{},"cohorts":{}}
+            """.trimIndent()
+        val cacheData: Map<String, Any?> = config.serializer.deserialize(definition.reader())
+        var storedData: Map<String, Any?>? = null
+        val provider =
+            object : PostHogBlockingFlagDefinitionCacheProvider() {
+                override fun getFlagDefinitionsBlocking() = cacheData
+
+                override fun shouldFetchFlagDefinitionsBlocking() = fetch
+
+                override fun onFlagDefinitionsReceivedBlocking(data: Map<String, Any?>) {
+                    storedData = data
+                }
+            }
+        val sut =
+            PostHogFeatureFlags(
+                config,
+                PostHogApi(config),
+                60000,
+                100,
+                localEvaluation = true,
+                personalApiKey = "null-properties-personal-test",
+                pollerEnabled = false,
+                flagDefinitionCacheProvider = provider,
+            )
+        try {
+            if (fetch) http.enqueue(MockResponse().setBody(definition))
+            sut.loadFeatureFlagDefinitions()
+            if (fetch) {
+                val request = assertNotNull(http.takeRequest(5, TimeUnit.SECONDS))
+                assertTrue(request.path!!.startsWith("/api/feature_flag/local_evaluation/"))
+                val flags = assertNotNull(storedData)["flags"] as List<*>
+                val filters = (flags.single() as Map<*, *>)["filters"] as Map<*, *>
+                assertEquals(listOf("x"), (filters["payloads"] as Map<*, *>)["true"])
+                assertEquals(1, http.requestCount)
+            } else {
+                // Local evaluation retains its existing list-to-string payload conversion.
+                assertEquals("[x]", sut.getFeatureFlagPayload("cached", distinctId = "user"))
+                assertEquals(0, http.requestCount)
+            }
+            val originalFilters = ((cacheData["flags"] as List<*>).single() as Map<*, *>)["filters"] as Map<*, *>
+            assertEquals(listOf(null, "x"), (originalFilters["payloads"] as Map<*, *>)["true"])
+        } finally {
+            sut.clear()
+            transport.connectionPool.evictAll()
+            transport.dispatcher.executorService.shutdown()
+            http.shutdown()
+        }
+    }
+
+    private fun assertHook(properties: JsonObject) {
+        assertFalse(properties.has("hookNull"))
+        assertEquals(JsonParser.parseString("""[null,{},[null],false,0,""]"""), properties["hookItems"])
+    }
+
+    private fun withServer(test: (PostHog, MockWebServer) -> Unit) {
+        val http = MockWebServer()
+        http.start(InetAddress.getByName("127.0.0.1"), 0)
+        val transport =
+            OkHttpClient.Builder()
+                .proxy(Proxy.NO_PROXY)
+                .followRedirects(false)
+                .followSslRedirects(false)
+                .addInterceptor { chain ->
+                    if (chain.request().url.host != "127.0.0.1") {
+                        throw IOException("Non-loopback SDK request forbidden: ${chain.request().url}")
+                    }
+                    chain.proceed(chain.request())
+                }
+                .build()
+        val serverConfig =
+            PostHogConfig(
+                "null-properties-server-test",
+                http.url("/").newBuilder().host("127.0.0.1").build().toString(),
+                preloadFeatureFlags = false,
+                flushAt = 100,
+            ).apply {
+                flushIntervalSeconds = 3600
+                addBeforeSend { event ->
+                    if (event.event == "Dropped") {
+                        null
+                    } else {
+                        // Java-style null entries remain legal in the existing nullable runtime API.
+                        @Suppress("UNCHECKED_CAST")
+                        val properties = event.properties as MutableMap<String, Any?>
+                        properties["hookNull"] = null
+                        properties["hookItems"] = listOf(null, mapOf("drop" to null), listOf(null), false, 0, "")
+                        event
+                    }
+                }
+            }
+        // Use the real server config conversion (including its memory queue), with guarded HTTP.
+        val config = serverConfig.asCoreConfig().apply { httpClient = transport }
+        val sut = PostHog()
+        try {
+            sut.setup(config)
+            test(sut, http)
+        } finally {
+            sut.close()
+            transport.connectionPool.evictAll()
+            transport.dispatcher.executorService.shutdown()
+            http.shutdown()
+        }
+    }
+}

posthog-server/src/test/resources/json/server-safe-properties.json

@@ -18,10 +18,12 @@
         },
         "ordered_array": [
           "alpha",
+          null,
           "omega"
         ],
         "ordered_list": [
           "first",
+          null,
           "third"
         ]
       },

posthog/src/main/java/com/posthog/internal/GsonSafeMapSerializer.kt

@@ -2,6 +2,7 @@ package com.posthog.internal
 
 import com.google.gson.JsonArray
 import com.google.gson.JsonElement
+import com.google.gson.JsonNull
 import com.google.gson.JsonObject
 import com.google.gson.JsonSerializationContext
 import com.google.gson.JsonSerializer
@@ -18,7 +19,10 @@ import java.lang.reflect.Type
  *
  * @property config the Config
  */
-internal class GsonSafeMapSerializer(private val config: PostHogConfig) : JsonSerializer<Map<String, Any?>> {
+internal class GsonSafeMapSerializer(
+    private val config: PostHogConfig,
+    private val preserveNullArrayElements: Boolean = false,
+) : JsonSerializer<Map<String, Any?>> {
     private val mapType: Type = object : TypeToken<Map<String, Any?>>() {}.type
 
     override fun serialize(
@@ -57,7 +61,12 @@ internal class GsonSafeMapSerializer(private val config: PostHogConfig) : JsonSe
             }
             else -> {
                 try {
-                    config.serializer.gson.toJsonTree(targetValue)
+                    if (preserveNullArrayElements) {
+                        // Keep event mode for maps reached through reflected custom values.
+                        context.serialize(targetValue)
+                    } else {
+                        config.serializer.gson.toJsonTree(targetValue)
+                    }
                 } catch (e: Throwable) {
                     config.logger.log(
                         "Property '$key' with value '$targetValue' cannot be serialized to JSON: $e. " +
@@ -70,7 +79,8 @@ internal class GsonSafeMapSerializer(private val config: PostHogConfig) : JsonSe
     }
 
     /**
-     * Safely serializes a list, filtering out unserializable elements.
+     * Safely serializes a list, preserving null positions only for event encoding.
+     * Unserializable elements are always filtered out.
      */
     private fun safeSerializeList(
         list: List<*>,
@@ -79,7 +89,11 @@ internal class GsonSafeMapSerializer(private val config: PostHogConfig) : JsonSe
         val jsonArray = JsonArray()
 
         list.forEach { element ->
-            if (element != null) {
+            if (element == null) {
+                if (preserveNullArrayElements) {
+                    jsonArray.add(JsonNull.INSTANCE)
+                }
+            } else {
                 val serialized = safeSerializeValue(element, "list-element", context)
                 if (serialized != null) {
                     jsonArray.add(serialized)

posthog/src/main/java/com/posthog/internal/PostHogSerializer.kt

@@ -3,9 +3,11 @@ package com.posthog.internal
 import com.google.gson.Gson
 import com.google.gson.GsonBuilder
 import com.google.gson.JsonIOException
+import com.google.gson.JsonSerializer
 import com.google.gson.JsonSyntaxException
 import com.google.gson.reflect.TypeToken
 import com.posthog.PostHogConfig
+import com.posthog.PostHogEvent
 import com.posthog.PostHogInternal
 import com.posthog.internal.replay.GsonRREventTypeSerializer
 import com.posthog.internal.replay.GsonRRIncrementalSourceSerializer
@@ -81,6 +83,26 @@ public class PostHogSerializer(private val config: PostHogConfig) {
             registerTypeAdapter(PropertyValue::class.java, GsonPropertyValueAdapter())
             registerTypeAdapter(PropertyOperator::class.java, GsonPropertyOperatorAdapter())
             registerTypeAdapter(PropertyType::class.java, GsonPropertyTypeAdapter())
+
+            // Only event encoding opts into null array slots; log storage and flag caches
+            // keep the general map adapter's existing behavior. Build before registering
+            // the event adapter so its reflected event encoding cannot recurse into itself.
+            val eventMapSerializer = GsonSafeMapSerializer(config, preserveNullArrayElements = true)
+            val eventGson =
+                create().newBuilder()
+                    .registerTypeAdapter(
+                        object : TypeToken<Map<String, Any?>>() {}.type,
+                        eventMapSerializer,
+                    )
+                    .registerTypeAdapter(
+                        object : TypeToken<MutableMap<String, Any?>>() {}.type,
+                        eventMapSerializer,
+                    )
+                    .create()
+            registerTypeAdapter(
+                PostHogEvent::class.java,
+                JsonSerializer<PostHogEvent> { event, _, _ -> eventGson.toJsonTree(event) },
+            )
         }.create()
 
     @Throws(JsonIOException::class, IOException::class)

posthog/src/test/java/com/posthog/internal/NullPropertyInputs.java

@@ -0,0 +1,29 @@
+package com.posthog.internal;
+
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/** Java callers can put null into the existing Map<String, Object> API. */
+public final class NullPropertyInputs {
+    private NullPropertyInputs() {}
+
+    public static Map<String, Object> properties() {
+        Map<String, Object> empty = new LinkedHashMap<>();
+        empty.put("drop", null);
+        Map<String, Object> properties = new LinkedHashMap<>();
+        properties.put("test", null);
+        properties.put("nested", empty);
+        properties.put("items", Arrays.asList("1", null, 2, empty, Arrays.asList((Object) null)));
+        properties.put("array", new Object[] {null, empty});
+        properties.put("$set", empty);
+        properties.put("$group_set", empty);
+        properties.put("empty", "");
+        properties.put("zero", 0);
+        properties.put("enabled", false);
+        properties.put("literal", "null");
+        properties.put("literalUndefined", "undefined");
+        properties.put("emptyArray", new Object[] {});
+        return properties;
+    }
+}