From 7e4837c2486e73beca3ddab59bd7f5459e14bf4c Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Wed, 9 Sep 2026 10:52:07 +0200 Subject: [PATCH 1/2] fix: preserve null array positions in event serialization --- .changeset/preserve-null-array-positions.md | 7 + .../posthog/server/PostHogNullPropertyTest.kt | 199 ++++++++++++++++++ .../posthog/internal/GsonSafeMapSerializer.kt | 22 +- .../com/posthog/internal/PostHogSerializer.kt | 22 ++ .../posthog/internal/NullPropertyInputs.java | 29 +++ .../PostHogNullPropertySerializationTest.kt | 134 ++++++++++++ .../internal/PostHogQueueNullPropertyTest.kt | 186 ++++++++++++++++ 7 files changed, 595 insertions(+), 4 deletions(-) create mode 100644 .changeset/preserve-null-array-positions.md create mode 100644 posthog-server/src/test/java/com/posthog/server/PostHogNullPropertyTest.kt create mode 100644 posthog/src/test/java/com/posthog/internal/NullPropertyInputs.java create mode 100644 posthog/src/test/java/com/posthog/internal/PostHogNullPropertySerializationTest.kt create mode 100644 posthog/src/test/java/com/posthog/internal/PostHogQueueNullPropertyTest.kt diff --git a/.changeset/preserve-null-array-positions.md b/.changeset/preserve-null-array-positions.md new file mode 100644 index 000000000..ba466ca3e --- /dev/null +++ b/.changeset/preserve-null-array-positions.md @@ -0,0 +1,7 @@ +--- +"posthog": patch +"posthog-android": patch +"posthog-server": patch +--- + +Preserve null array positions when serializing event properties, while continuing to omit null-valued object members recursively. diff --git a/posthog-server/src/test/java/com/posthog/server/PostHogNullPropertyTest.kt b/posthog-server/src/test/java/com/posthog/server/PostHogNullPropertyTest.kt new file mode 100644 index 000000000..d95e03460 --- /dev/null +++ b/posthog-server/src/test/java/com/posthog/server/PostHogNullPropertyTest.kt @@ -0,0 +1,199 @@ +package com.posthog.server + +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import com.posthog.internal.PostHogApi +import com.posthog.server.internal.PostHogFeatureFlags +import okhttp3.OkHttpClient +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import java.io.IOException +import java.net.InetAddress +import java.net.Proxy +import java.util.concurrent.TimeUnit +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +internal class PostHogNullPropertyTest { + @Test + fun `server capture exception and hook properties preserve null slots in memory queue batch`() { + withServer { sut, http -> + val properties = mapOf("nested" to mapOf("drop" to null), "items" to listOf("1", null, 2, mapOf("drop" to null), listOf(null))) + sut.capture("user", "Nullable", properties) + sut.captureException(IllegalStateException("synthetic"), "user", properties) + sut.capture("user", "Dropped", properties) + assertEquals(0, http.requestCount) + http.enqueue(MockResponse().setBody("{}")) + sut.flush() + val request = assertNotNull(http.takeRequest(5, TimeUnit.SECONDS)) + assertEquals("/batch", request.path) + val events = JsonParser.parseString(request.body.readUtf8()).asJsonObject.getAsJsonArray("batch") + assertEquals(setOf("Nullable", "${'$'}exception"), events.map { it.asJsonObject["event"].asString }.toSet()) + events.forEach { + val actual = it.asJsonObject.getAsJsonObject("properties") + assertEquals(JsonParser.parseString("""["1",null,2,{},[null]]"""), actual["items"]) + assertEquals(JsonObject(), actual["nested"]) + assertHook(actual) + } + assertTrue((properties["nested"] as Map<*, *>).containsKey("drop")) + assertEquals(5, (properties["items"] as List<*>).size) + } + } + + @Test + fun `fatal server memory queue serializes hook properties on blocking path`() { + withServer { sut, http -> + http.enqueue(MockResponse().setBody("{}")) + // Exercise the fatal queue marker without installing a handler or crashing the process. + sut.capture("user", "${'$'}exception", mapOf("${'$'}exception_level" to "fatal")) + assertEquals(1, http.requestCount) + val request = assertNotNull(http.takeRequest(5, TimeUnit.SECONDS)) + assertEquals("/batch", request.path) + val event = JsonParser.parseString(request.body.readUtf8()).asJsonObject.getAsJsonArray("batch").single().asJsonObject + assertEquals("fatal", event.getAsJsonObject("properties")["${'$'}exception_level"].asString) + assertHook(event.getAsJsonObject("properties")) + } + } + + @Test + fun `reading flag definitions from cache retains baseline null array compaction`() { + withFlagCache(fetch = false) + } + + @Test + fun `writing flag definitions to cache retains baseline null array compaction`() { + withFlagCache(fetch = true) + } + + private fun withFlagCache(fetch: Boolean) { + val http = MockWebServer() + http.start(InetAddress.getByName("127.0.0.1"), 0) + val transport = + OkHttpClient.Builder() + .proxy(Proxy.NO_PROXY) + .followRedirects(false) + .followSslRedirects(false) + .addInterceptor { chain -> + if (!fetch || chain.request().url.host != "127.0.0.1") { + throw IOException("Unexpected SDK request: ${chain.request().url}") + } + chain.proceed(chain.request()) + } + .build() + val config = + com.posthog.PostHogConfig( + "null-properties-cache-test", + http.url("/").newBuilder().host("127.0.0.1").build().toString(), + ).apply { httpClient = transport } + val definition = + """ + {"flags":[{"id":1,"name":"cached","key":"cached","active":true,"version":1, + "filters":{"groups":[{"properties":[],"rollout_percentage":100}],"payloads":{"true":[null,"x"]}}}], + "group_type_mapping":{},"cohorts":{}} + """.trimIndent() + val cacheData: Map = config.serializer.deserialize(definition.reader()) + var storedData: Map? = null + val provider = + object : PostHogBlockingFlagDefinitionCacheProvider() { + override fun getFlagDefinitionsBlocking() = cacheData + + override fun shouldFetchFlagDefinitionsBlocking() = fetch + + override fun onFlagDefinitionsReceivedBlocking(data: Map) { + storedData = data + } + } + val sut = + PostHogFeatureFlags( + config, + PostHogApi(config), + 60000, + 100, + localEvaluation = true, + personalApiKey = "null-properties-personal-test", + pollerEnabled = false, + flagDefinitionCacheProvider = provider, + ) + try { + if (fetch) http.enqueue(MockResponse().setBody(definition)) + sut.loadFeatureFlagDefinitions() + if (fetch) { + val request = assertNotNull(http.takeRequest(5, TimeUnit.SECONDS)) + assertTrue(request.path!!.startsWith("/api/feature_flag/local_evaluation/")) + val flags = assertNotNull(storedData)["flags"] as List<*> + val filters = (flags.single() as Map<*, *>)["filters"] as Map<*, *> + assertEquals(listOf("x"), (filters["payloads"] as Map<*, *>)["true"]) + assertEquals(1, http.requestCount) + } else { + // Local evaluation retains its existing list-to-string payload conversion. + assertEquals("[x]", sut.getFeatureFlagPayload("cached", distinctId = "user")) + assertEquals(0, http.requestCount) + } + val originalFilters = ((cacheData["flags"] as List<*>).single() as Map<*, *>)["filters"] as Map<*, *> + assertEquals(listOf(null, "x"), (originalFilters["payloads"] as Map<*, *>)["true"]) + } finally { + sut.clear() + transport.connectionPool.evictAll() + transport.dispatcher.executorService.shutdown() + http.shutdown() + } + } + + private fun assertHook(properties: JsonObject) { + assertFalse(properties.has("hookNull")) + assertEquals(JsonParser.parseString("""[null,{},[null],false,0,""]"""), properties["hookItems"]) + } + + private fun withServer(test: (PostHog, MockWebServer) -> Unit) { + val http = MockWebServer() + http.start(InetAddress.getByName("127.0.0.1"), 0) + val transport = + OkHttpClient.Builder() + .proxy(Proxy.NO_PROXY) + .followRedirects(false) + .followSslRedirects(false) + .addInterceptor { chain -> + if (chain.request().url.host != "127.0.0.1") { + throw IOException("Non-loopback SDK request forbidden: ${chain.request().url}") + } + chain.proceed(chain.request()) + } + .build() + val serverConfig = + PostHogConfig( + "null-properties-server-test", + http.url("/").newBuilder().host("127.0.0.1").build().toString(), + preloadFeatureFlags = false, + flushAt = 100, + ).apply { + flushIntervalSeconds = 3600 + addBeforeSend { event -> + if (event.event == "Dropped") { + null + } else { + // Java-style null entries remain legal in the existing nullable runtime API. + @Suppress("UNCHECKED_CAST") + val properties = event.properties as MutableMap + properties["hookNull"] = null + properties["hookItems"] = listOf(null, mapOf("drop" to null), listOf(null), false, 0, "") + event + } + } + } + // Use the real server config conversion (including its memory queue), with guarded HTTP. + val config = serverConfig.asCoreConfig().apply { httpClient = transport } + val sut = PostHog() + try { + sut.setup(config) + test(sut, http) + } finally { + sut.close() + transport.connectionPool.evictAll() + transport.dispatcher.executorService.shutdown() + http.shutdown() + } + } +} diff --git a/posthog/src/main/java/com/posthog/internal/GsonSafeMapSerializer.kt b/posthog/src/main/java/com/posthog/internal/GsonSafeMapSerializer.kt index 103f6d222..f164004f6 100644 --- a/posthog/src/main/java/com/posthog/internal/GsonSafeMapSerializer.kt +++ b/posthog/src/main/java/com/posthog/internal/GsonSafeMapSerializer.kt @@ -2,6 +2,7 @@ package com.posthog.internal import com.google.gson.JsonArray import com.google.gson.JsonElement +import com.google.gson.JsonNull import com.google.gson.JsonObject import com.google.gson.JsonSerializationContext import com.google.gson.JsonSerializer @@ -18,7 +19,10 @@ import java.lang.reflect.Type * * @property config the Config */ -internal class GsonSafeMapSerializer(private val config: PostHogConfig) : JsonSerializer> { +internal class GsonSafeMapSerializer( + private val config: PostHogConfig, + private val preserveNullArrayElements: Boolean = false, +) : JsonSerializer> { private val mapType: Type = object : TypeToken>() {}.type override fun serialize( @@ -57,7 +61,12 @@ internal class GsonSafeMapSerializer(private val config: PostHogConfig) : JsonSe } else -> { try { - config.serializer.gson.toJsonTree(targetValue) + if (preserveNullArrayElements) { + // Keep event mode for maps reached through reflected custom values. + context.serialize(targetValue) + } else { + config.serializer.gson.toJsonTree(targetValue) + } } catch (e: Throwable) { config.logger.log( "Property '$key' with value '$targetValue' cannot be serialized to JSON: $e. " + @@ -70,7 +79,8 @@ internal class GsonSafeMapSerializer(private val config: PostHogConfig) : JsonSe } /** - * Safely serializes a list, filtering out unserializable elements. + * Safely serializes a list, preserving null positions only for event encoding. + * Unserializable elements are always filtered out. */ private fun safeSerializeList( list: List<*>, @@ -79,7 +89,11 @@ internal class GsonSafeMapSerializer(private val config: PostHogConfig) : JsonSe val jsonArray = JsonArray() list.forEach { element -> - if (element != null) { + if (element == null) { + if (preserveNullArrayElements) { + jsonArray.add(JsonNull.INSTANCE) + } + } else { val serialized = safeSerializeValue(element, "list-element", context) if (serialized != null) { jsonArray.add(serialized) diff --git a/posthog/src/main/java/com/posthog/internal/PostHogSerializer.kt b/posthog/src/main/java/com/posthog/internal/PostHogSerializer.kt index 9def9342a..7e19d36a7 100644 --- a/posthog/src/main/java/com/posthog/internal/PostHogSerializer.kt +++ b/posthog/src/main/java/com/posthog/internal/PostHogSerializer.kt @@ -3,9 +3,11 @@ package com.posthog.internal import com.google.gson.Gson import com.google.gson.GsonBuilder import com.google.gson.JsonIOException +import com.google.gson.JsonSerializer import com.google.gson.JsonSyntaxException import com.google.gson.reflect.TypeToken import com.posthog.PostHogConfig +import com.posthog.PostHogEvent import com.posthog.PostHogInternal import com.posthog.internal.replay.GsonRREventTypeSerializer import com.posthog.internal.replay.GsonRRIncrementalSourceSerializer @@ -81,6 +83,26 @@ public class PostHogSerializer(private val config: PostHogConfig) { registerTypeAdapter(PropertyValue::class.java, GsonPropertyValueAdapter()) registerTypeAdapter(PropertyOperator::class.java, GsonPropertyOperatorAdapter()) registerTypeAdapter(PropertyType::class.java, GsonPropertyTypeAdapter()) + + // Only event encoding opts into null array slots; log storage and flag caches + // keep the general map adapter's existing behavior. Build before registering + // the event adapter so its reflected event encoding cannot recurse into itself. + val eventMapSerializer = GsonSafeMapSerializer(config, preserveNullArrayElements = true) + val eventGson = + create().newBuilder() + .registerTypeAdapter( + object : TypeToken>() {}.type, + eventMapSerializer, + ) + .registerTypeAdapter( + object : TypeToken>() {}.type, + eventMapSerializer, + ) + .create() + registerTypeAdapter( + PostHogEvent::class.java, + JsonSerializer { event, _, _ -> eventGson.toJsonTree(event) }, + ) }.create() @Throws(JsonIOException::class, IOException::class) diff --git a/posthog/src/test/java/com/posthog/internal/NullPropertyInputs.java b/posthog/src/test/java/com/posthog/internal/NullPropertyInputs.java new file mode 100644 index 000000000..e6b840927 --- /dev/null +++ b/posthog/src/test/java/com/posthog/internal/NullPropertyInputs.java @@ -0,0 +1,29 @@ +package com.posthog.internal; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Java callers can put null into the existing Map API. */ +public final class NullPropertyInputs { + private NullPropertyInputs() {} + + public static Map properties() { + Map empty = new LinkedHashMap<>(); + empty.put("drop", null); + Map properties = new LinkedHashMap<>(); + properties.put("test", null); + properties.put("nested", empty); + properties.put("items", Arrays.asList("1", null, 2, empty, Arrays.asList((Object) null))); + properties.put("array", new Object[] {null, empty}); + properties.put("$set", empty); + properties.put("$group_set", empty); + properties.put("empty", ""); + properties.put("zero", 0); + properties.put("enabled", false); + properties.put("literal", "null"); + properties.put("literalUndefined", "undefined"); + properties.put("emptyArray", new Object[] {}); + return properties; + } +} diff --git a/posthog/src/test/java/com/posthog/internal/PostHogNullPropertySerializationTest.kt b/posthog/src/test/java/com/posthog/internal/PostHogNullPropertySerializationTest.kt new file mode 100644 index 000000000..a9b34677b --- /dev/null +++ b/posthog/src/test/java/com/posthog/internal/PostHogNullPropertySerializationTest.kt @@ -0,0 +1,134 @@ +package com.posthog.internal + +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import com.posthog.PostHogConfig +import com.posthog.PostHogEvent +import java.io.StringWriter +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +internal class PostHogNullPropertySerializationTest { + private val serializer = PostHogConfig("null-properties-test", "http://127.0.0.1").serializer + + @Test + fun `event and batch omit null members without compacting Java arrays or mutating input`() { + val properties = NullPropertyInputs.properties() + val original = serializer.gson.toJsonTree(properties).deepCopy() + val event = PostHogEvent("Nullable", "user", properties) + val encoded = StringWriter() + serializer.serialize(event, encoded) + assertEquals(expectedProperties(), JsonParser.parseString(encoded.toString()).asJsonObject["properties"]) + + val batch = StringWriter() + serializer.serialize(PostHogBatchEvent("null-properties-test", listOf(event)), batch) + val batchEvent = JsonParser.parseString(batch.toString()).asJsonObject.getAsJsonArray("batch")[0].asJsonObject + assertEquals(expectedProperties(), batchEvent["properties"]) + assertEquals(original, serializer.gson.toJsonTree(properties)) + assertTrue(properties.containsKey("test")) + assertNull(properties["test"]) + } + + @Test + fun `nullable Kotlin containers preserve every null position`() { + val properties = + mutableMapOf( + "items" to listOf(null, mapOf("drop" to null), null, arrayOf(null, false, 0, ""), null), + ) + val encoded = StringWriter() + serializer.serialize(PostHogEvent("Nullable", "user", properties), encoded) + val actual = JsonParser.parseString(encoded.toString()).asJsonObject["properties"] + assertEquals(JsonParser.parseString("""{"items":[null,{},null,[null,false,0,""],null]}"""), actual) + } + + @Test + fun `null-only property object and absent typed metadata retain existing semantics`() { + val properties = NullPropertyInputs.properties().apply { keys.retainAll(setOf("test")) } + val encoded = StringWriter() + serializer.serialize(PostHogEvent("OnlyNull", "user", properties), encoded) + val actual = JsonParser.parseString(encoded.toString()).asJsonObject + assertEquals(JsonObject(), actual["properties"]) + assertEquals("OnlyNull", actual["event"].asString) + assertFalse(actual.has("api_key")) + assertFalse(actual.has("message_id")) + } + + @Test + fun `Gson tree and object conversions omit null members and preserve array positions`() { + class CustomProperty(val items: List, val absent: String? = null) + + val properties = + mutableMapOf( + "tree" to JsonParser.parseString("""{"drop":null,"items":[null,{"drop":null}]}"""), + "object" to CustomProperty(listOf(null, mapOf("drop" to null))), + ) + val encoded = StringWriter() + serializer.serialize(PostHogEvent("Converted", "user", properties), encoded) + assertEquals( + JsonParser.parseString("""{"tree":{"items":[null,{}]},"object":{"items":[null,{}]}}"""), + JsonParser.parseString(encoded.toString()).asJsonObject["properties"], + ) + } + + @Test + fun `event mode reaches maps inside reflected custom properties`() { + class CustomProperty(val nested: Map) + + val nested = mapOf("drop" to null, "items" to listOf(null, "x", mapOf("drop" to null))) + val properties = mutableMapOf("object" to CustomProperty(nested)) + val encoded = StringWriter() + serializer.serialize(PostHogEvent("Converted", "user", properties), encoded) + assertEquals( + JsonParser.parseString("""{"object":{"nested":{"items":[null,"x",{}]}}}"""), + JsonParser.parseString(encoded.toString()).asJsonObject["properties"], + ) + val general = StringWriter() + serializer.serialize(properties, general) + assertEquals( + JsonParser.parseString("""{"object":{"nested":{"items":["x",{}]}}}"""), + JsonParser.parseString(general.toString()), + ) + } + + @Test + fun `unsupported list elements are still dropped rather than replaced with null`() { + val properties = mutableMapOf("items" to listOf(null, Thread.currentThread(), "good", null)) + val encoded = StringWriter() + serializer.serialize(PostHogEvent("Unsupported", "user", properties), encoded) + assertEquals( + JsonParser.parseString("""{"items":[null,"good",null]}"""), + JsonParser.parseString(encoded.toString()).asJsonObject["properties"], + ) + } + + @Test + fun `event adapter does not change general map serialization before or after encoding`() { + val properties = mutableMapOf("items" to listOf(null, "x", mapOf("drop" to null))) + repeat(2) { + val mutableMap = StringWriter() + serializer.serialize(properties, mutableMap) + assertEquals(JsonParser.parseString("""{"items":["x",{}]}"""), JsonParser.parseString(mutableMap.toString())) + val map = StringWriter() + serializer.serialize>(properties, map) + assertEquals(mutableMap.toString(), map.toString()) + + val event = StringWriter() + serializer.serialize(PostHogEvent("Nullable", "user", properties), event) + assertEquals( + JsonParser.parseString("""{"items":[null,"x",{}]}"""), + JsonParser.parseString(event.toString()).asJsonObject["properties"], + ) + } + } + + private fun expectedProperties() = + JsonParser.parseString( + """ + {"nested":{},"items":["1",null,2,{},[null]],"array":[null,{}],"${'$'}set":{},"${'$'}group_set":{}, + "empty":"","zero":0,"enabled":false,"literal":"null","literalUndefined":"undefined","emptyArray":[]} + """.trimIndent(), + ) +} diff --git a/posthog/src/test/java/com/posthog/internal/PostHogQueueNullPropertyTest.kt b/posthog/src/test/java/com/posthog/internal/PostHogQueueNullPropertyTest.kt new file mode 100644 index 000000000..dce1ad56e --- /dev/null +++ b/posthog/src/test/java/com/posthog/internal/PostHogQueueNullPropertyTest.kt @@ -0,0 +1,186 @@ +package com.posthog.internal + +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import com.posthog.PostHogConfig +import com.posthog.PostHogEvent +import com.posthog.PostHogStateless +import com.posthog.logs.PostHogLogRecord +import okhttp3.OkHttpClient +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.junit.Rule +import org.junit.rules.TemporaryFolder +import java.io.File +import java.io.IOException +import java.net.InetAddress +import java.net.Proxy +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +internal class PostHogQueueNullPropertyTest { + @get:Rule + val tmpDir = TemporaryFolder(File("build").apply { mkdirs() }) + + @Test + fun `hook properties are normalized on disk and fresh queue restore to wire`() { + val http = MockWebServer() + http.start(InetAddress.getByName("127.0.0.1"), 0) + val executor = Executors.newSingleThreadScheduledExecutor() + val flagsExecutor = Executors.newSingleThreadScheduledExecutor() + val transport = loopbackClient() + val config = + PostHogConfig("null-properties-test", http.url("/").newBuilder().host("127.0.0.1").build().toString()).apply { + storagePrefix = tmpDir.root.absolutePath + flushAt = 100 + flushIntervalSeconds = 3600 + preloadFeatureFlags = false + httpClient = transport + addBeforeSend { event -> + if (event.event == "Dropped") { + null + } else { + event.properties!!["hook"] = mapOf("drop" to null, "items" to listOf(null, mapOf("drop" to null))) + event + } + } + } + val sut = + object : PostHogStateless(executor, flagsExecutor) { + fun clear() = queue?.clear() + } + var restored: PostHogQueue? = null + try { + sut.setup(config) + val input = NullPropertyInputs.properties() + sut.captureStateless("Nullable", "user", input) + sut.captureStateless("OnlyNull", "user", input.filterKeys { it == "test" }) + sut.captureExceptionStateless(IllegalStateException("synthetic"), "user", input) + sut.captureStateless("Dropped", "user", input) + executor.submit {}.get(5, TimeUnit.SECONDS) + + val files = File(tmpDir.root, config.apiKey).listFiles()!!.filter { it.extension == "event" } + assertEquals(3, files.size) + assertEquals(0, http.requestCount) + val diskEvents = files.map { JsonParser.parseString(it.readText()).asJsonObject } + diskEvents.forEach { assertProperties(it) } + assertTrue(input.containsKey("test")) + assertEquals(5, (input["items"] as List<*>).size) + assertTrue((input["nested"] as Map<*, *>).containsKey("drop")) + + // A new queue must discover persisted records, not reuse the capture queue's deque. + restored = PostHogQueue(config, EndpointSpec.batch(config, PostHogApi(config), config.storagePrefix), executor) + http.enqueue(MockResponse().setBody("{}")) + restored.flush() + executor.submit {}.get(5, TimeUnit.SECONDS) + val request = assertNotNull(http.takeRequest(5, TimeUnit.SECONDS)) + assertEquals("/batch", request.path) + val wireEvents = JsonParser.parseString(request.body.readUtf8()).asJsonObject.getAsJsonArray("batch") + assertEquals(diskEvents.toSet(), wireEvents.map { it.asJsonObject }.toSet()) + assertEquals(0, File(tmpDir.root, config.apiKey).listFiles()!!.size) + } finally { + restored?.clear() + restored?.stop() + sut.clear() + executor.submit {}.get(5, TimeUnit.SECONDS) + sut.close() + executor.shutdown() + flagsExecutor.shutdown() + assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)) + assertTrue(flagsExecutor.awaitTermination(5, TimeUnit.SECONDS)) + transport.connectionPool.evictAll() + transport.dispatcher.executorService.shutdown() + http.shutdown() + } + } + + @Test + fun `previously persisted null members are omitted on restored batch wire`() { + val http = MockWebServer() + http.start(InetAddress.getByName("127.0.0.1"), 0) + val executor = Executors.newSingleThreadScheduledExecutor() + val transport = loopbackClient() + val config = + PostHogConfig("null-properties-test", http.url("/").newBuilder().host("127.0.0.1").build().toString()).apply { + storagePrefix = tmpDir.root.absolutePath + httpClient = transport + } + val sut = PostHogQueue(config, EndpointSpec.batch(config, PostHogApi(config), config.storagePrefix), executor) + try { + val directory = File(tmpDir.root, config.apiKey).apply { mkdirs() } + File(directory, "restored.event").writeText( + """ + {"event":"Restored","distinct_id":"user","properties": + {"drop":null,"nested":{"drop":null},"items":["1",null,2,{"drop":null},[null]]}} + """.trimIndent(), + ) + http.enqueue(MockResponse().setBody("{}")) + sut.flush() + executor.submit {}.get(5, TimeUnit.SECONDS) + val request = assertNotNull(http.takeRequest(5, TimeUnit.SECONDS)) + assertEquals("/batch", request.path) + val properties = + JsonParser.parseString(request.body.readUtf8()).asJsonObject.getAsJsonArray("batch")[0] + .asJsonObject["properties"] + assertEquals(JsonParser.parseString("""{"nested":{},"items":["1",null,2,{},[null]]}"""), properties) + assertEquals(0, directory.listFiles()!!.size) + } finally { + sut.clear() + sut.stop() + executor.shutdown() + assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)) + transport.connectionPool.evictAll() + transport.dispatcher.executorService.shutdown() + http.shutdown() + } + } + + @Test + fun `log disk codec retains baseline null array compaction`() { + val transport = OkHttpClient.Builder().addInterceptor { throw IOException("SDK requests forbidden") }.build() + val config = PostHogConfig("null-properties-test", "http://127.0.0.1").apply { httpClient = transport } + val codec = EndpointSpec.logs(config, PostHogApi(config), tmpDir.root.absolutePath) + val record = PostHogLogRecord("example", attributes = mapOf("items" to listOf(null, "x"))) + try { + val file = tmpDir.newFile("record.log") + file.outputStream().use { codec.encode(record, it) } + val attributes = JsonParser.parseString(file.readText()).asJsonObject["attributes"] + assertEquals(JsonParser.parseString("""{"items":["x"]}"""), attributes) + val restored = file.inputStream().use { codec.decode(it) } + assertEquals(listOf("x"), assertNotNull(restored).attributes["items"]) + assertEquals(listOf(null, "x"), record.attributes["items"]) + } finally { + transport.connectionPool.evictAll() + transport.dispatcher.executorService.shutdown() + } + } + + private fun assertProperties(event: JsonObject) { + val properties = event.getAsJsonObject("properties") + assertFalse(properties.has("test")) + assertEquals(JsonParser.parseString("""{"items":[null,{}]}"""), properties["hook"]) + if (event["event"].asString != "OnlyNull") { + assertEquals(JsonParser.parseString("""["1",null,2,{},[null]]"""), properties["items"]) + assertEquals(JsonParser.parseString("""[null,{}]"""), properties["array"]) + assertEquals(JsonObject(), properties["nested"]) + } + } + + private fun loopbackClient(): OkHttpClient = + OkHttpClient.Builder() + .proxy(Proxy.NO_PROXY) + .followRedirects(false) + .followSslRedirects(false) + .addInterceptor { chain -> + if (chain.request().url.host != "127.0.0.1") { + throw IOException("Non-loopback SDK request forbidden: ${chain.request().url}") + } + chain.proceed(chain.request()) + } + .build() +} From 47bdf22084f2c5e0ef17858beaef1f2a4357d683 Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Wed, 9 Sep 2026 11:55:51 +0200 Subject: [PATCH 2/2] test: preserve null array slots in server wire snapshot --- .../src/test/resources/json/server-safe-properties.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/posthog-server/src/test/resources/json/server-safe-properties.json b/posthog-server/src/test/resources/json/server-safe-properties.json index 8f87e3a65..f195a8868 100644 --- a/posthog-server/src/test/resources/json/server-safe-properties.json +++ b/posthog-server/src/test/resources/json/server-safe-properties.json @@ -18,10 +18,12 @@ }, "ordered_array": [ "alpha", + null, "omega" ], "ordered_list": [ "first", + null, "third" ] },