From 4563f45f789efcc5fa8d3e36a39bfa065a879f13 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:04:56 -0700 Subject: [PATCH 1/3] fix(otel-collector): stop logging ParseJSON errors for non-JSON log bodies --- .changeset/collector-json-parse-noise.md | 8 ++++++++ docker/otel-collector/config.yaml | 10 ++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 .changeset/collector-json-parse-noise.md diff --git a/.changeset/collector-json-parse-noise.md b/.changeset/collector-json-parse-noise.md new file mode 100644 index 0000000000..071de98511 --- /dev/null +++ b/.changeset/collector-json-parse-noise.md @@ -0,0 +1,8 @@ +--- +'@hyperdx/otel-collector': patch +--- + +Stop the collector logging a `ParseJSON` syntax error for every log body that +contains braces but isn't JSON (e.g. Go struct or Python dict dumps). JSON +extraction now only matches objects that start with a quoted key, and remaining +parse failures in that statement group are silenced. diff --git a/docker/otel-collector/config.yaml b/docker/otel-collector/config.yaml index 18151391b2..fce203c602 100644 --- a/docker/otel-collector/config.yaml +++ b/docker/otel-collector/config.yaml @@ -3,14 +3,16 @@ processors: transform: log_statements: - context: log - error_mode: ignore + # OTTL has no JSON validity check, so ParseJSON failures on non-JSON + # brace-delimited bodies are expected and would otherwise be logged. + error_mode: silent statements: # JSON parsing: Extends log attributes with the fields from structured log body content, either as an OTEL map or # as a string containing JSON content. - - set(log.cache, ExtractPatterns(log.body, "(?P<0>(\\{.*\\}))")) where - IsString(log.body) + - set(log.cache, ExtractPatterns(log.body, "(?P<0>(\\{\\s*\".*\\}))")) + where IsString(log.body) - merge_maps(log.attributes, ParseJSON(log.cache["0"]), "upsert") - where IsMap(log.cache) + where IsMap(log.cache) and IsString(log.cache["0"]) - flatten(log.attributes) where IsMap(log.cache) - merge_maps(log.attributes, log.body, "upsert") where IsMap(log.body) - context: log From a997300d42a8df9eccd7e4f8dedb9f9790a103a3 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:13:06 -0700 Subject: [PATCH 2/3] test(otel-collector): cover JSON auto-parse regex edge cases in smoke tests --- .../otel-collector/auto-parse-json.bats | 16 +++++ .../brace-not-json/assert_query.sql | 1 + .../auto-parse/brace-not-json/expected.snap | 5 ++ .../data/auto-parse/brace-not-json/input.json | 59 +++++++++++++++++++ .../json-whitespace/assert_query.sql | 1 + .../auto-parse/json-whitespace/expected.snap | 3 + .../auto-parse/json-whitespace/input.json | 47 +++++++++++++++ 7 files changed, 132 insertions(+) create mode 100644 smoke-tests/otel-collector/data/auto-parse/brace-not-json/assert_query.sql create mode 100644 smoke-tests/otel-collector/data/auto-parse/brace-not-json/expected.snap create mode 100644 smoke-tests/otel-collector/data/auto-parse/brace-not-json/input.json create mode 100644 smoke-tests/otel-collector/data/auto-parse/json-whitespace/assert_query.sql create mode 100644 smoke-tests/otel-collector/data/auto-parse/json-whitespace/expected.snap create mode 100644 smoke-tests/otel-collector/data/auto-parse/json-whitespace/input.json diff --git a/smoke-tests/otel-collector/auto-parse-json.bats b/smoke-tests/otel-collector/auto-parse-json.bats index a4c346cb3e..410f99e69e 100644 --- a/smoke-tests/otel-collector/auto-parse-json.bats +++ b/smoke-tests/otel-collector/auto-parse-json.bats @@ -20,3 +20,19 @@ load 'test_helpers/assertions.bash' sleep 1 assert_test_data "data/auto-parse/default" } + +@test "brace-delimited non-JSON bodies are not merged and don't log parse errors" { + emit_otel_data "http://localhost:4318" "data/auto-parse/brace-not-json" + wait_for_rows 9000 "SELECT count() FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'brace-not-json'" 5 + assert_test_data "data/auto-parse/brace-not-json" + + run docker compose logs otel-collector + [[ "$output" != *"invalid character"* ]] + [[ "$output" != *"ParseJSON"* ]] +} + +@test "JSON objects with whitespace after the opening brace are parsed" { + emit_otel_data "http://localhost:4318" "data/auto-parse/json-whitespace" + wait_for_rows 9000 "SELECT count() FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'json-whitespace'" 3 + assert_test_data "data/auto-parse/json-whitespace" +} diff --git a/smoke-tests/otel-collector/data/auto-parse/brace-not-json/assert_query.sql b/smoke-tests/otel-collector/data/auto-parse/brace-not-json/assert_query.sql new file mode 100644 index 0000000000..c8ee94820d --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/brace-not-json/assert_query.sql @@ -0,0 +1 @@ +SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'brace-not-json' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/auto-parse/brace-not-json/expected.snap b/smoke-tests/otel-collector/data/auto-parse/brace-not-json/expected.snap new file mode 100644 index 0000000000..50ca6fef0c --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/brace-not-json/expected.snap @@ -0,0 +1,5 @@ +"{'suite-id':'auto-parse','test-id':'brace-not-json'}","{}" +"{'suite-id':'auto-parse','test-id':'brace-not-json'}","{}" +"{'suite-id':'auto-parse','test-id':'brace-not-json'}","{}" +"{'suite-id':'auto-parse','test-id':'brace-not-json'}","{}" +"{'suite-id':'auto-parse','test-id':'brace-not-json'}","{}" \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/auto-parse/brace-not-json/input.json b/smoke-tests/otel-collector/data/auto-parse/brace-not-json/input.json new file mode 100644 index 0000000000..4918b676c4 --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/brace-not-json/input.json @@ -0,0 +1,59 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "auto-parse" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "brace-not-json" + } + } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1901999580000000000", + "body": { + "stringValue": "{Name:x Code:1}" + } + }, + { + "timeUnixNano": "1901999580000000001", + "body": { + "stringValue": "{'a': 1}" + } + }, + { + "timeUnixNano": "1901999580000000002", + "body": { + "stringValue": "{}" + } + }, + { + "timeUnixNano": "1901999580000000003", + "body": { + "stringValue": "{ \"k\": unquoted }" + } + }, + { + "timeUnixNano": "1901999580000000004", + "body": { + "stringValue": "{\"k\": \"v\"" + } + } + ] + } + ] + } + ] +} diff --git a/smoke-tests/otel-collector/data/auto-parse/json-whitespace/assert_query.sql b/smoke-tests/otel-collector/data/auto-parse/json-whitespace/assert_query.sql new file mode 100644 index 0000000000..78ca65e931 --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/json-whitespace/assert_query.sql @@ -0,0 +1 @@ +SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'json-whitespace' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/auto-parse/json-whitespace/expected.snap b/smoke-tests/otel-collector/data/auto-parse/json-whitespace/expected.snap new file mode 100644 index 0000000000..522b2d2897 --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/json-whitespace/expected.snap @@ -0,0 +1,3 @@ +"{'suite-id':'auto-parse','test-id':'json-whitespace'}","{'a':'1'}" +"{'suite-id':'auto-parse','test-id':'json-whitespace'}","{'b':'true'}" +"{'suite-id':'auto-parse','test-id':'json-whitespace'}","{'c':'d'}" \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/auto-parse/json-whitespace/input.json b/smoke-tests/otel-collector/data/auto-parse/json-whitespace/input.json new file mode 100644 index 0000000000..073d15911e --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/json-whitespace/input.json @@ -0,0 +1,47 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "auto-parse" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "json-whitespace" + } + } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1901999580000000000", + "body": { + "stringValue": "{ \"a\": 1 }" + } + }, + { + "timeUnixNano": "1901999580000000001", + "body": { + "stringValue": "{\t\"b\": true}" + } + }, + { + "timeUnixNano": "1901999580000000002", + "body": { + "stringValue": "before {id:1} then { \"c\": \"d\" }" + } + } + ] + } + ] + } + ] +} From 47610d892dcc5e3c7f631070deb324b0ba187c53 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:31:00 -0700 Subject: [PATCH 3/3] test(otel-collector): fail parse-error check when collector logs can't be read --- smoke-tests/otel-collector/auto-parse-json.bats | 3 +++ 1 file changed, 3 insertions(+) diff --git a/smoke-tests/otel-collector/auto-parse-json.bats b/smoke-tests/otel-collector/auto-parse-json.bats index 410f99e69e..5065629cc4 100644 --- a/smoke-tests/otel-collector/auto-parse-json.bats +++ b/smoke-tests/otel-collector/auto-parse-json.bats @@ -27,6 +27,9 @@ load 'test_helpers/assertions.bash' assert_test_data "data/auto-parse/brace-not-json" run docker compose logs otel-collector + [ "$status" -eq 0 ] + # Prove we read the collector's output before asserting what's absent from it. + [[ "$output" == *"Everything is ready"* ]] [[ "$output" != *"invalid character"* ]] [[ "$output" != *"ParseJSON"* ]] }