From 59d3b4ab04cca8c986b1599bfc652a0d2740f602 Mon Sep 17 00:00:00 2001 From: Charles Paul Date: Mon, 23 Mar 2026 10:47:15 -0700 Subject: [PATCH 1/4] feat(contexts): Add TraceId by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We are beginning to store events in EAP, which — as a Trace-centric datastore — requires all TraceItems to be associated with a TraceId. This is problematic, since we currently don't require that events actually have TraceIds. That means that we're currently just silently dropping a bunch of events before we can successfully ingest them into EAP. This PR adds a random TraceId & SpanId to events that do not have a TraceContext. --- CHANGELOG.md | 1 + Cargo.lock | 1 + relay-event-normalization/src/event.rs | 9 ++++ relay-event-schema/Cargo.toml | 1 + .../src/protocol/contexts/trace.rs | 53 +++++++++++++++++++ 5 files changed, 65 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcb1426048c..84e80f9ede7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ - Playstation: Do not upload attachments if quota is 0. ([#5770](https://github.com/getsentry/relay/pull/5770)) - Add payload byte size to trace metrics. ([#5764](https://github.com/getsentry/relay/pull/5764)) - Mix kafka partition key with org id. ([#5772](https://github.com/getsentry/relay/pull/5772)) +- Set a trace_id on all events by default for internal use. ([#5759](https://github.com/getsentry/relay/pull/5759)) ## 26.3.1 diff --git a/Cargo.lock b/Cargo.lock index 8b79660dbab..886c4f19c7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4404,6 +4404,7 @@ dependencies = [ "insta", "minidump", "opentelemetry-proto", + "rand 0.9.2", "relay-base-schema", "relay-common", "relay-conventions", diff --git a/relay-event-normalization/src/event.rs b/relay-event-normalization/src/event.rs index 86433a943a7..d5ee5e0e56f 100644 --- a/relay-event-normalization/src/event.rs +++ b/relay-event-normalization/src/event.rs @@ -1308,12 +1308,21 @@ fn remove_logger_word(tokens: &mut Vec<&str>) { /// Normalizes incoming contexts for the downstream metric extraction. fn normalize_contexts(contexts: &mut Annotated) { + // We will always need a TraceContext. + let _ = contexts.get_or_insert_with(Contexts::new); + let _ = processor::apply(contexts, |contexts, _meta| { // Reprocessing context sent from SDKs must not be accepted, it is a Sentry-internal // construct. // [`normalize`] does not run on renormalization anyway. contexts.0.remove("reprocessing"); + // We need a TraceId to ingest the event into EAP. + // If the event lacks a TraceContext, add a random one. + if !contexts.contains::() { + contexts.add(TraceContext::random()) + } + for annotated in &mut contexts.0.values_mut() { if let Some(ContextInner(Context::Trace(context))) = annotated.value_mut() { context.status.get_or_insert_with(|| SpanStatus::Unknown); diff --git a/relay-event-schema/Cargo.toml b/relay-event-schema/Cargo.toml index 207a7c60d34..6c78c333be3 100644 --- a/relay-event-schema/Cargo.toml +++ b/relay-event-schema/Cargo.toml @@ -31,6 +31,7 @@ serde_json = { workspace = true } thiserror = { workspace = true } url = { workspace = true } uuid = { workspace = true } +rand.workspace = true [dev-dependencies] insta = { workspace = true } diff --git a/relay-event-schema/src/protocol/contexts/trace.rs b/relay-event-schema/src/protocol/contexts/trace.rs index 9157ce5c77d..e8145190815 100644 --- a/relay-event-schema/src/protocol/contexts/trace.rs +++ b/relay-event-schema/src/protocol/contexts/trace.rs @@ -1,3 +1,4 @@ +use rand::RngCore; use relay_protocol::{ Annotated, Array, Empty, Error, FromValue, HexId, IntoValue, Meta, Object, Remark, RemarkType, SkipSerialization, Val, Value, @@ -168,6 +169,14 @@ pub struct SpanId([u8; 8]); relay_common::impl_str_serde!(SpanId, "a span identifier"); +impl SpanId { + pub fn random() -> Self { + let mut span_bytes = [0u8; 8]; + rand::rng().fill_bytes(&mut span_bytes); + Self(span_bytes) + } +} + impl FromStr for SpanId { type Err = Error; @@ -324,6 +333,23 @@ pub struct TraceContext { pub other: Object, } +impl TraceContext { + /// Generates a random [`TraceId`] and random [`SpanId`]. + /// Leaves all other fields blank. + pub fn random() -> Self { + let mut trace_meta = Meta::default(); + trace_meta.add_remark(Remark::new(RemarkType::Substituted, "trace_id.missing")); + + let mut span_meta = Meta::default(); + span_meta.add_remark(Remark::new(RemarkType::Substituted, "span_id.missing")); + TraceContext { + trace_id: Annotated(Some(TraceId::random()), trace_meta), + span_id: Annotated(Some(SpanId::random()), span_meta), + ..Default::default() + } + } +} + impl super::DefaultContext for TraceContext { fn default_key() -> &'static str { "trace" @@ -615,4 +641,31 @@ mod tests { let remark = annotated.meta().iter_remarks().next().unwrap(); assert_eq!(remark.rule_id(), "trace_id.invalid"); } + + #[test] + fn test_random_trace_context() { + let rand_context = TraceContext::random(); + assert!(rand_context.trace_id.value().is_some()); + assert_eq!( + rand_context + .trace_id + .meta() + .iter_remarks() + .next() + .unwrap() + .rule_id(), + "trace_id.missing" + ); + assert!(rand_context.span_id.value().is_some()); + assert_eq!( + rand_context + .span_id + .meta() + .iter_remarks() + .next() + .unwrap() + .rule_id(), + "span_id.missing" + ); + } } From f4749a6bd69182c53e59f0da5d839e5cfe89b8c8 Mon Sep 17 00:00:00 2001 From: Charles Paul Date: Mon, 30 Mar 2026 11:05:39 -0700 Subject: [PATCH 2/4] suggestions --- py/tests/test_processing.py | 1 + relay-event-schema/Cargo.toml | 2 +- relay-event-schema/src/protocol/contexts/trace.rs | 5 ++--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/py/tests/test_processing.py b/py/tests/test_processing.py index aeaa5f28083..7a9589518aa 100644 --- a/py/tests/test_processing.py +++ b/py/tests/test_processing.py @@ -127,6 +127,7 @@ def test_normalize_user_agent(must_normalize): "type": "browser", }, "client_os": {"name": "Ubuntu", "os": "Ubuntu", "type": "os"}, + "trace": pytest.mock.ANY, } else: assert "contexts" not in event diff --git a/relay-event-schema/Cargo.toml b/relay-event-schema/Cargo.toml index 6c78c333be3..c8b84fe0564 100644 --- a/relay-event-schema/Cargo.toml +++ b/relay-event-schema/Cargo.toml @@ -20,6 +20,7 @@ debugid = { workspace = true, features = ["serde"] } enumset = { workspace = true } minidump = { workspace = true } opentelemetry-proto = { workspace = true, features = ["gen-tonic", "trace"] } +rand = { workspace = true } relay-common = { workspace = true } relay-conventions = { workspace = true } relay-base-schema = { workspace = true } @@ -31,7 +32,6 @@ serde_json = { workspace = true } thiserror = { workspace = true } url = { workspace = true } uuid = { workspace = true } -rand.workspace = true [dev-dependencies] insta = { workspace = true } diff --git a/relay-event-schema/src/protocol/contexts/trace.rs b/relay-event-schema/src/protocol/contexts/trace.rs index e8145190815..d397c08f864 100644 --- a/relay-event-schema/src/protocol/contexts/trace.rs +++ b/relay-event-schema/src/protocol/contexts/trace.rs @@ -171,9 +171,8 @@ relay_common::impl_str_serde!(SpanId, "a span identifier"); impl SpanId { pub fn random() -> Self { - let mut span_bytes = [0u8; 8]; - rand::rng().fill_bytes(&mut span_bytes); - Self(span_bytes) + let value: u64 = rand::random_range(1..=u64::MAX); + Self(value.to_ne_bytes()) } } From 43afc56885b62146ef07c90e8836d40636261666 Mon Sep 17 00:00:00 2001 From: Charles Paul Date: Wed, 1 Apr 2026 14:15:42 -0700 Subject: [PATCH 3/4] add feature flag --- relay-cabi/src/processing.rs | 1 + relay-dynamic-config/src/feature.rs | 3 + relay-event-normalization/src/event.rs | 142 +++++++++++++++++- .../src/protocol/contexts/trace.rs | 9 +- relay-server/src/processing/utils/event.rs | 1 + 5 files changed, 145 insertions(+), 11 deletions(-) diff --git a/relay-cabi/src/processing.rs b/relay-cabi/src/processing.rs index 5d5b662695b..e220abf706f 100644 --- a/relay-cabi/src/processing.rs +++ b/relay-cabi/src/processing.rs @@ -276,6 +276,7 @@ pub unsafe extern "C" fn relay_store_normalizer_normalize_event( span_allowed_hosts: &[], // only supported in relay span_op_defaults: Default::default(), // only supported in relay performance_issues_spans: Default::default(), + should_add_trace_id_by_default: Default::default(), }; normalize_event(&mut event, &normalization_config); diff --git a/relay-dynamic-config/src/feature.rs b/relay-dynamic-config/src/feature.rs index b758ba1175f..84213d85a9c 100644 --- a/relay-dynamic-config/src/feature.rs +++ b/relay-dynamic-config/src/feature.rs @@ -118,6 +118,9 @@ pub enum Feature { /// Upload non-prosperodmp playstation attachments via the upload-endpoint. #[serde(rename = "projects:relay-playstation-uploads")] PlaystationUploads, + /// Add a random trace ID to events that lack one. + #[serde(rename = "organizations:add-default-trace-id-relay")] + AddDefaultTraceID, /// Enables OTLP spans to use the Span V2 processing pipeline in Relay. /// diff --git a/relay-event-normalization/src/event.rs b/relay-event-normalization/src/event.rs index d5ee5e0e56f..b3bf697729b 100644 --- a/relay-event-normalization/src/event.rs +++ b/relay-event-normalization/src/event.rs @@ -166,6 +166,9 @@ pub struct NormalizationConfig<'a> { /// Set a flag to enable performance issue detection on spans. pub performance_issues_spans: bool, + + /// Should add a random trace ID to events that lack one. + pub should_add_trace_id_by_default: bool, } impl Default for NormalizationConfig<'_> { @@ -201,6 +204,7 @@ impl Default for NormalizationConfig<'_> { span_allowed_hosts: Default::default(), span_op_defaults: Default::default(), performance_issues_spans: Default::default(), + should_add_trace_id_by_default: Default::default(), } } } @@ -333,8 +337,11 @@ fn normalize(event: &mut Event, meta: &mut Meta, config: &NormalizationConfig) { Ok(()) }); + // We know this exists thanks to normalize_default_attributes. + let event_id = event.id.value().unwrap().0; + // Some contexts need to be normalized before metrics extraction takes place. - normalize_contexts(&mut event.contexts); + normalize_contexts(&mut event.contexts, event_id, config); if config.normalize_spans && event.ty.value() == Some(&EventType::Transaction) { span::reparent_broken_spans::reparent_broken_spans(event); @@ -1307,9 +1314,15 @@ fn remove_logger_word(tokens: &mut Vec<&str>) { } /// Normalizes incoming contexts for the downstream metric extraction. -fn normalize_contexts(contexts: &mut Annotated) { - // We will always need a TraceContext. - let _ = contexts.get_or_insert_with(Contexts::new); +fn normalize_contexts( + contexts: &mut Annotated, + event_id: Uuid, + config: &NormalizationConfig, +) { + if config.should_add_trace_id_by_default { + // We will always need a TraceContext. + let _ = contexts.get_or_insert_with(Contexts::new); + } let _ = processor::apply(contexts, |contexts, _meta| { // Reprocessing context sent from SDKs must not be accepted, it is a Sentry-internal @@ -1319,8 +1332,9 @@ fn normalize_contexts(contexts: &mut Annotated) { // We need a TraceId to ingest the event into EAP. // If the event lacks a TraceContext, add a random one. - if !contexts.contains::() { - contexts.add(TraceContext::random()) + + if config.should_add_trace_id_by_default && !contexts.contains::() { + contexts.add(TraceContext::random(event_id)) } for annotated in &mut contexts.0.values_mut() { @@ -4012,6 +4026,122 @@ mod tests { "###); } + #[test] + fn test_normalize_adds_trace_id() { + let json = r#" + { + "type": "transaction", + "timestamp": "2021-04-26T08:00:05+0100", + "start_timestamp": "2021-04-26T08:00:00+0100", + "measurements": { + "inp": {"value": 120.0} + } + } + "#; + + let mut event = Annotated::::from_json(json).unwrap().0.unwrap(); + + let performance_score: PerformanceScoreConfig = serde_json::from_value(json!({ + "profiles": [ + { + "name": "Desktop", + "scoreComponents": [ + { + "measurement": "inp", + "weight": 1.0, + "p10": 0.1, + "p50": 0.25 + }, + ], + "condition": { + "op":"and", + "inner": [] + }, + "version": "beta" + } + ] + })) + .unwrap(); + + normalize( + &mut event, + &mut Meta::default(), + &NormalizationConfig { + performance_score: Some(&performance_score), + should_add_trace_id_by_default: true, + ..Default::default() + }, + ); + + insta::assert_ron_snapshot!(SerializableAnnotated(&event.contexts), { + ".event_id" => "[event-id]", + ".trace.trace_id" => "[trace-id]", + ".trace.span_id" => "[span-id]" + }, @r#" + { + "performance_score": { + "score_profile_version": "beta", + "type": "performancescore", + }, + "trace": { + "trace_id": "[trace-id]", + "span_id": "[span-id]", + "status": "unknown", + "exclusive_time": 5000.0, + "type": "trace", + }, + "_meta": { + "trace": { + "span_id": { + "": Meta(Some(MetaInner( + rem: [ + [ + "span_id.missing", + s, + ], + ], + ))), + }, + "trace_id": { + "": Meta(Some(MetaInner( + rem: [ + [ + "trace_id.missing", + s, + ], + ], + ))), + }, + }, + }, + } + "#); + insta::assert_ron_snapshot!(SerializableAnnotated(&event.measurements), {}, @r###" + { + "inp": { + "value": 120.0, + "unit": "millisecond", + }, + "score.inp": { + "value": 0.0, + "unit": "ratio", + }, + "score.ratio.inp": { + "value": 0.0, + "unit": "ratio", + }, + "score.total": { + "value": 0.0, + "unit": "ratio", + }, + "score.weight.inp": { + "value": 1.0, + "unit": "ratio", + }, + } + "###); + } + #[test] fn test_computes_standalone_cls_performance_score() { let json = r#" diff --git a/relay-event-schema/src/protocol/contexts/trace.rs b/relay-event-schema/src/protocol/contexts/trace.rs index d397c08f864..6d5a64dd2db 100644 --- a/relay-event-schema/src/protocol/contexts/trace.rs +++ b/relay-event-schema/src/protocol/contexts/trace.rs @@ -1,4 +1,3 @@ -use rand::RngCore; use relay_protocol::{ Annotated, Array, Empty, Error, FromValue, HexId, IntoValue, Meta, Object, Remark, RemarkType, SkipSerialization, Val, Value, @@ -333,16 +332,16 @@ pub struct TraceContext { } impl TraceContext { - /// Generates a random [`TraceId`] and random [`SpanId`]. + /// Generates a random [`SpanId`] and takes `[TraceId]` from the event's UUID. /// Leaves all other fields blank. - pub fn random() -> Self { + pub fn random(event_id: Uuid) -> Self { let mut trace_meta = Meta::default(); trace_meta.add_remark(Remark::new(RemarkType::Substituted, "trace_id.missing")); let mut span_meta = Meta::default(); span_meta.add_remark(Remark::new(RemarkType::Substituted, "span_id.missing")); TraceContext { - trace_id: Annotated(Some(TraceId::random()), trace_meta), + trace_id: Annotated(Some(TraceId::from(event_id)), trace_meta), span_id: Annotated(Some(SpanId::random()), span_meta), ..Default::default() } @@ -643,7 +642,7 @@ mod tests { #[test] fn test_random_trace_context() { - let rand_context = TraceContext::random(); + let rand_context = TraceContext::random(Uuid::new_v4()); assert!(rand_context.trace_id.value().is_some()); assert_eq!( rand_context diff --git a/relay-server/src/processing/utils/event.rs b/relay-server/src/processing/utils/event.rs index eaa9764dd34..544621de8c6 100644 --- a/relay-server/src/processing/utils/event.rs +++ b/relay-server/src/processing/utils/event.rs @@ -303,6 +303,7 @@ pub fn normalize( performance_issues_spans: ctx .project_info .has_feature(Feature::PerformanceIssuesSpans), + should_add_trace_id_by_default: project_info.has_feature(Feature::AddDefaultTraceID), }; metric!(timer(RelayTimers::EventProcessingNormalization), { From 9927eccb53864ad63f2c934e3099b10920964993 Mon Sep 17 00:00:00 2001 From: Charles Paul Date: Wed, 8 Apr 2026 23:30:40 -0700 Subject: [PATCH 4/4] rename --- py/tests/test_processing.py | 1 - relay-cabi/src/processing.rs | 2 +- relay-dynamic-config/src/feature.rs | 2 +- relay-event-normalization/src/event.rs | 10 +++++----- relay-server/src/processing/utils/event.rs | 2 +- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/py/tests/test_processing.py b/py/tests/test_processing.py index 7a9589518aa..aeaa5f28083 100644 --- a/py/tests/test_processing.py +++ b/py/tests/test_processing.py @@ -127,7 +127,6 @@ def test_normalize_user_agent(must_normalize): "type": "browser", }, "client_os": {"name": "Ubuntu", "os": "Ubuntu", "type": "os"}, - "trace": pytest.mock.ANY, } else: assert "contexts" not in event diff --git a/relay-cabi/src/processing.rs b/relay-cabi/src/processing.rs index e220abf706f..df3050accc0 100644 --- a/relay-cabi/src/processing.rs +++ b/relay-cabi/src/processing.rs @@ -276,7 +276,7 @@ pub unsafe extern "C" fn relay_store_normalizer_normalize_event( span_allowed_hosts: &[], // only supported in relay span_op_defaults: Default::default(), // only supported in relay performance_issues_spans: Default::default(), - should_add_trace_id_by_default: Default::default(), + derive_trace_id: Default::default(), }; normalize_event(&mut event, &normalization_config); diff --git a/relay-dynamic-config/src/feature.rs b/relay-dynamic-config/src/feature.rs index 84213d85a9c..22c4c9999ae 100644 --- a/relay-dynamic-config/src/feature.rs +++ b/relay-dynamic-config/src/feature.rs @@ -119,7 +119,7 @@ pub enum Feature { #[serde(rename = "projects:relay-playstation-uploads")] PlaystationUploads, /// Add a random trace ID to events that lack one. - #[serde(rename = "organizations:add-default-trace-id-relay")] + #[serde(rename = "organizations:relay-default-trace-id")] AddDefaultTraceID, /// Enables OTLP spans to use the Span V2 processing pipeline in Relay. diff --git a/relay-event-normalization/src/event.rs b/relay-event-normalization/src/event.rs index b3bf697729b..4459e1f9a96 100644 --- a/relay-event-normalization/src/event.rs +++ b/relay-event-normalization/src/event.rs @@ -168,7 +168,7 @@ pub struct NormalizationConfig<'a> { pub performance_issues_spans: bool, /// Should add a random trace ID to events that lack one. - pub should_add_trace_id_by_default: bool, + pub derive_trace_id: bool, } impl Default for NormalizationConfig<'_> { @@ -204,7 +204,7 @@ impl Default for NormalizationConfig<'_> { span_allowed_hosts: Default::default(), span_op_defaults: Default::default(), performance_issues_spans: Default::default(), - should_add_trace_id_by_default: Default::default(), + derive_trace_id: Default::default(), } } } @@ -1319,7 +1319,7 @@ fn normalize_contexts( event_id: Uuid, config: &NormalizationConfig, ) { - if config.should_add_trace_id_by_default { + if config.derive_trace_id { // We will always need a TraceContext. let _ = contexts.get_or_insert_with(Contexts::new); } @@ -1333,7 +1333,7 @@ fn normalize_contexts( // We need a TraceId to ingest the event into EAP. // If the event lacks a TraceContext, add a random one. - if config.should_add_trace_id_by_default && !contexts.contains::() { + if config.derive_trace_id && !contexts.contains::() { contexts.add(TraceContext::random(event_id)) } @@ -4068,7 +4068,7 @@ mod tests { &mut Meta::default(), &NormalizationConfig { performance_score: Some(&performance_score), - should_add_trace_id_by_default: true, + derive_trace_id: true, ..Default::default() }, ); diff --git a/relay-server/src/processing/utils/event.rs b/relay-server/src/processing/utils/event.rs index 544621de8c6..73d04e5ca4f 100644 --- a/relay-server/src/processing/utils/event.rs +++ b/relay-server/src/processing/utils/event.rs @@ -303,7 +303,7 @@ pub fn normalize( performance_issues_spans: ctx .project_info .has_feature(Feature::PerformanceIssuesSpans), - should_add_trace_id_by_default: project_info.has_feature(Feature::AddDefaultTraceID), + derive_trace_id: project_info.has_feature(Feature::AddDefaultTraceID), }; metric!(timer(RelayTimers::EventProcessingNormalization), {