From 70756e8ed6d49dbb16b681c0e2daeaa2dd073e11 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Thu, 16 Oct 2025 14:25:59 +0200 Subject: [PATCH 01/16] feat(eap): Normalize deprecated attributes This implements the normalization (backfilling/replacement) of deprecated attribute names based on `sentry-conventions`. The feature applies to spans V2 and logs alike. See the documentation of `normalize_attribute_names` and the `test_normalize_attributes` test for the exact normalization logic. --- relay-event-normalization/src/eap/mod.rs | 171 +++++++++++++++++- relay-event-schema/src/protocol/attributes.rs | 29 +++ relay-server/src/processing/logs/process.rs | 1 + relay-server/src/processing/logs/utils.rs | 2 +- relay-server/src/processing/spans/process.rs | 3 +- tests/integration/test_ourlogs.py | 43 ++--- tests/integration/test_spansv2.py | 11 ++ 7 files changed, 227 insertions(+), 33 deletions(-) diff --git a/relay-event-normalization/src/eap/mod.rs b/relay-event-normalization/src/eap/mod.rs index b7e009c4cf5..518f981034b 100644 --- a/relay-event-normalization/src/eap/mod.rs +++ b/relay-event-normalization/src/eap/mod.rs @@ -5,11 +5,11 @@ use chrono::{DateTime, Utc}; use relay_common::time::UnixTimestamp; use relay_conventions::{ - BROWSER_NAME, BROWSER_VERSION, OBSERVED_TIMESTAMP_NANOS, USER_GEO_CITY, USER_GEO_COUNTRY_CODE, - USER_GEO_REGION, USER_GEO_SUBDIVISION, + AttributeInfo, BROWSER_NAME, BROWSER_VERSION, OBSERVED_TIMESTAMP_NANOS, USER_GEO_CITY, + USER_GEO_COUNTRY_CODE, USER_GEO_REGION, USER_GEO_SUBDIVISION, WriteBehavior, }; use relay_event_schema::protocol::{AttributeType, Attributes, BrowserContext, Geo}; -use relay_protocol::{Annotated, ErrorKind, Value}; +use relay_protocol::{Annotated, ErrorKind, Remark, RemarkType, Value}; use crate::{ClientHints, FromUserAgentInfo as _, RawUserAgentInfo}; @@ -132,6 +132,58 @@ pub fn normalize_user_geo( attributes.insert_if_missing(USER_GEO_REGION, || geo.region); } +/// Normalizes deprecated attributes according to `sentry-conventions`. +/// +/// Attributes with a status of `"normalize"` will be moved to their replacement name. +/// If there is already a value present under the replacement name, it will be left alone, +/// but the deprecated attribute is removed anyway. +/// +/// Attributes with a status of `"backfill"` will be copied to their replacement name if the +/// replacement name is not present. In any case, the original name is left alone. +pub fn normalize_attribute_names(attributes: &mut Annotated) { + normalize_attribute_names_inner(attributes, relay_conventions::attribute_info) +} + +fn normalize_attribute_names_inner( + attributes: &mut Annotated, + attribute_info: fn(&str) -> Option<&'static AttributeInfo>, +) { + let attributes = attributes.get_or_insert_with(Default::default); + let attribute_names: Vec<_> = attributes.keys().cloned().collect(); + + for name in attribute_names { + let Some(attribute_info) = attribute_info(&name) else { + continue; + }; + + match attribute_info.write_behavior { + WriteBehavior::CurrentName => continue, + WriteBehavior::NewName(new_name) => { + let Some(old_attribute) = attributes.get_raw_mut(&name) else { + continue; + }; + + let new_attribute = old_attribute.clone(); + old_attribute.set_value(None); + old_attribute + .meta_mut() + .add_remark(Remark::new(RemarkType::Removed, "attribute.deprecated")); + + if !attributes.contains_key(new_name) { + attributes.insert_raw(new_name.to_owned(), new_attribute); + } + } + WriteBehavior::BothNames(new_name) => { + if !attributes.contains_key(new_name) + && let Some(current_attribute) = attributes.get_raw(&name).cloned() + { + attributes.insert_raw(new_name.to_owned(), current_attribute); + } + } + } + } +} + #[cfg(test)] mod tests { use relay_protocol::SerializableAnnotated; @@ -464,4 +516,117 @@ mod tests { "#, ); } + + #[test] + fn test_normalize_attributes() { + fn mock_attribute_info(name: &str) -> Option<&'static AttributeInfo> { + use relay_conventions::Pii; + + match name { + "replace.empty" => Some(&AttributeInfo { + write_behavior: WriteBehavior::NewName("replaced"), + pii: Pii::Maybe, + aliases: &["replaced"], + }), + "replace.existing" => Some(&AttributeInfo { + write_behavior: WriteBehavior::NewName("not.replaced"), + pii: Pii::Maybe, + aliases: &["not.replaced"], + }), + "backfill.empty" => Some(&AttributeInfo { + write_behavior: WriteBehavior::BothNames("backfilled"), + pii: Pii::Maybe, + aliases: &["backfilled"], + }), + "backfill.existing" => Some(&AttributeInfo { + write_behavior: WriteBehavior::BothNames("not.backfilled"), + pii: Pii::Maybe, + aliases: &["not.backfilled"], + }), + _ => None, + } + } + + let mut attributes = Annotated::new(Attributes::from([ + ( + "replace.empty".to_owned(), + Annotated::new("Should be moved".to_owned().into()), + ), + ( + "replace.existing".to_owned(), + Annotated::new("Should be removed".to_owned().into()), + ), + ( + "not.replaced".to_owned(), + Annotated::new("Should be left alone".to_owned().into()), + ), + ( + "backfill.empty".to_owned(), + Annotated::new("Should be copied".to_owned().into()), + ), + ( + "backfill.existing".to_owned(), + Annotated::new("Should be left alone".to_owned().into()), + ), + ( + "not.backfilled".to_owned(), + Annotated::new("Should be left alone".to_owned().into()), + ), + ])); + + normalize_attribute_names_inner(&mut attributes, mock_attribute_info); + + insta::assert_json_snapshot!(SerializableAnnotated(&attributes), @r###" + { + "backfill.empty": { + "type": "string", + "value": "Should be copied" + }, + "backfill.existing": { + "type": "string", + "value": "Should be left alone" + }, + "backfilled": { + "type": "string", + "value": "Should be copied" + }, + "not.backfilled": { + "type": "string", + "value": "Should be left alone" + }, + "not.replaced": { + "type": "string", + "value": "Should be left alone" + }, + "replace.empty": null, + "replace.existing": null, + "replaced": { + "type": "string", + "value": "Should be moved" + }, + "_meta": { + "replace.empty": { + "": { + "rem": [ + [ + "attribute.deprecated", + "x" + ] + ] + } + }, + "replace.existing": { + "": { + "rem": [ + [ + "attribute.deprecated", + "x" + ] + ] + } + } + } + } + "###); + } } diff --git a/relay-event-schema/src/protocol/attributes.rs b/relay-event-schema/src/protocol/attributes.rs index 39a65170662..58203a4b2f5 100644 --- a/relay-event-schema/src/protocol/attributes.rs +++ b/relay-event-schema/src/protocol/attributes.rs @@ -236,6 +236,24 @@ impl Attributes { Some(&self.0.get(key)?.value()?.value.value) } + /// Returns the annotated attribute with the given key. + pub fn get_raw(&self, key: &Q) -> Option<&Annotated> + where + String: Borrow, + Q: Ord + ?Sized, + { + self.0.get(key) + } + + /// Mutably returns the annotated attribute with the given key. + pub fn get_raw_mut(&mut self, key: &Q) -> Option<&mut Annotated> + where + String: Borrow, + Q: Ord + ?Sized, + { + self.0.get_mut(key) + } + /// Inserts an attribute with the given value into this collection. pub fn insert, V: Into>(&mut self, key: K, value: V) { fn inner(slf: &mut Attributes, key: String, value: AttributeValue) { @@ -296,6 +314,17 @@ impl Attributes { ) -> std::collections::btree_map::IterMut<'_, String, Annotated> { self.0.iter_mut() } + + pub fn keys(&self) -> std::collections::btree_map::Keys<'_, String, Annotated> { + self.0.keys() + } + + pub fn entry( + &mut self, + key: String, + ) -> std::collections::btree_map::Entry<'_, String, Annotated> { + self.0.entry(key) + } } impl IntoIterator for Attributes { diff --git a/relay-server/src/processing/logs/process.rs b/relay-server/src/processing/logs/process.rs index 22131e93809..cde5271f8c5 100644 --- a/relay-server/src/processing/logs/process.rs +++ b/relay-server/src/processing/logs/process.rs @@ -117,6 +117,7 @@ fn normalize_log(log: &mut Annotated, meta: &RequestMeta) -> Result<()> eap::normalize_received(&mut log.attributes, meta.received_at()); eap::normalize_user_agent(&mut log.attributes, meta.user_agent(), meta.client_hints()); eap::normalize_attribute_types(&mut log.attributes); + eap::normalize_attribute_names(&mut log.attributes); Ok(()) } diff --git a/relay-server/src/processing/logs/utils.rs b/relay-server/src/processing/logs/utils.rs index 323b65db7f9..8b18c936e32 100644 --- a/relay-server/src/processing/logs/utils.rs +++ b/relay-server/src/processing/logs/utils.rs @@ -4,7 +4,7 @@ use crate::envelope::WithHeader; /// Returns the calculated size of a [`OurLog`]. /// -/// Unlike [`relay_ourlogs::calculate_size`], this access the already manifested byte size +/// Unlike [`relay_ourlogs::calculate_size`], this accesses the already manifested byte size /// of the log, instead of calculating it. /// /// When compiled with debug assertion the function asserts the presence of a manifested byte size. diff --git a/relay-server/src/processing/spans/process.rs b/relay-server/src/processing/spans/process.rs index 3a304de0f72..e0254e98cab 100644 --- a/relay-server/src/processing/spans/process.rs +++ b/relay-server/src/processing/spans/process.rs @@ -71,10 +71,11 @@ fn normalize_span( if let Some(span) = span.value_mut() { eap::normalize_received(&mut span.attributes, meta.received_at()); eap::normalize_user_agent(&mut span.attributes, meta.user_agent(), meta.client_hints()); - eap::normalize_attribute_types(&mut span.attributes); eap::normalize_user_geo(&mut span.attributes, || { meta.client_addr().and_then(|ip| geo_lookup.lookup(ip)) }); + eap::normalize_attribute_types(&mut span.attributes); + eap::normalize_attribute_names(&mut span.attributes); // TODO: ai model costs } else { diff --git a/tests/integration/test_ourlogs.py b/tests/integration/test_ourlogs.py index dd12949260f..d8cb1459921 100644 --- a/tests/integration/test_ourlogs.py +++ b/tests/integration/test_ourlogs.py @@ -46,6 +46,9 @@ def timestamps(ts: datetime): "sentry.observed_timestamp_nanos": { "stringValue": time_within(ts, expect_resolution="ns") }, + "sentry._internal.observed_timestamp_nanos": { + "stringValue": time_within(ts, expect_resolution="ns") + }, "sentry.timestamp_nanos": { "stringValue": time_within_delta( ts, delta=timedelta(seconds=0), expect_resolution="ns", precision="us" @@ -130,16 +133,16 @@ def test_ourlog_multiple_containers_not_allowed( @pytest.mark.parametrize( "external_mode,expected_byte_size", [ - # 260 here is a billing relevant metric, do not arbitrarily change it, + # 296 here is a billing relevant metric, do not arbitrarily change it, # this value is supposed to be static and purely based on data received, # independent of any normalization. - (None, 260), + (None, 296), # Same applies as above, a proxy Relay does not need to run normalization. - ("proxy", 260), + ("proxy", 296), # If an external Relay/Client makes modifications, sizes can change, # this is fuzzy due to slight changes in sizes due to added timestamps # and may need to be adjusted when changing normalization. - ("managed", 454), + ("managed", 641), ], ) def test_ourlog_extraction_with_sentry_logs( @@ -200,6 +203,7 @@ def test_ourlog_extraction_with_sentry_logs( "string.attribute": {"value": "some string", "type": "string"}, "pii": {"value": "4242 4242 4242 4242", "type": "string"}, "sentry.severity_text": {"value": "info", "type": "string"}, + "http.response_content_length": {"value": 17, "type": "integer"}, "unknown_type": {"value": "info", "type": "unknown"}, "broken_type": {"value": "info", "type": "not_a_real_type"}, "mismatched_type": {"value": "some string", "type": "boolean"}, @@ -263,6 +267,8 @@ def test_ourlog_extraction_with_sentry_logs( "sentry.browser.version": {"stringValue": "2.32"}, "sentry.severity_text": {"stringValue": "info"}, "sentry.payload_size_bytes": {"intValue": mock.ANY}, + "http.response_content_length": {"intValue": "17"}, + "http.response.body.size": {"intValue": "17"}, "sentry.span_id": {"stringValue": "eee19b7ec3c1b174"}, "string.attribute": {"stringValue": "some string"}, "valid_string_with_other": {"stringValue": "test"}, @@ -355,6 +361,10 @@ def test_ourlog_extraction_with_string_pii_scrubbing( "type": "string", "value": time_within(ts, expect_resolution="ns"), }, + "sentry._internal.observed_timestamp_nanos": { + "type": "string", + "value": time_within(ts, expect_resolution="ns"), + }, }, "__header": {"byte_size": mock.ANY}, "_meta": { @@ -496,36 +506,13 @@ def test_ourlog_extraction_default_pii_scrubbing_does_not_scrub_default_attribut "custom_field": {"stringValue": "[REDACTED]"}, "sentry.body": {"stringValue": "[REDACTED]"}, "sentry.severity_text": {"stringValue": "info"}, - "sentry.observed_timestamp_nanos": { - "stringValue": time_within_delta( - ts, - delta=timedelta(seconds=1), - expect_resolution="ns", - precision="us", - ) - }, "sentry.span_id": {"stringValue": "eee19b7ec3c1b174"}, "sentry.payload_size_bytes": mock.ANY, "sentry.browser.name": {"stringValue": "Python Requests"}, "sentry._meta.fields.body": { "stringValue": '{"meta":{"":{"rem":[["remove_custom_field","s",0,10]],"len":8}}}' }, - "sentry.timestamp_nanos": { - "stringValue": time_within_delta( - ts, - delta=timedelta(seconds=0), - expect_resolution="ns", - precision="us", - ) - }, - "sentry.timestamp_precise": { - "intValue": time_within_delta( - ts, - delta=timedelta(seconds=0), - expect_resolution="ns", - precision="us", - ) - }, + **timestamps(ts) }, "clientSampleRate": 1.0, "itemId": mock.ANY, diff --git a/tests/integration/test_spansv2.py b/tests/integration/test_spansv2.py index e1db57756f5..8189af9ad58 100644 --- a/tests/integration/test_spansv2.py +++ b/tests/integration/test_spansv2.py @@ -73,6 +73,7 @@ def test_spansv2_basic( "attributes": { "foo": {"value": "bar", "type": "string"}, "invalid": {"value": True, "type": "string"}, + "http.response_content_length": {"value": 17, "type": "integer"}, }, } ) @@ -84,6 +85,8 @@ def test_spansv2_basic( "span_id": "eee19b7ec3c1b175", "attributes": { "foo": {"type": "string", "value": "bar"}, + "http.response_content_length": {"value": 17, "type": "integer"}, + "http.response.body.size": {"value": 17, "type": "integer"}, "invalid": None, "sentry.browser.name": {"type": "string", "value": "Python Requests"}, "sentry.browser.version": {"type": "string", "value": "2.32"}, @@ -91,6 +94,10 @@ def test_spansv2_basic( "type": "string", "value": time_within(ts, expect_resolution="ns"), }, + "sentry._internal.observed_timestamp_nanos": { + "type": "string", + "value": time_within(ts, expect_resolution="ns"), + }, }, "_meta": { "attributes": { @@ -659,6 +666,10 @@ def test_spanv2_with_string_pii_scrubbing( "type": "string", "value": time_within(ts, expect_resolution="ns"), }, + "sentry._internal.observed_timestamp_nanos": { + "type": "string", + "value": time_within(ts, expect_resolution="ns"), + }, }, "_meta": { "attributes": { From 95de740e8b06a744cc14b1f277fecabe8473dd06 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Thu, 16 Oct 2025 15:12:18 +0200 Subject: [PATCH 02/16] Missed docstrings --- relay-event-schema/src/protocol/attributes.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/relay-event-schema/src/protocol/attributes.rs b/relay-event-schema/src/protocol/attributes.rs index 58203a4b2f5..2f362903713 100644 --- a/relay-event-schema/src/protocol/attributes.rs +++ b/relay-event-schema/src/protocol/attributes.rs @@ -315,10 +315,12 @@ impl Attributes { self.0.iter_mut() } + /// Returns an iterator over the keys in this collection. pub fn keys(&self) -> std::collections::btree_map::Keys<'_, String, Annotated> { self.0.keys() } + /// Provides mutable access to an entry in this collection. pub fn entry( &mut self, key: String, From aeea4a387cc9300b93f95da24db312e8617522ba Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Thu, 16 Oct 2025 15:13:55 +0200 Subject: [PATCH 03/16] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfed5655575..3b8dbdcf6e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - Add event merging logic for Playstation crashes. ([#5228](https://github.com/getsentry/relay/pull/5228)) - Implement PII scrubbing for V2 spans. ([#5168](https://github.com/getsentry/relay/pull/5168)) - Add vercel log drain endpoint. ([#5212](https://github.com/getsentry/relay/pull/5212)) +- Normalize deprecated attribute names according to `sentry-convetions` for logs and V2 spans. ([#5257](https://github.com/getsentry/relay/pull/5257)) **Bug Fixes**: From 9d485d42ab7a7705953abba192785edce3ffb34e Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Thu, 16 Oct 2025 15:15:16 +0200 Subject: [PATCH 04/16] Fix changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b8dbdcf6e4..66e2bbaee7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +**Features**: + +- Normalize deprecated attribute names according to `sentry-convetions` for logs and V2 spans. ([#5257](https://github.com/getsentry/relay/pull/5257)) + ## 25.10.0 **Features**: @@ -22,7 +28,6 @@ - Add event merging logic for Playstation crashes. ([#5228](https://github.com/getsentry/relay/pull/5228)) - Implement PII scrubbing for V2 spans. ([#5168](https://github.com/getsentry/relay/pull/5168)) - Add vercel log drain endpoint. ([#5212](https://github.com/getsentry/relay/pull/5212)) -- Normalize deprecated attribute names according to `sentry-convetions` for logs and V2 spans. ([#5257](https://github.com/getsentry/relay/pull/5257)) **Bug Fixes**: From 61e6a130685919612b6c21a30830e691fc6d7a05 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Thu, 16 Oct 2025 15:27:01 +0200 Subject: [PATCH 05/16] Formatting --- tests/integration/test_ourlogs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_ourlogs.py b/tests/integration/test_ourlogs.py index d8cb1459921..1ce494c1d25 100644 --- a/tests/integration/test_ourlogs.py +++ b/tests/integration/test_ourlogs.py @@ -512,7 +512,7 @@ def test_ourlog_extraction_default_pii_scrubbing_does_not_scrub_default_attribut "sentry._meta.fields.body": { "stringValue": '{"meta":{"":{"rem":[["remove_custom_field","s",0,10]],"len":8}}}' }, - **timestamps(ts) + **timestamps(ts), }, "clientSampleRate": 1.0, "itemId": mock.ANY, From 43b10d4b20af8bdcdfe8c52b962c52504deb4cba Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Thu, 16 Oct 2025 15:51:03 +0200 Subject: [PATCH 06/16] Fix vercel log tests --- tests/integration/test_vercel_logs.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_vercel_logs.py b/tests/integration/test_vercel_logs.py index 57cceb3b9bd..641ed3a1d17 100644 --- a/tests/integration/test_vercel_logs.py +++ b/tests/integration/test_vercel_logs.py @@ -74,9 +74,10 @@ "vercel.project_name": {"stringValue": "my-app"}, "sentry.severity_text": {"stringValue": "info"}, "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, + "sentry._internal.observed_timestamp_nanos": {"stringValue": mock.ANY}, "sentry.timestamp_precise": {"intValue": "1573817187330000000"}, "vercel.build_id": {"stringValue": "bld_cotnkcr76"}, - "sentry.payload_size_bytes": {"intValue": "436"}, + "sentry.payload_size_bytes": {"intValue": "496"}, "sentry.browser.name": {"stringValue": "Python Requests"}, "vercel.project_id": {"stringValue": "gdufoJxB6b9b1fEqr1jUtFkyavUU"}, "sentry._meta.fields.trace_id": { @@ -128,8 +129,9 @@ "sentry.body": {"stringValue": "API request processed"}, "vercel.proxy.status_code": {"intValue": "200"}, "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, + "sentry._internal.observed_timestamp_nanos": {"stringValue": mock.ANY}, "sentry.timestamp_precise": {"intValue": "1573817250283000000"}, - "sentry.payload_size_bytes": {"intValue": "886"}, + "sentry.payload_size_bytes": {"intValue": "946"}, "vercel.proxy.region": {"stringValue": "sfo1"}, }, "clientSampleRate": 1.0, @@ -186,7 +188,7 @@ def test_vercel_logs_json_array( "org_id": 1, "outcome": 0, "project_id": 42, - "quantity": 1322, + "quantity": 1442, }, ] @@ -234,6 +236,6 @@ def test_vercel_logs_ndjson( "org_id": 1, "outcome": 0, "project_id": 42, - "quantity": 1322, + "quantity": 1442, }, ] From 273078bd2d0ab33b229788ac893349cf427a4b4b Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Thu, 16 Oct 2025 16:28:01 +0200 Subject: [PATCH 07/16] Update CHANGELOG.md Co-authored-by: Joris Bayer --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66e2bbaee7b..aa6e276d207 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ **Features**: -- Normalize deprecated attribute names according to `sentry-convetions` for logs and V2 spans. ([#5257](https://github.com/getsentry/relay/pull/5257)) +- Normalize deprecated attribute names according to `sentry-conventions` for logs and V2 spans. ([#5257](https://github.com/getsentry/relay/pull/5257)) ## 25.10.0 From 8d5bf197e9ef0fa84cf7530fad9581223a54d9b6 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Fri, 17 Oct 2025 10:03:21 +0200 Subject: [PATCH 08/16] Normalize deprecated attributes first --- relay-conventions/src/consts.rs | 3 ++- relay-event-normalization/src/eap/mod.rs | 28 +++++++++++++------- relay-server/src/processing/logs/process.rs | 4 +-- relay-server/src/processing/spans/process.rs | 4 +-- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/relay-conventions/src/consts.rs b/relay-conventions/src/consts.rs index daaa3617527..bf02b0cfa92 100644 --- a/relay-conventions/src/consts.rs +++ b/relay-conventions/src/consts.rs @@ -28,7 +28,8 @@ convention_attributes!( HTTP_TARGET => "http.target", MESSAGING_SYSTEM => "messaging.system", ENVIRONMENT => "sentry.environment", - OBSERVED_TIMESTAMP_NANOS => "sentry.observed_timestamp_nanos", + OBSERVED_TIMESTAMP_NANOS => "sentry._internal.observed_timestamp_nanos", + OBSERVED_TIMESTAMP_NANOS_DEPRECATED => "sentry.observed_timestamp_nanos", OP => "sentry.op", ORIGIN => "sentry.origin", PLATFORM => "sentry.platform", diff --git a/relay-event-normalization/src/eap/mod.rs b/relay-event-normalization/src/eap/mod.rs index 518f981034b..5126c5137bb 100644 --- a/relay-event-normalization/src/eap/mod.rs +++ b/relay-event-normalization/src/eap/mod.rs @@ -5,8 +5,9 @@ use chrono::{DateTime, Utc}; use relay_common::time::UnixTimestamp; use relay_conventions::{ - AttributeInfo, BROWSER_NAME, BROWSER_VERSION, OBSERVED_TIMESTAMP_NANOS, USER_GEO_CITY, - USER_GEO_COUNTRY_CODE, USER_GEO_REGION, USER_GEO_SUBDIVISION, WriteBehavior, + AttributeInfo, BROWSER_NAME, BROWSER_VERSION, OBSERVED_TIMESTAMP_NANOS, + OBSERVED_TIMESTAMP_NANOS_DEPRECATED, USER_GEO_CITY, USER_GEO_COUNTRY_CODE, USER_GEO_REGION, + USER_GEO_SUBDIVISION, WriteBehavior, }; use relay_event_schema::protocol::{AttributeType, Attributes, BrowserContext, Geo}; use relay_protocol::{Annotated, ErrorKind, Remark, RemarkType, Value}; @@ -64,14 +65,21 @@ pub fn normalize_attribute_types(attributes: &mut Annotated) { /// Adds the `received` time to the attributes. pub fn normalize_received(attributes: &mut Annotated, received: DateTime) { - attributes - .get_or_insert_with(Default::default) - .insert_if_missing(OBSERVED_TIMESTAMP_NANOS, || { - received - .timestamp_nanos_opt() - .unwrap_or_else(|| UnixTimestamp::now().as_nanos() as i64) - .to_string() - }); + let attributes = attributes.get_or_insert_with(Default::default); + + attributes.insert_if_missing(OBSERVED_TIMESTAMP_NANOS, || { + received + .timestamp_nanos_opt() + .unwrap_or_else(|| UnixTimestamp::now().as_nanos() as i64) + .to_string() + }); + + attributes.insert_if_missing(OBSERVED_TIMESTAMP_NANOS_DEPRECATED, || { + received + .timestamp_nanos_opt() + .unwrap_or_else(|| UnixTimestamp::now().as_nanos() as i64) + .to_string() + }); } /// Normalizes the user agent/client information into [`Attributes`]. diff --git a/relay-server/src/processing/logs/process.rs b/relay-server/src/processing/logs/process.rs index cde5271f8c5..84f95007e47 100644 --- a/relay-server/src/processing/logs/process.rs +++ b/relay-server/src/processing/logs/process.rs @@ -114,10 +114,10 @@ fn normalize_log(log: &mut Annotated, meta: &RequestMeta) -> Result<()> return Err(Error::Invalid(DiscardReason::NoData)); }; - eap::normalize_received(&mut log.attributes, meta.received_at()); - eap::normalize_user_agent(&mut log.attributes, meta.user_agent(), meta.client_hints()); eap::normalize_attribute_types(&mut log.attributes); eap::normalize_attribute_names(&mut log.attributes); + eap::normalize_received(&mut log.attributes, meta.received_at()); + eap::normalize_user_agent(&mut log.attributes, meta.user_agent(), meta.client_hints()); Ok(()) } diff --git a/relay-server/src/processing/spans/process.rs b/relay-server/src/processing/spans/process.rs index e0254e98cab..e15d1b7bca9 100644 --- a/relay-server/src/processing/spans/process.rs +++ b/relay-server/src/processing/spans/process.rs @@ -69,13 +69,13 @@ fn normalize_span( // TODO: `validate_span()` (start/end timestamps) if let Some(span) = span.value_mut() { + eap::normalize_attribute_types(&mut span.attributes); + eap::normalize_attribute_names(&mut span.attributes); eap::normalize_received(&mut span.attributes, meta.received_at()); eap::normalize_user_agent(&mut span.attributes, meta.user_agent(), meta.client_hints()); eap::normalize_user_geo(&mut span.attributes, || { meta.client_addr().and_then(|ip| geo_lookup.lookup(ip)) }); - eap::normalize_attribute_types(&mut span.attributes); - eap::normalize_attribute_names(&mut span.attributes); // TODO: ai model costs } else { From 2c907f04d68d27056811ffa000200be488b30005 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 21 Oct 2025 09:32:03 +0200 Subject: [PATCH 09/16] Remove _internal.observed_timestamp_nanos --- relay-conventions/src/consts.rs | 3 +-- relay-event-normalization/src/eap/mod.rs | 28 +++++++++--------------- tests/integration/test_ourlogs.py | 7 ------ tests/integration/test_spansv2.py | 8 ------- tests/integration/test_vercel_logs.py | 2 -- 5 files changed, 11 insertions(+), 37 deletions(-) diff --git a/relay-conventions/src/consts.rs b/relay-conventions/src/consts.rs index bf02b0cfa92..daaa3617527 100644 --- a/relay-conventions/src/consts.rs +++ b/relay-conventions/src/consts.rs @@ -28,8 +28,7 @@ convention_attributes!( HTTP_TARGET => "http.target", MESSAGING_SYSTEM => "messaging.system", ENVIRONMENT => "sentry.environment", - OBSERVED_TIMESTAMP_NANOS => "sentry._internal.observed_timestamp_nanos", - OBSERVED_TIMESTAMP_NANOS_DEPRECATED => "sentry.observed_timestamp_nanos", + OBSERVED_TIMESTAMP_NANOS => "sentry.observed_timestamp_nanos", OP => "sentry.op", ORIGIN => "sentry.origin", PLATFORM => "sentry.platform", diff --git a/relay-event-normalization/src/eap/mod.rs b/relay-event-normalization/src/eap/mod.rs index 5126c5137bb..518f981034b 100644 --- a/relay-event-normalization/src/eap/mod.rs +++ b/relay-event-normalization/src/eap/mod.rs @@ -5,9 +5,8 @@ use chrono::{DateTime, Utc}; use relay_common::time::UnixTimestamp; use relay_conventions::{ - AttributeInfo, BROWSER_NAME, BROWSER_VERSION, OBSERVED_TIMESTAMP_NANOS, - OBSERVED_TIMESTAMP_NANOS_DEPRECATED, USER_GEO_CITY, USER_GEO_COUNTRY_CODE, USER_GEO_REGION, - USER_GEO_SUBDIVISION, WriteBehavior, + AttributeInfo, BROWSER_NAME, BROWSER_VERSION, OBSERVED_TIMESTAMP_NANOS, USER_GEO_CITY, + USER_GEO_COUNTRY_CODE, USER_GEO_REGION, USER_GEO_SUBDIVISION, WriteBehavior, }; use relay_event_schema::protocol::{AttributeType, Attributes, BrowserContext, Geo}; use relay_protocol::{Annotated, ErrorKind, Remark, RemarkType, Value}; @@ -65,21 +64,14 @@ pub fn normalize_attribute_types(attributes: &mut Annotated) { /// Adds the `received` time to the attributes. pub fn normalize_received(attributes: &mut Annotated, received: DateTime) { - let attributes = attributes.get_or_insert_with(Default::default); - - attributes.insert_if_missing(OBSERVED_TIMESTAMP_NANOS, || { - received - .timestamp_nanos_opt() - .unwrap_or_else(|| UnixTimestamp::now().as_nanos() as i64) - .to_string() - }); - - attributes.insert_if_missing(OBSERVED_TIMESTAMP_NANOS_DEPRECATED, || { - received - .timestamp_nanos_opt() - .unwrap_or_else(|| UnixTimestamp::now().as_nanos() as i64) - .to_string() - }); + attributes + .get_or_insert_with(Default::default) + .insert_if_missing(OBSERVED_TIMESTAMP_NANOS, || { + received + .timestamp_nanos_opt() + .unwrap_or_else(|| UnixTimestamp::now().as_nanos() as i64) + .to_string() + }); } /// Normalizes the user agent/client information into [`Attributes`]. diff --git a/tests/integration/test_ourlogs.py b/tests/integration/test_ourlogs.py index 1ce494c1d25..e19b906fd33 100644 --- a/tests/integration/test_ourlogs.py +++ b/tests/integration/test_ourlogs.py @@ -46,9 +46,6 @@ def timestamps(ts: datetime): "sentry.observed_timestamp_nanos": { "stringValue": time_within(ts, expect_resolution="ns") }, - "sentry._internal.observed_timestamp_nanos": { - "stringValue": time_within(ts, expect_resolution="ns") - }, "sentry.timestamp_nanos": { "stringValue": time_within_delta( ts, delta=timedelta(seconds=0), expect_resolution="ns", precision="us" @@ -361,10 +358,6 @@ def test_ourlog_extraction_with_string_pii_scrubbing( "type": "string", "value": time_within(ts, expect_resolution="ns"), }, - "sentry._internal.observed_timestamp_nanos": { - "type": "string", - "value": time_within(ts, expect_resolution="ns"), - }, }, "__header": {"byte_size": mock.ANY}, "_meta": { diff --git a/tests/integration/test_spansv2.py b/tests/integration/test_spansv2.py index 8189af9ad58..b2bf6cbd9fb 100644 --- a/tests/integration/test_spansv2.py +++ b/tests/integration/test_spansv2.py @@ -94,10 +94,6 @@ def test_spansv2_basic( "type": "string", "value": time_within(ts, expect_resolution="ns"), }, - "sentry._internal.observed_timestamp_nanos": { - "type": "string", - "value": time_within(ts, expect_resolution="ns"), - }, }, "_meta": { "attributes": { @@ -666,10 +662,6 @@ def test_spanv2_with_string_pii_scrubbing( "type": "string", "value": time_within(ts, expect_resolution="ns"), }, - "sentry._internal.observed_timestamp_nanos": { - "type": "string", - "value": time_within(ts, expect_resolution="ns"), - }, }, "_meta": { "attributes": { diff --git a/tests/integration/test_vercel_logs.py b/tests/integration/test_vercel_logs.py index 641ed3a1d17..cbb1e06c8ff 100644 --- a/tests/integration/test_vercel_logs.py +++ b/tests/integration/test_vercel_logs.py @@ -74,7 +74,6 @@ "vercel.project_name": {"stringValue": "my-app"}, "sentry.severity_text": {"stringValue": "info"}, "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, - "sentry._internal.observed_timestamp_nanos": {"stringValue": mock.ANY}, "sentry.timestamp_precise": {"intValue": "1573817187330000000"}, "vercel.build_id": {"stringValue": "bld_cotnkcr76"}, "sentry.payload_size_bytes": {"intValue": "496"}, @@ -129,7 +128,6 @@ "sentry.body": {"stringValue": "API request processed"}, "vercel.proxy.status_code": {"intValue": "200"}, "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, - "sentry._internal.observed_timestamp_nanos": {"stringValue": mock.ANY}, "sentry.timestamp_precise": {"intValue": "1573817250283000000"}, "sentry.payload_size_bytes": {"intValue": "946"}, "vercel.proxy.region": {"stringValue": "sfo1"}, From 82f80418c2cd87c17ca3c4f3e91633f0226d4295 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 28 Oct 2025 15:05:53 +0100 Subject: [PATCH 10/16] Revert "Remove _internal.observed_timestamp_nanos" This reverts commit 2c907f04d68d27056811ffa000200be488b30005. --- relay-conventions/src/consts.rs | 3 ++- relay-event-normalization/src/eap/mod.rs | 28 +++++++++++++++--------- tests/integration/test_ourlogs.py | 10 ++++++--- tests/integration/test_spansv2.py | 8 +++++++ tests/integration/test_vercel_logs.py | 8 ++++--- 5 files changed, 40 insertions(+), 17 deletions(-) diff --git a/relay-conventions/src/consts.rs b/relay-conventions/src/consts.rs index daaa3617527..5a9fd6001c6 100644 --- a/relay-conventions/src/consts.rs +++ b/relay-conventions/src/consts.rs @@ -28,7 +28,8 @@ convention_attributes!( HTTP_TARGET => "http.target", MESSAGING_SYSTEM => "messaging.system", ENVIRONMENT => "sentry.environment", - OBSERVED_TIMESTAMP_NANOS => "sentry.observed_timestamp_nanos", + OBSERVED_TIMESTAMP_NANOS => "sentry._internal.observed_timestamp_nanos", + OBSERVED_TIMESTAMP_NANOS_INTERNAL => "sentry.observed_timestamp_nanos", OP => "sentry.op", ORIGIN => "sentry.origin", PLATFORM => "sentry.platform", diff --git a/relay-event-normalization/src/eap/mod.rs b/relay-event-normalization/src/eap/mod.rs index 518f981034b..2ef76d91ef4 100644 --- a/relay-event-normalization/src/eap/mod.rs +++ b/relay-event-normalization/src/eap/mod.rs @@ -5,8 +5,9 @@ use chrono::{DateTime, Utc}; use relay_common::time::UnixTimestamp; use relay_conventions::{ - AttributeInfo, BROWSER_NAME, BROWSER_VERSION, OBSERVED_TIMESTAMP_NANOS, USER_GEO_CITY, - USER_GEO_COUNTRY_CODE, USER_GEO_REGION, USER_GEO_SUBDIVISION, WriteBehavior, + AttributeInfo, BROWSER_NAME, BROWSER_VERSION, OBSERVED_TIMESTAMP_NANOS, + OBSERVED_TIMESTAMP_NANOS_INTERNAL, USER_GEO_CITY, USER_GEO_COUNTRY_CODE, USER_GEO_REGION, + USER_GEO_SUBDIVISION, WriteBehavior, }; use relay_event_schema::protocol::{AttributeType, Attributes, BrowserContext, Geo}; use relay_protocol::{Annotated, ErrorKind, Remark, RemarkType, Value}; @@ -64,14 +65,21 @@ pub fn normalize_attribute_types(attributes: &mut Annotated) { /// Adds the `received` time to the attributes. pub fn normalize_received(attributes: &mut Annotated, received: DateTime) { - attributes - .get_or_insert_with(Default::default) - .insert_if_missing(OBSERVED_TIMESTAMP_NANOS, || { - received - .timestamp_nanos_opt() - .unwrap_or_else(|| UnixTimestamp::now().as_nanos() as i64) - .to_string() - }); + let attributes = attributes.get_or_insert_with(Default::default); + + attributes.insert_if_missing(OBSERVED_TIMESTAMP_NANOS, || { + received + .timestamp_nanos_opt() + .unwrap_or_else(|| UnixTimestamp::now().as_nanos() as i64) + .to_string() + }); + + attributes.insert_if_missing(OBSERVED_TIMESTAMP_NANOS_INTERNAL, || { + received + .timestamp_nanos_opt() + .unwrap_or_else(|| UnixTimestamp::now().as_nanos() as i64) + .to_string() + }); } /// Normalizes the user agent/client information into [`Attributes`]. diff --git a/tests/integration/test_ourlogs.py b/tests/integration/test_ourlogs.py index 214644b6a17..bab1c79a65e 100644 --- a/tests/integration/test_ourlogs.py +++ b/tests/integration/test_ourlogs.py @@ -46,6 +46,9 @@ def timestamps(ts: datetime): "sentry.observed_timestamp_nanos": { "stringValue": time_within(ts, expect_resolution="ns") }, + "sentry._internal.observed_timestamp_nanos": { + "stringValue": time_within(ts, expect_resolution="ns") + }, "sentry.timestamp_nanos": { "stringValue": time_within_delta( ts, delta=timedelta(seconds=0), expect_resolution="ns", precision="us" @@ -358,6 +361,10 @@ def test_ourlog_extraction_with_string_pii_scrubbing( "type": "string", "value": time_within(ts, expect_resolution="ns"), }, + "sentry._internal.observed_timestamp_nanos": { + "type": "string", + "value": time_within(ts, expect_resolution="ns"), + }, }, "__header": {"byte_size": mock.ANY}, "_meta": { @@ -551,9 +558,6 @@ def test_ourlog_extraction_default_pii_scrubbing_does_not_scrub_default_attribut "sentry.span_id": {"stringValue": "eee19b7ec3c1b174"}, "sentry.payload_size_bytes": mock.ANY, "sentry.browser.name": {"stringValue": "Python Requests"}, - "sentry._meta.fields.body": { - "stringValue": '{"meta":{"":{"rem":[["remove_custom_field","s",0,10]],"len":8}}}' - }, **timestamps(ts), }, "clientSampleRate": 1.0, diff --git a/tests/integration/test_spansv2.py b/tests/integration/test_spansv2.py index dd65db8a8bd..cba615fda16 100644 --- a/tests/integration/test_spansv2.py +++ b/tests/integration/test_spansv2.py @@ -97,6 +97,10 @@ def test_spansv2_basic( "type": "string", "value": time_within(ts, expect_resolution="ns"), }, + "sentry._internal.observed_timestamp_nanos": { + "type": "string", + "value": time_within(ts, expect_resolution="ns"), + }, }, "_meta": { "attributes": { @@ -762,6 +766,10 @@ def test_spanv2_with_string_pii_scrubbing( "type": "string", "value": time_within(ts, expect_resolution="ns"), }, + "sentry._internal.observed_timestamp_nanos": { + "type": "string", + "value": time_within(ts, expect_resolution="ns"), + }, }, "_meta": { "attributes": { diff --git a/tests/integration/test_vercel_logs.py b/tests/integration/test_vercel_logs.py index 17db2aff540..dffe3eecb88 100644 --- a/tests/integration/test_vercel_logs.py +++ b/tests/integration/test_vercel_logs.py @@ -74,6 +74,7 @@ "vercel.project_name": {"stringValue": "my-app"}, "sentry.severity_text": {"stringValue": "info"}, "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, + "sentry._internal.observed_timestamp_nanos": {"stringValue": mock.ANY}, "sentry.timestamp_precise": {"intValue": "1573817187330000000"}, "vercel.build_id": {"stringValue": "bld_cotnkcr76"}, "sentry.payload_size_bytes": {"intValue": "496"}, @@ -128,8 +129,9 @@ "sentry.body": {"stringValue": "API request processed"}, "vercel.proxy.status_code": {"intValue": "200"}, "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, + "sentry._internal.observed_timestamp_nanos": {"stringValue": mock.ANY}, "sentry.timestamp_precise": {"intValue": "1573817250283000000"}, - "sentry.payload_size_bytes": {"intValue": "946"}, + "sentry.payload_size_bytes": {"intValue": "949"}, "vercel.proxy.region": {"stringValue": "sfo1"}, }, "clientSampleRate": 1.0, @@ -186,7 +188,7 @@ def test_vercel_logs_json_array( "org_id": 1, "outcome": 0, "project_id": 42, - "quantity": 1442, + "quantity": 1445, }, ] @@ -234,6 +236,6 @@ def test_vercel_logs_ndjson( "org_id": 1, "outcome": 0, "project_id": 42, - "quantity": 1442, + "quantity": 1445, }, ] From 7ae5f0be482bae0fd394edb953501d2b18af8da6 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 28 Oct 2025 17:00:00 +0100 Subject: [PATCH 11/16] Fix some tests --- tests/integration/test_nel.py | 4 ++++ tests/integration/test_otlp_logs.py | 15 ++++++++++++--- tests/integration/test_spans_standalone.py | 4 ++++ tests/integration/test_spansv2_otel.py | 4 ++++ tests/integration/test_trace_metrics.py | 16 ++++++++++++++++ 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_nel.py b/tests/integration/test_nel.py index 9c701cec897..cc402c686d4 100644 --- a/tests/integration/test_nel.py +++ b/tests/integration/test_nel.py @@ -69,6 +69,10 @@ def test_nel_converted_to_logs(mini_sentry, relay): "type": "string", "value": time_within_delta(expect_resolution="ns"), }, + "sentry._internal.observed_timestamp_nanos": { + "type": "string", + "value": time_within_delta(expect_resolution="ns"), + }, }, "body": "The user agent successfully received a response, but it had a 500 status code", "level": "warn", diff --git a/tests/integration/test_otlp_logs.py b/tests/integration/test_otlp_logs.py index 7f4a2c5e812..a6a1e7a1f0a 100644 --- a/tests/integration/test_otlp_logs.py +++ b/tests/integration/test_otlp_logs.py @@ -139,8 +139,11 @@ def test_otlp_logs_conversion( "sentry.observed_timestamp_nanos": { "stringValue": time_within(ts, expect_resolution="ns") }, + "sentry._internal.observed_timestamp_nanos": { + "stringValue": time_within(ts, expect_resolution="ns") + }, "sentry.origin": {"stringValue": "auto.otlp.logs"}, - "sentry.payload_size_bytes": {"intValue": "385"}, + "sentry.payload_size_bytes": {"intValue": "445"}, "sentry.severity_text": {"stringValue": "info"}, "sentry.span_id": {"stringValue": "eee19b7ec3c1b174"}, "sentry.timestamp_nanos": { @@ -191,7 +194,7 @@ def test_otlp_logs_conversion( "org_id": 1, "outcome": 0, "project_id": 42, - "quantity": 385, + "quantity": 445, }, ] @@ -259,6 +262,9 @@ def test_otlp_logs_multiple_records( "sentry.observed_timestamp_nanos": { "stringValue": time_within(ts, expect_resolution="ns") }, + "sentry._internal.observed_timestamp_nanos": { + "stringValue": time_within(ts, expect_resolution="ns") + }, "sentry.origin": {"stringValue": "auto.otlp.logs"}, "sentry.payload_size_bytes": {"intValue": mock.ANY}, "sentry.severity_text": {"stringValue": "error"}, @@ -300,6 +306,9 @@ def test_otlp_logs_multiple_records( "sentry.observed_timestamp_nanos": { "stringValue": time_within(ts, expect_resolution="ns") }, + "sentry._internal.observed_timestamp_nanos": { + "stringValue": time_within(ts, expect_resolution="ns") + }, "sentry.origin": {"stringValue": "auto.otlp.logs"}, "sentry.payload_size_bytes": {"intValue": mock.ANY}, "sentry.severity_text": {"stringValue": "debug"}, @@ -351,6 +360,6 @@ def test_otlp_logs_multiple_records( "org_id": 1, "outcome": 0, "project_id": 42, - "quantity": 305, + "quantity": 425, }, ] diff --git a/tests/integration/test_spans_standalone.py b/tests/integration/test_spans_standalone.py index dfb8514a78c..c79d275fb82 100644 --- a/tests/integration/test_spans_standalone.py +++ b/tests/integration/test_spans_standalone.py @@ -46,6 +46,10 @@ def lcp_cls_inp_differences(mode): "type": "string", "value": time_within_delta(expect_resolution="ns"), }, + "sentry._internal.observed_timestamp_nanos": { + "type": "string", + "value": time_within_delta(expect_resolution="ns"), + }, # Maybe should not exist. Segment information in legacy processing is removed. "sentry.segment.id": {"type": "string", "value": "8a6626cc9bdd5d9b"}, } diff --git a/tests/integration/test_spansv2_otel.py b/tests/integration/test_spansv2_otel.py index e61ef3c060b..89276645957 100644 --- a/tests/integration/test_spansv2_otel.py +++ b/tests/integration/test_spansv2_otel.py @@ -93,6 +93,10 @@ def test_span_ingestion( "resource.company": {"type": "string", "value": "Relay Corp"}, "sentry.browser.name": {"type": "string", "value": "Python Requests"}, "sentry.browser.version": {"type": "string", "value": "2.32"}, + "sentry._internal.observed_timestamp_nanos": { + "type": "string", + "value": time_within(ts, expect_resolution="ns"), + }, "sentry.observed_timestamp_nanos": { "type": "string", "value": time_within(ts, expect_resolution="ns"), diff --git a/tests/integration/test_trace_metrics.py b/tests/integration/test_trace_metrics.py index 710af45e7ce..5f3c61ffcce 100644 --- a/tests/integration/test_trace_metrics.py +++ b/tests/integration/test_trace_metrics.py @@ -94,6 +94,14 @@ def test_trace_metric_extraction( precision="us", ) }, + "sentry._internal.observed_timestamp_nanos": { + "stringValue": time_within_delta( + start, + delta=timedelta(seconds=2), + expect_resolution="ns", + precision="us", + ) + }, "sentry.span_id": {"stringValue": "eee19b7ec3c1b175"}, "sentry.browser.name": {"stringValue": mock.ANY}, "sentry.browser.version": {"stringValue": mock.ANY}, @@ -240,6 +248,14 @@ def test_trace_metric_pii_scrubbing( precision="us", ) }, + "sentry._internal.observed_timestamp_nanos": { + "stringValue": time_within_delta( + start, + delta=timedelta(seconds=2), + expect_resolution="ns", + precision="us", + ) + }, "sentry.browser.name": {"stringValue": mock.ANY}, "sentry.browser.version": {"stringValue": mock.ANY}, "safe.attribute": {"stringValue": "keep this"}, From 130391099dd8b2efea273b886ac010acd77d46e8 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 28 Oct 2025 17:09:02 +0100 Subject: [PATCH 12/16] Switch constant names --- relay-conventions/src/consts.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relay-conventions/src/consts.rs b/relay-conventions/src/consts.rs index 13c572413a0..ae66f2d796b 100644 --- a/relay-conventions/src/consts.rs +++ b/relay-conventions/src/consts.rs @@ -28,8 +28,8 @@ convention_attributes!( HTTP_TARGET => "http.target", MESSAGING_SYSTEM => "messaging.system", ENVIRONMENT => "sentry.environment", - OBSERVED_TIMESTAMP_NANOS => "sentry._internal.observed_timestamp_nanos", - OBSERVED_TIMESTAMP_NANOS_INTERNAL => "sentry.observed_timestamp_nanos", + OBSERVED_TIMESTAMP_NANOS => "sentry.observed_timestamp_nanos", + OBSERVED_TIMESTAMP_NANOS_INTERNAL => "sentry._internal.observed_timestamp_nanos", OP => "sentry.op", ORIGIN => "sentry.origin", PLATFORM => "sentry.platform", From 6b00628a7da49b0fe360fccd9a3f1fe4f30fe1f5 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Wed, 29 Oct 2025 14:34:09 +0100 Subject: [PATCH 13/16] Update sentry-conventions --- relay-conventions/sentry-conventions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay-conventions/sentry-conventions b/relay-conventions/sentry-conventions index 7b0c420974f..4c7cda093a2 160000 --- a/relay-conventions/sentry-conventions +++ b/relay-conventions/sentry-conventions @@ -1 +1 @@ -Subproject commit 7b0c420974f5def95fe5ae4829101d22ee340a2c +Subproject commit 4c7cda093a241389073046f9f4e15a8ac2ccd0da From c4459f27099ef477f8ab62167d18a674e13d1458 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Wed, 29 Oct 2025 11:25:43 +0100 Subject: [PATCH 14/16] Remove sentry._internal.observed_timestamp_nanos --- relay-conventions/src/consts.rs | 1 - relay-event-normalization/src/eap/mod.rs | 40 ++++++---------------- tests/integration/test_nel.py | 4 --- tests/integration/test_otlp_logs.py | 15 ++------ tests/integration/test_ourlogs.py | 9 +---- tests/integration/test_spans_standalone.py | 4 --- tests/integration/test_spansv2.py | 8 ----- tests/integration/test_spansv2_otel.py | 4 --- tests/integration/test_trace_metrics.py | 16 --------- tests/integration/test_vercel_logs.py | 10 +++--- 10 files changed, 18 insertions(+), 93 deletions(-) diff --git a/relay-conventions/src/consts.rs b/relay-conventions/src/consts.rs index ae66f2d796b..7e4fc6fc760 100644 --- a/relay-conventions/src/consts.rs +++ b/relay-conventions/src/consts.rs @@ -29,7 +29,6 @@ convention_attributes!( MESSAGING_SYSTEM => "messaging.system", ENVIRONMENT => "sentry.environment", OBSERVED_TIMESTAMP_NANOS => "sentry.observed_timestamp_nanos", - OBSERVED_TIMESTAMP_NANOS_INTERNAL => "sentry._internal.observed_timestamp_nanos", OP => "sentry.op", ORIGIN => "sentry.origin", PLATFORM => "sentry.platform", diff --git a/relay-event-normalization/src/eap/mod.rs b/relay-event-normalization/src/eap/mod.rs index 95c06f3f5ae..f173bcd39a8 100644 --- a/relay-event-normalization/src/eap/mod.rs +++ b/relay-event-normalization/src/eap/mod.rs @@ -5,9 +5,8 @@ use chrono::{DateTime, Utc}; use relay_common::time::UnixTimestamp; use relay_conventions::{ - AttributeInfo, BROWSER_NAME, BROWSER_VERSION, OBSERVED_TIMESTAMP_NANOS, - OBSERVED_TIMESTAMP_NANOS_INTERNAL, USER_AGENT_ORIGINAL, USER_GEO_CITY, USER_GEO_COUNTRY_CODE, - USER_GEO_REGION, USER_GEO_SUBDIVISION, WriteBehavior, + AttributeInfo, BROWSER_NAME, BROWSER_VERSION, OBSERVED_TIMESTAMP_NANOS, USER_AGENT_ORIGINAL, + USER_GEO_CITY, USER_GEO_COUNTRY_CODE, USER_GEO_REGION, USER_GEO_SUBDIVISION, WriteBehavior, }; use relay_event_schema::protocol::{AttributeType, Attributes, BrowserContext, Geo}; use relay_protocol::{Annotated, ErrorKind, Remark, RemarkType, Value}; @@ -65,21 +64,14 @@ pub fn normalize_attribute_types(attributes: &mut Annotated) { /// Adds the `received` time to the attributes. pub fn normalize_received(attributes: &mut Annotated, received: DateTime) { - let attributes = attributes.get_or_insert_with(Default::default); - - attributes.insert_if_missing(OBSERVED_TIMESTAMP_NANOS, || { - received - .timestamp_nanos_opt() - .unwrap_or_else(|| UnixTimestamp::now().as_nanos() as i64) - .to_string() - }); - - attributes.insert_if_missing(OBSERVED_TIMESTAMP_NANOS_INTERNAL, || { - received - .timestamp_nanos_opt() - .unwrap_or_else(|| UnixTimestamp::now().as_nanos() as i64) - .to_string() - }); + attributes + .get_or_insert_with(Default::default) + .insert_if_missing(OBSERVED_TIMESTAMP_NANOS, || { + received + .timestamp_nanos_opt() + .unwrap_or_else(|| UnixTimestamp::now().as_nanos() as i64) + .to_string() + }); } /// Normalizes the user agent/client information into [`Attributes`]. @@ -215,10 +207,6 @@ mod tests { insta::assert_json_snapshot!(SerializableAnnotated(&attributes), @r#" { - "sentry._internal.observed_timestamp_nanos": { - "type": "string", - "value": "1234201337" - }, "sentry.observed_timestamp_nanos": { "type": "string", "value": "1234201337" @@ -231,10 +219,6 @@ mod tests { fn test_normalize_received_existing() { let mut attributes = Annotated::from_json( r#"{ - "sentry._internal.observed_timestamp_nanos": { - "type": "string", - "value": "111222333" - }, "sentry.observed_timestamp_nanos": { "type": "string", "value": "111222333" @@ -250,10 +234,6 @@ mod tests { insta::assert_json_snapshot!(SerializableAnnotated(&attributes), @r###" { - "sentry._internal.observed_timestamp_nanos": { - "type": "string", - "value": "111222333" - }, "sentry.observed_timestamp_nanos": { "type": "string", "value": "111222333" diff --git a/tests/integration/test_nel.py b/tests/integration/test_nel.py index cc402c686d4..9c701cec897 100644 --- a/tests/integration/test_nel.py +++ b/tests/integration/test_nel.py @@ -69,10 +69,6 @@ def test_nel_converted_to_logs(mini_sentry, relay): "type": "string", "value": time_within_delta(expect_resolution="ns"), }, - "sentry._internal.observed_timestamp_nanos": { - "type": "string", - "value": time_within_delta(expect_resolution="ns"), - }, }, "body": "The user agent successfully received a response, but it had a 500 status code", "level": "warn", diff --git a/tests/integration/test_otlp_logs.py b/tests/integration/test_otlp_logs.py index 759fe84d80f..6d8e9c543b7 100644 --- a/tests/integration/test_otlp_logs.py +++ b/tests/integration/test_otlp_logs.py @@ -139,11 +139,8 @@ def test_otlp_logs_conversion( "sentry.observed_timestamp_nanos": { "stringValue": time_within(ts, expect_resolution="ns") }, - "sentry._internal.observed_timestamp_nanos": { - "stringValue": time_within(ts, expect_resolution="ns") - }, "sentry.origin": {"stringValue": "auto.otlp.logs"}, - "sentry.payload_size_bytes": {"intValue": "445"}, + "sentry.payload_size_bytes": {"intValue": "385"}, "sentry.severity_text": {"stringValue": "info"}, "sentry.span_id": {"stringValue": "eee19b7ec3c1b174"}, "sentry.timestamp_precise": { @@ -186,7 +183,7 @@ def test_otlp_logs_conversion( "org_id": 1, "outcome": 0, "project_id": 42, - "quantity": 445, + "quantity": 385, }, ] @@ -254,9 +251,6 @@ def test_otlp_logs_multiple_records( "sentry.observed_timestamp_nanos": { "stringValue": time_within(ts, expect_resolution="ns") }, - "sentry._internal.observed_timestamp_nanos": { - "stringValue": time_within(ts, expect_resolution="ns") - }, "sentry.origin": {"stringValue": "auto.otlp.logs"}, "sentry.payload_size_bytes": {"intValue": mock.ANY}, "sentry.severity_text": {"stringValue": "error"}, @@ -290,9 +284,6 @@ def test_otlp_logs_multiple_records( "sentry.observed_timestamp_nanos": { "stringValue": time_within(ts, expect_resolution="ns") }, - "sentry._internal.observed_timestamp_nanos": { - "stringValue": time_within(ts, expect_resolution="ns") - }, "sentry.origin": {"stringValue": "auto.otlp.logs"}, "sentry.payload_size_bytes": {"intValue": mock.ANY}, "sentry.severity_text": {"stringValue": "debug"}, @@ -336,6 +327,6 @@ def test_otlp_logs_multiple_records( "org_id": 1, "outcome": 0, "project_id": 42, - "quantity": 425, + "quantity": 305, }, ] diff --git a/tests/integration/test_ourlogs.py b/tests/integration/test_ourlogs.py index 9cd8b4003b9..deee9c415b6 100644 --- a/tests/integration/test_ourlogs.py +++ b/tests/integration/test_ourlogs.py @@ -46,9 +46,6 @@ def timestamps(ts: datetime): "sentry.observed_timestamp_nanos": { "stringValue": time_within(ts, expect_resolution="ns") }, - "sentry._internal.observed_timestamp_nanos": { - "stringValue": time_within(ts, expect_resolution="ns") - }, "sentry.timestamp_precise": { "intValue": time_within_delta( ts, delta=timedelta(seconds=0), expect_resolution="ns", precision="us" @@ -137,7 +134,7 @@ def test_ourlog_multiple_containers_not_allowed( # If an external Relay/Client makes modifications, sizes can change, # this is fuzzy due to slight changes in sizes due to added timestamps # and may need to be adjusted when changing normalization. - ("managed", 641), + ("managed", 521), ], ) def test_ourlog_extraction_with_sentry_logs( @@ -356,10 +353,6 @@ def test_ourlog_extraction_with_string_pii_scrubbing( "type": "string", "value": time_within(ts, expect_resolution="ns"), }, - "sentry._internal.observed_timestamp_nanos": { - "type": "string", - "value": time_within(ts, expect_resolution="ns"), - }, }, "__header": {"byte_size": mock.ANY}, "_meta": { diff --git a/tests/integration/test_spans_standalone.py b/tests/integration/test_spans_standalone.py index c79d275fb82..dfb8514a78c 100644 --- a/tests/integration/test_spans_standalone.py +++ b/tests/integration/test_spans_standalone.py @@ -46,10 +46,6 @@ def lcp_cls_inp_differences(mode): "type": "string", "value": time_within_delta(expect_resolution="ns"), }, - "sentry._internal.observed_timestamp_nanos": { - "type": "string", - "value": time_within_delta(expect_resolution="ns"), - }, # Maybe should not exist. Segment information in legacy processing is removed. "sentry.segment.id": {"type": "string", "value": "8a6626cc9bdd5d9b"}, } diff --git a/tests/integration/test_spansv2.py b/tests/integration/test_spansv2.py index cba615fda16..dd65db8a8bd 100644 --- a/tests/integration/test_spansv2.py +++ b/tests/integration/test_spansv2.py @@ -97,10 +97,6 @@ def test_spansv2_basic( "type": "string", "value": time_within(ts, expect_resolution="ns"), }, - "sentry._internal.observed_timestamp_nanos": { - "type": "string", - "value": time_within(ts, expect_resolution="ns"), - }, }, "_meta": { "attributes": { @@ -766,10 +762,6 @@ def test_spanv2_with_string_pii_scrubbing( "type": "string", "value": time_within(ts, expect_resolution="ns"), }, - "sentry._internal.observed_timestamp_nanos": { - "type": "string", - "value": time_within(ts, expect_resolution="ns"), - }, }, "_meta": { "attributes": { diff --git a/tests/integration/test_spansv2_otel.py b/tests/integration/test_spansv2_otel.py index 89276645957..e61ef3c060b 100644 --- a/tests/integration/test_spansv2_otel.py +++ b/tests/integration/test_spansv2_otel.py @@ -93,10 +93,6 @@ def test_span_ingestion( "resource.company": {"type": "string", "value": "Relay Corp"}, "sentry.browser.name": {"type": "string", "value": "Python Requests"}, "sentry.browser.version": {"type": "string", "value": "2.32"}, - "sentry._internal.observed_timestamp_nanos": { - "type": "string", - "value": time_within(ts, expect_resolution="ns"), - }, "sentry.observed_timestamp_nanos": { "type": "string", "value": time_within(ts, expect_resolution="ns"), diff --git a/tests/integration/test_trace_metrics.py b/tests/integration/test_trace_metrics.py index 96f8f059878..c349724754d 100644 --- a/tests/integration/test_trace_metrics.py +++ b/tests/integration/test_trace_metrics.py @@ -86,14 +86,6 @@ def test_trace_metric_extraction( precision="us", ) }, - "sentry._internal.observed_timestamp_nanos": { - "stringValue": time_within_delta( - start, - delta=timedelta(seconds=2), - expect_resolution="ns", - precision="us", - ) - }, "sentry.span_id": {"stringValue": "eee19b7ec3c1b175"}, "sentry.browser.name": {"stringValue": mock.ANY}, "sentry.browser.version": {"stringValue": mock.ANY}, @@ -232,14 +224,6 @@ def test_trace_metric_pii_scrubbing( precision="us", ) }, - "sentry._internal.observed_timestamp_nanos": { - "stringValue": time_within_delta( - start, - delta=timedelta(seconds=2), - expect_resolution="ns", - precision="us", - ) - }, "sentry.browser.name": {"stringValue": mock.ANY}, "sentry.browser.version": {"stringValue": mock.ANY}, "safe.attribute": {"stringValue": "keep this"}, diff --git a/tests/integration/test_vercel_logs.py b/tests/integration/test_vercel_logs.py index 5cfc9f92c70..ece2a9e3ff4 100644 --- a/tests/integration/test_vercel_logs.py +++ b/tests/integration/test_vercel_logs.py @@ -73,10 +73,9 @@ "vercel.project_name": {"stringValue": "my-app"}, "sentry.severity_text": {"stringValue": "info"}, "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, - "sentry._internal.observed_timestamp_nanos": {"stringValue": mock.ANY}, "sentry.timestamp_precise": {"intValue": "1573817187330000000"}, "vercel.build_id": {"stringValue": "bld_cotnkcr76"}, - "sentry.payload_size_bytes": {"intValue": "496"}, + "sentry.payload_size_bytes": {"intValue": "436"}, "sentry.browser.name": {"stringValue": "Python Requests"}, "vercel.project_id": {"stringValue": "gdufoJxB6b9b1fEqr1jUtFkyavUU"}, "sentry._meta.fields.trace_id": { @@ -127,9 +126,8 @@ "sentry.body": {"stringValue": "API request processed"}, "vercel.proxy.status_code": {"intValue": "200"}, "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, - "sentry._internal.observed_timestamp_nanos": {"stringValue": mock.ANY}, "sentry.timestamp_precise": {"intValue": "1573817250283000000"}, - "sentry.payload_size_bytes": {"intValue": "949"}, + "sentry.payload_size_bytes": {"intValue": "889"}, "vercel.proxy.region": {"stringValue": "sfo1"}, }, "clientSampleRate": 1.0, @@ -186,7 +184,7 @@ def test_vercel_logs_json_array( "org_id": 1, "outcome": 0, "project_id": 42, - "quantity": 1445, + "quantity": 1325, }, ] @@ -234,6 +232,6 @@ def test_vercel_logs_ndjson( "org_id": 1, "outcome": 0, "project_id": 42, - "quantity": 1445, + "quantity": 1325, }, ] From 508887ba53ed70f79bb2a7af52ef5fa42a9a023c Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Wed, 29 Oct 2025 15:16:52 +0100 Subject: [PATCH 15/16] Don't clone attribute when renaming --- relay-event-normalization/src/eap/mod.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/relay-event-normalization/src/eap/mod.rs b/relay-event-normalization/src/eap/mod.rs index 8d84cb6bda3..521fe1b227f 100644 --- a/relay-event-normalization/src/eap/mod.rs +++ b/relay-event-normalization/src/eap/mod.rs @@ -12,7 +12,7 @@ use relay_conventions::{ USER_GEO_SUBDIVISION, WriteBehavior, }; use relay_event_schema::protocol::{AttributeType, Attributes, BrowserContext, Geo}; -use relay_protocol::{Annotated, ErrorKind, Remark, RemarkType, Value}; +use relay_protocol::{Annotated, ErrorKind, Meta, Remark, RemarkType, Value}; use crate::{ClientHints, FromUserAgentInfo as _, RawUserAgentInfo}; @@ -196,11 +196,10 @@ fn normalize_attribute_names_inner( continue; }; - let new_attribute = old_attribute.clone(); - old_attribute.set_value(None); - old_attribute - .meta_mut() - .add_remark(Remark::new(RemarkType::Removed, "attribute.deprecated")); + let mut meta = Meta::default(); + // TODO: Possibly add a new RemarkType for "renamed/moved" + meta.add_remark(Remark::new(RemarkType::Removed, "attribute.deprecated")); + let new_attribute = std::mem::replace(old_attribute, Annotated(None, meta)); if !attributes.contains_key(new_name) { attributes.insert_raw(new_name.to_owned(), new_attribute); From 9552f46e726c1383fbf1621552dccb6eb54e8f36 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Thu, 30 Oct 2025 09:59:02 +0100 Subject: [PATCH 16/16] Don't pointlessly insert empty attribute object --- relay-event-normalization/src/eap/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/relay-event-normalization/src/eap/mod.rs b/relay-event-normalization/src/eap/mod.rs index 521fe1b227f..065f4f9c26a 100644 --- a/relay-event-normalization/src/eap/mod.rs +++ b/relay-event-normalization/src/eap/mod.rs @@ -181,7 +181,10 @@ fn normalize_attribute_names_inner( attributes: &mut Annotated, attribute_info: fn(&str) -> Option<&'static AttributeInfo>, ) { - let attributes = attributes.get_or_insert_with(Default::default); + let Some(attributes) = attributes.value_mut() else { + return; + }; + let attribute_names: Vec<_> = attributes.keys().cloned().collect(); for name in attribute_names {