Merge branch 'master' into jferg/uptime-checks-1

.envrc

@@ -19,4 +19,6 @@ else
     unset PS1
 fi
 
+export PATH="$PWD/snuba/admin/node_modules/.bin/:$PATH"
+
 . scripts/rust-envvars

.gitignore

@@ -12,6 +12,7 @@ snuba.egg-info/
 node_modules
 .vscode/*.log
 snuba/admin/dist/bundle.js*
+snuba/admin/dist/bundle.css*
 tmp/
 gocd/templates/vendor/
 gocd/generated-pipelines/

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## 24.12.1
+
+### Various fixes & improvements
+
+- Revert "fix(eap-spans): Add an index on project_id (#6695)" (9f5c3f57) by @getsentry-bot
+- fix(eap-spans): Add an index on project_id (#6695) by @phacops
+- 984: scrub eap_spans_str_attrs (#6694) by @kylemumma
+- fix(inc984): scrub the correct bucket for sentry.user (#6693) by @xurui-c
+- fix(inc984): scrub the correct bucket for sentry.user.ip (#6692) by @volokluev
+- inc984: scrub `user` from eap_spans (#6691) by @xurui-c
+- inc984: scrub `user` from spans (#6689) by @xurui-c
+- fix(rust): Honor exit code (#6674) by @untitaker
+- ref(admin): lw delete related system queries (#6685) by @MeredithAnya
+- chore: Bump rust-toolchain (#6688) by @untitaker
+- chore(devservices): Bumping the version of devservices to latest (#6682) by @IanWoodard
+- fix(devservices): Add orchestrator devservices label to clickhouse (#6687) by @hubertdeng123
+- fix(eap): Fix divide by 0 errors caused when the sample count is 0 (#6681) by @davidtsuk
+- meta: Bump new development version (44657332)
+- ref(lw-deletes): add project_id killswitch and some logging (#6677) by @MeredithAnya
+- docs: update CH supported versions (#6683) by @MeredithAnya
+- feat(inc-984): Cleanup EAP spans  (#6676) by @xurui-c
+- fix(admin): Allow special characters in SYSTEM/OPTIMIZE queries (#6680) by @evanh
+- feat(eap): Use weighted average instead of simple average for calculating average sampling rate (#6678) by @davidtsuk
+- ref(lw-deletes): concurrent allocation policy requires org id (#6679) by @MeredithAnya
+
 ## 24.12.0
 
 ### Various fixes & improvements

requirements.txt

@@ -6,7 +6,7 @@ click==8.1.7
 clickhouse-driver==0.2.6
 confluent-kafka==2.3.0
 datadog==0.21.0
-devservices==1.0.6
+devservices==1.0.8
 flake8==7.0.0
 Flask==2.2.5
 google-cloud-storage==2.18.0
@@ -29,6 +29,7 @@ python-rapidjson==1.8
 redis==4.5.4
 sentry-arroyo==2.19.4
 sentry-kafka-schemas==0.1.122
+sentry-protos==0.1.41
 sentry-redis-tools==0.3.0
 sentry-relay==0.9.4
 sentry-sdk==2.18.0
@@ -46,4 +47,3 @@ sqlparse==0.5.0
 google-api-python-client==2.88.0
 sentry-usage-accountant==0.0.11
 freezegun==1.2.2
-sentry-protos==0.1.40

rust_snuba/Cargo.toml

@@ -54,6 +54,7 @@ zstd = "0.12.3"
 serde_with = "3.8.1"
 seq-macro = "0.3"
 sentry_arroyo = "2.19.5"
+regex = "1.11.1"
 
 
 [dev-dependencies]

rust_snuba/benches/processors.rs

@@ -98,7 +98,6 @@ fn create_factory(
         },
         stop_at_timestamp: None,
         batch_write_timeout: None,
-        max_bytes_before_external_group_by: None,
     };
     Box::new(factory)
 }

rust_snuba/src/consumer.rs

@@ -45,7 +45,6 @@ pub fn consumer(
     health_check_file: Option<&str>,
     stop_at_timestamp: Option<i64>,
     batch_write_timeout_ms: Option<u64>,
-    max_bytes_before_external_group_by: Option<usize>,
     max_dlq_buffer_length: Option<usize>,
 ) -> usize {
     py.allow_threads(|| {
@@ -64,7 +63,6 @@ pub fn consumer(
             health_check_file,
             stop_at_timestamp,
             batch_write_timeout_ms,
-            max_bytes_before_external_group_by,
             mutations_mode,
             max_dlq_buffer_length,
         )
@@ -87,7 +85,6 @@ pub fn consumer_impl(
     health_check_file: Option<&str>,
     stop_at_timestamp: Option<i64>,
     batch_write_timeout_ms: Option<u64>,
-    max_bytes_before_external_group_by: Option<usize>,
     mutations_mode: bool,
     max_dlq_buffer_length: Option<usize>,
 ) -> usize {
@@ -278,7 +275,6 @@ pub fn consumer_impl(
             accountant_topic_config: consumer_config.accountant_topic,
             stop_at_timestamp,
             batch_write_timeout,
-            max_bytes_before_external_group_by,
         };
 
         StreamProcessor::with_kafka(config, factory, topic, dlq_policy)

rust_snuba/src/factory.rs

@@ -57,7 +57,6 @@ pub struct ConsumerStrategyFactory {
     pub accountant_topic_config: config::TopicConfig,
     pub stop_at_timestamp: Option<i64>,
     pub batch_write_timeout: Option<Duration>,
-    pub max_bytes_before_external_group_by: Option<usize>,
 }
 
 impl ProcessingStrategyFactory<KafkaPayload> for ConsumerStrategyFactory {
@@ -103,10 +102,7 @@ impl ProcessingStrategyFactory<KafkaPayload> for ConsumerStrategyFactory {
             };
 
         // Write to clickhouse
-        let next_step = Box::new(ClickhouseWriterStep::new(
-            next_step,
-            &self.clickhouse_concurrency,
-        ));
+        let next_step = ClickhouseWriterStep::new(next_step, &self.clickhouse_concurrency);
 
         let next_step = SetJoinTimeout::new(next_step, None);
 
@@ -121,7 +117,6 @@ impl ProcessingStrategyFactory<KafkaPayload> for ConsumerStrategyFactory {
             &self.storage_config.clickhouse_cluster.password,
             self.async_inserts,
             self.batch_write_timeout,
-            self.max_bytes_before_external_group_by,
         );
 
         let accumulator = Arc::new(
@@ -210,15 +205,13 @@ impl ProcessingStrategyFactory<KafkaPayload> for ConsumerStrategyFactory {
                 let schema = get_schema(&self.logical_topic_name, self.enforce_schema);
 
                 Box::new(RunTaskInThreads::new(
-                    Box::new(
-                        PythonTransformStep::new(
-                            next_step,
-                            self.storage_config.message_processor.clone(),
-                            self.processing_concurrency.concurrency,
-                            self.python_max_queue_depth,
-                        )
-                        .unwrap(),
-                    ),
+                    PythonTransformStep::new(
+                        next_step,
+                        self.storage_config.message_processor.clone(),
+                        self.processing_concurrency.concurrency,
+                        self.python_max_queue_depth,
+                    )
+                    .unwrap(),
                     Box::new(SchemaValidator {
                         schema,
                         enforce_schema: self.enforce_schema,

rust_snuba/src/mutations/clickhouse.rs

@@ -211,6 +211,7 @@ struct MutationRow {
 #[cfg(test)]
 mod tests {
     use std::env;
+    use std::time::{SystemTime, UNIX_EPOCH};
     use uuid::Uuid;
 
     use crate::mutations::parser::Update;
@@ -284,7 +285,11 @@ mod tests {
         let mut update = Update::default();
 
         let organization_id = 69;
-        let _sort_timestamp = 1727466947;
+        let curr_time_unix = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("Time went backwards")
+            .as_secs();
+        let _sort_timestamp = curr_time_unix as u32; //TODO year 2038 problem
         let trace_id = Uuid::parse_str("deadbeef-dead-beef-dead-beefdeadbeef").unwrap();
         let span_id = 16045690984833335023;
 

rust_snuba/src/mutations/factory.rs

@@ -45,7 +45,7 @@ impl ProcessingStrategyFactory<KafkaPayload> for MutConsumerStrategyFactory {
         let next_step = CommitOffsets::new(Duration::from_secs(1));
 
         let next_step = RunTaskInThreads::new(
-            Box::new(next_step),
+            next_step,
             Box::new(ClickhouseWriter::new(
                 &self.storage_config.clickhouse_cluster.host,
                 self.storage_config.clickhouse_cluster.http_port,
@@ -86,7 +86,7 @@ impl ProcessingStrategyFactory<KafkaPayload> for MutConsumerStrategyFactory {
         );
 
         let next_step = RunTaskInThreads::new(
-            Box::new(next_step),
+            next_step,
             Box::new(MutationParser),
             &self.processing_concurrency,
             Some("parse"),