Skip to content

Commit

Permalink
fix: telemetry init channel unsuccessful leads to panic (#20160)
Browse files Browse the repository at this point in the history
Co-authored-by: tabversion <[email protected]>
  • Loading branch information
tabVersion and tabversion authored Jan 16, 2025
1 parent bc429d5 commit 439956e
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 2 deletions.
12 changes: 10 additions & 2 deletions src/common/src/telemetry/report.rs
Original file line number Diff line number Diff line change
Expand Up @@ -101,25 +101,33 @@ where
});

let (tx, mut event_rx) = tokio::sync::mpsc::unbounded_channel::<PbEventMessage>();

let mut enable_event_report = true;
TELEMETRY_EVENT_REPORT_TX.set(tx).unwrap_or_else(|_| {
tracing::warn!(
"Telemetry failed to set event reporting tx, event reporting will be disabled"
);
// possible failure:
// When running in standalone mode, the static TELEMETRY_EVENT_REPORT_TX is shared
// and can be set by meta/compute nodes.
// In such case, the one first set the static will do the event reporting and others'
// event report is disabled.
enable_event_report = false;
});
let mut event_stash = Vec::new();

loop {
tokio::select! {
_ = interval.tick() => {},
event = event_rx.recv() => {
event = event_rx.recv(), if enable_event_report => {
debug_assert!(event.is_some());
event_stash.push(event.unwrap());
if event_stash.len() >= TELEMETRY_EVENT_REPORT_STASH_SIZE {
do_telemetry_event_report(&mut event_stash).await;
}
continue;
}
_ = event_interval.tick() => {
_ = event_interval.tick(), if enable_event_report => {
do_telemetry_event_report(&mut event_stash).await;
continue;
},
Expand Down
4 changes: 4 additions & 0 deletions src/common/telemetry_event/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ pub fn get_telemetry_risingwave_cloud_uuid() -> Option<String> {
}

pub async fn do_telemetry_event_report(event_stash: &mut Vec<PbEventMessage>) {
if event_stash.is_empty() {
return;
}

const TELEMETRY_EVENT_REPORT_TYPE: &str = "events"; // the batch report url
let url = (TELEMETRY_REPORT_URL.to_owned() + "/" + TELEMETRY_EVENT_REPORT_TYPE).to_owned();
let batch_message = PbBatchEventMessage {
Expand Down

0 comments on commit 439956e

Please sign in to comment.