Add sensitive param to get_from_env_opt (#5910)

Author: Abdul-Andha

quickwit/quickwit-cli/src/logger.rs

@@ -152,7 +152,7 @@ impl EventFormat<'_> {
     /// Gets the log format from the environment variable `QW_LOG_FORMAT`. Returns a JSON
     /// formatter if the variable is set to `json`, otherwise returns a full formatter.
     fn get_from_env() -> Self {
-        if get_from_env_opt::<String>("QW_LOG_FORMAT")
+        if get_from_env_opt::<String>("QW_LOG_FORMAT", false)
             .map(|log_format| log_format.eq_ignore_ascii_case("json"))
             .unwrap_or(false)
         {

quickwit/quickwit-cli/src/main.rs

@@ -33,7 +33,11 @@ use tracing::error;
 /// QW_RUNTIME_NUM_THREADS environment variable.
 fn get_main_runtime_num_threads() -> usize {
     let default_num_runtime_threads: usize = quickwit_common::num_cpus().div_ceil(3);
-    quickwit_common::get_from_env("QW_TOKIO_RUNTIME_NUM_THREADS", default_num_runtime_threads)
+    quickwit_common::get_from_env(
+        "QW_TOKIO_RUNTIME_NUM_THREADS",
+        default_num_runtime_threads,
+        false,
+    )
 }
 
 fn main() -> anyhow::Result<()> {

quickwit/quickwit-common/src/cpus.rs

@@ -88,7 +88,7 @@ fn parse_cpu_to_mcpu(cpu_string: &str) -> Result<usize, &'static str> {
 //
 // We then get the number of vCPUs by ceiling any non integer value.
 fn get_num_cpus_from_env(env_key: &str) -> Option<usize> {
-    let k8s_cpu_limit_str: String = crate::get_from_env_opt(env_key)?;
+    let k8s_cpu_limit_str: String = crate::get_from_env_opt(env_key, false)?;
     let mcpus = parse_cpu_to_mcpu(&k8s_cpu_limit_str)
         .inspect_err(|err_msg| {
             warn!(

quickwit/quickwit-common/src/lib.rs

@@ -87,31 +87,36 @@ pub fn split_file(split_id: impl Display) -> String {
 fn get_from_env_opt_aux<T: Debug>(
     key: &str,
     parse_fn: impl FnOnce(&str) -> Option<T>,
+    sensitive: bool,
 ) -> Option<T> {
     let value_str = std::env::var(key).ok()?;
     let Some(value) = parse_fn(&value_str) else {
         error!(value=%value_str, "failed to parse environment variable `{key}` value");
         return None;
     };
-    info!(value=?value, "using environment variable `{key}` value");
+    if sensitive {
+        info!("using environment variable `{key}` value");
+    } else {
+        info!(value=?value, "using environment variable `{key}` value");
+    }
     Some(value)
 }
 
-pub fn get_from_env<T: FromStr + Debug>(key: &str, default_value: T) -> T {
-    if let Some(value) = get_from_env_opt(key) {
+pub fn get_from_env<T: FromStr + Debug>(key: &str, default_value: T, sensitive: bool) -> T {
+    if let Some(value) = get_from_env_opt(key, sensitive) {
         value
     } else {
         info!(default_value=?default_value, "using environment variable `{key}` default value");
         default_value
     }
 }
 
-pub fn get_from_env_opt<T: FromStr + Debug>(key: &str) -> Option<T> {
-    get_from_env_opt_aux(key, |val_str| val_str.parse().ok())
+pub fn get_from_env_opt<T: FromStr + Debug>(key: &str, sensitive: bool) -> Option<T> {
+    get_from_env_opt_aux(key, |val_str| val_str.parse().ok(), sensitive)
 }
 
 pub fn get_bool_from_env_opt(key: &str) -> Option<bool> {
-    get_from_env_opt_aux(key, parse_bool_lenient)
+    get_from_env_opt_aux(key, parse_bool_lenient, false)
 }
 
 pub fn get_bool_from_env(key: &str, default_value: bool) -> bool {
@@ -305,11 +310,11 @@ mod tests {
         // way, we are keeping it that way
 
         const TEST_KEY: &str = "TEST_KEY";
-        assert_eq!(super::get_from_env(TEST_KEY, 10), 10);
+        assert_eq!(super::get_from_env(TEST_KEY, 10, false), 10);
         unsafe { std::env::set_var(TEST_KEY, "15") };
-        assert_eq!(super::get_from_env(TEST_KEY, 10), 15);
+        assert_eq!(super::get_from_env(TEST_KEY, 10, false), 15);
         unsafe { std::env::set_var(TEST_KEY, "1invalidnumber") };
-        assert_eq!(super::get_from_env(TEST_KEY, 10), 10);
+        assert_eq!(super::get_from_env(TEST_KEY, 10, false), 10);
     }
 
     #[test]

quickwit/quickwit-common/src/shared_consts.rs

@@ -40,6 +40,7 @@ pub fn split_deletion_grace_period() -> Duration {
         let deletion_grace_period_secs: u64 = crate::get_from_env(
             "QW_SPLIT_DELETION_GRACE_PERIOD_SECS",
             DEFAULT_DELETION_GRACE_PERIOD.as_secs(),
+            false,
         );
         let deletion_grace_period_secs_clamped: u64 = deletion_grace_period_secs.clamp(
             MINIMUM_DELETION_GRACE_PERIOD.as_secs(),

quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs

@@ -187,6 +187,7 @@ fn get_default_load_per_shard() -> NonZeroU32 {
         let default_load_per_shard = quickwit_common::get_from_env(
             "QW_DEFAULT_LOAD_PER_SHARD",
             PIPELINE_FULL_CAPACITY.cpu_millis() / 4,
+            false,
         );
         NonZeroU32::new(default_load_per_shard).unwrap()
     })

quickwit/quickwit-index-management/src/garbage_collection.rs

@@ -292,14 +292,15 @@ async fn list_splits_metadata(
 fn get_maximum_split_deletion_rate_per_sec() -> Option<usize> {
     static MAX_SPLIT_DELETION_RATE_PER_SEC: OnceLock<Option<usize>> = OnceLock::new();
     *MAX_SPLIT_DELETION_RATE_PER_SEC.get_or_init(|| {
-        quickwit_common::get_from_env_opt::<usize>("QW_MAX_SPLIT_DELETION_RATE_PER_SEC")
+        quickwit_common::get_from_env_opt::<usize>("QW_MAX_SPLIT_DELETION_RATE_PER_SEC", false)
     })
 }
 
 fn get_index_gc_concurrency() -> Option<usize> {
     static INDEX_GC_CONCURRENCY: OnceLock<Option<usize>> = OnceLock::new();
-    *INDEX_GC_CONCURRENCY
-        .get_or_init(|| quickwit_common::get_from_env_opt::<usize>("QW_INDEX_GC_CONCURRENCY"))
+    *INDEX_GC_CONCURRENCY.get_or_init(|| {
+        quickwit_common::get_from_env_opt::<usize>("QW_INDEX_GC_CONCURRENCY", false)
+    })
 }
 
 /// Removes any splits marked for deletion which haven't been

quickwit/quickwit-ingest/src/ingest_v2/ingester.rs

@@ -101,7 +101,7 @@ const DEFAULT_BATCH_NUM_BYTES: usize = 1024 * 1024; // 1 MiB
 fn get_batch_num_bytes() -> usize {
     static BATCH_NUM_BYTES_CELL: OnceCell<usize> = OnceCell::new();
     *BATCH_NUM_BYTES_CELL.get_or_init(|| {
-        quickwit_common::get_from_env("QW_INGEST_BATCH_NUM_BYTES", DEFAULT_BATCH_NUM_BYTES)
+        quickwit_common::get_from_env("QW_INGEST_BATCH_NUM_BYTES", DEFAULT_BATCH_NUM_BYTES, false)
     })
 }
 

quickwit/quickwit-ingest/src/ingest_v2/router.rs

@@ -66,6 +66,7 @@ fn ingest_request_timeout() -> Duration {
         let duration_ms = quickwit_common::get_from_env(
             "QW_INGEST_REQUEST_TIMEOUT_MS",
             DEFAULT_INGEST_REQUEST_TIMEOUT.as_millis() as u64,
+            false,
         );
         let minimum_ingest_request_timeout: Duration =
             PERSIST_REQUEST_TIMEOUT * (MAX_PERSIST_ATTEMPTS as u32) + Duration::from_secs(5);

quickwit/quickwit-search/src/list_fields.rs

@@ -49,7 +49,7 @@ use crate::{
 /// when building the response. This is a workaround until a way is found to
 /// prune the long tail of rare fields.
 static FIELD_LIST_SIZE_LIMIT: LazyLock<usize> =
-    LazyLock::new(|| quickwit_common::get_from_env("QW_FIELD_LIST_SIZE_LIMIT", 100_000));
+    LazyLock::new(|| quickwit_common::get_from_env("QW_FIELD_LIST_SIZE_LIMIT", 100_000, false));
 
 const DYNAMIC_FIELD_PREFIX: &str = "_dynamic.";