fix: fixed errors on unknown level in configuration files (#787)

Author: pamburus

Cargo.lock

@@ -4,7 +4,7 @@ members = [".", "crate/encstr", "crate/heapopt"]
 [workspace.package]
 repository = "https://github.com/pamburus/hl"
 authors = ["Pavel Ivanov <[email protected]>"]
-version = "0.30.4-alpha.2"
+version = "0.30.4-alpha.3"
 edition = "2021"
 license = "MIT"
 
@@ -72,6 +72,7 @@ regex = "1"
 rust-embed = "8"
 serde = { version = "1", features = ["derive"] }
 serde_json = { version = "1", features = ["raw_value"] }
+serde_plain = "1"
 serde_yml = "0"
 serde-logfmt = { path = "./crate/serde-logfmt" }
 sha2 = "0"

Cargo.toml

@@ -49,6 +49,7 @@ function test() {
     ${MAIN_EXECUTABLE:?} --config - --man-page > /dev/null
     ${MAIN_EXECUTABLE:?} --config - --list-themes > /dev/null
     ${MAIN_EXECUTABLE:?} --config - sample/prometheus.log -P > /dev/null
+    HL_DEBUG_LOG=info ${MAIN_EXECUTABLE:?} --config - sample/prometheus.log -P > /dev/null
     echo "" | ${MAIN_EXECUTABLE:?} --config - --concurrency 4 > /dev/null
 } 
 

build/ci/coverage.sh

@@ -1078,7 +1078,12 @@ mod tests {
 
     // local imports
     use crate::{
-        filtering::MatchOptions, level::Level, model::FieldFilterSet, settings, themecfg::testing, LinuxDateFormat,
+        filtering::MatchOptions,
+        level::{InfallibleLevel, Level},
+        model::FieldFilterSet,
+        settings,
+        themecfg::testing,
+        LinuxDateFormat,
     };
 
     #[test]
@@ -1397,13 +1402,13 @@ mod tests {
             settings: Fields {
                 predefined: settings::PredefinedFields {
                     level: settings::LevelField {
-                        variants: vec![settings::LevelFieldVariant {
+                        variants: vec![settings::RawLevelFieldVariant {
                             names: vec!["level".to_string()],
                             values: hashmap! {
-                                Level::Debug => vec!["dbg".to_string()],
-                                Level::Info => vec!["INF".to_string()],
-                                Level::Warning => vec!["wrn".to_string()],
-                                Level::Error => vec!["ERR".to_string()],
+                                InfallibleLevel::new(Level::Debug) => vec!["dbg".to_string()],
+                                InfallibleLevel::new(Level::Info) => vec!["INF".to_string()],
+                                InfallibleLevel::new(Level::Warning) => vec!["wrn".to_string()],
+                                InfallibleLevel::new(Level::Error) => vec!["ERR".to_string()],
                             },
                             level: None,
                         }],

src/app.rs

@@ -128,7 +128,7 @@ mod tests {
 
     use maplit::hashmap;
 
-    use crate::level::Level;
+    use crate::level::{InfallibleLevel, Level};
 
     #[test]
     fn test_default() {
@@ -152,11 +152,11 @@ mod tests {
         assert_eq!(
             variant.values,
             hashmap! {
-                Level::Debug => vec!["dbg".to_owned()],
+                InfallibleLevel::new(Level::Debug) => vec!["dbg".to_owned()],
                 // TODO: replace `"inf"` with `"INF"` when https://github.com/mehcode/config-rs/issues/568 is fixed
-                Level::Info => vec!["inf".to_owned()],
-                Level::Warning => vec!["wrn".to_owned()],
-                Level::Error => vec!["ERR".to_owned()],
+                InfallibleLevel::new(Level::Info) => vec!["inf".to_owned()],
+                InfallibleLevel::new(Level::Warning) => vec!["wrn".to_owned()],
+                InfallibleLevel::new(Level::Error) => vec!["ERR".to_owned()],
             }
         );
     }

src/config.rs

@@ -41,6 +41,27 @@ pub enum Level {
 
 // ---
 
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[serde(untagged)]
+pub enum InfallibleLevel {
+    Valid(Level),
+    Invalid(String),
+}
+
+impl InfallibleLevel {
+    pub const fn new(level: Level) -> Self {
+        Self::Valid(level)
+    }
+}
+
+impl From<Level> for InfallibleLevel {
+    fn from(value: Level) -> Self {
+        InfallibleLevel::Valid(value)
+    }
+}
+
+// ---
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct ParseError;
 

src/level.rs

@@ -11,7 +11,7 @@ use std::{
 // third-party imports
 use chrono::Utc;
 use clap::{CommandFactory, Parser};
-use env_logger::{self as logger, fmt::TimestampPrecision};
+use env_logger::{self as logger};
 use itertools::Itertools;
 
 // local imports
@@ -36,10 +36,13 @@ const HL_DEBUG_LOG: &str = "HL_DEBUG_LOG";
 
 fn bootstrap() -> Result<Settings> {
     if std::env::var(HL_DEBUG_LOG).is_ok() {
-        logger::Builder::from_env(HL_DEBUG_LOG)
-            .format_timestamp(Some(TimestampPrecision::Micros))
-            .init();
+        logger::Builder::from_env(HL_DEBUG_LOG).format_timestamp_micros().init();
         log::debug!("logging initialized");
+    } else {
+        logger::Builder::new()
+            .filter_level(log::LevelFilter::Warn)
+            .format_timestamp_millis()
+            .init()
     }
 
     let opt = cli::BootstrapOpt::parse().args;

src/main.rs

@@ -26,7 +26,7 @@ use serde_logfmt::logfmt;
 use crate::{
     app::{InputFormat, UnixTimestampUnit},
     error::{Error, Result},
-    level,
+    level::{self},
     serdex::StreamDeserializerWithOffsets,
     settings::PredefinedFields,
     timestamp::Timestamp,
@@ -529,6 +529,10 @@ impl ParserSettings {
 
         let mut j = 0;
         for variant in &pf.level.variants {
+            let Some(variant) = variant.resolve() else {
+                continue;
+            };
+
             let mut mapping = HashMap::new();
             for (level, values) in &variant.values {
                 for value in values {

src/model.rs

@@ -1,6 +1,6 @@
 // std imports
 use std::{
-    collections::{BTreeMap, HashMap},
+    collections::{BTreeMap, HashMap, HashSet},
     include_str,
     path::{Path, PathBuf},
 };
@@ -15,7 +15,7 @@ use strum::IntoEnumIterator;
 
 // local imports
 use crate::error::Error;
-use crate::level::Level;
+use crate::level::{InfallibleLevel, Level};
 
 // ---
 
@@ -163,17 +163,17 @@ impl Default for TimeField {
 #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub struct LevelField {
     pub show: FieldShowOption,
-    pub variants: Vec<LevelFieldVariant>,
+    pub variants: Vec<RawLevelFieldVariant>,
 }
 
 impl Default for LevelField {
     fn default() -> Self {
         Self {
             show: FieldShowOption::default(),
-            variants: vec![LevelFieldVariant {
+            variants: vec![RawLevelFieldVariant {
                 names: vec!["level".into()],
                 values: Level::iter()
-                    .map(|level| (level, vec![level.as_ref().to_lowercase().into()]))
+                    .map(|level| (level.into(), vec![level.as_ref().to_lowercase().into()]))
                     .collect(),
                 level: None,
             }],
@@ -183,6 +183,56 @@ impl Default for LevelField {
 
 // ---
 
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub struct RawLevelFieldVariant {
+    pub names: Vec<String>,
+    #[serde(default, serialize_with = "ordered_map_serialize")]
+    pub values: HashMap<InfallibleLevel, Vec<String>>,
+    pub level: Option<InfallibleLevel>,
+}
+
+impl RawLevelFieldVariant {
+    pub fn resolve(&self) -> Option<LevelFieldVariant> {
+        let mut unknowns = HashSet::new();
+        let mut values = HashMap::new();
+        let mut valid = true;
+
+        for (level, names) in &self.values {
+            match level {
+                InfallibleLevel::Valid(level) => {
+                    values.insert(level.clone(), names.clone());
+                }
+                InfallibleLevel::Invalid(name) => {
+                    unknowns.insert(name.clone());
+                }
+            }
+        }
+
+        let level = self.level.clone().and_then(|level| match level {
+            InfallibleLevel::Valid(level) => Some(level),
+            InfallibleLevel::Invalid(name) => {
+                unknowns.insert(name);
+                valid = false;
+                None
+            }
+        });
+
+        for name in unknowns {
+            log::warn!("unknown level: {:?}", name);
+        }
+
+        if valid && !values.is_empty() {
+            Some(LevelFieldVariant {
+                names: self.names.clone(),
+                values,
+                level,
+            })
+        } else {
+            None
+        }
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub struct LevelFieldVariant {
     pub names: Vec<String>,
@@ -410,4 +460,53 @@ mod tests {
         assert_eq!(settings.time_zone, chrono_tz::UTC);
         assert_eq!(settings.theme, "universal");
     }
+
+    #[test]
+    fn test_unknown_level_values() {
+        let variant = RawLevelFieldVariant {
+            names: vec!["level".into()],
+            values: vec![
+                (InfallibleLevel::Valid(Level::Info), vec!["info".into()]),
+                (InfallibleLevel::Invalid("unknown".into()), vec!["unknown".into()]),
+            ]
+            .into_iter()
+            .collect(),
+            level: None,
+        };
+
+        assert_eq!(
+            variant.resolve(),
+            Some(LevelFieldVariant {
+                names: vec!["level".into()],
+                values: vec![(Level::Info, vec!["info".into()])].into_iter().collect(),
+                level: None,
+            })
+        );
+    }
+
+    #[test]
+    fn test_unknown_level_main() {
+        let variant = RawLevelFieldVariant {
+            names: vec!["level".into()],
+            values: vec![(InfallibleLevel::Valid(Level::Info), vec!["info".into()])]
+                .into_iter()
+                .collect(),
+            level: Some(InfallibleLevel::Invalid("unknown".into())),
+        };
+
+        assert_eq!(variant.resolve(), None);
+    }
+
+    #[test]
+    fn test_unknown_level_all_unknown() {
+        let variant = RawLevelFieldVariant {
+            names: vec!["level".into()],
+            values: vec![(InfallibleLevel::Invalid("unknown".into()), vec!["unknown".into()])]
+                .into_iter()
+                .collect(),
+            level: Some(InfallibleLevel::Valid(Level::Info)),
+        };
+
+        assert_eq!(variant.resolve(), None);
+    }
 }

src/settings.rs

@@ -10,7 +10,8 @@ use crate::{
     error::*,
     eseq::{Brightness, Color, ColorCode, Mode, Sequence, StyleCode},
     fmtx::Push,
-    level, themecfg,
+    level::{self, InfallibleLevel},
+    themecfg,
 };
 
 // ---
@@ -79,6 +80,13 @@ impl<S: Borrow<themecfg::Theme>> From<S> for Theme {
         let default = StylePack::load(&s.elements);
         let mut packs = EnumMap::default();
         for (level, pack) in &s.levels {
+            let level = match level {
+                InfallibleLevel::Valid(level) => level,
+                InfallibleLevel::Invalid(s) => {
+                    log::warn!("unknown level: {:?}", s);
+                    continue;
+                }
+            };
             packs[*level] = StylePack::load(&s.elements.clone().merged(pack.clone()));
         }
         Self {
@@ -357,5 +365,19 @@ mod tests {
         theme.apply(&mut buf, &Some(Level::Debug), |s| {
             s.element(Element::Message, |s| s.batch(|buf| buf.extend_from_slice(b"hello!")));
         });
+        assert_eq!(buf, b"hello!");
+    }
+
+    #[test]
+    fn test_unknown_level() {
+        let mut cfg = themecfg::Theme::default();
+        cfg.levels
+            .insert(InfallibleLevel::Invalid("unknown".to_string()), Default::default());
+        let theme = Theme::from(&cfg);
+        let mut buf = Vec::new();
+        theme.apply(&mut buf, &Some(Level::Debug), |s| {
+            s.element(Element::Message, |s| s.batch(|buf| buf.extend_from_slice(b"hello!")));
+        });
+        assert_eq!(buf, b"hello!");
     }
 }