diff --git a/Cargo.lock b/Cargo.lock index 8c371588..16859d06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -864,6 +864,49 @@ dependencies = [ "libc", ] +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -979,7 +1022,7 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hl" -version = "0.31.0-alpha.3" +version = "0.31.0-alpha.4" dependencies = [ "bincode", "byte-strings", @@ -1029,6 +1072,7 @@ dependencies = [ "pest", "pest_derive", "regex", + "rstest", "rust-embed", "serde", "serde-logfmt", @@ -1532,6 +1576,18 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.31" @@ -1592,6 +1648,15 @@ dependencies = [ "termtree", ] +[[package]] +name = "proc-macro-crate" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.93" @@ -1694,6 +1759,12 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "roff" version = "0.2.2" @@ -1712,6 +1783,36 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "rstest" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e905296805ab93e13c1ec3a03f4b6c4f35e9498a3d5fa96dc626d22c03cd89" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef0053bbffce09062bee4bcc499b0fbe7a57b879f1efe088d6d8d4c7adcdef9b" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn", + "unicode-ident", +] + [[package]] name = "rust-embed" version = "8.5.0" @@ -1757,6 +1858,15 @@ dependencies = [ "trim-in-place", ] +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.43" @@ -1791,6 +1901,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "semver" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" + [[package]] name = "serde" version = "1.0.217" @@ -1802,7 +1918,7 @@ dependencies = [ [[package]] name = "serde-logfmt" -version = "0.1.0" +version = "0.1.1" dependencies = [ "serde", ] @@ -1906,6 +2022,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "snap" version = "1.1.1" diff --git a/Cargo.toml b/Cargo.toml index b450842f..fcef3603 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = [".", "crate/encstr", "crate/heapopt"] [workspace.package] repository = "https://github.com/pamburus/hl" authors = ["Pavel Ivanov "] -version = "0.31.0-alpha.3" +version = "0.31.0-alpha.4" edition = "2021" license = "MIT" @@ -99,6 +99,7 @@ mockall = "0" stats_alloc = "0" regex = "1" wildmatch = "2" +rstest = "0" [profile.release] debug = false diff --git a/crate/serde-logfmt/Cargo.toml b/crate/serde-logfmt/Cargo.toml index 570d75a7..d888fa1a 100644 --- a/crate/serde-logfmt/Cargo.toml +++ b/crate/serde-logfmt/Cargo.toml @@ -1,7 +1,7 @@ [package] description = "Logfmt decoding based on serde" name = "serde-logfmt" -version = "0.1.0" +version = "0.1.1" workspace = "../.." repository.workspace = true edition.workspace = true diff --git a/crate/serde-logfmt/src/logfmt/de.rs b/crate/serde-logfmt/src/logfmt/de.rs index 156b5bba..b23ed293 100644 --- a/crate/serde-logfmt/src/logfmt/de.rs +++ b/crate/serde-logfmt/src/logfmt/de.rs @@ -540,8 +540,6 @@ impl<'de> Parser<'de> { } fn parse_unquoted_value(&mut self) -> Result<&'de str> { - self.skip_garbage(); - let start = self.index; let mut unicode = false; @@ -1027,4 +1025,24 @@ mod tests { let val: TestStruct = from_str("v=B").unwrap(); assert_eq!(val, TestStruct { v: TestEnum::B }); } + + #[test] + fn test_empty_value() { + #[derive(Deserialize, PartialEq, Debug)] + struct Test { + int: u32, + str1: String, + str2: String, + str3: String, + } + + let j = r#"int=0 str1="" str2= str3="#; + let expected = Test { + int: 0, + str1: "".to_string(), + str2: "".to_string(), + str3: "".to_string(), + }; + assert_eq!(expected, from_str(j).unwrap()); + } } diff --git a/src/model.rs b/src/model.rs index 13501b89..f457e125 100644 --- a/src/model.rs +++ b/src/model.rs @@ -371,6 +371,7 @@ pub trait RecordWithSourceConstructor<'r, 's> { // --- +#[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Caller<'a> { Text(&'a str), FileLine(&'a str, &'a str), @@ -737,10 +738,18 @@ impl FieldSettings { match *self { Self::Time => { let s = value.raw_str(); - let s = if s.as_bytes()[0] == b'"' { &s[1..s.len() - 1] } else { s }; - let ts = Timestamp::new(s).with_unix_unit(ps.unix_ts_unit); - to.ts = Some(ts); - true + let s = if s.len() > 0 && s.as_bytes()[0] == b'"' { + &s[1..s.len() - 1] + } else { + s + }; + if !s.is_empty() { + let ts = Timestamp::new(s).with_unix_unit(ps.unix_ts_unit); + to.ts = Some(ts); + true + } else { + false + } } Self::Level(i) => { let value = value.parse().ok().unwrap_or_else(|| value.raw_str()); @@ -753,7 +762,7 @@ impl FieldSettings { } } Self::Logger => { - to.logger = value.parse().ok(); + to.logger = value.parse::<&str>().ok().filter(|s| !s.is_empty()); true } Self::Message => { @@ -761,16 +770,24 @@ impl FieldSettings { true } Self::Caller => { - to.caller = value.parse().ok().map(|x| Caller::Text(x)); + to.caller = value + .parse::<&str>() + .ok() + .filter(|x| !x.is_empty()) + .map(|x| Caller::Text(x)); true } Self::CallerFile => match &mut to.caller { None => { - to.caller = value.parse().ok().map(|x| Caller::FileLine(x, "")); + to.caller = value + .parse::<&str>() + .ok() + .filter(|x| !x.is_empty()) + .map(|x| Caller::FileLine(x, "")); to.caller.is_some() } Some(Caller::FileLine(file, _)) => { - if let Some(value) = value.parse().ok() { + if let Some(value) = value.parse::<&str>().ok().filter(|x| !x.is_empty()) { *file = value; true } else { @@ -779,28 +796,26 @@ impl FieldSettings { } _ => false, }, - Self::CallerLine => match &mut to.caller { - None => { - to.caller = Some(Caller::FileLine("", value.raw_str())); - true - } - Some(Caller::FileLine(_, line)) => match value { - RawValue::Number(value) => { - *line = value; - true - } + Self::CallerLine => { + let value = match value { + RawValue::Number(value) => value, RawValue::String(_) => { - if let Some(value) = value.parse().ok() { - *line = value; - true + if let Some(value) = value.parse::<&str>().ok().filter(|x| !x.is_empty()) { + value } else { - false + return false; } } - _ => false, - }, - _ => false, - }, + _ => return false, + }; + + match &mut to.caller { + None => to.caller = Some(Caller::FileLine("", value)), + Some(Caller::FileLine(_, line)) => *line = value, + Some(Caller::Text(_)) => return false, + } + true + } Self::Nested(_) => false, } } @@ -1749,6 +1764,7 @@ const RAW_RECORD_FIELDS_CAPACITY: usize = RECORD_EXTRA_CAPACITY + MAX_PREDEFINED #[cfg(test)] mod tests { use super::*; + use rstest::rstest; use chrono::TimeZone; use maplit::hashmap; @@ -2264,8 +2280,10 @@ mod tests { assert!(matches!(result, Err(Error::JsonParseError(_)))); } - #[test] - fn test_nested_predefined_fields() { + #[rstest] + #[case(br#"{"some":{"deep":{"message":"test"}}}"#, Some(r#""test""#))] + #[case(br#"{"some":{"deep":[{"message":"test"}]}}"#, None)] + fn test_nested_predefined_fields(#[case] input: &[u8], #[case] expected: Option<&str>) { let predefined = PredefinedFields { message: Field { names: vec!["some.deep.message".into()], @@ -2277,16 +2295,113 @@ mod tests { let settings = ParserSettings::new(&predefined, [], None); let parser = Parser::new(settings); - let cases: &[(&[u8], Option<&str>)] = &[ - (br#"{"some":{"deep":{"message":"test"}}}"#, Some(r#""test""#)), - (br#"{"some":{"deep":[{"message":"test"}]}}"#, None), - ]; + let record = RawRecord::parser().parse(input).next().unwrap().unwrap(); + let record = parser.parse(&record.record); + assert_eq!(record.message.map(|x| x.raw_str()), expected); + } - for (input, expected) in cases.iter() { - let record = RawRecord::parser().parse(*input).next().unwrap().unwrap(); - let record = parser.parse(&record.record); - assert_eq!(record.message.map(|x| x.raw_str()), *expected); + #[rstest] + #[case(br#"{"ts":""}"#, None)] + #[case(br#"{"ts":"3"}"#, Some("3"))] + #[case(br#"ts="""#, None)] + #[case(br#"ts="#, None)] + #[case(br#"ts=1"#, Some("1"))] + #[case(br#"ts="2""#, Some("2"))] + fn test_timestamp(#[case] input: &[u8], #[case] expected: Option<&str>) { + let parser = Parser::new(ParserSettings::default()); + let record = RawRecord::parser().parse(input).next().unwrap().unwrap(); + let record = parser.parse(&record.record); + assert_eq!(record.ts.map(|x| x.raw()), expected); + } + + #[rstest] + #[case(br#"{"level":""}"#, None)] + #[case(br#"{"level":"info"}"#, Some(Level::Info))] + #[case(br#"level="""#, None)] + #[case(br#"level="#, None)] + #[case(br#"level=info"#, Some(Level::Info))] + #[case(br#"level="info""#, Some(Level::Info))] + fn test_level(#[case] input: &[u8], #[case] expected: Option) { + let parser = Parser::new(ParserSettings::default()); + let record = RawRecord::parser().parse(input).next().unwrap().unwrap(); + let record = parser.parse(&record.record); + assert_eq!(record.level, expected); + } + + #[rstest] + #[case(br#"{"logger":""}"#, None)] + #[case(br#"{"logger":"x"}"#, Some("x"))] + #[case(br#"logger="""#, None)] + #[case(br#"logger="#, None)] + #[case(br#"logger=x"#, Some("x"))] + #[case(br#"logger="x""#, Some("x"))] + fn test_logger(#[case] input: &[u8], #[case] expected: Option<&str>) { + let parser = Parser::new(ParserSettings::default()); + let record = RawRecord::parser().parse(input).next().unwrap().unwrap(); + let record = parser.parse(&record.record); + assert_eq!(record.logger, expected); + } + + #[rstest] + #[case(br#"{"caller":""}"#, None)] + #[case(br#"{"caller":"x"}"#, Some(Caller::Text("x")))] + #[case(br#"caller="""#, None)] + #[case(br#"caller="#, None)] + #[case(br#"caller=x"#, Some(Caller::Text("x")))] + #[case(br#"caller="x""#, Some(Caller::Text("x")))] + fn test_caller(#[case] input: &[u8], #[case] expected: Option) { + let parser = Parser::new(ParserSettings::default()); + let record = RawRecord::parser().parse(input).next().unwrap().unwrap(); + let record = parser.parse(&record.record); + assert_eq!(record.caller, expected); + } + + #[rstest] + #[case(br#"{"file":""}"#, None)] // 1 + #[case(br#"{"file":"x"}"#, Some(Caller::FileLine("x", "")))] // 2 + #[case(br#"file="""#, None)] // 3 + #[case(br#"file="#, None)] // 4 + #[case(br#"file=x"#, Some(Caller::FileLine("x", "")))] // 5 + #[case(br#"file="x""#, Some(Caller::FileLine("x", "")))] // 6 + #[case(br#"{"line":""}"#, None)] // 7 + #[case(br#"{"line":"8"}"#, Some(Caller::FileLine("", "8")))] // 8 + #[case(br#"line="""#, None)] // 9 + #[case(br#"line="#, None)] // 10 + #[case(br#"line=11"#, Some(Caller::FileLine("", "11")))] // 11 + #[case(br#"line="12""#, Some(Caller::FileLine("", "12")))] // 12 + #[case(br#"{"file":"","line":""}"#, None)] // 13 + #[case(br#"{"file":"x","line":"14"}"#, Some(Caller::FileLine("x", "14")))] // 14 + #[case(br#"file="" line="""#, None)] // 15 + #[case(br#"file= line="#, None)] // 16 + #[case(br#"file=x line=17"#, Some(Caller::FileLine("x", "17")))] // 17 + #[case(br#"file="x" line="18""#, Some(Caller::FileLine("x", "18")))] // 18 + #[case(br#"{"file":"","line":"19"}"#, Some(Caller::FileLine("", "19")))] // 19 + #[case(br#"{"file":"x","line":""}"#, Some(Caller::FileLine("x", "")))] // 20 + #[case(br#"file="" line="21""#, Some(Caller::FileLine("", "21")))] // 21 + #[case(br#"file= line=22"#, Some(Caller::FileLine("", "22")))] // 22 + #[case(br#"file=x line="#, Some(Caller::FileLine("x", "")))] // 23 + #[case(br#"file="x" line="#, Some(Caller::FileLine("x", "")))] // 24 + #[case(br#"file="x" line=21 line=25"#, Some(Caller::FileLine("x", "25")))] // 25 + #[case(br#"file=x line=26 file=y"#, Some(Caller::FileLine("y", "26")))] // 26 + #[case(br#"{"file":123, "file": {}, "line":27}"#, Some(Caller::FileLine("123", "27")))] // 27 + #[case(br#"{"caller":"a", "file": "b", "line":28}"#, Some(Caller::Text("a")))] // 28 + #[case(br#"{"file": "b", "line":{}}"#, Some(Caller::FileLine("b", "")))] // 29 + fn test_caller_file_line(#[case] input: &[u8], #[case] expected: Option) { + let mut predefined = PredefinedFields::default(); + predefined.caller_file = Field { + names: vec!["file".into()], + show: FieldShowOption::Always, + } + .into(); + predefined.caller_line = Field { + names: vec!["line".into()], + show: FieldShowOption::Always, } + .into(); + let parser = Parser::new(ParserSettings::new(&predefined, [], None)); + let record = RawRecord::parser().parse(input).next().unwrap().unwrap(); + let record = parser.parse(&record.record); + assert_eq!(record.caller, expected); } fn parse(s: &str) -> Record { diff --git a/src/settings.rs b/src/settings.rs index 2025eec7..4cd92bd8 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -149,7 +149,7 @@ impl Default for &PredefinedFields { // --- -#[derive(Debug, Serialize, Deserialize, Deref, Clone, PartialEq, Eq)] +#[derive(Debug, Serialize, Deserialize, Deref, Clone, PartialEq, Eq, From)] pub struct TimeField(pub Field); impl Default for TimeField {