fix(parser): don't panic on multibyte char in consecutive comments

changelog.d/lexer_comment_multibyte_panic.fix.md

@@ -0,0 +1,5 @@
+The lexer no longer panics on a program with consecutive comment lines inside a block when a
+later comment contains a non-ASCII character, and no longer fails to lex such a program when a
+later comment contains a quote.
+
+authors: DeviousCardi

lib/tests/tests/issues/1989_multibyte_char_in_consecutive_comments.vrl

@@ -0,0 +1,11 @@
+# issue: https://github.com/vectordotdev/vrl/issues/1989
+# result: 1
+
+x = {}
+if is_object(x) {
+# a
+# b — c
+# it's "quoted
+  .a = 1
+}
+.a

src/parser/lex.rs

@@ -915,20 +915,28 @@ impl<'input> Lexer<'input> {
                     break;
                 }
             }
-            return chars.peek().map(|(_, ch)| *ch).ok_or_else(|| {
-                Error::UnexpectedParseError("Expected characters at end of comment.".to_string())
-            });
+            // The comment may be directly followed by another comment or a
+            // literal, which must be skipped as a whole as well.
+            return match chars.peek() {
+                Some(&(next, _)) => self.delimited_query_char(next, chars),
+                None => Err(Error::UnexpectedParseError(
+                    "Expected characters at end of comment.".to_string(),
+                )),
+            };
         }
 
-        let mut lexer = Lexer::new(&self.input[pos + 1..]);
+        // Only slice past the current character once we know it is the
+        // (single-byte) opening of a literal: `pos + 1` is not a char boundary
+        // when the current character is multi-byte.
+        let literal_lexer = || Lexer::new(&self.input[pos + 1..]);
         let (result, description) = if input.starts_with('"') {
-            (lexer.string_literal(0), "string")
+            (literal_lexer().string_literal(0), "string")
         } else if input.starts_with("s'") {
-            (lexer.raw_string_literal(0), "raw string")
+            (literal_lexer().raw_string_literal(0), "raw string")
         } else if input.starts_with("r'") {
-            (lexer.regex_literal(0), "regex")
+            (literal_lexer().regex_literal(0), "regex")
         } else if input.starts_with("t'") {
-            (lexer.timestamp_literal(0), "timestamp")
+            (literal_lexer().timestamp_literal(0), "timestamp")
         } else {
             return Ok(chars.peek().expect("query character is present").1);
         };
@@ -2547,6 +2555,43 @@ mod test {
         );
     }
 
+    #[test]
+    fn consecutive_comments_with_multibyte_char_in_block() {
+        // https://github.com/vectordotdev/vrl/issues/1989
+        assert_eq!(
+            token_kinds("if !is_object(x) {\n# a\n# b — c\n  .a = 1\n}"),
+            token_kinds("if !is_object(x) {\n# a\n# b - c\n  .a = 1\n}"),
+        );
+    }
+
+    #[test]
+    fn consecutive_comments_with_quote_in_block() {
+        assert_eq!(
+            token_kinds("if x {\n# a\n# it's \"quoted\n.a = 1\n}\n.a"),
+            vec![
+                If,
+                Identifier("x"),
+                LBrace,
+                Newline,
+                Newline,
+                Newline,
+                LQuery,
+                Dot,
+                Identifier("a"),
+                RQuery,
+                Equals,
+                IntegerLiteral(1),
+                Newline,
+                RBrace,
+                Newline,
+                LQuery,
+                Dot,
+                Identifier("a"),
+                RQuery,
+            ],
+        );
+    }
+
     #[test]
     fn newline_kept_when_followed_by_else_prefixed_identifier() {
         // `elsewhere` is an identifier, not the `else` keyword — newline stays.