[passes ui tests] Add a skip field to Delimiter::Invisible.

Currently it's always set to `true`, which doesn't change existing
behaviour.

Author: nnethercote

compiler/rustc_ast/src/ast.rs

@@ -1653,7 +1653,7 @@ impl MacDelimiter {
             Delimiter::Parenthesis => Some(MacDelimiter::Parenthesis),
             Delimiter::Bracket => Some(MacDelimiter::Bracket),
             Delimiter::Brace => Some(MacDelimiter::Brace),
-            Delimiter::Invisible => None,
+            Delimiter::Invisible { .. } => None,
         }
     }
 }

compiler/rustc_ast/src/attr/mod.rs

@@ -551,7 +551,7 @@ impl MetaItemKind {
         tokens: &mut impl Iterator<Item = TokenTree>,
     ) -> Option<MetaItemKind> {
         match tokens.next() {
-            Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
+            Some(TokenTree::Delimited(_, Delimiter::Invisible { .. }, inner_tokens)) => {
                 MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
             }
             Some(TokenTree::Token(token)) => {
@@ -621,7 +621,7 @@ impl NestedMetaItem {
                 tokens.next();
                 return Some(NestedMetaItem::Literal(lit));
             }
-            Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
+            Some(TokenTree::Delimited(_, Delimiter::Invisible { .. }, inner_tokens)) => {
                 let inner_tokens = inner_tokens.clone();
                 tokens.next();
                 return NestedMetaItem::from_tokens(&mut inner_tokens.into_trees().peekable());

compiler/rustc_ast/src/token.rs

@@ -56,7 +56,9 @@ pub enum Delimiter {
     /// `$var * 3` where `$var` is `1 + 2`.
     /// Invisible delimiters are not directly writable in normal Rust code except as comments.
     /// Therefore, they might not survive a roundtrip of a token stream through a string.
-    Invisible,
+    ///
+    /// njn: comment about `skip`
+    Invisible { skip: bool },
 }
 
 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -599,7 +599,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
                 self.end();
                 self.bclose(span, empty);
             }
-            Some(Delimiter::Invisible) => {
+            Some(Delimiter::Invisible { .. }) => {
                 self.word("/*«*/");
                 let empty = tts.is_empty();
                 if !empty {
@@ -786,8 +786,8 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::CloseDelim(Delimiter::Bracket) => "]".into(),
             token::OpenDelim(Delimiter::Brace) => "{".into(),
             token::CloseDelim(Delimiter::Brace) => "}".into(),
-            token::OpenDelim(Delimiter::Invisible) => "/*«*/".into(),
-            token::CloseDelim(Delimiter::Invisible) => "/*»*/".into(),
+            token::OpenDelim(Delimiter::Invisible { .. }) => "/*«*/".into(),
+            token::CloseDelim(Delimiter::Invisible { .. }) => "/*»*/".into(),
             token::Pound => "#".into(),
             token::Dollar => "$".into(),
             token::Question => "?".into(),

compiler/rustc_expand/src/mbe/quoted.rs

@@ -147,11 +147,14 @@ fn parse_tree(
     match tree {
         // `tree` is a `$` token. Look at the next token in `trees`
         tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => {
-            // FIXME: Handle `Invisible`-delimited groups in a more systematic way
-            // during parsing.
             let mut next = outer_trees.next();
             let mut trees: Box<dyn Iterator<Item = tokenstream::TokenTree>>;
-            if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts)) = next {
+            if let Some(tokenstream::TokenTree::Delimited(
+                _,
+                Delimiter::Invisible { skip: true },
+                tts,
+            )) = next
+            {
                 trees = Box::new(tts.into_trees());
                 next = trees.next();
             } else {

compiler/rustc_expand/src/proc_macro_server.rs

@@ -34,7 +34,7 @@ impl FromInternal<token::Delimiter> for Delimiter {
             token::Delimiter::Parenthesis => Delimiter::Parenthesis,
             token::Delimiter::Brace => Delimiter::Brace,
             token::Delimiter::Bracket => Delimiter::Bracket,
-            token::Delimiter::Invisible => Delimiter::None,
+            token::Delimiter::Invisible { .. } => Delimiter::None,
         }
     }
 }
@@ -45,7 +45,7 @@ impl ToInternal<token::Delimiter> for Delimiter {
             Delimiter::Parenthesis => token::Delimiter::Parenthesis,
             Delimiter::Brace => token::Delimiter::Brace,
             Delimiter::Bracket => token::Delimiter::Bracket,
-            Delimiter::None => token::Delimiter::Invisible,
+            Delimiter::None => token::Delimiter::Invisible { skip: true },
         }
     }
 }

compiler/rustc_parse/src/parser/attr_wrapper.rs

@@ -414,11 +414,11 @@ fn make_token_stream(
             FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => {
                 // HACK: If we encounter a mismatched `Invisible` delimiter at the top
                 // level, just ignore it.
-                if matches!(delim, Delimiter::Invisible)
+                if matches!(delim, Delimiter::Invisible { .. })
                     && (stack.len() == 1
                         || !matches!(
                             stack.last_mut().unwrap().open_delim_sp.unwrap().0,
-                            Delimiter::Invisible
+                            Delimiter::Invisible { .. }
                         ))
                 {
                     token_and_spacing = iter.next();
@@ -431,8 +431,8 @@ fn make_token_stream(
                 // HACK: If our current frame has a mismatched opening `Invisible` delimiter,
                 // merge our current frame with the one above it. That is, transform
                 // `[ { < first second } third ]` into `[ { first second } third ]`
-                if !matches!(delim, Delimiter::Invisible)
-                    && matches!(frame_data.open_delim_sp.unwrap().0, Delimiter::Invisible)
+                if !matches!(delim, Delimiter::Invisible { .. })
+                    && matches!(frame_data.open_delim_sp.unwrap().0, Delimiter::Invisible { .. })
                 {
                     stack.last_mut().unwrap().inner.extend(frame_data.inner);
                     // Process our closing delimiter again, this time at the previous
@@ -475,7 +475,8 @@ fn make_token_stream(
     // HACK: If we don't have a closing `Invisible` delimiter for our last
     // frame, merge the frame with the top-level frame. That is,
     // turn `< first second` into `first second`
-    if stack.len() == 2 && stack[1].open_delim_sp.unwrap().0 == Delimiter::Invisible {
+    if stack.len() == 2 && matches!(stack[1].open_delim_sp.unwrap().0, Delimiter::Invisible { .. })
+    {
         let temp_buf = stack.pop().unwrap();
         stack.last_mut().unwrap().inner.extend(temp_buf.inner);
     }

compiler/rustc_parse/src/parser/diagnostics.rs

@@ -2109,7 +2109,9 @@ impl<'a> Parser<'a> {
                     brace_depth -= 1;
                     continue;
                 }
-            } else if self.token == token::Eof || self.eat(&token::CloseDelim(Delimiter::Invisible))
+            // njn: ??
+            } else if self.token == token::Eof
+                || self.eat(&token::CloseDelim(Delimiter::Invisible { skip: false }))
             {
                 return;
             } else {

compiler/rustc_parse/src/parser/mod.rs

@@ -277,14 +277,14 @@ impl TokenCursor {
                         // Set `open_delim` to true here because we deal with it immediately.
                         let frame = TokenCursorFrame::new(Some((delim, sp)), tts.clone());
                         self.stack.push(mem::replace(&mut self.frame, frame));
-                        if delim != Delimiter::Invisible {
+                        if delim != (Delimiter::Invisible { skip: true }) {
                             return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone);
                         }
                         // No open delimeter to return; continue on to the next iteration.
                     }
                 };
             } else if let Some(frame) = self.stack.pop() {
-                if let Some((delim, span)) = self.frame.delim_sp && delim != Delimiter::Invisible {
+                if let Some((delim, span)) = self.frame.delim_sp && delim != (Delimiter::Invisible { skip: true }) {
                     self.frame = frame;
                     return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone);
                 }
@@ -1004,7 +1004,8 @@ impl<'a> Parser<'a> {
         }
         debug_assert!(!matches!(
             next.0.kind,
-            token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
+            token::OpenDelim(Delimiter::Invisible { skip: true })
+                | token::CloseDelim(Delimiter::Invisible { skip: true })
         ));
         self.inlined_bump_with(next)
     }
@@ -1017,10 +1018,10 @@ impl<'a> Parser<'a> {
         }
 
         let frame = &self.token_cursor.frame;
-        if let Some((delim, span)) = frame.delim_sp && delim != Delimiter::Invisible {
+        if let Some((delim, span)) = frame.delim_sp && (delim != Delimiter::Invisible { skip: true }) {
             let all_normal = (0..dist).all(|i| {
                 let token = frame.tree_cursor.look_ahead(i);
-                !matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible, _)))
+                !matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible { skip: true }, _)))
             });
             if all_normal {
                 return match frame.tree_cursor.look_ahead(dist - 1) {
@@ -1042,7 +1043,8 @@ impl<'a> Parser<'a> {
             token = cursor.next(/* desugar_doc_comments */ false).0;
             if matches!(
                 token.kind,
-                token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
+                token::OpenDelim(Delimiter::Invisible { skip: true })
+                    | token::CloseDelim(Delimiter::Invisible { skip: true })
             ) {
                 continue;
             }

compiler/rustc_session/src/utils.rs

@@ -137,7 +137,7 @@ impl<'a> FlattenNonterminals<'a> {
                 let tts = (self.nt_to_tokenstream)(&nt, self.parse_sess, self.synthesize_tokens);
                 TokenTree::Delimited(
                     DelimSpan::from_single(token.span),
-                    Delimiter::Invisible,
+                    Delimiter::Invisible { skip: true }, // njn: ?
                     self.process_token_stream(tts),
                 )
                 .into()