diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index 3e85565beb6d1..495ad176542c1 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -18,8 +18,7 @@ use errors::DiagnosticBuilder;
use ext::expand::{self, Invocation, Expansion};
use ext::hygiene::Mark;
use fold::{self, Folder};
-use parse;
-use parse::parser::{self, Parser};
+use parse::{self, parser};
use parse::token;
use parse::token::{InternedString, str_to_ident};
use ptr::P;
@@ -188,146 +187,6 @@ impl<F> AttrProcMacro for F
}
}
-pub struct TokResult<'a> {
- pub parser: Parser<'a>,
- pub span: Span,
-}
-
-impl<'a> TokResult<'a> {
- // There is quite a lot of overlap here with ParserAnyMacro in ext/tt/macro_rules.rs
- // We could probably share more code.
- // FIXME(#36641) Unify TokResult and ParserAnyMacro.
- fn ensure_complete_parse(&mut self, allow_semi: bool) {
- let macro_span = &self.span;
- self.parser.ensure_complete_parse(allow_semi, |parser| {
- let token_str = parser.this_token_to_string();
- let msg = format!("macro expansion ignores token `{}` and any following", token_str);
- let span = parser.span;
- parser.diagnostic()
- .struct_span_err(span, &msg)
- .span_note(*macro_span, "caused by the macro expansion here")
- .emit();
- });
- }
-}
-
-impl<'a> MacResult for TokResult<'a> {
- fn make_items(mut self: Box<Self>) -> Option<SmallVector<P<ast::Item>>> {
- if self.parser.sess.span_diagnostic.has_errors() {
- return Some(SmallVector::zero());
- }
-
- let mut items = SmallVector::zero();
- loop {
- match self.parser.parse_item() {
- Ok(Some(item)) => items.push(item),
- Ok(None) => {
- self.ensure_complete_parse(false);
- return Some(items);
- }
- Err(mut e) => {
- e.emit();
- return Some(SmallVector::zero());
- }
- }
- }
- }
-
- fn make_impl_items(mut self: Box<Self>) -> Option<SmallVector<ast::ImplItem>> {
- let mut items = SmallVector::zero();
- loop {
- if self.parser.token == token::Eof {
- break;
- }
- match self.parser.parse_impl_item() {
- Ok(item) => items.push(item),
- Err(mut e) => {
- e.emit();
- return Some(SmallVector::zero());
- }
- }
- }
- self.ensure_complete_parse(false);
- Some(items)
- }
-
- fn make_trait_items(mut self: Box<Self>) -> Option<SmallVector<ast::TraitItem>> {
- let mut items = SmallVector::zero();
- loop {
- if self.parser.token == token::Eof {
- break;
- }
- match self.parser.parse_trait_item() {
- Ok(item) => items.push(item),
- Err(mut e) => {
- e.emit();
- return Some(SmallVector::zero());
- }
- }
- }
- self.ensure_complete_parse(false);
- Some(items)
- }
-
- fn make_expr(mut self: Box<Self>) -> Option<P<ast::Expr>> {
- match self.parser.parse_expr() {
- Ok(e) => {
- self.ensure_complete_parse(true);
- Some(e)
- }
- Err(mut e) => {
- e.emit();
- Some(DummyResult::raw_expr(self.span))
- }
- }
- }
-
- fn make_pat(mut self: Box<Self>) -> Option<P<ast::Pat>> {
- match self.parser.parse_pat() {
- Ok(e) => {
- self.ensure_complete_parse(false);
- Some(e)
- }
- Err(mut e) => {
- e.emit();
- Some(P(DummyResult::raw_pat(self.span)))
- }
- }
- }
-
- fn make_stmts(mut self: Box<Self>) -> Option<SmallVector<ast::Stmt>> {
- let mut stmts = SmallVector::zero();
- loop {
- if self.parser.token == token::Eof {
- break;
- }
- match self.parser.parse_full_stmt(false) {
- Ok(Some(stmt)) => stmts.push(stmt),
- Ok(None) => { /* continue */ }
- Err(mut e) => {
- e.emit();
- return Some(SmallVector::zero());
- }
- }
- }
- self.ensure_complete_parse(false);
- Some(stmts)
- }
-
- fn make_ty(mut self: Box<Self>) -> Option<P<ast::Ty>> {
- match self.parser.parse_ty() {
- Ok(e) => {
- self.ensure_complete_parse(false);
- Some(e)
- }
- Err(mut e) => {
- e.emit();
- Some(DummyResult::raw_ty(self.span))
- }
- }
- }
-}
-
/// Represents a thing that maps token trees to Macro Results
pub trait TTMacroExpander {
fn expand<'cx>(&self,
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 7359c21ecccac..7c43f1f307698 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -9,7 +9,7 @@
// except according to those terms.
use ast::{Block, Crate, Ident, Mac_, PatKind};
-use ast::{MacStmtStyle, StmtKind, ItemKind};
+use ast::{Name, MacStmtStyle, StmtKind, ItemKind};
use ast;
use ext::hygiene::Mark;
use ext::placeholders::{placeholder, PlaceholderExpander};
@@ -21,9 +21,9 @@ use ext::base::*;
use feature_gate::{self, Features};
use fold;
use fold::*;
-use parse::{ParseSess, lexer};
+use parse::{ParseSess, PResult, lexer};
use parse::parser::Parser;
-use parse::token::{intern, keywords};
+use parse::token::{self, intern, keywords};
use print::pprust;
use ptr::P;
use tokenstream::{TokenTree, TokenStream};
@@ -38,12 +38,12 @@ macro_rules! expansions {
($($kind:ident: $ty:ty [$($vec:ident, $ty_elt:ty)*], $kind_name:expr, .$make:ident,
$(.$fold:ident)* $(lift .$fold_elt:ident)*,
$(.$visit:ident)* $(lift .$visit_elt:ident)*;)*) => {
- #[derive(Copy, Clone)]
+ #[derive(Copy, Clone, PartialEq, Eq)]
pub enum ExpansionKind { OptExpr, $( $kind, )* }
pub enum Expansion { OptExpr(Option<P<ast::Expr>>), $( $kind($ty), )* }
impl ExpansionKind {
- fn name(self) -> &'static str {
+ pub fn name(self) -> &'static str {
match self {
ExpansionKind::OptExpr => "expression",
$( ExpansionKind::$kind => $kind_name, )*
@@ -106,6 +106,12 @@ macro_rules! expansions {
self.expand(Expansion::$kind(SmallVector::one(node))).$make()
})*)*
}
+
+ impl<'a> MacResult for ::ext::tt::macro_rules::ParserAnyMacro<'a> {
+ $(fn $make(self: Box<::ext::tt::macro_rules::ParserAnyMacro<'a>>) -> Option<$ty> {
+ Some(self.make(ExpansionKind::$kind).$make())
+ })*
+ }
}
}
@@ -293,10 +299,11 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
};
attr::mark_used(&attr);
+ let name = intern(&attr.name());
self.cx.bt_push(ExpnInfo {
call_site: attr.span,
callee: NameAndSpan {
- format: MacroAttribute(intern(&attr.name())),
+ format: MacroAttribute(name),
span: Some(attr.span),
allow_internal_unstable: false,
}
@@ -319,14 +326,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
let item_toks = TokenStream::from_tts(tts_for_item(&item, &self.cx.parse_sess));
let tok_result = mac.expand(self.cx, attr.span, attr_toks, item_toks);
- let parser = self.cx.new_parser_from_tts(&tok_result.to_tts());
- let result = Box::new(TokResult { parser: parser, span: attr.span });
-
- kind.make_from(result).unwrap_or_else(|| {
- let msg = format!("macro could not be expanded into {} position", kind.name());
- self.cx.span_err(attr.span, &msg);
- kind.dummy(attr.span)
- })
+ self.parse_expansion(tok_result, kind, name, attr.span)
}
_ => unreachable!(),
}
@@ -423,14 +423,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
},
});
-
- let tok_result = expandfun.expand(self.cx,
- span,
- TokenStream::from_tts(marked_tts));
- let parser = self.cx.new_parser_from_tts(&tok_result.to_tts());
- let result = Box::new(TokResult { parser: parser, span: span });
- // FIXME better span info.
- kind.make_from(result).map(|i| i.fold_with(&mut ChangeSpan { span: span }))
+ let toks = TokenStream::from_tts(marked_tts);
+ let tok_result = expandfun.expand(self.cx, span, toks);
+ Some(self.parse_expansion(tok_result, kind, extname, span))
}
};
@@ -448,6 +443,75 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
expn_id: Some(self.cx.backtrace()),
})
}
+
+ fn parse_expansion(&mut self, toks: TokenStream, kind: ExpansionKind, name: Name, span: Span)
+ -> Expansion {
+ let mut parser = self.cx.new_parser_from_tts(&toks.to_tts());
+ let expansion = match parser.parse_expansion(kind, false) {
+ Ok(expansion) => expansion,
+ Err(mut err) => {
+ err.emit();
+ return kind.dummy(span);
+ }
+ };
+ parser.ensure_complete_parse(name, kind.name(), span);
+ // FIXME better span info
+ expansion.fold_with(&mut ChangeSpan { span: span })
+ }
+}
+
+impl<'a> Parser<'a> {
+ pub fn parse_expansion(&mut self, kind: ExpansionKind, macro_legacy_warnings: bool)
+ -> PResult<'a, Expansion> {
+ Ok(match kind {
+ ExpansionKind::Items => {
+ let mut items = SmallVector::zero();
+ while let Some(item) = self.parse_item()? {
+ items.push(item);
+ }
+ Expansion::Items(items)
+ }
+ ExpansionKind::TraitItems => {
+ let mut items = SmallVector::zero();
+ while self.token != token::Eof {
+ items.push(self.parse_trait_item()?);
+ }
+ Expansion::TraitItems(items)
+ }
+ ExpansionKind::ImplItems => {
+ let mut items = SmallVector::zero();
+ while self.token != token::Eof {
+ items.push(self.parse_impl_item()?);
+ }
+ Expansion::ImplItems(items)
+ }
+ ExpansionKind::Stmts => {
+ let mut stmts = SmallVector::zero();
+ while self.token != token::Eof {
+ if let Some(stmt) = self.parse_full_stmt(macro_legacy_warnings)? {
+ stmts.push(stmt);
+ }
+ }
+ Expansion::Stmts(stmts)
+ }
+ ExpansionKind::Expr => Expansion::Expr(self.parse_expr()?),
+ ExpansionKind::OptExpr => Expansion::OptExpr(Some(self.parse_expr()?)),
+ ExpansionKind::Ty => Expansion::Ty(self.parse_ty()?),
+ ExpansionKind::Pat => Expansion::Pat(self.parse_pat()?),
+ })
+ }
+
+ pub fn ensure_complete_parse(&mut self, macro_name: ast::Name, kind_name: &str, span: Span) {
+ if self.token != token::Eof {
+ let msg = format!("macro expansion ignores token `{}` and any following",
+ self.this_token_to_string());
+ let mut err = self.diagnostic().struct_span_err(self.span, &msg);
+ let msg = format!("caused by the macro expansion here; the usage \
+ of `{}!` is likely invalid in {} context",
+ macro_name, kind_name);
+ err.span_note(span, &msg).emit();
+ }
+ }
}
struct InvocationCollector<'a, 'b: 'a> {
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index 0eed3e5898c00..d222de2dd36c6 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -12,6 +12,7 @@ use {ast, attr};
use syntax_pos::{Span, DUMMY_SP};
use ext::base::{DummyResult, ExtCtxt, MacEager, MacResult, SyntaxExtension};
use ext::base::{IdentMacroExpander, NormalTT, TTMacroExpander};
+use ext::expand::{Expansion, ExpansionKind};
use ext::placeholders;
use ext::tt::macro_parser::{Success, Error, Failure};
use ext::tt::macro_parser::{MatchedSeq, MatchedNonterminal};
@@ -22,18 +23,14 @@ use parse::parser::{Parser, Restrictions};
use parse::token::{self, gensym_ident, NtTT, Token};
use parse::token::Token::*;
use print;
-use ptr::P;
use tokenstream::{self, TokenTree};
-use util::small_vector::SmallVector;
-
-use std::cell::RefCell;
use std::collections::{HashMap};
use std::collections::hash_map::{Entry};
use std::rc::Rc;
-struct ParserAnyMacro<'a> {
- parser: RefCell<Parser<'a>>,
+pub struct ParserAnyMacro<'a> {
+ parser: Parser<'a>,
/// Span of the expansion site of the macro this parser is for
site_span: Span,
@@ -42,106 +39,20 @@ struct ParserAnyMacro<'a> {
}
impl<'a> ParserAnyMacro<'a> {
- /// Make sure we don't have any tokens left to parse, so we don't
- /// silently drop anything. `allow_semi` is so that "optional"
- /// semicolons at the end of normal expressions aren't complained
- /// about e.g. the semicolon in `macro_rules! kapow { () => {
- /// panic!(); } }` doesn't get picked up by .parse_expr(), but it's
- /// allowed to be there.
- fn ensure_complete_parse(&self, allow_semi: bool, context: &str) {
- let mut parser = self.parser.borrow_mut();
- parser.ensure_complete_parse(allow_semi, |parser| {
- let token_str = parser.this_token_to_string();
- let msg = format!("macro expansion ignores token `{}` and any \
- following",
- token_str);
- let span = parser.span;
- let mut err = parser.diagnostic().struct_span_err(span, &msg);
- let msg = format!("caused by the macro expansion here; the usage \
- of `{}!` is likely invalid in {} context",
- self.macro_ident, context);
- err.span_note(self.site_span, &msg)
- .emit();
- });
- }
-}
-
-impl<'a> MacResult for ParserAnyMacro<'a> {
- fn make_expr(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Expr>> {
- let ret = panictry!(self.parser.borrow_mut().parse_expr());
- self.ensure_complete_parse(true, "expression");
- Some(ret)
- }
- fn make_pat(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Pat>> {
- let ret = panictry!(self.parser.borrow_mut().parse_pat());
- self.ensure_complete_parse(false, "pattern");
- Some(ret)
- }
- fn make_items(self: Box<ParserAnyMacro<'a>>) -> Option<SmallVector<P<ast::Item>>> {
- let mut ret = SmallVector::zero();
- while let Some(item) = panictry!(self.parser.borrow_mut().parse_item()) {
- ret.push(item);
- }
- self.ensure_complete_parse(false, "item");
- Some(ret)
- }
-
- fn make_impl_items(self: Box<ParserAnyMacro<'a>>)
- -> Option<SmallVector<ast::ImplItem>> {
- let mut ret = SmallVector::zero();
- loop {
- let mut parser = self.parser.borrow_mut();
- match parser.token {
- token::Eof => break,
- _ => ret.push(panictry!(parser.parse_impl_item()))
- }
- }
- self.ensure_complete_parse(false, "item");
- Some(ret)
- }
-
- fn make_trait_items(self: Box<ParserAnyMacro<'a>>)
- -> Option<SmallVector<ast::TraitItem>> {
- let mut ret = SmallVector::zero();
- loop {
- let mut parser = self.parser.borrow_mut();
- match parser.token {
- token::Eof => break,
- _ => ret.push(panictry!(parser.parse_trait_item()))
- }
+ pub fn make(mut self: Box<ParserAnyMacro<'a>>, kind: ExpansionKind) -> Expansion {
+ let ParserAnyMacro { site_span, macro_ident, ref mut parser } = *self;
+ let expansion = panictry!(parser.parse_expansion(kind, true));
+
+ // We allow semicolons at the end of expressions -- e.g. the semicolon in
+ // `macro_rules! m { () => { panic!(); } }` isn't parsed by `.parse_expr()`,
+ // but `m!()` is allowed in expression positions (c.f. issue #34706).
+ if kind == ExpansionKind::Expr && parser.token == token::Semi {
+ parser.bump();
}
- self.ensure_complete_parse(false, "item");
- Some(ret)
- }
-
-
- fn make_stmts(self: Box<ParserAnyMacro<'a>>)
- -> Option<SmallVector<ast::Stmt>> {
- let mut ret = SmallVector::zero();
- loop {
- let mut parser = self.parser.borrow_mut();
- match parser.token {
- token::Eof => break,
- _ => match parser.parse_full_stmt(true) {
- Ok(maybe_stmt) => match maybe_stmt {
- Some(stmt) => ret.push(stmt),
- None => (),
- },
- Err(mut e) => {
- e.emit();
- break;
- }
- }
- }
- }
- self.ensure_complete_parse(false, "statement");
- Some(ret)
- }
- fn make_ty(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Ty>> {
- let ret = panictry!(self.parser.borrow_mut().parse_ty());
- self.ensure_complete_parse(false, "type");
- Some(ret)
+ // Make sure we don't have any tokens left to parse so we don't silently drop anything.
+ parser.ensure_complete_parse(macro_ident.name, kind.name(), site_span);
+ expansion
}
}
@@ -219,7 +130,7 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
// Let the context choose how to interpret the result.
// Weird, but useful for X-macros.
return Box::new(ParserAnyMacro {
- parser: RefCell::new(p),
+ parser: p,
// Pass along the original expansion site and the name of the macro
// so we can print a useful error message if the parse of the expanded
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 23085fadc5e60..e83d003ab74dc 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -6171,15 +6171,4 @@ impl<'a> Parser<'a> {
_ => Err(self.fatal("expected string literal"))
}
}
-
- pub fn ensure_complete_parse<F>(&mut self, allow_semi: bool, on_err: F)
- where F: FnOnce(&Parser)
- {
- if allow_semi && self.token == token::Semi {
- self.bump();
- }
- if self.token != token::Eof {
- on_err(self);
- }
- }
}