Update TokenTree's, remove Quasiquote

Updated AST as per <rust-lang/rust#39419>,
thereby closing #19.

Since those changes involved removing MathcNt and SubstNt, I've
removed the unsafe parts of Quote that relied on that. Quote is now a
safe module (where '$x' and '$x:ty' don't work), enabled by default.

Difference tests now work on 'ast.rs' as well as 'parser.rs'. Additional
work was done on cleaning up tests around macros and tokens. Closes #16.

Author: harpocrates

README.md

@@ -22,19 +22,28 @@ from nightly as they come out, but in general will only target compatibility wit
 
 ## Bugs
 
+### Parser
+
 Any difference between what is accepted by the `rustc` parser and the `language-rust` parser
 indicates
 
   * a bug in `language-rust` (this is almost always the case)
   * a bug in `rustc`
   * that there is a newer version of `rustc` which made a breaking change to this syntax
 
+If the AST/parser of `rustc` changes, the `rustc-tests` test suite should start failing - it
+compares the JSON AST debug output of `rustc` to our parsed AST.
+
+### Pretty-printer
+
 For the pretty-printer, bugs are a bit tougher to list exhaustively. Suggestions for better layout
-algorithms are most welcome!
+algorithms are most welcome! The [`fmt-rfcs`][6] repo is loosely used as the reference for "correct"
+pretty-printing.
 
 [0]: https://www.rust-lang.org/en-US/
 [1]: https://docs.haskellstack.org/en/stable/README/
 [2]: https://hackage.haskell.org/package/alex
 [3]: https://hackage.haskell.org/package/happy
 [4]: https://travis-ci.org/harpocrates/language-rust.svg?branch=master
 [5]: https://travis-ci.org/harpocrates/language-rust
+[6]: https://github.com/rust-lang-nursery/fmt-rfcs

language-rust.cabal

@@ -25,7 +25,7 @@ flag useByteStrings
 
 flag enableQuasiquotes
     description:     Provide the experimental 'Language.Rust.Quote' module
-    default:         False
+    default:         True
 
 library
   hs-source-dirs:      src

sample-sources/expressions.rs

@@ -34,8 +34,8 @@ fn main() {
   let x = &a;
   let x = &mut a;
   let x = return 1;
-//  let x = asm!("NOP");
-//  let x = println!("hi");
+  let x = asm!("NOP");
+  let x = println!("hi");
   let x = Foo { x: 1, y: 2 };
   let x = Foo { x: 1, ..base };
   let x = [1; 5];

sample-sources/items.rs

@@ -2,9 +2,9 @@
 extern crate foo;
 extern crate foo_bar as foo;
 
-// use foo;
-// use foo::bar;
-// use foo::bar as FooBar;
+use foo;
+use foo::bar;
+use foo::bar as FooBar;
 
 static FOO: i32 = 42;
 static mut FOO: i32 = 42;
@@ -34,14 +34,20 @@ mod bar {
     const ID1: i32;
     const ID2: i32 = 1;
 
+    fn area1(self) -> f64;
+    fn area2(mut self) -> f64 { 1f64 }
     fn area1(&self) -> f64;
-    fn area2(&self) -> f64 { 1f64 }
+    fn area2(&mut self) -> f64 { 1f64 }
+    fn area1(&'lt self) -> f64;
+    fn area2(&'lt mut self) -> f64 { 1f64 }
+    fn area1(self: Foo<T>) -> f64;
+    fn area2(mut self: Foo<T>) -> f64 { 1f64 }
 
     type N;
     type N: fmt::Display;
     type N: fmt::Display = i32;
 
-   // foo!{}
+    foo!{}
   }
 
   fn foo<T: ?Sized>(x: &T) { }
@@ -56,14 +62,13 @@ mod bar {
     const ID: i32 = 1;
     fn area(&self) -> f64 { 1f64 }
     type N = i32;
-    // foo!()
-
+    foo!();
   }
   impl<A> Trait for Foo<A> { }
   impl<A> !Trait for Foo<A> { }
 
- // macro_rules! foo { }
- // foo!();
+  macro_rules! foo { }
+  foo!();
 
   enum Foo {
     Baz {

sample-sources/macros.rs

@@ -0,0 +1,66 @@
+#!/bin/sh
+
+// Every token
+token!{
+  [= < > & | ! ~ + - * / % ^]
+  (>= >>= && || << >> == != <= <<= -= &= |= += *= /= ^= %=)
+  {@ . .. ... , ; : :: -> <- => # $ ?}
+  ( ) [ ] { } 1.0foo x _ 'lt /*! doc comment */ #! $x
+}
+
+// Check new way of tokenization
+tokentrees!{
+  $x
+  $x:ty
+  $($xs:expr),+
+  $(1+2),+ 
+  br##"hello "#
+     world!"###suf
+}
+
+// literals
+literals!{
+  b'a' b'\n' b'a'suffix
+  'a' '\n' 'a'suffix
+  
+  123 123i32
+  0b1100_1101 0b1100_1101isize
+  0o3170 0o3170i64
+  0xAFAC 0xAFACu32
+  
+  123.1 123.1f32 
+  // 123.f32 123e-9f32 0e+10
+ 
+  strings!{
+    "hello \n world!"
+    
+    "hello \n world!"suffix
+    
+    r"hello 
+     world!"
+    
+    r"hello 
+     world!"suffix
+    
+    b"hello \n world!"
+    
+    b"hello \n world!"suffix
+    
+    br"hello 
+     world!"
+    
+    br"hello 
+     world!"suffix
+    
+    
+    "hello \
+          world!"
+    
+    b"hello \
+          world!"
+    
+    br##"hello "#
+     world!"##suf
+  }
+}
+

sample-sources/patterns.rs

@@ -20,7 +20,7 @@ fn main() {
     [a, b, i.., y, z] => 0,
     [a, b, .., y, z] => 0,
     [a, b, c] => 0,
- //   LinkedList!(1,2,3) => 0,
+    LinkedList!(1,2,3) => 0,
   }
 }
 

sample-sources/statements.rs

@@ -15,5 +15,5 @@ fn main() {
   2 + { 1 };
 
   // Mac
- // println!("hi")
+  println!("hi")
 }

sample-sources/types.rs

@@ -8,8 +8,8 @@ fn main() {
   let x: &'a i32;
   let x: &i32;
   let x: fn() -> i32;
-//  let x: fn(i32) -> i32;
-//  let x: fn(i32,i32);
+  let x: fn(i32) -> i32;
+  let x: fn(i32,i32);
   let x: !;
   let x: (i32,);
   let x: (i32,!);
@@ -20,7 +20,7 @@ fn main() {
   let x: (i32);
   let x: typeof(1i32);
   let x: _;
-//  let x: HList![i32,(),u8];
+  let x: HList![i32,(),u8];
 }
 
 fn foo() -> impl Bound1 + Bound2 + Bound3 { }

src/Language/Rust/Parser/Internal.y

@@ -209,10 +209,6 @@ import Text.Read (readMaybe)
   -- Lifetimes.
   LIFETIME       { Spanned (LifetimeTok _) _ }
 
-  -- macro related
-  substNt        { Spanned (SubstNt _ ) _ }
-  matchNt        { Spanned (MatchNt _ _) _ }
-
   -- Interpolated
   ntItem         { Spanned (Interpolated (NtItem $$)) _ }
   ntBlock        { Spanned (Interpolated (NtBlock $$)) _ }
@@ -1380,26 +1376,21 @@ token_tree :: { TokenTree }
   -- # Delimited
   | '(' many(token_tree) ')'                              { Delimited mempty Paren mempty $2 mempty }
   | '{' many(token_tree) '}'                              { Delimited mempty Brace mempty $2 mempty }
-  | '[' many(token_tree) ']'                              { Delimited mempty Bracket mempty $2 mempty }
-  -- # Sequence
-  | '$' '(' many(token_tree) ')' token_not_plus_star '+'  { Sequence mempty $3 (Just (unspan $5)) OneOrMore }
-  | '$' '(' many(token_tree) ')' token_not_plus_star '*'  { Sequence mempty $3 (Just (unspan $5)) ZeroOrMore }
-  | '$' '(' many(token_tree) ')' '+'                      { Sequence mempty $3 Nothing OneOrMore }
-  | '$' '(' many(token_tree) ')' '*'                      { Sequence mempty $3 Nothing ZeroOrMore }
+  | '[' many(token_tree) ']'                              { Delimited mempty Bracket mempty $2 mempty } 
   -- # Token
   -- Expression-operator symbols.
-  | token_not_plus_star                                   { mkTokenTree $1 }
-  | '+'                                                   { mkTokenTree $1 }
-  | '*'                                                   { mkTokenTree $1 }
+  | token                                                 { let Spanned t s = $1 in Token s t }
 
-token_not_plus_star :: { Spanned Token }
+token :: { Spanned Token }
   : '='        { $1 }
   | '<'        { $1 }
   | '>'        { $1 }
   | '!'        { $1 }
   | '~'        { $1 }
   | '-'        { $1 }
   | '/'        { $1 }
+  | '+'        { $1 }
+  | '*'        { $1 }
   | '%'        { $1 }
   | '^'        { $1 }
   | '&'        { $1 }
@@ -1437,6 +1428,7 @@ token_not_plus_star :: { Spanned Token }
   | '#'        { $1 }
   | '$' %prec DOLLAR  { $1 }
   | '?'        { $1 }
+  | '#!'       { $1 }
   -- Literals.
   | byte       { $1 }
   | char       { $1 }
@@ -1511,9 +1503,6 @@ token_not_plus_star :: { Spanned Token }
   | '_'        { $1 }
   -- Lifetimes.
   | LIFETIME   { $1 }
-  -- Macro related
-  | substNt    { $1 }
-  | matchNt    { $1 }
 
 
 {
@@ -1603,10 +1592,6 @@ addAttrs as (ParenExpr as' e s)      = ParenExpr (as ++ as') e s
 addAttrs as (Try as' e s)            = Try (as ++ as') e s
 
 
--- | Given a spanned token, convert it to a token tree. Basically just move the Span
-mkTokenTree :: Spanned Token -> TokenTree
-mkTokenTree (Spanned t s) = Token s t
-
 -- | Given a 'Doc' token, convert it into an attribute
 mkDocAttribute :: Spanned Token -> Attribute Span
 mkDocAttribute (Spanned (Doc docStr sty) s) = Attribute sty' doc True s

src/Language/Rust/Parser/Lexer.x

@@ -974,7 +974,6 @@ $hexit             = [0-9a-fA-F]
 -- Macro related
 
 @subst_nt          = "$" @ident
-@match_nt          = @subst_nt ":" @ident
 
 tokens :-
 
@@ -1076,13 +1075,6 @@ $white+         { \s -> pure (Space Whitespace s)  }
 @line_comment     { \c -> pure (Space Comment (drop 2 c)) }
 @inline_comment   { \_ -> Space Comment <$> nestedComment }
 
-"#!"              { token Shebang } 
-
-@subst_nt         { \(_:i) -> pure (SubstNt (mkIdent i)) }
-@match_nt         { \(_:s) -> let (i,':':n) = Prelude.span (/= ':') s
-                              in pure (MatchNt (mkIdent i) (mkIdent n))
-                  }
-
 {
 
 -- | Make a token.
@@ -1106,8 +1098,7 @@ literal lit = do
     _ -> pure (LiteralTok lit Nothing)
 
 -- | Parses a raw string, the closing quotation, and the appropriate number of
--- '#' characters. Note that there can be more closing '#' characters than
--- opening ones (this is as per Rust's standard).
+-- '#' characters.
 rawString :: Int -> P String
 rawString n = do
   c_m <- nextChar
@@ -1117,8 +1108,8 @@ rawString n = do
 
     -- The string has a chance of being closed
     Just '"' -> do
-      n' <- greedyChar '#'
-      if n' >= n
+      n' <- greedyChar '#' n
+      if n' == n
         then pure ""
         else (('"' : replicate n' '#') ++) <$> rawString n 
 
@@ -1174,12 +1165,13 @@ peekChar = do
          in pure (Just c)
 
 -- | Greedily try to eat as many of a given character as possible (and return
--- how many characters were eaten).
-greedyChar :: Char -> P Int
-greedyChar c = do
+-- how many characters were eaten). The second argument is an upper limit.
+greedyChar :: Char -> Int -> P Int
+greedyChar _ 0 = pure 0
+greedyChar c limit = do
   c_m <- peekChar
   case c_m of
-    Just c' | c == c' -> do { _ <- nextChar; n <- greedyChar c; pure (n+1) }
+    Just c' | c == c' -> do { _ <- nextChar; n <- greedyChar c (limit - 1); pure (n+1) }
     _ -> pure 0
 
 -- | Signal a lexical error.
@@ -1257,19 +1249,13 @@ lexTokens lexer = do
     _ -> (tok :) <$> lexTokens lexer
 
 -- | Lex the first line, if it immediately starts with @#!@ (but not @#![@ - that should be an
--- inner attribute). If this fails to find a shebang line, it consumes no input (in reality it does
--- consume one token, but it pushed it back).
+-- inner attribute). If this fails to find a shebang line, it consumes no input.
 lexShebangLine :: P (Maybe String)
 lexShebangLine = do
-  tok <- lexNonSpace
-  case unspan tok of
-    Shebang -> do
-      c <- peekChar
-      case c of
-        Just '[' -> pushToken tok *> pure Nothing
-        _ -> Just <$> toNewline
-    _ -> pushToken tok *> pure Nothing 
-
+  inp <- getInput
+  case takeChars 3 inp of
+    '#':'!':r | r /= "[" -> Just <$> toNewline
+    _ -> pure Nothing
   where
   -- Lexes a string until a newline
   toNewline :: P String

src/Language/Rust/Pretty/Internal.hs

@@ -22,7 +22,7 @@ import Language.Rust.Syntax.Token
 import Language.Rust.Syntax.Ident
 
 import Text.PrettyPrint.Annotated.WL (
-    hcat, cat, punctuate, group, angles, flatten, align, fillSep, text, vcat, char, annotate, 
+    hcat, punctuate, group, angles, flatten, align, fillSep, text, vcat, char, annotate, 
     noAnnotate, flatAlt, parens, brackets, (<>), Doc
   )
 import qualified Text.PrettyPrint.Annotated.WL as WL
@@ -197,10 +197,6 @@ printMac (Mac path tts x) d = annotate x (printPath path False <> "!" <> body)
 printTt :: TokenTree -> Doc a
 printTt (Token _ t) = printToken t 
 printTt (Delimited _ d _ tts _) = block d True mempty mempty [ fillSep (printTt <$> tts) ]
-printTt (Sequence _ tts s op) = "$" <> parens body <> perhaps printToken s <> suf
-  where body = cat [ printTt tt | tt <- tts ]
-        suf = case op of ZeroOrMore -> "*"
-                         OneOrMore -> "+"
 
 -- | Print a token (@token_to_string@)
 -- Single character expression-operator symbols.
@@ -274,8 +270,6 @@ printToken (Doc d OuterDoc) = "/**" <> text d <> "*/"
 printToken Shebang = "#!"
 -- Macro related 
 printToken (Interpolated n) = noAnnotate (printNonterminal n)
-printToken (MatchNt i s) = "$" <> printIdent i <> ":" <> printIdent s
-printToken (SubstNt s) = "$" <> printIdent s
 -- Other
 printToken t = error $ "printToken: " ++ show t