New quasiquote implementation

shared/src/main/scala/mlscript/JSBackend.scala

@@ -66,7 +66,7 @@ class JSBackend(allowUnresolvedSymbols: Boolean) {
       translatePattern(base)
     case Inst(bod) => translatePattern(bod)
     case _: Lam | _: App | _: Sel | _: Let | _: Blk | _: Bind | _: Test | _: With | _: CaseOf | _: Subs | _: Assign
-        | If(_, _) | New(_, _) | _: Splc | _: Forall | _: Where | _: Super | _: Eqn | _: AdtMatchWith =>
+        | If(_, _) | New(_, _) | _: Splc | _: Forall | _: Where | _: Super | _: Eqn | _: AdtMatchWith | _: Quoted | _: Unquoted =>
       throw CodeGenError(s"term ${inspect(t)} is not a valid pattern")
   }
 
@@ -305,7 +305,7 @@ class JSBackend(allowUnresolvedSymbols: Boolean) {
     case TyApp(base, _) => translateTerm(base)
     case Eqn(Var(name), _) =>
       throw CodeGenError(s"assignment of $name is not supported outside a constructor")
-    case _: Bind | _: Test | If(_, _)  | _: Splc | _: Where | _: AdtMatchWith =>
+    case _: Bind | _: Test | If(_, _)  | _: Splc | _: Where | _: AdtMatchWith | _: Quoted | _: Unquoted =>
       throw CodeGenError(s"cannot generate code for term ${inspect(term)}")
   }
 

shared/src/main/scala/mlscript/NewLexer.scala

@@ -91,11 +91,17 @@ class NewLexer(origin: Origin, raise: Diagnostic => Unit, dbg: Bool) {
   def loc(start: Int, end: Int): Loc = Loc(start, end, origin)
 
   @tailrec final
-  def lex(i: Int, ind: Ls[Int], acc: Ls[TokLoc]): Ls[TokLoc] = if (i >= length) acc.reverse else {
+  def lex(i: Int, ind: Ls[Int], acc: Ls[TokLoc])(implicit qqList: Ls[BracketKind]): Ls[TokLoc] = if (i >= length) acc.reverse else {
     val c = bytes(i)
     def pe(msg: Message): Unit =
       // raise(ParseError(false, msg -> S(loc(i, i + 1)) :: Nil))
       raise(ErrorReport(msg -> S(loc(i, i + 1)) :: Nil, newDefs = true, source = Lexing))
+    def fit(i: Int, syntax: Str): Bool =
+      i + syntax.length <= length && bytes.slice(i, i + syntax.length).mkString === syntax
+    def isQuasiquoteOpening(i: Int): Bool = fit(i, BracketKind.Quasiquote.beg)
+    def isQuasiquoteTripleOpening(i: Int): Bool =  fit(i, BracketKind.QuasiquoteTriple.beg)
+    def isUnquoteOpening(i: Int): Bool = fit(i, BracketKind.Unquote.beg)
+    def isQuasiquoteTripleClosing(i: Int): Bool = fit(i, BracketKind.QuasiquoteTriple.end)
     // @inline 
     // def go(j: Int, tok: Token) = lex(j, ind, (tok, loc(i, j)) :: acc)
     def next(j: Int, tok: Token) = (tok, loc(i, j)) :: acc
@@ -108,15 +114,37 @@ class NewLexer(origin: Origin, raise: Diagnostic => Unit, dbg: Bool) {
         val j = i + 1
         // go(j, COMMA)
         lex(j, ind, next(j, COMMA))
+      case 'c' if isQuasiquoteOpening(i) || isQuasiquoteTripleOpening(i) =>
+        val isTripleQuoteQQ = isQuasiquoteTripleOpening(i)
+        val bracket_kind = if (isTripleQuoteQQ)
+          BracketKind.QuasiquoteTriple
+        else
+          BracketKind.Quasiquote
+        val len = bracket_kind.beg.length
+        lex(i + len, ind, next(i + len, OPEN_BRACKET(bracket_kind)))(bracket_kind :: qqList)
+      case '$' if isUnquoteOpening(i) =>
+        lex(i + 2, ind, next(i + 2, OPEN_BRACKET(BracketKind.Unquote)))
       case '"' =>
-        val j = i + 1
-        val (chars, k) = str(j, false)
-        val k2 = if (bytes.lift(k) === Some('"')) k + 1 else {
-          pe(msg"unclosed quotation mark")
-          k
+        val (isTripleQQ, cons) = qqList match {
+          case h :: t => (h === BracketKind.QuasiquoteTriple, t)
+          case Nil => (false, Nil)
+        }
+        if (isTripleQQ && isQuasiquoteTripleClosing(i)) {
+          val length = BracketKind.QuasiquoteTriple.end.length
+          lex(i + length, ind, next(i + length, CLOSE_BRACKET(BracketKind.QuasiquoteTriple)))(cons)
+        } else if (!isTripleQQ && qqList.nonEmpty) {
+          lex(i + 1, ind, next(i + 1, CLOSE_BRACKET(BracketKind.Quasiquote)))(cons)
+        } else {
+          val j = i + 1
+          val (chars, k) = str(j, false)
+          val k2 = if (bytes.lift(k) === Some('"')) k + 1 else {
+            pe(msg"unclosed quotation mark")
+            k
+          }
+          // go(k2, LITVAL(StrLit(chars)))
+          lex(k2, ind, next(k2, LITVAL(StrLit(chars))))
         }
-        // go(k2, LITVAL(StrLit(chars)))
-        lex(k2, ind, next(k2, LITVAL(StrLit(chars))))
+
       case '/' if bytes.lift(i + 1).contains('/') =>
         val j = i + 2
         val (txt, k) =
@@ -205,7 +233,7 @@ class NewLexer(origin: Origin, raise: Diagnostic => Unit, dbg: Bool) {
   }
 
 
-  lazy val tokens: Ls[Token -> Loc] = lex(0, Nil, Nil)
+  lazy val tokens: Ls[Token -> Loc] = lex(0, Nil, Nil)(Nil)
 
   /** Converts the lexed tokens into structured tokens. */
   lazy val bracketedTokens: Ls[Stroken -> Loc] = {
@@ -223,7 +251,7 @@ class NewLexer(origin: Origin, raise: Diagnostic => Unit, dbg: Bool) {
               // * Ignore empty indented blocks:
               go(rest, false, stack, oldAcc)
             case ((k0, l0), oldAcc) :: stack =>
-              if (k0 =/= k1)
+              if (k0 =/= k1 && !(k0 === Unquote && k1 === Curly))
                 raise(ErrorReport(msg"Mistmatched closing ${k1.name}" -> S(l1) ::
                   msg"does not correspond to opening ${k0.name}" -> S(l0) :: Nil, newDefs = true,
                   source = Parsing))
@@ -339,12 +367,12 @@ object NewLexer {
     case (KEYWORD(name: String), _) => "#" + name
     case (IDENT(name: String, symbolic: Bool), _) => name
     case (SELECT(name: String), _) => "." + name
-    case (OPEN_BRACKET(k), _) => k.beg.toString
-    case (CLOSE_BRACKET(k), _) => k.end.toString
+    case (OPEN_BRACKET(k), _) => k.beg
+    case (CLOSE_BRACKET(k), _) => k.end
     case (BRACKETS(k @ BracketKind.Indent, contents), _) =>
-      k.beg.toString + printTokens(contents) + k.end.toString
+      k.beg + printTokens(contents) + k.end
     case (BRACKETS(k, contents), _) =>
-      k.beg.toString + printTokens(contents) + k.end.toString
+      k.beg + printTokens(contents) + k.end
     case (COMMENT(text: String), _) => "/*" + text + "*/"
   }
   def printTokens(ts: Ls[TokLoc]): Str =

shared/src/main/scala/mlscript/NewParser.scala

@@ -604,6 +604,14 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo
       case (IDENT(nme, false), l0) :: _ =>
         consume
         exprCont(Var(nme).withLoc(S(l0)), prec, allowNewlines = false)
+      case (br @ BRACKETS(Quasiquote | QuasiquoteTriple, toks), loc) :: _ =>
+        consume
+        val body = rec(toks, S(br.innerLoc), br.describe).concludeWith(_.expr(0))
+        exprCont(Quoted(body).withLoc(S(loc)), prec, allowNewlines = false)
+      case (br @ BRACKETS(Unquote, toks), loc) :: _ =>
+        consume 
+        val body = rec(toks, S(br.innerLoc), br.describe).concludeWith(_.expr(0))
+        exprCont(Unquoted(body).withLoc(S(loc)), prec, allowNewlines = false)
       case (KEYWORD("super"), l0) :: _ =>
         consume
         exprCont(Super().withLoc(S(l0)), prec, allowNewlines = false)

shared/src/main/scala/mlscript/Token.scala

@@ -48,18 +48,24 @@ final case class COMMENT(text: String) extends Token with Stroken
 sealed abstract class BracketKind {
   import BracketKind._
   lazy val (beg, end) = this match {
-    case Round => '(' -> ')'
-    case Curly => '{' -> '}'
-    case Square => '[' -> ']'
-    case Angle => '‹' -> '›'
-    case Indent => '→' -> '←'
+    case Round => "(" -> ")"
+    case Curly => "{" -> "}"
+    case Square => "[" -> "]"
+    case Angle => "‹" -> "›"
+    case Indent => "→" -> "←"
+    case Quasiquote => "code\"" -> "\""
+    case QuasiquoteTriple => "code\"\"\"" -> "\"\"\""
+    case Unquote => "${" -> "}"
   }
   def name: Str = this match {
     case Round => "parenthesis"
     case Curly => "curly brace"
     case Square => "square bracket"
     case Angle => "angle bracket"
     case Indent => "indentation"
+    case Quasiquote => "quasiquote"
+    case QuasiquoteTriple => "quasiquote triple"
+    case Unquote => "unquote"
   }
 }
 
@@ -69,6 +75,9 @@ object BracketKind {
   case object Square extends BracketKind
   case object Angle extends BracketKind
   case object Indent extends BracketKind
+  case object Quasiquote extends BracketKind
+  case object QuasiquoteTriple extends BracketKind
+  case object Unquote extends BracketKind
 
   def unapply(c: Char): Opt[Either[BracketKind, BracketKind]] = c |>? {
     case '(' => Left(Round)

shared/src/main/scala/mlscript/Typer.scala

@@ -62,15 +62,48 @@ class Typer(var dbg: Boolean, var verbose: Bool, var explainErrors: Bool, val ne
       env: MutMap[Str, TypeInfo],
       mthEnv: MutMap[(Str, Str) \/ (Opt[Str], Str), MethodType],
       lvl: Int,
+      qenv: MutMap[Str, SkolemTag], // * SkolemTag for variables in quasiquotes
+      fvars: MutSet[ST], // * Free variables
+      quotedLvl: Int, // * Level of quasiquotes
       inPattern: Bool,
       tyDefs: Map[Str, TypeDef],
       tyDefs2: MutMap[Str, DelayedTypeInfo],
       inRecursiveDef: Opt[Var], // TODO rm
       extrCtx: ExtrCtx,
   ) {
-    def +=(b: Str -> TypeInfo): Unit = env += b
+    def +=(b: Str -> TypeInfo): Unit = {
+      env += b
+      if (quotedLvl > 0) {
+        val tag = SkolemTag(freshVar(NoProv, N)(lvl))(NoProv)
+        println(s"Create skolem tag $tag for ${b._2} in quasiquote.")
+        qenv += b._1 -> tag
+      }
+    }
     def ++=(bs: IterableOnce[Str -> TypeInfo]): Unit = bs.iterator.foreach(+=)
     def get(name: Str): Opt[TypeInfo] = env.get(name) orElse parent.dlof(_.get(name))(N)
+    def qget(name: Str, qlvl: Int = quotedLvl): Opt[SkolemTag] =
+      if (qlvl === quotedLvl) qenv.get(name) orElse parent.dlof(_.qget(name, qlvl))(N)
+      else parent.dlof(_.qget(name, qlvl))(N)
+    def wrapCode: Ls[(Str, TypeInfo)] = qenv.flatMap {
+      case (name, tag) =>
+        get(name) match {
+          case S(VarSymbol(ty, _)) =>
+            name -> VarSymbol(TypeRef(TypeName("Code"), ty :: tag :: Nil)(noProv), Var(name)) :: Nil
+          case S(_: AbstractConstructor) | S(_: LazyTypeInfo) => die // * Abstract ctors and type defs are not allowed
+          case N => Nil // * In the same quasiquote but not the same scope
+        }
+    }.toList
+    def unwrap[T](names: Ls[Str], f: () => T): T = { // * Revert ctx modification temporarily
+      val cache: MutMap[Str, TypeInfo] = MutMap.empty
+      names.foreach(name => {
+        cache += name -> env.getOrElse(name, die)
+        env -= name
+      })
+      val res = f()
+      cache.foreach(env += _)
+      res
+    }
+    def traceFV(fv: ST): Unit = fvars += fv
     def contains(name: Str): Bool = env.contains(name) || parent.exists(_.contains(name))
     def addMth(parent: Opt[Str], nme: Str, ty: MethodType): Unit = mthEnv += R(parent, nme) -> ty
     def addMthDefn(parent: Str, nme: Str, ty: MethodType): Unit = mthEnv += L(parent, nme) -> ty
@@ -147,6 +180,9 @@ class Typer(var dbg: Boolean, var verbose: Bool, var explainErrors: Bool, val ne
       env = MutMap.from(builtinBindings.iterator.map(nt => nt._1 -> VarSymbol(nt._2, Var(nt._1)))),
       mthEnv = MutMap.empty,
       lvl = MinLevel,
+      qenv = MutMap.empty,
+      fvars = MutSet.empty,
+      quotedLvl = 0,
       inPattern = false,
       tyDefs = Map.from(builtinTypes.map(t => t.nme.name -> t)),
       tyDefs2 = MutMap.empty,
@@ -239,6 +275,7 @@ class Typer(var dbg: Boolean, var verbose: Bool, var explainErrors: Bool, val ne
     NuTypeDef(Cls, TN("Str"), Nil, N, N, N, Nil, N, N, TypingUnit(Nil))(N, S(preludeLoc)),
     NuTypeDef(Als, TN("undefined"), Nil, N, N, S(Literal(UnitLit(true))), Nil, N, N, TypingUnit(Nil))(N, S(preludeLoc)),
     NuTypeDef(Als, TN("null"), Nil, N, N, S(Literal(UnitLit(false))), Nil, N, N, TypingUnit(Nil))(N, S(preludeLoc)),
+    NuTypeDef(Cls, TN("Code"), (S(VarianceInfo.co) -> TN("T")) :: (S(VarianceInfo.co) -> TN("C")) :: Nil, N, N, N, Nil, N, N, TypingUnit(Nil))(N, S(preludeLoc))
   )
   val builtinTypes: Ls[TypeDef] =
     TypeDef(Cls, TN("?"), Nil, TopType, Nil, Nil, Set.empty, N, Nil) :: // * Dummy for pretty-printing unknown type locations
@@ -370,6 +407,11 @@ class Typer(var dbg: Boolean, var verbose: Bool, var explainErrors: Bool, val ne
         val v = freshVar(noProv, N)(1)
         PolymorphicType(0, ArrayType(FieldType(S(v), v)(noProv))(noProv))
       },
+      "run" -> {
+        val tv = freshVar(noProv, N)(1)
+        PolymorphicType(0, fun(singleTup(TypeRef(TypeName("Code"), tv :: BotType :: Nil)(noProv)), tv)(noProv))
+      },
+      "Const" -> fun(singleTup(IntType), TypeRef(TypeName("Code"), IntType :: BotType :: Nil)(noProv))(noProv),
     ) ++ (if (!newDefs) primTypes ++ primTypes.map(p => p._1.capitalize -> p._2) // TODO settle on naming convention...
       else Nil)
   }
@@ -834,7 +876,15 @@ class Typer(var dbg: Boolean, var verbose: Bool, var explainErrors: Bool, val ne
             res
           }
       case v @ ValidVar(name) =>
-        val ty = ctx.get(name).fold(err("identifier not found: " + name, term.toLoc): ST) {
+        val tyOpt = if (ctx.quotedLvl > 0 && !builtinBindings.contains(name)) {
+          ctx.qget(name) match {
+            case S(ctxTy) =>
+              if (!ctx.qenv.contains(name)) ctx.traceFV(ctxTy)
+              ctx.get(name)
+            case _ => N
+          }
+        } else ctx.get(name)
+        val ty = tyOpt.fold(err("identifier not found: " + name, term.toLoc): ST) {
           case AbstractConstructor(absMths, traitWithMths) =>
             val td = ctx.tyDefs(name)
             err((msg"Instantiation of an abstract type is forbidden" -> term.toLoc)
@@ -1382,6 +1432,34 @@ class Typer(var dbg: Boolean, var verbose: Bool, var explainErrors: Bool, val ne
         res
       case Eqn(lhs, rhs) =>
         err(msg"Unexpected equation in this position", term.toLoc)
+      case Quoted(body) =>
+        val newCtx =
+          ctx.nest.copy(quotedLvl = ctx.quotedLvl + 1, qenv = MutMap.empty, fvars = MutSet.empty)
+        val bodyType = ctx.parent match {
+          case S(p) if p.quotedLvl > ctx.quotedLvl =>
+            ctx.unwrap(p.wrapCode.map(_._1), () => typePolymorphicTerm(body)(newCtx, raise, vars))
+          case _ => typePolymorphicTerm(body)(newCtx, raise, vars)
+        }
+        val ctxTy = newCtx.fvars.foldLeft[ST](BotType)((res, ty) => res | ty)
+        TypeRef(TypeName("Code"), bodyType :: ctxTy :: Nil)(noProv) // TODO: trace the unbound free vars
+      case Unquoted(body) =>
+        if (ctx.quotedLvl > 0) {
+          val newCtx = ctx.nest.copy(quotedLvl = 0)
+          val wrappedCodes = ctx.wrapCode
+          println("Map qenv to env in unquote...")
+          wrappedCodes.foreach(c => {
+            println(s"Create ${c._2} in newCtx")
+            newCtx += c
+          })
+          val bodyType = typePolymorphicTerm(body)(newCtx, raise, vars)
+          val res = freshVar(noTyProv, N)
+          val ctxTy = freshVar(noTyProv, N)
+          val ty =
+            con(bodyType, TypeRef(TypeName("Code"), res :: ctx.qenv.foldLeft[ST](ctxTy)((res, ty) => ty._2 | res) :: Nil)(noProv), res)
+          ctx.traceFV(ctxTy)
+          ty
+        }
+        else err("Unquotes should be enclosed with a quasiquote.", body.toLoc)(raise)
     }
   }(r => s"$lvl. : ${r}")
 

shared/src/main/scala/mlscript/codegen/Helpers.scala

@@ -60,6 +60,10 @@ object Helpers {
     case Super() => "Super()"
     case AdtMatchWith(cond, arms) =>
       s"match ${inspect(cond)} with ${arms.map(patmat => s"${inspect(patmat.pat)} -> ${inspect(patmat.rhs)}").mkString(" | ")}"
+    case Quoted(body) =>
+      s"Quoted(${inspect(body)})"
+    case Unquoted(body) =>
+      s"Unquoted(${inspect(body)})"
   }
 
   def inspect(body: IfBody): Str = body match {

shared/src/main/scala/mlscript/helpers.scala

@@ -548,6 +548,8 @@ trait TermImpl extends StatementImpl { self: Term =>
       case Super() => "super"
       case Eqn(lhs, rhs) => "assign for ctor"
       case AdtMatchWith(cond, arms) => "ADT pattern matching"
+      case Quoted(_) => "quasiquote"
+      case Unquoted(_) => "unquote"
     }
   }
 
@@ -603,6 +605,8 @@ trait TermImpl extends StatementImpl { self: Term =>
     case Eqn(lhs, rhs) => s"${lhs} = ${rhs}"
     case AdtMatchWith(cond, arms) =>
       s"match ${cond} with ${arms.map (patmat => s"${patmat.pat} -> ${patmat.rhs}").mkString (" | ") }"
+    case Quoted(b) => s"code\"$b\""
+    case Unquoted(b) => s"$${$b}"
   }}
 
   def toTypeRaise(implicit raise: Raise): Type = toType match {
@@ -975,6 +979,8 @@ trait StatementImpl extends Located { self: Statement =>
     case NuTypeDef(k, nme, tps, ps, ctor, sig, pars, sup, ths, bod) =>
       nme :: tps.map(_._2) ::: ps.toList ::: pars ::: ths.toList ::: bod :: Nil
     case AdtMatchWith(cond, _) => cond :: Nil // FIXME discards branches...
+    case Quoted(body) => body :: Nil
+    case Unquoted(body) => body :: Nil
   }
 
 

shared/src/main/scala/mlscript/syntax.scala

@@ -89,6 +89,8 @@ final case class Forall(params: Ls[TypeVar], body: Term)             extends Ter
 final case class Inst(body: Term)                                    extends Term
 final case class Super()                                             extends Term
 final case class Eqn(lhs: Var, rhs: Term)                            extends Term // equations such as x = y, notably used in constructors; TODO: make lhs a Term
+final case class Quoted(body: Term)                                  extends Term 
+final case class Unquoted(body: Term)                                extends Term 
 
 final case class AdtMatchWith(cond: Term, arms: Ls[AdtMatchPat])     extends Term
 final case class AdtMatchPat(pat: Term, rhs: Term)                   extends AdtMatchPatImpl

shared/src/test/diff/basics/Simplesub1.fun

@@ -439,28 +439,28 @@ let rec x = (let y = (x x); (z => z))
 //│   where
 //│     forall 'c 'd. 'd -> 'c
 //│       where
-//│         'e <: (forall 'f 'g. 'f -> 'g
+//│         'e <: (forall 'f 'g. 'g -> 'f
 //│           where
-//│             'd <: 'd -> 'f -> 'g) -> 'c <: (forall 'c 'd. 'd -> 'c
+//│             'd <: 'd -> 'g -> 'f) -> 'c <: (forall 'c 'd. 'd -> 'c
 //│       where
-//│         'e <: (forall 'f 'g. 'f -> 'g
+//│         'e <: (forall 'f 'g. 'g -> 'f
 //│           where
-//│             'd <: 'd -> 'f -> 'g) -> 'c) -> 'a -> 'b) -> 'h & 'e) -> 'h
+//│             'd <: 'd -> 'g -> 'f) -> 'c) -> 'a -> 'b) -> 'h & 'e) -> 'h
 
 // * Function that takes arbitrarily many arguments:
 // :e // Works thanks to inconsistent constrained types...
 (f => (x => f (v => (x x) v)) (x => f (v => (x x) v))) (f => x => f)
 //│ res: anything -> (forall 'a 'b. 'a -> 'b
 //│   where
-//│     forall 'c 'd. 'c -> 'd
+//│     forall 'c 'd. 'd -> 'c
 //│       where
 //│         forall 'e. 'e -> anything -> 'e <: (forall 'f 'g. 'f -> 'g
 //│           where
-//│             'c <: 'c -> 'f -> 'g) -> 'd <: (forall 'c 'd. 'c -> 'd
+//│             'd <: 'd -> 'f -> 'g) -> 'c <: (forall 'c 'd. 'd -> 'c
 //│       where
 //│         forall 'e. 'e -> anything -> 'e <: (forall 'f 'g. 'f -> 'g
 //│           where
-//│             'c <: 'c -> 'f -> 'g) -> 'd) -> 'a -> 'b)
+//│             'd <: 'd -> 'f -> 'g) -> 'c) -> 'a -> 'b)
 
 
 

shared/src/test/diff/codegen/IndirectRecursion.mls

@@ -155,7 +155,7 @@ def z =
 //│ ((anything -> nothing) -> anything) -> error
 //│   <:  z:
 //│ (('a -> 'b) -> ('a -> 'b & 'c)) -> 'c
-//│ ╔══[ERROR] Subtyping constraint of the form `?a -> ?b <: (forall ?c ?d. ?d -> ?c) -> ?e` exceeded recursion depth limit (250)
+//│ ╔══[ERROR] Subtyping constraint of the form `?a -> ?b <: (forall ?c ?d. ?c -> ?d) -> ?e` exceeded recursion depth limit (250)
 //│ ║  l.154: 	  (fun f -> (fun x -> f (fun v -> (x x) v)) (fun x -> f (fun v -> (x x) v)))
 //│ ║         	            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 //│ ╙── Note: use flag `:ex` to see internal error info.