Add TASTY quote pickling for trees

* Pickle/unpickle quotes
* Add Splicer
* Add tree interpreter for splicing staged expression
* Add concrete implementations of Quoted

Author: nicolasstucki

compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala

@@ -38,14 +38,18 @@ class DottyUnpickler(bytes: Array[Byte]) extends ClassfileParser.Embedded {
 
   val unpickler = new TastyUnpickler(bytes)
   private val posUnpicklerOpt = unpickler.unpickle(new PositionsSectionUnpickler)
-  private val treeUnpickler = unpickler.unpickle(new TreeSectionUnpickler(posUnpicklerOpt)).get
+  private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt)).get
 
   /** Enter all toplevel classes and objects into their scopes
    *  @param roots          a set of SymDenotations that should be overwritten by unpickling
    */
   def enter(roots: Set[SymDenotation])(implicit ctx: Context): Unit =
     treeUnpickler.enterTopLevel(roots)
 
+  protected def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler]): TreeSectionUnpickler = {
+    new TreeSectionUnpickler(posUnpicklerOpt)
+  }
+
   /** Only used if `-Yretain-trees` is set. */
   private[this] var myBody: List[Tree] = _
   /** The unpickled trees, and the source file they come from. */

compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala

@@ -67,7 +67,7 @@ class TastyPrinter(bytes: Array[Byte])(implicit ctx: Context) {
               printName(); printTrees()
             case REFINEDtype =>
               printName(); printTree(); printTrees()
-            case RETURN =>
+            case RETURN | HOLE =>
               printNat(); printTrees()
             case METHODtype | POLYtype | TYPELAMBDAtype =>
               printTree()

compiler/src/dotty/tools/dotc/core/tasty/TastyString.scala

@@ -0,0 +1,20 @@
+package dotty.tools.dotc.core.tasty
+
+/** Utils for String representation of TASTY */
+object TastyString {
+
+  /** Decode the TASTY String into TASTY bytes */
+  def stringToTasty(str: String): Array[Byte] = {
+    val bytes = new Array[Byte](str.length)
+    for (i <- str.indices) bytes(i) = str.charAt(i).toByte
+    bytes
+  }
+
+  /** Encode TASTY bytes into a TASTY String */
+  def tastyToString(bytes: Array[Byte]): String = {
+    val chars = new Array[Char](bytes.length)
+    for (i <- bytes.indices) chars(i) = (bytes(i) & 0xff).toChar
+    new String(chars)
+  }
+
+}

compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

@@ -300,7 +300,7 @@ class TreePickler(pickler: TastyPickler) {
       pickleName(sym.name)
       pickleParams
       tpt match {
-        case templ: Template => pickleTree(tpt)
+        case _: Template | _: Hole => pickleTree(tpt)
         case _ if tpt.isType => pickleTpt(tpt)
       }
       pickleTreeUnlessEmpty(rhs)

compiler/src/dotty/tools/dotc/interpreter/Interpreter.scala

@@ -0,0 +1,129 @@
+package dotty.tools.dotc
+package interpreter
+
+import dotty.tools.dotc.ast.tpd
+import dotty.tools.dotc.ast.Trees._
+import dotty.tools.dotc.core.Constants._
+import dotty.tools.dotc.core.Contexts._
+import dotty.tools.dotc.core.Flags._
+import dotty.tools.dotc.core.Decorators._
+import dotty.tools.dotc.core.Symbols._
+
+import scala.reflect.ClassTag
+import java.net.URLClassLoader
+
+/** Tree interpreter that can interpret
+ *   * Literal constants
+ *   * Calls to static methods
+ *   * New objects with explicit `new` keyword
+ *   * Quoted code
+ *
+ *  The interpreter assumes that all calls in the trees are to code that was
+ *  previously compiled and is present in the classpath of the current context.
+ */
+class Interpreter(implicit ctx: Context) {
+  import tpd._
+
+  private[this] val classLoader = {
+    val urls = ctx.settings.classpath.value.split(':').map(cp => java.nio.file.Paths.get(cp).toUri.toURL)
+    new URLClassLoader(urls, getClass.getClassLoader)
+  }
+
+  /** Returns the interpreted result of interpreting the code represented by the tree.
+   *  Return Some of the result or None if some error happen during the interpretation.
+   */
+  def interpretTree[T](tree: Tree)(implicit ct: ClassTag[T]): Option[T] = {
+    try {
+      interpretTreeImpl(tree) match {
+        case obj: T => Some(obj)
+        case obj =>
+          ctx.error(s"Interpreted tree returned a result of an unexpected type. Expected ${ct.runtimeClass} but was ${obj.getClass}", tree.pos)
+          throw new StopInterpretation
+      }
+    } catch {
+      case _: StopInterpretation => None
+    }
+  }
+
+  /** Returns the interpreted result of interpreting the code represented by the tree.
+   *  Returns the result of the interpreted tree.
+   *
+   *  If some error is encountered while interpreting a ctx.error is emited and a StopInterpretation is thrown.
+   */
+  private def interpretTreeImpl(tree: Tree): Object = {
+    try {
+      tree match {
+        case Apply(_, quote :: Nil) if tree.symbol eq defn.quoteMethod =>
+          new RawExpr(quote)
+        case TypeApply(_, quote :: Nil) if tree.symbol eq defn.typeQuoteMethod =>
+          new RawType(quote)
+
+        case Literal(Constant(c)) =>
+          c.asInstanceOf[AnyRef]
+
+        case Apply(fn, args) if fn.symbol.isConstructor =>
+          val cls = fn.symbol.owner
+          val clazz = classLoader.loadClass(cls.symbol.fullName.toString)
+          val paramClasses = paramsSig(fn.symbol)
+          val args1: List[Object] = args.map(arg => interpretTreeImpl(arg))
+          clazz.getConstructor(paramClasses: _*).newInstance(args1: _*).asInstanceOf[Object]
+
+        case Apply(fun, args) if fun.symbol.isStatic =>
+          val clazz = classLoader.loadClass(fun.symbol.owner.companionModule.fullName.toString)
+          val paramClasses = paramsSig(fun.symbol)
+          val args1: List[Object] = args.map(arg => interpretTreeImpl(arg))
+          val method = clazz.getMethod(fun.symbol.name.toString, paramClasses: _*)
+          method.invoke(null, args1: _*)
+
+        case tree: RefTree if tree.symbol.isStatic =>
+          val clazz = classLoader.loadClass(tree.symbol.owner.companionModule.fullName.toString)
+          val method = clazz.getMethod(tree.name.toString)
+          method.invoke(null)
+
+        case tree: RefTree if tree.symbol.is(Module) =>
+          ??? // TODO
+
+        case Inlined(_, bindings, expansion) =>
+          if (bindings.nonEmpty) ??? // TODO evaluate bindings and add environment
+          interpretTreeImpl(expansion)
+        case _ =>
+          val msg =
+            if (tree.tpe.derivesFrom(defn.QuotedExprClass)) "Quote needs to be explicit or a call to a static method"
+            else "Value needs to be a explicit or a call to a static method"
+          ctx.error(msg, tree.pos)
+          throw new StopInterpretation
+      }
+    } catch {
+      case ex: NoSuchMethodException =>
+        ctx.error("Could not find interpreted method in classpath: " + ex.getMessage, tree.pos)
+        throw new StopInterpretation
+      case ex: ClassNotFoundException =>
+        ctx.error("Could not find interpreted class in classpath: " + ex.getMessage, tree.pos)
+        throw new StopInterpretation
+      case ex: RuntimeException =>
+        ex.printStackTrace()
+        ctx.error("A runtime exception occurred while interpreting: " + ex.getMessage, tree.pos)
+        throw new StopInterpretation
+    }
+  }
+
+  /** List of classes of the parameters of the signature of `sym` */
+  private def paramsSig(sym: Symbol): List[Class[_]] = {
+    sym.signature.paramsSig.map { param =>
+      val paramString = param.toString
+      if (paramString == defn.BooleanClass.showFullName) classOf[Boolean]
+      else if (paramString == defn.ByteClass.showFullName) classOf[Byte]
+      else if (paramString == defn.CharClass.showFullName) classOf[Char]
+      else if (paramString == defn.ShortClass.showFullName) classOf[Short]
+      else if (paramString == defn.IntClass.showFullName) classOf[Int]
+      else if (paramString == defn.LongClass.showFullName) classOf[Long]
+      else if (paramString == defn.DoubleClass.showFullName) classOf[Float]
+      else if (paramString == defn.DoubleClass.showFullName) classOf[Double]
+      else classLoader.loadClass(paramString)
+    }
+  }
+
+  /** Exception that stops interpretation if some issue is found */
+  private class StopInterpretation extends Exception
+
+}

compiler/src/dotty/tools/dotc/interpreter/RawExpr.scala

@@ -0,0 +1,7 @@
+package dotty.tools.dotc.interpreter
+
+import dotty.tools.dotc.ast.tpd
+
+class RawExpr(val tree: tpd.Tree) extends quoted.Expr[Any] with RawQuoted {
+  override def toString: String = s"RawExpr(${tree.toString})"
+}

compiler/src/dotty/tools/dotc/interpreter/RawQuoted.scala

@@ -0,0 +1,7 @@
+package dotty.tools.dotc.interpreter
+
+import dotty.tools.dotc.ast.tpd
+
+trait RawQuoted extends quoted.Quoted {
+  def tree: tpd.Tree
+}

compiler/src/dotty/tools/dotc/interpreter/RawType.scala

@@ -0,0 +1,7 @@
+package dotty.tools.dotc.interpreter
+
+import dotty.tools.dotc.ast.tpd
+
+class RawType(val tree: tpd.Tree) extends quoted.Type[Any] with RawQuoted {
+  override def toString: String = s"RawType(${tree.toString})"
+}

compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala

@@ -0,0 +1,95 @@
+package dotty.tools.dotc.quoted
+
+import dotty.tools.dotc.ast.Trees._
+import dotty.tools.dotc.ast.tpd
+import dotty.tools.dotc.config.Printers._
+import dotty.tools.dotc.core.Constants.Constant
+import dotty.tools.dotc.core.Contexts._
+import dotty.tools.dotc.core.Decorators._
+import dotty.tools.dotc.core.Flags._
+import dotty.tools.dotc.core.Symbols._
+import dotty.tools.dotc.core.tasty._
+import dotty.tools.dotc.interpreter.RawQuoted
+
+object PickledQuotes {
+  import tpd._
+
+  /** Pickle the quote into a TASTY string */
+  def pickleQuote(tree: Tree)(implicit ctx: Context): String = {
+    if (ctx.reporter.hasErrors) "<error>"
+    else {
+      val encapsulated = encapsulateQuote(tree)
+      val pickled = pickle(encapsulated)
+      TastyString.tastyToString(pickled)
+    }
+  }
+
+  /** Transform the expression into it's fully spliced Tree */
+  def quotedToTree(expr: quoted.Quoted)(implicit ctx: Context): Tree = expr match {
+    case expr: quoted.TastyQuoted => unpickleQuote(expr)
+    case expr: quoted.ValueExpr[_] => Literal(Constant(expr.value))
+    case expr: RawQuoted => expr.tree
+  }
+
+  /** Unpickle the tree contained in the TastyQuoted */
+  private def unpickleQuote(expr: quoted.TastyQuoted)(implicit ctx: Context): Tree = {
+    val tastyBytes = TastyString.stringToTasty(expr.tasty)
+    val splices = expr.args.map {
+      case arg: quoted.Quoted => quotedToTree(arg)
+      case arg => arg
+    }
+    val unpickled = unpickle(tastyBytes, splices)
+    unpickled match { // Expects `package _root_ { val ': Any = <tree> }`
+      case PackageDef(_, (vdef: ValDef) :: Nil) => vdef.rhs
+    }
+  }
+
+  /** Encapsulate the tree in a top level `val`
+   *    `<tree>` ==> `package _root_ { val ': Any = <tree> }`
+   *
+   *  Note: Trees for types are also encapsulated this way to preserve the holes in the tree.
+   *        Encapsulating the type of the tree in a `type ' = <tree.tpe>` can potentially
+   *        contain references to the outer environment.
+   */
+  private def encapsulateQuote(tree: Tree)(implicit ctx: Context): Tree = {
+    val sym = ctx.newSymbol(ctx.owner, "'".toTermName, Synthetic, defn.AnyType, coord = tree.pos)
+    val quotedVal = ValDef(sym, tree).withPos(tree.pos)
+    PackageDef(ref(defn.RootPackage).asInstanceOf[Ident], quotedVal :: Nil).withPos(tree.pos)
+  }
+
+  // TASTY picklingtests/pos/quoteTest.scala
+
+  /** Pickle tree into it's TASTY bytes s*/
+  private def pickle(tree: Tree)(implicit ctx: Context): Array[Byte] = {
+    val pickler = new TastyPickler(defn.RootClass)
+    val treePkl = pickler.treePkl
+    treePkl.pickle(tree :: Nil)
+    treePkl.compactify()
+    pickler.addrOfTree = treePkl.buf.addrOfTree
+    pickler.addrOfSym = treePkl.addrOfSym
+    // if (tree.pos.exists)
+    //   new PositionPickler(pickler, treePkl.buf.addrOfTree).picklePositions(tree :: Nil)
+
+    // other pickle sections go here.
+    val pickled = pickler.assembleParts()
+
+    if (pickling ne noPrinter) {
+      println(i"**** pickled quote of \n${tree.show}")
+      new TastyPrinter(pickled).printContents()
+    }
+
+    pickled
+  }
+
+  /** Unpickle TASTY bytes into it's tree */
+  private def unpickle(bytes: Array[Byte], splices: Seq[Any])(implicit ctx: Context): Tree = {
+    val unpickler = new QuoteUnpickler(bytes, splices)
+    unpickler.enter(roots = Set(defn.RootPackage))
+    val tree = unpickler.body.head
+    if (pickling ne noPrinter) {
+      println(i"**** unpickled quote for \n${tree.show}")
+      new TastyPrinter(bytes).printContents()
+    }
+    tree
+  }
+}

compiler/src/dotty/tools/dotc/quoted/QuoteUnpickler.scala

@@ -0,0 +1,23 @@
+package dotty.tools.dotc.quoted
+
+import dotty.tools.dotc.core.tasty._
+
+object QuoteUnpickler {
+  class QuotedTreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], splices: Seq[Any])
+    extends DottyUnpickler.TreeSectionUnpickler(posUnpickler) {
+    override def unpickle(reader: TastyReader, nameAtRef: TastyUnpickler.NameTable) =
+      new TreeUnpickler(reader, nameAtRef, posUnpickler, splices)
+  }
+}
+
+/** A class for unpickling quoted Tasty trees and symbols.
+ *  @param bytes         the bytearray containing the Tasty file from which we unpickle
+ *  @param splices       splices that will fill the holes in the quote
+ */
+class QuoteUnpickler(bytes: Array[Byte], splices: Seq[Any]) extends DottyUnpickler(bytes) {
+  import DottyUnpickler._
+  import QuoteUnpickler._
+
+  protected override def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler]): TreeSectionUnpickler =
+    new QuotedTreeSectionUnpickler(posUnpicklerOpt, splices)
+}