Always pickle uncooked comments

The comments are now always pickled as raw comments, as seen in the
source file. It's the job of Dotty doc to trigger expansion of the
comments after they are unpickled.

This approach simplifies incremental generation of the documentation, as
we don't need to teach Zinc how to extract dependencies in the
documentation.

Author: Duhemm

compiler/src/dotty/tools/dotc/core/Comments.scala

@@ -40,14 +40,21 @@ object Comments {
     *
     * The `Comment` contains functionality to create versions of itself without
     * `@usecase` sections as well as functionality to map the `raw` docstring
+    *
+    * @param pos The position of this `Comment`.
+    * @param raw The raw comment, as seen in the source code, without any expansion.
     */
   abstract case class Comment(pos: Position, raw: String) { self =>
+
+    /** The expansion of this comment */
+    def expanded: Option[String]
+
     /** Has this comment been cooked or expanded? */
-    def isExpanded: Boolean
+    final def isExpanded: Boolean = expanded.isDefined
 
-    /** The body of this comment, without the `@usecase` and `@define` sections. */
-    lazy val body: String =
-      removeSections(raw, "@usecase", "@define")
+    /** The body of this comment, without the `@usecase` and `@define` sections, after expansion. */
+    lazy val expandedBody: Option[String] =
+      expanded.map(removeSections(_, "@usecase", "@define"))
 
     /**
      * The `@usecase` sections of this comment.
@@ -57,23 +64,26 @@ object Comments {
 
     val isDocComment = raw.startsWith("/**")
 
-    def expand(f: String => String): Comment = new Comment(pos, f(raw)) {
-      val isExpanded = true
+    def expand(f: String => String): Comment = new Comment(pos, raw) {
+      val expanded = Some(f(raw))
       val usecases = self.usecases
     }
 
     def withUsecases(implicit ctx: Context): Comment = new Comment(pos, raw) {
-      val isExpanded = self.isExpanded
+      assert(self.isExpanded)
+      val expanded = self.expanded
       val usecases = parseUsecases
     }
 
     private[this] def parseUsecases(implicit ctx: Context): List[UseCase] =
       if (!isDocComment) {
         Nil
       } else {
-        tagIndex(raw)
-          .filter { startsWithTag(raw, _, "@usecase") }
-          .map { case (start, end) => decomposeUseCase(start, end) }
+        expanded.map { body =>
+          tagIndex(body)
+            .filter { startsWithTag(body, _, "@usecase") }
+            .map { case (start, end) => decomposeUseCase(body, start, end) }
+        }.getOrElse(Nil)
       }
 
     /** Turns a usecase section into a UseCase, with code changed to:
@@ -84,7 +94,7 @@ object Comments {
      *  def foo: A = ???
      *  }}}
      */
-    private[this] def decomposeUseCase(start: Int, end: Int)(implicit ctx: Context): UseCase = {
+    private[this] def decomposeUseCase(body: String, start: Int, end: Int)(implicit ctx: Context): UseCase = {
       def subPos(start: Int, end: Int) =
         if (pos == NoPosition) NoPosition
         else {
@@ -93,19 +103,19 @@ object Comments {
           pos withStart start1 withPoint start1 withEnd end1
         }
 
-      val codeStart    = skipWhitespace(raw, start + "@usecase".length)
-      val codeEnd      = skipToEol(raw, codeStart)
-      val code         = raw.substring(codeStart, codeEnd) + " = ???"
-      val codePos      = subPos(codeStart, codeEnd)
+      val codeStart = skipWhitespace(body, start + "@usecase".length)
+      val codeEnd   = skipToEol(body, codeStart)
+      val code      = body.substring(codeStart, codeEnd) + " = ???"
+      val codePos   = subPos(codeStart, codeEnd)
 
       UseCase(code, codePos)
     }
   }
 
   object Comment {
-    def apply(pos: Position, raw: String, expanded: Boolean = false, usc: List[UseCase] = Nil): Comment =
+    def apply(pos: Position, raw: String, expandedComment: Option[String] = None, usc: List[UseCase] = Nil): Comment =
       new Comment(pos, raw) {
-        val isExpanded = expanded
+        val expanded = expandedComment
         val usecases = usc
       }
   }

compiler/src/dotty/tools/dotc/core/tasty/CommentPickler.scala

@@ -24,7 +24,6 @@ class CommentPickler(pickler: TastyPickler, addrOfTree: tpd.Tree => Option[Addr]
       buf.writeNat(length)
       buf.writeBytes(bytes, length)
       buf.writeLongInt(cmt.pos.coords)
-      buf.writeByte(if (cmt.isExpanded) 1 else 0)
     case other =>
       ()
   }

compiler/src/dotty/tools/dotc/core/tasty/CommentUnpickler.scala

@@ -20,9 +20,8 @@ class CommentUnpickler(reader: TastyReader) {
       if (length > 0) {
         val bytes = readBytes(length)
         val position = new Position(readLongInt())
-        val expanded = readByte() == 1
         val rawComment = new String(bytes, Charset.forName("UTF-8"))
-        comments(addr) = Comment(position, rawComment, expanded = expanded)
+        comments(addr) = Comment(position, rawComment)
       }
     }
     comments.toMap

compiler/src/dotty/tools/dotc/core/tasty/TastyFormat.scala

@@ -224,8 +224,7 @@ Standard Section: "Positions" Assoc*
 
 Standard Section: "Comments" Comment*
 
-  Comment       = Length Bytes LongInt Byte // Raw comment's bytes encoded as UTF-8, followed by the comment's coordinates,
-                                            // plus a byte indicating whether the comment is expanded (1) or not expanded (0)
+  Comment       = Length Bytes LongInt      // Raw comment's bytes encoded as UTF-8, followed by the comment's coordinates.
 
 
 **************************************************************************************/

compiler/src/dotty/tools/dotc/typer/Docstrings.scala

@@ -9,29 +9,33 @@ import ast.tpd
 
 trait Docstrings { self: Typer =>
 
-  /** The Docstrings typer will handle the expansion of `@define` and
-    * `@inheritdoc` if there is a `DocContext` present as a property in the
-    * supplied `ctx`.
-    *
-    * It will also type any `@usecase` available in function definitions.
-    */
-  def cookComments(syms: List[Symbol], owner: Symbol)(implicit ctx: Context): Unit =
-    ctx.docCtx.foreach { docbase =>
-      val relevantSyms = syms.filter(docbase.docstring(_).exists(!_.isExpanded))
-      relevantSyms.foreach { sym =>
-        expandParentDocs(sym)
-        val usecases = docbase.docstring(sym).map(_.usecases).getOrElse(Nil)
-
-        usecases.foreach { usecase =>
-          enterSymbol(createSymbol(usecase.untpdCode))
-
-          typedStats(usecase.untpdCode :: Nil, owner) match {
-            case List(df: tpd.DefDef) => usecase.tpdCode = df
-            case _ => ctx.error("`@usecase` was not a valid definition", usecase.codePos)
-          }
+  /**
+   * Expands or cooks the documentation for `sym` in class `owner`.
+   * The expanded comment will directly replace the original comment in the doc context.
+   *
+   * The expansion registers `@define` sections, and will replace `@inheritdoc` and variable
+   * occurrences in the comments.
+   *
+   * If the doc comments contain `@usecase` sections, they will be typed.
+   *
+   * @param sym   The symbol for which the comment is being cooked.
+   * @param owner The class for which comments are being cooked.
+   */
+  def cookComment(sym: Symbol, owner: Symbol)(implicit ctx: Context): Unit = {
+    for {
+      docbase <- ctx.docCtx
+      comment <- docbase.docstring(sym).filter(!_.isExpanded)
+    } {
+      expandParentDocs(sym)
+      docbase.docstring(sym).get.usecases.foreach { usecase =>
+        enterSymbol(createSymbol(usecase.untpdCode))
+        typedStats(usecase.untpdCode :: Nil, owner) match {
+          case List(df: tpd.DefDef) => usecase.tpdCode = df
+          case _ => ctx.error("`@usecase` was not a valid definition", usecase.codePos)
         }
       }
     }
+  }
 
   private def expandParentDocs(sym: Symbol)(implicit ctx: Context): Unit =
     ctx.docCtx.foreach { docCtx =>

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -1540,7 +1540,11 @@ class Typer extends Namer
 
       // Expand comments and type usecases if `-Ycook-comments` is set.
       if (ctx.settings.YcookComments.value) {
-        cookComments(cls :: body1.map(_.symbol), self1.symbol)(ctx.localContext(cdef, cls).setNewScope)
+        val cookingCtx = ctx.localContext(cdef, cls).setNewScope
+        body1.foreach { stat =>
+          cookComment(stat.symbol, self1.symbol)(cookingCtx)
+        }
+        cookComment(cls, cls)(cookingCtx)
       }
 
       checkNoDoubleDeclaration(cls)

doc-tool/src/dotty/tools/dottydoc/DocFrontEnd.scala

@@ -2,13 +2,18 @@ package dotty.tools
 package dottydoc
 
 import dotc.fromtasty.ReadTastyTreesFromClasses
-import dotc.typer.FrontEnd
+import dotc.typer.{FrontEnd, Typer}
 import dotc.core.Contexts.Context
 import dotc.CompilationUnit
 
+import util.syntax.ContextWithContextDottydoc
+
 /** `DocFrontEnd` uses the Dotty `FrontEnd` without discarding the AnyVal
  *  interfaces for Boolean, Int, Char, Long, Byte etc.
  *
+ *  If `-from-tasty` is set, then the trees and documentation will be loaded
+ *  from TASTY. The comments will be cooked after being unpickled.
+ *
  *  It currently still throws away Java sources by overriding
  *  `discardAfterTyper`.
  */
@@ -17,7 +22,18 @@ class DocFrontEnd extends FrontEnd {
   override def runOn(units: List[CompilationUnit])(implicit ctx: Context): List[CompilationUnit] = {
     if (ctx.settings.fromTasty.value) {
       val fromTastyFrontend = new ReadTastyTreesFromClasses
-      fromTastyFrontend.runOn(units)
+      val unpickledUnits = fromTastyFrontend.runOn(units)
+
+      val typer = new Typer()
+      if (ctx.settings.YcookComments.value) {
+        ctx.docbase.docstrings.keys.foreach { sym =>
+            val owner = sym.owner
+            val cookingCtx = ctx.withOwner(owner)
+            typer.cookComment(sym, owner)(cookingCtx)
+        }
+      }
+
+      unpickledUnits
     } else {
       super.runOn(units)
     }

doc-tool/src/dotty/tools/dottydoc/core/DocstringPhase.scala

@@ -15,7 +15,7 @@ import util.syntax._
 /** Phase to add docstrings to the Dottydoc AST */
 class DocstringPhase extends DocMiniPhase with CommentParser with CommentCleaner {
 
-  private def getComment(sym: Symbol)(implicit ctx: Context): Option[CompilerComment] =
+  private def getComment(sym: Symbol)(implicit ctx: Context): Option[CompilerComment] = {
     ctx.docbase.docstring(sym)
     .orElse {
       // If the symbol doesn't have a docstring, look for an overridden
@@ -26,17 +26,20 @@ class DocstringPhase extends DocMiniPhase with CommentParser with CommentCleaner
       }
       .flatMap(ctx.docbase.docstring)
     }
+  }
 
-  private def parsedComment(ent: Entity)(implicit ctx: Context): Option[Comment] =
-    getComment(ent.symbol).map { cmt =>
-      val text = cmt.body
-      val parsed = parse(ent, ctx.docbase.packages, clean(text), text, cmt.pos)
-
+  private def parsedComment(ent: Entity)(implicit ctx: Context): Option[Comment] = {
+    for {
+      comment <- getComment(ent.symbol)
+      text <- comment.expandedBody
+    } yield {
+      val parsed = parse(ent, ctx.docbase.packages, clean(text), text, comment.pos)
       if (ctx.settings.wikiSyntax.value)
-        WikiComment(ent, parsed, cmt.pos).comment
+        WikiComment(ent, parsed, comment.pos).comment
       else
-        MarkdownComment(ent, parsed, cmt.pos).comment
+        MarkdownComment(ent, parsed, comment.pos).comment
     }
+  }
 
   override def transformPackage(implicit ctx: Context) = { case ent: PackageImpl =>
     ent.copy(comment = parsedComment(ent))

doc-tool/src/dotty/tools/dottydoc/core/UsecasePhase.scala

@@ -13,7 +13,7 @@ import util.syntax._
 
 class UsecasePhase extends DocMiniPhase {
   private def defdefToDef(d: tpd.DefDef, sym: Symbol)(implicit ctx: Context) = {
-    val name = d.name.show.split("\\$").head // UseCase defs get $pos appended to their names
+    val name = d.name.show.split("\\$").head
     DefImpl(
       sym,
       annotations(sym),
@@ -27,6 +27,10 @@ class UsecasePhase extends DocMiniPhase {
   }
 
   override def transformDef(implicit ctx: Context) = { case df: DefImpl =>
-    ctx.docbase.docstring(df.symbol).flatMap(_.usecases.headOption.map(_.tpdCode)).map(defdefToDef(_, df.symbol)).getOrElse(df)
+    ctx.docbase
+      .docstring(df.symbol)
+      .flatMap(_.usecases.headOption.map(_.tpdCode))
+      .map(defdefToDef(_, df.symbol))
+      .getOrElse(df)
   }
 }

doc-tool/test/ConstructorTest.scala

@@ -25,7 +25,7 @@ abstract class ConstructorsBase extends DottyDocTest {
 
     val className = "scala.Class"
 
-    check(className :: Nil, source :: Nil) { packages =>
+    check(className :: Nil, source :: Nil) { (ctx, packages) =>
       packages("scala") match {
         case PackageImpl(_, _, _, List(cls: Class), _, _, _, _) =>
           cls.constructors.headOption match {
@@ -49,7 +49,7 @@ abstract class ConstructorsBase extends DottyDocTest {
 
     val className = "scala.Class"
 
-    check(className :: Nil, source :: Nil) { packages =>
+    check(className :: Nil, source :: Nil) { (ctx, packages) =>
       packages("scala") match {
         case PackageImpl(_, _,  _, List(cls: Class), _, _, _, _) =>
           cls.constructors match {
@@ -76,7 +76,7 @@ abstract class ConstructorsBase extends DottyDocTest {
 
     val className = "scala.Class"
 
-    check(className :: Nil, source :: Nil) { packages =>
+    check(className :: Nil, source :: Nil) { (ctx, packages) =>
       packages("scala") match {
         case PackageImpl(_, _, _, List(cls: Class), _, _, _, _) =>
           cls.constructors match {
@@ -110,7 +110,7 @@ abstract class ConstructorsBase extends DottyDocTest {
 
     val className = "scala.Class"
 
-    check(className :: Nil, source :: Nil) { packages =>
+    check(className :: Nil, source :: Nil) { (ctx, packages) =>
       packages("scala") match {
         case PackageImpl(_, _, _, List(cls: Class), _, _, _, _) =>
           cls.constructors match {
@@ -150,7 +150,7 @@ abstract class ConstructorsBase extends DottyDocTest {
 
     val className = "scala.Class"
 
-    check(className :: Nil, source :: Nil) { packages =>
+    check(className :: Nil, source :: Nil) { (ctx, packages) =>
       packages("scala") match {
         case PackageImpl(_, _, _, List(cls: CaseClass, obj: Object), _, _, _, _) =>
           cls.constructors match {
@@ -185,7 +185,7 @@ abstract class ConstructorsBase extends DottyDocTest {
 
     val className = "scala.Trait"
 
-    check(className :: Nil, source :: Nil) { packages =>
+    check(className :: Nil, source :: Nil) { (ctx, packages) =>
       packages("scala") match {
         case PackageImpl(_, _, _, List(trt: Trait), _, _, _, _) =>
           trt.traitParams match {