Record progress in the current run

Test that the callbacks are called with expected values

Author: bishabosha

compiler/src/dotty/tools/dotc/Run.scala

@@ -12,7 +12,9 @@ import typer.Typer
 import typer.ImportInfo.withRootImports
 import Decorators._
 import io.AbstractFile
-import Phases.unfusedPhases
+import Phases.{unfusedPhases, Phase}
+
+import sbt.interfaces.ProgressCallback
 
 import util._
 import reporting.{Suppression, Action, Profile, ActiveProfile, NoProfile}
@@ -32,6 +34,9 @@ import scala.collection.mutable
 import scala.util.control.NonFatal
 import scala.io.Codec
 
+import Run.Progress
+import scala.compiletime.uninitialized
+
 /** A compiler run. Exports various methods to compile source files */
 class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with ConstraintRunInfo {
 
@@ -155,14 +160,51 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
   }
 
   /** The source files of all late entered symbols, as a set */
-  private var lateFiles = mutable.Set[AbstractFile]()
+  private val lateFiles = mutable.Set[AbstractFile]()
 
   /** A cache for static references to packages and classes */
   val staticRefs = util.EqHashMap[Name, Denotation](initialCapacity = 1024)
 
   /** Actions that need to be performed at the end of the current compilation run */
   private var finalizeActions = mutable.ListBuffer[() => Unit]()
 
+  private var _progress: Progress | Null = null // Set if progress reporting is enabled
+
+  /** Only safe to call if progress is being tracked. */
+  private inline def trackProgress(using Context)(inline op: Context ?=> Progress => Unit): Unit =
+    val local = _progress
+    if local != null then
+      op(using ctx)(local)
+
+  def doBeginUnit(unit: CompilationUnit)(using Context): Unit =
+    trackProgress: progress =>
+      progress.informUnitStarting(unit)
+
+  def doAdvanceUnit()(using Context): Unit =
+    trackProgress: progress =>
+      progress.unitc += 1 // trace that we completed a unit in the current phase
+      progress.refreshProgress()
+
+  def doAdvanceLate()(using Context): Unit =
+    trackProgress: progress =>
+      progress.latec += 1 // trace that we completed a late compilation
+      progress.refreshProgress()
+
+  private def doEnterPhase(currentPhase: Phase)(using Context): Unit =
+    trackProgress: progress =>
+      progress.enterPhase(currentPhase)
+
+  private def doAdvancePhase(currentPhase: Phase, wasRan: Boolean)(using Context): Unit =
+    trackProgress: progress =>
+      progress.unitc = 0 // reset unit count in current phase
+      progress.seen += 1 // trace that we've seen a phase
+      if wasRan then
+        // add an extra traversal now that we completed a phase
+        progress.traversalc += 1
+      else
+        // no phase was ran, remove a traversal from expected total
+        progress.runnablePhases -= 1
+
   /** Will be set to true if any of the compiled compilation units contains
    *  a pureFunctions language import.
    */
@@ -233,13 +275,15 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
     if ctx.settings.YnoDoubleBindings.value then
       ctx.base.checkNoDoubleBindings = true
 
-    def runPhases(using Context) = {
+    def runPhases(allPhases: Array[Phase])(using Context) = {
       var lastPrintedTree: PrintedTree = NoPrintedTree
       val profiler = ctx.profiler
       var phasesWereAdjusted = false
 
-      for (phase <- ctx.base.allPhases)
-        if (phase.isRunnable)
+      for phase <- allPhases do
+        doEnterPhase(phase)
+        val phaseWillRun = phase.isRunnable
+        if phaseWillRun then
           Stats.trackTime(s"phase time ms/$phase") {
             val start = System.currentTimeMillis
             val profileBefore = profiler.beforePhase(phase)
@@ -261,14 +305,21 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
             if !Feature.ccEnabledSomewhere then
               ctx.base.unlinkPhaseAsDenotTransformer(Phases.checkCapturesPhase.prev)
               ctx.base.unlinkPhaseAsDenotTransformer(Phases.checkCapturesPhase)
-
+            end if
+          end if
+        end if
+        doAdvancePhase(phase, wasRan = phaseWillRun)
+      end for
       profiler.finished()
     }
 
     val runCtx = ctx.fresh
     runCtx.setProfiler(Profiler())
     unfusedPhases.foreach(_.initContext(runCtx))
-    runPhases(using runCtx)
+    val fusedPhases = runCtx.base.allPhases
+    runCtx.withProgressCallback: cb =>
+      _progress = Progress(cb, this, fusedPhases.length)
+    runPhases(allPhases = fusedPhases)(using runCtx)
     if (!ctx.reporter.hasErrors)
       Rewrites.writeBack()
     suppressions.runFinished(hasErrors = ctx.reporter.hasErrors)
@@ -294,10 +345,9 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
         .withRootImports
 
       def process()(using Context) =
-        ctx.typer.lateEnterUnit(doTypeCheck =>
-          if typeCheck then
-            if compiling then finalizeActions += doTypeCheck
-            else doTypeCheck()
+        ctx.typer.lateEnterUnit(typeCheck)(doTypeCheck =>
+          if compiling then finalizeActions += doTypeCheck
+          else doTypeCheck()
         )
 
       process()(using unitCtx)
@@ -400,7 +450,66 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
 }
 
 object Run {
+
+  /**Computes the next MegaPhase for the given phase.*/
+  def nextMegaPhase(phase: Phase)(using Context): Phase = phase.megaPhase.next.megaPhase
+
+  private class Progress(cb: ProgressCallback, private val run: Run, val initialPhases: Int):
+    private[Run] var runnablePhases: Int = initialPhases // track how many phases we expect to run
+    private[Run] var unitc: Int = 0 // current unit count in the current phase
+    private[Run] var latec: Int = 0 // current late unit count
+    private[Run] var traversalc: Int = 0 // completed traversals over all files
+    private[Run] var seen: Int = 0 // how many phases we've seen so far
+
+    private var currPhase: Phase = uninitialized  // initialized by enterPhase
+    private var currPhaseName: String = uninitialized // initialized by enterPhase
+    private var nextPhaseName: String = uninitialized // initialized by enterPhase
+
+    private def phaseNameFor(phase: Phase): String =
+      if phase.exists then phase.phaseName
+      else "<end>"
+
+    private[Run] def enterPhase(newPhase: Phase)(using Context): Unit =
+      if newPhase ne currPhase then
+        currPhase = newPhase
+        currPhaseName = phaseNameFor(newPhase)
+        nextPhaseName = phaseNameFor(Run.nextMegaPhase(newPhase))
+        if seen > 0 then
+          refreshProgress()
+
+
+    /** Counts the number of completed full traversals over files, plus the number of units in the current phase */
+    private def currentProgress()(using Context): Int =
+      traversalc * run.files.size + unitc + latec
+
+    /**Total progress is computed as the sum of
+     * - the number of traversals we expect to make over all files
+     * - the number of late compilations
+     */
+    private def totalProgress()(using Context): Int =
+      runnablePhases * run.files.size + run.lateFiles.size
+
+    private def requireInitialized(): Unit =
+      require((currPhase: Phase | Null) != null, "enterPhase was not called")
+
+    private[Run] def informUnitStarting(unit: CompilationUnit)(using Context): Unit =
+      requireInitialized()
+      cb.informUnitStarting(currPhaseName, unit)
+
+    private[Run] def refreshProgress()(using Context): Unit =
+      requireInitialized()
+      cb.progress(currentProgress(), totalProgress(), currPhaseName, nextPhaseName)
+
   extension (run: Run | Null)
+    def beginUnit(unit: CompilationUnit)(using Context): Unit =
+      if run != null then run.doBeginUnit(unit)
+
+    def advanceUnit()(using Context): Unit =
+      if run != null then run.doAdvanceUnit()
+
+    def advanceLate()(using Context): Unit =
+      if run != null then run.doAdvanceLate()
+
     def enrichedErrorMessage: Boolean = if run == null then false else run.myEnrichedErrorMessage
     def enrichErrorMessage(errorMessage: String)(using Context): String =
       if run == null then

compiler/src/dotty/tools/dotc/core/Phases.scala

@@ -335,10 +335,12 @@ object Phases {
     def runOn(units: List[CompilationUnit])(using runCtx: Context): List[CompilationUnit] =
       units.map { unit =>
         given unitCtx: Context = runCtx.fresh.setPhase(this.start).setCompilationUnit(unit).withRootImports
+        ctx.run.beginUnit(unit)
         try run
         catch case ex: Throwable if !ctx.run.enrichedErrorMessage =>
           println(ctx.run.enrichErrorMessage(s"unhandled exception while running $phaseName on $unit"))
           throw ex
+        finally ctx.run.advanceUnit()
         unitCtx.compilationUnit
       }
 

compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala

@@ -22,7 +22,12 @@ class ReadTasty extends Phase {
     ctx.settings.fromTasty.value
 
   override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] =
-    withMode(Mode.ReadPositions)(units.flatMap(readTASTY(_)))
+    withMode(Mode.ReadPositions)(units.flatMap(applyPhase(_)))
+
+  private def applyPhase(unit: CompilationUnit)(using Context): Option[CompilationUnit] =
+    ctx.run.beginUnit(unit)
+    try readTASTY(unit)
+    finally ctx.run.advanceUnit()
 
   def readTASTY(unit: CompilationUnit)(using Context): Option[CompilationUnit] = unit match {
     case unit: TASTYCompilationUnit =>

compiler/src/dotty/tools/dotc/parsing/ParserPhase.scala

@@ -43,10 +43,13 @@ class Parser extends Phase {
   override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] = {
     val unitContexts =
       for unit <- units yield
+        ctx.run.beginUnit(unit)
         report.inform(s"parsing ${unit.source}")
         ctx.fresh.setCompilationUnit(unit).withRootImports
 
-    unitContexts.foreach(parse(using _))
+    for given Context <- unitContexts do
+      try parse
+      finally ctx.run.advanceUnit()
     record("parsedTrees", ast.Trees.ntrees)
 
     unitContexts.map(_.compilationUnit)

compiler/src/dotty/tools/dotc/transform/init/Checker.scala

@@ -18,6 +18,7 @@ import Phases._
 import scala.collection.mutable
 
 import Semantic._
+import dotty.tools.unsupported
 
 class Checker extends Phase:
 
@@ -33,7 +34,10 @@ class Checker extends Phase:
   override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] =
     val checkCtx = ctx.fresh.setPhase(this.start)
     val traverser = new InitTreeTraverser()
-    units.foreach { unit => traverser.traverse(unit.tpdTree) }
+    for unit <- units do
+      checkCtx.run.beginUnit(unit)
+      try traverser.traverse(unit.tpdTree)
+      finally ctx.run.advanceUnit()
     val classes = traverser.getClasses()
 
     if ctx.settings.YcheckInit.value then
@@ -44,9 +48,7 @@ class Checker extends Phase:
 
     units
 
-  def run(using Context): Unit =
-    // ignore, we already called `Semantic.check()` in `runOn`
-    ()
+  def run(using Context): Unit = unsupported("run")
 
   class InitTreeTraverser extends TreeTraverser:
     private val classes: mutable.ArrayBuffer[ClassSymbol] = new mutable.ArrayBuffer

compiler/src/dotty/tools/dotc/typer/Namer.scala

@@ -724,20 +724,27 @@ class Namer { typer: Typer =>
    *  Will call the callback with an implementation of type checking
    *  That will set the tpdTree and root tree for the compilation unit.
    */
-  def lateEnterUnit(typeCheckCB: (() => Unit) => Unit)(using Context) =
+  def lateEnterUnit(typeCheck: Boolean)(typeCheckCB: (() => Unit) => Unit)(using Context) =
     val unit = ctx.compilationUnit
 
     /** Index symbols in unit.untpdTree with lateCompile flag = true */
     def lateEnter()(using Context): Context =
       val saved = lateCompile
       lateCompile = true
-      try index(unit.untpdTree :: Nil) finally lateCompile = saved
+      try
+        index(unit.untpdTree :: Nil)
+      finally
+        lateCompile = saved
+        if !typeCheck then ctx.run.advanceLate()
 
     /** Set the tpdTree and root tree of the compilation unit */
     def lateTypeCheck()(using Context) =
-      unit.tpdTree = typer.typedExpr(unit.untpdTree)
-      val phase = new transform.SetRootTree()
-      phase.run
+      try
+        unit.tpdTree = typer.typedExpr(unit.untpdTree)
+        val phase = new transform.SetRootTree()
+        phase.run
+      finally
+        if typeCheck then ctx.run.advanceLate()
 
     unit.untpdTree =
       if (unit.isJava) new JavaParser(unit.source).parse()
@@ -748,9 +755,10 @@ class Namer { typer: Typer =>
         // inline body annotations are set in namer, capturing the current context
         // we need to prepare the context for inlining.
         lateEnter()
-        typeCheckCB { () =>
-          lateTypeCheck()
-        }
+        if typeCheck then
+          typeCheckCB { () =>
+            lateTypeCheck()
+          }
       }
     }
   end lateEnterUnit

compiler/src/dotty/tools/dotc/typer/TyperPhase.scala

@@ -63,13 +63,15 @@ class TyperPhase(addRootImports: Boolean = true) extends Phase {
       for unit <- units yield
         val newCtx0 = ctx.fresh.setPhase(this.start).setCompilationUnit(unit)
         val newCtx = PrepareInlineable.initContext(newCtx0)
+        newCtx.run.beginUnit(unit)
         report.inform(s"typing ${unit.source}")
         if (addRootImports)
           newCtx.withRootImports
         else
           newCtx
 
-    unitContexts.foreach(enterSyms(using _))
+    for given Context <- unitContexts do
+      enterSyms
 
     ctx.base.parserPhase match {
       case p: ParserPhase =>
@@ -81,9 +83,13 @@ class TyperPhase(addRootImports: Boolean = true) extends Phase {
       case _ =>
     }
 
-    unitContexts.foreach(typeCheck(using _))
+    for given Context <- unitContexts do
+      typeCheck
+
     record("total trees after typer", ast.Trees.ntrees)
-    unitContexts.foreach(javaCheck(using _)) // after typechecking to avoid cycles
+    for given Context <- unitContexts do
+      try javaCheck // after typechecking to avoid cycles
+      finally ctx.run.advanceUnit()
 
     val newUnits = unitContexts.map(_.compilationUnit).filterNot(discardAfterTyper)
     ctx.run.nn.checkSuspendedUnits(newUnits)

compiler/test/dotty/tools/DottyTest.scala

@@ -44,9 +44,14 @@ trait DottyTest extends ContextEscapeDetection {
     fc.setProperty(ContextDoc, new ContextDocstrings)
   }
 
+  protected def defaultCompiler: Compiler = Compiler()
+
   private def compilerWithChecker(phase: String)(assertion: (tpd.Tree, Context) => Unit) = new Compiler {
+
+    private val baseCompiler = defaultCompiler
+
     override def phases = {
-      val allPhases = super.phases
+      val allPhases = baseCompiler.phases
       val targetPhase = allPhases.flatten.find(p => p.phaseName == phase).get
       val groupsBefore = allPhases.takeWhile(x => !x.contains(targetPhase))
       val lastGroup = allPhases.find(x => x.contains(targetPhase)).get.takeWhile(x => !(x eq targetPhase))
@@ -67,6 +72,15 @@ trait DottyTest extends ContextEscapeDetection {
     run.runContext
   }
 
+  def checkAfterCompile(checkAfterPhase: String, sources: List[String])(assertion: Context => Unit): Context = {
+    val c = defaultCompiler
+    val run = c.newRun
+    run.compileFromStrings(sources)
+    val rctx = run.runContext
+    assertion(rctx)
+    rctx
+  }
+
   def checkTypes(source: String, typeStrings: String*)(assertion: (List[Type], Context) => Unit): Unit =
     checkTypes(source, List(typeStrings.toList)) { (tpess, ctx) => (tpess: @unchecked) match {
       case List(tpes) => assertion(tpes, ctx)