Merge pull request #1739 from dotty-staging/topic/environmentally-friendly-tests

Environmentally friendly tests

Author: felixmulder

compiler/src/dotty/tools/dotc/core/Symbols.scala

@@ -428,7 +428,7 @@ object Symbols {
     final def entered(implicit ctx: Context): this.type = {
       assert(this.owner.isClass, s"symbol ($this) entered the scope of non-class owner ${this.owner}") // !!! DEBUG
       this.owner.asClass.enter(this)
-      if (this.is(Module, butNot = Package)) this.owner.asClass.enter(this.moduleClass)
+      if (this is Module) this.owner.asClass.enter(this.moduleClass)
       this
     }
 

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -555,8 +555,8 @@ class TreeUnpickler(reader: TastyReader, tastyName: TastyName.Table, posUnpickle
             val tp = readType()
             val lazyAnnotTree = readLater(end, rdr => ctx => rdr.readTerm()(ctx))
             annots += Annotation.deferredSymAndTree(tp.typeSymbol, _ => lazyAnnotTree.complete)
-          case _ =>
-            assert(false, s"illegal modifier tag at $currentAddr")
+          case tag =>
+            assert(false, s"illegal modifier tag $tag at $currentAddr, end = $end")
         }
       }
       (flags, annots.toList, privateWithin)

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -120,15 +120,20 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit
         }
       else false
 
-    /** A symbol qualifies if it really exists. In addition,
-     *  if we are in a constructor of a pattern, we ignore all definitions
+    /** A symbol qualifies if it really exists and is not a package class.
+     *  In addition, if we are in a constructor of a pattern, we ignore all definitions
      *  which are methods and not accessors (note: if we don't do that
      *  case x :: xs in class List would return the :: method).
+     *
+     *  Package classes are part of their parent's scope, because otherwise
+     *  we could not reload them via `_.member`. On the other hand, accessing a
+     *  package as a type from source is always an error.
      */
     def qualifies(denot: Denotation): Boolean =
-      reallyExists(denot) && !(
-        pt.isInstanceOf[UnapplySelectionProto] &&
-          (denot.symbol is (Method, butNot = Accessor)))
+      reallyExists(denot) &&
+        !(pt.isInstanceOf[UnapplySelectionProto] &&
+          (denot.symbol is (Method, butNot = Accessor))) &&
+        !(denot.symbol is PackageClass)
 
     /** Find the denotation of enclosing `name` in given context `ctx`.
      *  @param previous    A denotation that was found in a more deeply nested scope,

compiler/test/dotc/tests.scala

@@ -1,5 +1,6 @@
 package dotc
 
+import dotty.Jars
 import dotty.tools.dotc.CompilerTest
 import org.junit.{Before, Test}
 
@@ -32,18 +33,28 @@ class tests extends CompilerTest {
   )
 
   val classPath = {
-    val paths = List(
-      "../library/target/scala-2.11/dotty-library_2.11-0.1-SNAPSHOT.jar",
-      "./target/scala-2.11/dotty-compiler_2.11-0.1-SNAPSHOT.jar",
-      "../interfaces/target/dotty-interfaces-0.1-SNAPSHOT.jar"
-    ).map { p =>
+    val paths = Jars.dottyTestDeps map { p =>
       val file = new JFile(p)
       assert(
         file.exists,
-        s"""File "$p" couldn't be found. Run `packageAll` from build tool before testing"""
+        s"""|File "$p" couldn't be found. Run `packageAll` from build tool before
+            |testing.
+            |
+            |If running without sbt, test paths need to be setup environment variables:
+            |
+            | - DOTTY_LIBRARY
+            | - DOTTY_COMPILER
+            | - DOTTY_INTERFACES
+            | - DOTTY_EXTRAS
+            |
+            |Where these all contain locations, except extras which is a colon
+            |separated list of jars.
+            |
+            |When compiling with eclipse, you need the sbt-interfaces jar, put
+            |it in extras."""
       )
       file.getAbsolutePath
-    }.mkString(":")
+    } mkString (":")
 
     List("-classpath", paths)
   }
@@ -342,6 +353,8 @@ class tests extends CompilerTest {
     // first compile dotty
     compileDir(dottyDir, ".", List("-deep", "-Ycheck-reentrant", "-strict"))(allowDeepSubtypes)
 
+    compileDir(libDir, "dotty", "-deep" :: opt)
+    compileDir(libDir, "scala", "-deep" :: opt)
     compileDir(dottyDir, "tools", opt)
     compileDir(toolsDir, "dotc", opt)
     compileDir(dotcDir, "ast", opt)

compiler/test/dotty/Jars.scala

@@ -0,0 +1,24 @@
+package dotty
+
+/** Jars used when compiling test, defaults to sbt locations */
+object Jars {
+  val dottyLib: String = sys.env.get("DOTTY_LIB") getOrElse {
+    "../library/target/scala-2.11/dotty-library_2.11-0.1-SNAPSHOT.jar"
+  }
+
+  val dottyCompiler: String = sys.env.get("DOTTY_COMPILER") getOrElse {
+    "./target/scala-2.11/dotty-compiler_2.11-0.1-SNAPSHOT.jar"
+  }
+
+  val dottyInterfaces: String = sys.env.get("DOTTY_INTERFACE") getOrElse {
+    "../interfaces/target/dotty-interfaces-0.1-SNAPSHOT.jar"
+  }
+
+  val dottyExtras: List[String] = sys.env.get("DOTTY_EXTRAS")
+    .map(_.split(":").toList).getOrElse(Nil)
+
+  val dottyReplDeps: List[String] = dottyLib :: dottyExtras
+
+  val dottyTestDeps: List[String] =
+    dottyLib :: dottyCompiler :: dottyInterfaces :: dottyExtras
+}

compiler/test/dotty/partest/DPConsoleRunner.scala

@@ -88,27 +88,43 @@ extends SuiteRunner(testSourcePath, fileManager, updateCheck, failed, javaCmdPat
     """.stripMargin
   }
 
-  /** Tests which are compiled with one or more of the flags in this list will be run
-   *  one by one, without any other test running at the same time.
-   *  This is necessary because some test flags require a lot of memory when running
-   *  the compiler and may exhaust the available memory when run in parallel with other tests.
-   */
-  def sequentialFlags = List("-Ytest-pickler")
+  /** Some tests require a limitation of resources, tests which are compiled
+    * with one or more of the flags in this list will be run with
+    * `limitedThreads`. This is necessary because some test flags require a lot
+    * of memory when running the compiler and may exhaust the available memory
+    * when run in parallel with too many other tests.
+    *
+    * This number could be increased on the CI, but might fail locally if
+    * scaled too extreme - override with:
+    *
+    * ```
+    * -Ddotty.tests.limitedThreads=X
+    * ```
+    */
+  def limitResourceFlags = List("-Ytest-pickler")
+  private val limitedThreads = sys.props.get("dotty.tests.limitedThreads").getOrElse("2")
 
   override def runTestsForFiles(kindFiles: Array[File], kind: String): Array[TestState] = {
-    val (sequentialTests, parallelTests) =
+    val (limitResourceTests, parallelTests) =
       kindFiles partition { kindFile =>
         val flags = kindFile.changeExtension("flags").fileContents
-        sequentialFlags.exists(seqFlag => flags.contains(seqFlag))
+        limitResourceFlags.exists(seqFlag => flags.contains(seqFlag))
       }
 
     val seqResults =
-      if (!sequentialTests.isEmpty) {
+      if (!limitResourceTests.isEmpty) {
         val savedThreads = sys.props("partest.threads")
-        sys.props("partest.threads") = "2"
+        sys.props("partest.threads") = {
+          assert(
+            savedThreads == null || limitedThreads.toInt <= savedThreads.toInt,
+            """|Should not use more threads than the default, when the point
+               |is to limit the amount of resources""".stripMargin
+          )
+          limitedThreads
+        }
 
-        NestUI.echo(s"## we will run ${sequentialTests.length} tests using ${PartestDefaults.numThreads} thread(s)")
-        val res = super.runTestsForFiles(sequentialTests, kind)
+        NestUI.echo(s"## we will run ${limitResourceTests.length} tests using ${PartestDefaults.numThreads} thread(s) in parallel")
+        val res = super.runTestsForFiles(limitResourceTests, kind)
 
         if (savedThreads != null)
           sys.props("partest.threads") = savedThreads
@@ -383,13 +399,9 @@ class DPTestRunner(testFile: File, suiteRunner: DPSuiteRunner) extends nest.Runn
     suiteRunner.fileManager.asInstanceOf[DottyFileManager].extraJarList ::: super.extraClasspath
 
   // override to keep class files if failed and delete clog if ok
-  override def cleanup = if (lastState.isOk) try {
+  override def cleanup = if (lastState.isOk) {
     logFile.delete
     cLogFile.delete
     Directory(outDir).deleteRecursively
-  } catch {
-    case t: Throwable =>
-      println("whhhhhhhhhhhhhhhhhhhhhhhhhhhaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaat")
-      throw t
   }
 }

compiler/test/dotty/tools/DottyTest.scala

@@ -23,10 +23,7 @@ class DottyTest extends ContextEscapeDetection{
     import base.settings._
     val ctx = base.initialCtx.fresh
     ctx.setSetting(ctx.settings.encoding, "UTF8")
-    ctx.setSetting(
-      ctx.settings.classpath,
-      "../library/target/scala-2.11/dotty-library_2.11-0.1-SNAPSHOT.jar"
-    )
+    ctx.setSetting(ctx.settings.classpath, Jars.dottyLib)
     // when classpath is changed in ctx, we need to re-initialize to get the
     // correct classpath from PathResolver
     base.initialize()(ctx)

compiler/test/dotty/tools/ShowClassTests.scala

@@ -1,4 +1,5 @@
-package dotty.tools
+package dotty
+package tools
 
 import dotc.core._
 import dotc.core.Contexts._
@@ -18,8 +19,7 @@ class ShowClassTests extends DottyTest {
     ctx.setSetting(ctx.settings.encoding, "UTF8")
     ctx.setSetting(
       ctx.settings.classpath,
-      "../library/target/scala-2.11/dotty-library_2.11-0.1-SNAPSHOT.jar" +
-      ":../interfaces/target/dotty-interfaces-0.1-SNAPSHOT.jar"
+      Jars.dottyLib + ":" + Jars.dottyInterfaces
     )
     base.initialize()(ctx)
     ctx

compiler/test/dotty/tools/dotc/CompilerTest.scala

@@ -264,13 +264,14 @@ abstract class CompilerTest {
   private def compileArgs(args: Array[String], expectedErrorsPerFile: List[ErrorsInFile])
       (implicit defaultOptions: List[String]): Unit = {
     val allArgs = args ++ defaultOptions
+    val verbose = allArgs.contains("-verbose")
     //println(s"""all args: ${allArgs.mkString("\n")}""")
     val processor = if (allArgs.exists(_.startsWith("#"))) Bench else Main
     val storeReporter = new Reporter with UniqueMessagePositions with HideNonSensicalMessages {
       private val consoleReporter = new ConsoleReporter()
       private val innerStoreReporter = new StoreReporter(consoleReporter)
       def doReport(m: MessageContainer)(implicit ctx: Context): Unit = {
-        if (m.level == ERROR) {
+        if (m.level == ERROR || verbose) {
           innerStoreReporter.flush()
           consoleReporter.doReport(m)
         }

compiler/test/dotty/tools/dotc/EntryPointsTest.scala

@@ -1,4 +1,5 @@
-package dotty.tools
+package dotty
+package tools
 package dotc
 
 import org.junit.Test
@@ -19,14 +20,7 @@ import scala.collection.mutable.ListBuffer
 class EntryPointsTest {
   private val sources =
     List("../tests/pos/HelloWorld.scala").map(p => new java.io.File(p).getPath())
-  private val dottyInterfaces =
-    new java.io.File("../interfaces/dotty-interfaces-0.1-SNAPSHOT.jar").getPath
-  private val dottyLibrary =
-    new java.io.File("../library/target/scala-2.11/dotty-library_2.11-0.1-SNAPSHOT.jar").getPath
-  private val args =
-    sources ++
-    List("-d", "../out/") ++
-    List("-classpath", dottyInterfaces + ":" + dottyLibrary)
+  private val args = sources ++ List("-d", "../out/", "-usejavacp")
 
   @Test def runCompiler = {
     val reporter = new CustomReporter