Further refactoring of stackoverflow handling

This now handles all errors from #4368 to #4372 and also #318.

Author: odersky

compiler/src/dotty/tools/dotc/core/TypeApplications.scala

@@ -168,7 +168,7 @@ class TypeApplications(val self: Type) extends AnyVal {
    *  any type parameter that is-rebound by the refinement.
    */
   final def typeParams(implicit ctx: Context): List[TypeParamInfo] = /*>|>*/ track("typeParams") /*<|<*/ {
-    self match {
+    try self match {
       case self: TypeRef =>
         val tsym = self.symbol
         if (tsym.isClass) tsym.typeParams
@@ -193,6 +193,9 @@ class TypeApplications(val self: Type) extends AnyVal {
       case _ =>
         Nil
     }
+    catch {
+      case ex: Throwable => handleRecursive("type parameters of", self.show, ex)
+    }
   }
 
   /** If `self` is a higher-kinded type, its type parameters, otherwise Nil */

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -121,7 +121,11 @@ class TypeComparer(initctx: Context) extends DotClass with ConstraintHandling {
   protected def isSubType(tp1: Type, tp2: Type, a: ApproxState): Boolean = {
     val saved = approx
     this.approx = a
-    try recur(tp1, tp2) finally this.approx = saved
+    try recur(tp1, tp2)
+    catch {
+      case ex: Throwable => handleRecursive("subtype", i"$tp1 <:< $tp2", ex, weight = 2)
+    }
+    finally this.approx = saved
   }
 
   protected def isSubType(tp1: Type, tp2: Type): Boolean = isSubType(tp1, tp2, NoApprox)
@@ -161,7 +165,8 @@ class TypeComparer(initctx: Context) extends DotClass with ConstraintHandling {
         try {
           pendingSubTypes += p
           firstTry
-        } finally {
+        }
+        finally {
           pendingSubTypes -= p
         }
       }

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -652,7 +652,15 @@ object Types {
       catch {
         case ex: Throwable =>
           core.println(s"findMember exception for $this member $name, pre = $pre, recCount = $recCount")
-          throw new RecursionOverflow("find-member", pre, name, ex)
+
+          def showPrefixSafely(pre: Type)(implicit ctx: Context): String = pre.stripTypeVar match {
+            case pre: TermRef => i"${pre.termSymbol.name}."
+            case pre: TypeRef => i"${pre.typeSymbol.name}#"
+            case pre: TypeProxy => showPrefixSafely(pre.underlying)
+            case _ => if (pre.typeSymbol.exists) i"${pre.typeSymbol.name}#" else "."
+          }
+
+          handleRecursive("find-member", i"${showPrefixSafely(pre)}$name", ex)
       }
       finally {
         if (recCount >= Config.LogPendingFindMemberThreshold)

compiler/src/dotty/tools/dotc/typer/VarianceChecker.scala

@@ -82,21 +82,25 @@ class VarianceChecker()(implicit ctx: Context) {
      *  same is true of the parameters (ValDefs).
      */
     def apply(status: Option[VarianceError], tp: Type): Option[VarianceError] = trace(s"variance checking $tp of $base at $variance", variances) {
-      if (status.isDefined) status
-      else tp match {
-        case tp: TypeRef =>
-          val sym = tp.symbol
-          if (sym.variance != 0 && base.isContainedIn(sym.owner)) checkVarianceOfSymbol(sym)
-          else if (sym.isAliasType) this(status, sym.info.bounds.hi)
-          else foldOver(status, tp)
-        case tp: MethodOrPoly =>
-          this(status, tp.resultType) // params will be checked in their TypeDef or ValDef nodes.
-        case AnnotatedType(_, annot) if annot.symbol == defn.UncheckedVarianceAnnot =>
-          status
-        //case tp: ClassInfo =>
-        //  ???  not clear what to do here yet. presumably, it's all checked at local typedefs
-        case _ =>
-          foldOver(status, tp)
+      try
+        if (status.isDefined) status
+        else tp match {
+          case tp: TypeRef =>
+            val sym = tp.symbol
+            if (sym.variance != 0 && base.isContainedIn(sym.owner)) checkVarianceOfSymbol(sym)
+            else if (sym.isAliasType) this(status, sym.info.bounds.hi)
+            else foldOver(status, tp)
+          case tp: MethodOrPoly =>
+            this(status, tp.resultType) // params will be checked in their TypeDef or ValDef nodes.
+          case AnnotatedType(_, annot) if annot.symbol == defn.UncheckedVarianceAnnot =>
+            status
+          //case tp: ClassInfo =>
+          //  ???  not clear what to do here yet. presumably, it's all checked at local typedefs
+          case _ =>
+            foldOver(status, tp)
+        }
+      catch {
+        case ex: Throwable => handleRecursive("variance check of", tp.show, ex)
       }
     }
 

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -186,6 +186,7 @@ class CompilationTests extends ParallelTesting {
     compileFile("tests/neg-custom-args/noimports.scala", defaultOptions.and("-Yno-imports")) +
     compileFile("tests/neg-custom-args/noimports2.scala", defaultOptions.and("-Yno-imports")) +
     compileFile("tests/neg-custom-args/i3882.scala", allowDeepSubtypes) +
+    compileFile("tests/neg-custom-args/i4372.scala", allowDeepSubtypes) +
     compileFile("tests/neg-custom-args/i1754.scala", allowDeepSubtypes) +
     compileFilesInDir("tests/neg-custom-args/isInstanceOf", allowDeepSubtypes and "-Xfatal-warnings") +
     compileFile("tests/neg-custom-args/i3627.scala", allowDeepSubtypes)

tests/neg-custom-args/i4372.scala

@@ -0,0 +1,3 @@
+object i4372 {
+  class X[A >: X[_ <: X[_]] <: X[A]]
+}

tests/neg/i4368.scala

@@ -106,18 +106,16 @@ object Test7 {
     type Nat = Fix[Option]#T
   }
 }
-/*
 object Test8 {
 
   class A {
-    type T = B#U
+    type T = B#U  // error: cyclic
   }
 
   class B {
     type U = A#T
   }
 }
-*/
 object Test9 {
   trait W {
     type A
@@ -147,4 +145,29 @@ object Test9 {
       // scalac reports a volatility error, but the dotty equivalent (checkRealizable)
       // is checked too late.
   }
+}
+object i4369 {
+  trait X { self =>
+    type R <: Z
+    type Z >: X { type R = self.R; type Z = self.R }
+  }
+  class Foo extends X { type R = Foo; type Z = Foo } // error: too deep
+}
+object i4370 {
+  class Foo { type R = A } // error: cyclic
+  type A = List[Foo#R]
+}
+object i4371 {
+  class Foo { type A = Boo#B } // error: cyclic
+  class Boo { type B = Foo#A }
+}
+object i318 {
+  trait Y {
+    type A <: { type T >: B }
+    type B >: { type T >: A }
+  }
+
+  val y: Y = ???
+  val a: y.A = ??? // error: too deep
+  val b: y.B = a   // error: too deep
 }