Better utilizing GADT bounds for HKTs

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -172,7 +172,7 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
 
   private inline def inFrozenGadtIf[T](cond: Boolean)(inline op: T): T = {
     val savedFrozenGadt = frozenGadt
-    frozenGadt = cond
+    frozenGadt ||= cond
     try op finally frozenGadt = savedFrozenGadt
   }
 
@@ -1167,13 +1167,25 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
         else
           fallback(tycon2bounds.lo)
 
+      def byGadtBounds: Boolean =
+        {
+          tycon2 match
+            case tycon2: TypeRef =>
+              val tycon2sym = tycon2.symbol
+              tycon2sym.onGadtBounds { bounds2 =>
+                inFrozenGadt { compareLower(bounds2, tyconIsTypeRef = false) }
+              }
+            case _ => false
+        } && { GADTused = true; true }
+
       tycon2 match {
         case param2: TypeParamRef =>
           isMatchingApply(tp1) ||
           canConstrain(param2) && canInstantiate(param2) ||
           compareLower(bounds(param2), tyconIsTypeRef = false)
         case tycon2: TypeRef =>
           isMatchingApply(tp1) ||
+          byGadtBounds ||
           defn.isCompiletimeAppliedType(tycon2.symbol) && compareCompiletimeAppliedType(tp2, tp1, fromBelow = true) || {
             tycon2.info match {
               case info2: TypeBounds =>
@@ -1213,11 +1225,18 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
             isSubType(bounds(param1).hi.applyIfParameterized(args1), tp2, approx.addLow)
         case tycon1: TypeRef =>
           val sym = tycon1.symbol
+
+          def byGadtBounds: Boolean =
+            sym.onGadtBounds { bounds1 =>
+              inFrozenGadt { isSubType(bounds1.hi.applyIfParameterized(args1), tp2, approx.addLow) }
+            } && { GADTused = true; true }
+
+
           !sym.isClass && {
             defn.isCompiletimeAppliedType(sym) && compareCompiletimeAppliedType(tp1, tp2, fromBelow = false) ||
             recur(tp1.superType, tp2) ||
             tryLiftedToThis1
-          }
+          }|| byGadtBounds
         case tycon1: TypeProxy =>
           recur(tp1.superType, tp2)
         case _ =>

tests/pos/gadt-hkt-hi-bounds.scala

@@ -0,0 +1,8 @@
+type Const = [X] =>> Int
+
+trait Expr[-F[_]]
+case class ConstExpr() extends Expr[Const]
+
+def foo[F[_], A](e: Expr[F]) = e match
+  case _: ConstExpr =>
+    val i: Int = ??? : F[A]

tests/pos/gadt-hkt-lo-bounds.scala

@@ -0,0 +1,7 @@
+type Const = [X] =>> Int
+
+trait Expr[+F[_]]
+case class ConstExpr() extends Expr[Const]
+
+def foo[F[_], A](e: Expr[F]): F[A] = e match
+  case _: ConstExpr => 0