Add an explicit nulls mode to Dotty

This commit adds a `-Yexplicit-nulls` experimental flag which, if enabled,
modifies the type system so that reference types are non-nullable.

Nullability is then expressed explicitly via type unions (e.g. `val s: String|Null = null`).

At a high level, the changes to the compiler are:
  * A new type hierarchy for `Null`, which is now a subtype of `Any`
    directly, as opposed to being a subtype of _every_ refernce type.
  * A "translation layer" from Java types to Scala types so that Java
    interop can happen soundly yet conveniently.
  * An implementation of flow-typing so we can support a more natural style
    of programming with explicit nulls.

Author: abeln

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -156,6 +156,7 @@ class ScalaSettings extends Settings.SettingGroup {
 
   // Extremely experimental language features
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting("-Yno-kind-polymorphism", "Enable kind polymorphism (see http://dotty.epfl.ch/docs/reference/kind-polymorphism.html). Potentially unsound.")
+  val YexplicitNulls: Setting[Boolean] = BooleanSetting("-Yexplicit-nulls", "Make reference types non-nullable. Nullable types can be expressed with unions: e.g. String|Null.")
 
   /** Area-specific debug output */
   val YexplainLowlevel: Setting[Boolean] = BooleanSetting("-Yexplain-lowlevel", "When explaining type errors, show types at a lower level.")

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -37,6 +37,8 @@ import xsbti.AnalysisCallback
 import plugins._
 import java.util.concurrent.atomic.AtomicInteger
 
+import dotty.tools.dotc.core.FlowTyper.FlowFacts
+
 object Contexts {
 
   private val (compilerCallbackLoc, store1) = Store.empty.newLocation[CompilerCallback]()
@@ -47,7 +49,8 @@ object Contexts {
   private val (compilationUnitLoc,  store6) = store5.newLocation[CompilationUnit]()
   private val (runLoc,              store7) = store6.newLocation[Run]()
   private val (profilerLoc,         store8) = store7.newLocation[Profiler]()
-  private val initialStore = store8
+  private val (flowFactsLoc,        store9) = store8.newLocation[FlowFacts](FlowTyper.emptyFlowFacts)
+  private val initialStore = store9
 
   /** A context is passed basically everywhere in dotc.
    *  This is convenient but carries the risk of captured contexts in
@@ -143,6 +146,9 @@ object Contexts {
     protected def gadt_=(gadt: GadtConstraint): Unit = _gadt = gadt
     final def gadt: GadtConstraint = _gadt
 
+    /** The terms currently known to be non-null (in spite of their declared type) */
+    def flowFacts: FlowFacts = store(flowFactsLoc)
+
     /** The history of implicit searches that are currently active */
     private[this] var _searchHistory: SearchHistory = null
     protected def searchHistory_= (searchHistory: SearchHistory): Unit = _searchHistory = searchHistory
@@ -420,6 +426,9 @@ object Contexts {
     def useColors: Boolean =
       base.settings.color.value == "always"
 
+    /** Is the explicit nulls option set? */
+    def explicitNulls: Boolean = base.settings.YexplicitNulls.value
+
     protected def init(outer: Context, origin: Context): this.type = {
       util.Stats.record("Context.fresh")
       _outer = outer
@@ -536,6 +545,10 @@ object Contexts {
     def setImportInfo(importInfo: ImportInfo): this.type = { this.importInfo = importInfo; this }
     def setGadt(gadt: GadtConstraint): this.type = { this.gadt = gadt; this }
     def setFreshGADTBounds: this.type = setGadt(gadt.fresh)
+    def addFlowFacts(facts: FlowFacts): this.type = {
+      assert(settings.YexplicitNulls.value)
+      updateStore(flowFactsLoc, store(flowFactsLoc) ++ facts)
+    }
     def setSearchHistory(searchHistory: SearchHistory): this.type = { this.searchHistory = searchHistory; this }
     def setSource(source: SourceFile): this.type = { this.source = source; this }
     def setTypeComparerFn(tcfn: Context => TypeComparer): this.type = { this.typeComparer = tcfn(this); this }

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -4,7 +4,7 @@ package core
 
 import scala.annotation.threadUnsafe
 import Types._, Contexts._, Symbols._, SymDenotations._, StdNames._, Names._
-import Flags._, Scopes._, Decorators._, NameOps._, Periods._
+import Flags._, Scopes._, Decorators._, NameOps._, Periods._, NullOpsDecorator._
 import unpickleScala2.Scala2Unpickler.ensureConstructor
 import scala.collection.mutable
 import collection.mutable
@@ -293,7 +293,7 @@ class Definitions {
   @threadUnsafe lazy val ObjectClass: ClassSymbol = {
     val cls = ctx.requiredClass("java.lang.Object")
     assert(!cls.isCompleted, "race for completing java.lang.Object")
-    cls.info = ClassInfo(cls.owner.thisType, cls, AnyClass.typeRef :: Nil, newScope)
+    cls.info = ClassInfo(cls.owner.thisType, cls, AnyType :: Nil, newScope)
     cls.setFlag(NoInits)
 
     // The companion object doesn't really exist, `NoType` is the general
@@ -309,8 +309,18 @@ class Definitions {
   @threadUnsafe lazy val AnyRefAlias: TypeSymbol = enterAliasType(tpnme.AnyRef, ObjectType)
   def AnyRefType: TypeRef = AnyRefAlias.typeRef
 
-    @threadUnsafe lazy val Object_eq: TermSymbol = enterMethod(ObjectClass, nme.eq, methOfAnyRef(BooleanType), Final)
-    @threadUnsafe lazy val Object_ne: TermSymbol = enterMethod(ObjectClass, nme.ne, methOfAnyRef(BooleanType), Final)
+    @threadUnsafe lazy val Object_eq: TermSymbol = {
+      // If explicit nulls is enabled, then we want to allow `(x: String).eq(null)`, so we need
+      // to adjust the signature of `eq` accordingly.
+      val tpe = if (ctx.explicitNulls) methOfAnyRefOrNull(BooleanType) else methOfAnyRef(BooleanType)
+      enterMethod(ObjectClass, nme.eq, tpe, Final)
+    }
+    @threadUnsafe lazy val Object_ne: TermSymbol = {
+      // If explicit nulls is enabled, then we want to allow `(x: String).ne(null)`, so we need
+      // to adjust the signature of `ne` accordingly.
+      val tpe = if (ctx.explicitNulls) methOfAnyRefOrNull(BooleanType) else methOfAnyRef(BooleanType)
+      enterMethod(ObjectClass, nme.ne, tpe, Final)
+    }
     @threadUnsafe lazy val Object_synchronized: TermSymbol = enterPolyMethod(ObjectClass, nme.synchronized_, 1,
         pt => MethodType(List(pt.paramRefs(0)), pt.paramRefs(0)), Final)
     @threadUnsafe lazy val Object_clone: TermSymbol = enterMethod(ObjectClass, nme.clone_, MethodType(Nil, ObjectType), Protected)
@@ -344,18 +354,42 @@ class Definitions {
       pt => MethodType(List(FunctionOf(Nil, pt.paramRefs(0))), pt.paramRefs(0)))
 
   /** Method representing a throw */
-  @threadUnsafe lazy val throwMethod: TermSymbol = enterMethod(OpsPackageClass, nme.THROWkw,
-      MethodType(List(ThrowableType), NothingType))
+  @threadUnsafe lazy val throwMethod: TermSymbol = {
+    val argTpe = if (ctx.explicitNulls) OrType(ThrowableType, NullType) else ThrowableType
+    enterMethod(OpsPackageClass, nme.THROWkw, MethodType(List(argTpe), NothingType))
+  }
 
   @threadUnsafe lazy val NothingClass: ClassSymbol = enterCompleteClassSymbol(
     ScalaPackageClass, tpnme.Nothing, AbstractFinal, List(AnyClass.typeRef))
   def NothingType: TypeRef = NothingClass.typeRef
   @threadUnsafe lazy val RuntimeNothingModuleRef: TermRef = ctx.requiredModuleRef("scala.runtime.Nothing")
-  @threadUnsafe lazy val NullClass: ClassSymbol = enterCompleteClassSymbol(
-    ScalaPackageClass, tpnme.Null, AbstractFinal, List(ObjectClass.typeRef))
+
+  @threadUnsafe lazy val NullClass: ClassSymbol = {
+    val parents = List(if (ctx.explicitNulls) AnyType else ObjectType)
+    enterCompleteClassSymbol(ScalaPackageClass, tpnme.Null, AbstractFinal, parents)
+  }
   def NullType: TypeRef = NullClass.typeRef
   @threadUnsafe lazy val RuntimeNullModuleRef: TermRef = ctx.requiredModuleRef("scala.runtime.Null")
 
+  /** An alias for null values that originate in Java code.
+   *  This type gets special treatment in the Typer. Specifically, `JavaNull` can be selected through:
+   *  e.g.
+   *  ```
+   *  // x: String|Null
+   *  x.length // error: `Null` has no `length` field
+   *  // x2: String|JavaNull
+   *  x2.length // allowed by the Typer, but unsound (might throw NPE)
+   *  ```
+   */
+  lazy val JavaNullAlias: TypeSymbol = {
+    assert(ctx.explicitNulls)
+    enterAliasType(tpnme.JavaNull, NullType)
+  }
+  def JavaNullAliasType: TypeRef = {
+    assert(ctx.explicitNulls)
+    JavaNullAlias.typeRef
+  }
+
   @threadUnsafe lazy val ImplicitScrutineeTypeSym =
     newSymbol(ScalaPackageClass, tpnme.IMPLICITkw, EmptyFlags, TypeBounds.empty).entered
   def ImplicitScrutineeTypeRef: TypeRef = ImplicitScrutineeTypeSym.typeRef
@@ -591,12 +625,16 @@ class Definitions {
   @threadUnsafe lazy val BoxedNumberClass: ClassSymbol          = ctx.requiredClass("java.lang.Number")
   @threadUnsafe lazy val ClassCastExceptionClass: ClassSymbol   = ctx.requiredClass("java.lang.ClassCastException")
     @threadUnsafe lazy val ClassCastExceptionClass_stringConstructor: TermSymbol  = ClassCastExceptionClass.info.member(nme.CONSTRUCTOR).suchThat(_.info.firstParamTypes match {
-      case List(pt) => (pt isRef StringClass)
+      case List(pt) =>
+        val pt1 = if (ctx.explicitNulls) pt.stripNull else pt
+        pt1 isRef StringClass
       case _ => false
     }).symbol.asTerm
   @threadUnsafe lazy val ArithmeticExceptionClass: ClassSymbol  = ctx.requiredClass("java.lang.ArithmeticException")
     @threadUnsafe lazy val ArithmeticExceptionClass_stringConstructor: TermSymbol  = ArithmeticExceptionClass.info.member(nme.CONSTRUCTOR).suchThat(_.info.firstParamTypes match {
-      case List(pt) => (pt isRef StringClass)
+      case List(pt) =>
+        val pt1 = if (ctx.explicitNulls) pt.stripNull else pt
+        pt1 isRef StringClass
       case _ => false
     }).symbol.asTerm
 
@@ -967,6 +1005,7 @@ class Definitions {
   def methOfAny(tp: Type): MethodType = MethodType(List(AnyType), tp)
   def methOfAnyVal(tp: Type): MethodType = MethodType(List(AnyValType), tp)
   def methOfAnyRef(tp: Type): MethodType = MethodType(List(ObjectType), tp)
+  def methOfAnyRefOrNull(tp: Type): MethodType = MethodType(List(OrType(ObjectType, NullType)), tp)
 
   // Derived types
 
@@ -1128,10 +1167,23 @@ class Definitions {
       name.length > prefix.length &&
       name.drop(prefix.length).forall(_.isDigit))
 
-  def isBottomClass(cls: Symbol): Boolean =
+  def isBottomClass(cls: Symbol): Boolean = {
+    if (ctx.explicitNulls && !ctx.phase.erasedTypes) cls == NothingClass
+    else isBottomClassAfterErasure(cls)
+  }
+
+  def isBottomClassAfterErasure(cls: Symbol): Boolean = {
     cls == NothingClass || cls == NullClass
-  def isBottomType(tp: Type): Boolean =
+  }
+
+  def isBottomType(tp: Type): Boolean = {
+    if (ctx.explicitNulls && !ctx.phase.erasedTypes) tp.derivesFrom(NothingClass)
+    else isBottomTypeAfterErasure(tp)
+  }
+
+  def isBottomTypeAfterErasure(tp: Type): Boolean = {
     tp.derivesFrom(NothingClass) || tp.derivesFrom(NullClass)
+  }
 
   /** Is a function class.
    *   - FunctionXXL
@@ -1225,9 +1277,12 @@ class Definitions {
     () => ScalaPackageVal.termRef
   )
 
-  val PredefImportFns: List[() => TermRef] = List[() => TermRef](
+  lazy val PredefImportFns: List[() => TermRef] = List[() => TermRef](
     () => ScalaPredefModuleRef,
-    () => DottyPredefModuleRef
+    () => DottyPredefModuleRef,
+    // TODO(abeln): is this in the right place?
+    // And is it ok to import this unconditionally?
+    () => ctx.requiredModuleRef("scala.ExplicitNullsOps")
   )
 
   @threadUnsafe lazy val RootImportFns: List[() => TermRef] =
@@ -1475,18 +1530,22 @@ class Definitions {
   // ----- Initialization ---------------------------------------------------
 
   /** Lists core classes that don't have underlying bytecode, but are synthesized on-the-fly in every reflection universe */
-  @threadUnsafe lazy val syntheticScalaClasses: List[TypeSymbol] = List(
-    AnyClass,
-    AnyRefAlias,
-    AnyKindClass,
-    andType,
-    orType,
-    RepeatedParamClass,
-    ByNameParamClass2x,
-    AnyValClass,
-    NullClass,
-    NothingClass,
-    SingletonClass)
+  @threadUnsafe lazy val syntheticScalaClasses: List[TypeSymbol] = {
+    val synth = List(
+      AnyClass,
+      AnyRefAlias,
+      AnyKindClass,
+      andType,
+      orType,
+      RepeatedParamClass,
+      ByNameParamClass2x,
+      AnyValClass,
+      NullClass,
+      NothingClass,
+      SingletonClass)
+
+    if (ctx.explicitNulls) synth :+ JavaNullAlias else synth
+  }
 
   @threadUnsafe lazy val syntheticCoreClasses: List[Symbol] = syntheticScalaClasses ++ List(
     EmptyPackageVal,

compiler/src/dotty/tools/dotc/core/Flags.scala

@@ -358,9 +358,6 @@ object Flags {
   /** An inferable (`given`) parameter */
   final val Given = commonFlag(29, "given")
 
-  /** Symbol is defined by a Java class */
-  final val JavaDefined: FlagSet = commonFlag(30, "<java>")
-
   /** Symbol is implemented as a Java static */
   final val JavaStatic: FlagSet = commonFlag(31, "<static>")
   final val JavaStaticTerm: FlagSet = JavaStatic.toTermFlags
@@ -397,6 +394,12 @@ object Flags {
   /** Symbol is an enum class or enum case (if used with case) */
   final val Enum: FlagSet = commonFlag(40, "<enum>")
 
+  /** Symbol is defined by a Java class */
+  final val JavaDefined: FlagSet = commonFlag(30, "<java>")
+
+  /** A Java enum value */
+  final val JavaEnumValue: FlagConjunction = allOf(StableRealizable, JavaStatic, JavaDefined, Enum)
+
   /** An export forwarder */
   final val Exported: FlagSet = commonFlag(41, "exported")
 
@@ -488,7 +491,7 @@ object Flags {
   final val FromStartFlags: FlagSet =
     Module | Package | Deferred | Method.toCommonFlags | Case |
     HigherKinded.toCommonFlags | Param | ParamAccessor.toCommonFlags |
-    Scala2ExistentialCommon | MutableOrOpaque | Touched | JavaStatic |
+    Scala2ExistentialCommon | MutableOrOpaque | Touched | JavaStatic | JavaDefined | JavaEnumValue.toCommonFlags |
     CovariantOrOuter | ContravariantOrLabel | CaseAccessor.toCommonFlags |
     Extension.toCommonFlags | NonMember | Implicit | Given | Implied | Permanent | Synthetic |
     SuperAccessorOrScala2x | Inline
@@ -690,9 +693,6 @@ object Flags {
   /** A Java enum trait */
   final val JavaEnumTrait: FlagConjunction = allOf(JavaDefined, Enum)
 
-  /** A Java enum value */
-  final val JavaEnumValue: FlagConjunction = allOf(StableRealizable, JavaStatic, JavaDefined, Enum)
-
   /** An enum value */
   final val EnumValue: FlagConjunction = allOf(StableRealizable, JavaStatic, Enum)