Reland: [cfe] Embed and use allowed experiments in CFE

This CL embeds the sdk_nnbd/lib/_internal/allowed_experiments.json
into the CFE and uses this to allow experiments enabled on a
per library basis. The file is embedded through generated code to
avoid reliance on access to the file itself.

A presubmit check is also added to ensure that the json file and
the generated code are in sync.

This work is in preparation for #41538

This was reverted in https://dart-review.googlesource.com/c/sdk/+/149620
because flutter wasn't prepared for auto-enabling nnbd in dart:* libraries.
This has now been fixed in flutter/engine#18714

Closes #42162

Change-Id: I686ed6feaef8ee066b426068fe2a0f5080bf1713
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/149588
Reviewed-by: Jens Johansen <[email protected]>
Commit-Queue: Johnni Winther <[email protected]>

Author: johnniwinther

pkg/_fe_analyzer_shared/lib/src/sdk/allowed_experiments.dart

@@ -27,7 +27,7 @@ class AllowedExperiments {
   /// are enabled for all files of this package.
   final Map<String, List<String>> packageExperiments;
 
-  const AllowedExperiments({
+  AllowedExperiments({
     @required this.sdkDefaultExperiments,
     @required this.sdkLibraryExperiments,
     @required this.packageExperiments,

pkg/front_end/lib/src/api_prototype/compiler_options.dart

@@ -16,6 +16,7 @@ import '../base/nnbd_mode.dart';
 
 import 'experimental_flags.dart'
     show
+        AllowedExperimentalFlags,
         defaultExperimentalFlags,
         ExperimentalFlag,
         expiredExperimentalFlags,
@@ -129,6 +130,8 @@ class CompilerOptions {
   /// Features not mentioned in the map will have their default value.
   Map<ExperimentalFlag, bool> experimentalFlags = <ExperimentalFlag, bool>{};
 
+  AllowedExperimentalFlags allowedExperimentalFlags;
+
   /// Environment map used when evaluating `bool.fromEnvironment`,
   /// `int.fromEnvironment` and `String.fromEnvironment` during constant
   /// evaluation. If the map is `null`, all environment constants will be left

pkg/front_end/lib/src/api_prototype/experimental_flags.dart

@@ -1,86 +1,117 @@
-// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-// NOTE: THIS FILE IS GENERATED. DO NOT EDIT.
-//
-// Instead modify 'tools/experimental_features.yaml' and run
-// 'dart pkg/front_end/tool/fasta.dart generate-experimental-flags' to update.
-
-import 'package:_fe_analyzer_shared/src/sdk/allowed_experiments.dart';
 import 'package:kernel/kernel.dart' show Version;
 
-enum ExperimentalFlag {
-  constantUpdate2018,
-  controlFlowCollections,
-  extensionMethods,
-  nonNullable,
-  nonfunctionTypeAliases,
-  setLiterals,
-  spreadCollections,
-  tripleShift,
-  variance,
-}
+part 'experimental_flags_generated.dart';
+
+/// The set of experiments enabled for SDK and packages.
+///
+/// This are derived from an `allowed_experiments.json` file whose default is
+/// located in `sdk_nnbd/lib/_internal/allowed_experiments.json`.
+class AllowedExperimentalFlags {
+  /// The set of experiments that are enabled for all SDK libraries other than
+  /// for those which are specified in [sdkLibraryExperiments].
+  final Set<ExperimentalFlag> sdkDefaultExperiments;
+
+  /// Mapping from individual SDK libraries, e.g. 'core', to the set of
+  /// experiments that are enabled for this library.
+  final Map<String, Set<ExperimentalFlag>> sdkLibraryExperiments;
 
-const Version enableConstantUpdate2018Version = const Version(2, 4);
-const Version enableControlFlowCollectionsVersion = const Version(2, 2);
-const Version enableExtensionMethodsVersion = const Version(2, 6);
-const Version enableNonNullableVersion = const Version(2, 9);
-const Version enableNonfunctionTypeAliasesVersion = const Version(2, 9);
-const Version enableSetLiteralsVersion = const Version(2, 2);
-const Version enableSpreadCollectionsVersion = const Version(2, 2);
-const Version enableTripleShiftVersion = const Version(2, 9);
-const Version enableVarianceVersion = const Version(2, 9);
+  /// Mapping from package names, e.g. 'path', to the set of experiments that
+  /// are enabled for all files of this package.
+  final Map<String, Set<ExperimentalFlag>> packageExperiments;
 
-ExperimentalFlag parseExperimentalFlag(String flag) {
-  switch (flag) {
-    case "constant-update-2018":
-      return ExperimentalFlag.constantUpdate2018;
-    case "control-flow-collections":
-      return ExperimentalFlag.controlFlowCollections;
-    case "extension-methods":
-      return ExperimentalFlag.extensionMethods;
-    case "non-nullable":
-      return ExperimentalFlag.nonNullable;
-    case "nonfunction-type-aliases":
-      return ExperimentalFlag.nonfunctionTypeAliases;
-    case "set-literals":
-      return ExperimentalFlag.setLiterals;
-    case "spread-collections":
-      return ExperimentalFlag.spreadCollections;
-    case "triple-shift":
-      return ExperimentalFlag.tripleShift;
-    case "variance":
-      return ExperimentalFlag.variance;
+  const AllowedExperimentalFlags({
+    this.sdkDefaultExperiments: const {},
+    this.sdkLibraryExperiments: const {},
+    this.packageExperiments: const {},
+  });
+
+  /// Return the set of enabled experiments for the package with the [name],
+  /// e.g. "path", possibly `null`.
+  Set<ExperimentalFlag> forPackage(String name) {
+    return packageExperiments[name];
   }
-  return null;
-}
 
-const Map<ExperimentalFlag, bool> defaultExperimentalFlags = {
-  ExperimentalFlag.constantUpdate2018: true,
-  ExperimentalFlag.controlFlowCollections: true,
-  ExperimentalFlag.extensionMethods: true,
-  ExperimentalFlag.nonNullable: false,
-  ExperimentalFlag.nonfunctionTypeAliases: false,
-  ExperimentalFlag.setLiterals: true,
-  ExperimentalFlag.spreadCollections: true,
-  ExperimentalFlag.tripleShift: false,
-  ExperimentalFlag.variance: false,
-};
+  /// Return the set of enabled experiments for the library with the [name],
+  /// e.g. "core".
+  Set<ExperimentalFlag> forSdkLibrary(String name) {
+    return sdkLibraryExperiments[name] ?? sdkDefaultExperiments;
+  }
+}
 
-const Map<ExperimentalFlag, bool> expiredExperimentalFlags = {
-  ExperimentalFlag.constantUpdate2018: true,
-  ExperimentalFlag.controlFlowCollections: true,
-  ExperimentalFlag.extensionMethods: false,
-  ExperimentalFlag.nonNullable: false,
-  ExperimentalFlag.nonfunctionTypeAliases: false,
-  ExperimentalFlag.setLiterals: true,
-  ExperimentalFlag.spreadCollections: true,
-  ExperimentalFlag.tripleShift: false,
-  ExperimentalFlag.variance: false,
-};
+/// Returns `true` if [flag] is enabled using global [experimentalFlags].
+///
+/// If [experimentalFlags] is `null` or doesn't contain [flag], the default
+/// value from [defaultExperimentalFlags] is returned.
+///
+/// If [flag] is marked as expired in [expiredExperimentalFlags], the value from
+/// [defaultExperimentalFlags] is always returned.
+bool isExperimentEnabled(ExperimentalFlag flag,
+    {Map<ExperimentalFlag, bool> experimentalFlags}) {
+  assert(defaultExperimentalFlags.containsKey(flag),
+      "No default value for $flag.");
+  assert(expiredExperimentalFlags.containsKey(flag),
+      "No expired value for $flag.");
+  if (expiredExperimentalFlags[flag]) {
+    return defaultExperimentalFlags[flag];
+  }
+  bool enabled;
+  if (experimentalFlags != null) {
+    enabled = experimentalFlags[flag];
+  }
+  enabled ??= defaultExperimentalFlags[flag];
+  return enabled;
+}
 
-const AllowedExperiments allowedExperiments =
-    const AllowedExperiments(sdkDefaultExperiments: [
-  "non-nullable",
-], sdkLibraryExperiments: {}, packageExperiments: {});
+/// Returns `true` if [flag] is enabled in the library with the [canonicalUri]
+/// either globally using [experimentalFlags] or per library using
+/// [allowedExperimentalFlags].
+///
+/// If [experimentalFlags] is `null` or doesn't contain [flag], the default
+/// value from [defaultExperimentalFlags] used as the global flag state.
+///
+/// If [allowedExperimentalFlags] is `null` [defaultAllowedExperimentalFlags] is
+/// used for the per library flag state.
+///
+/// If [flag] is marked as expired in [expiredExperimentalFlags], the value from
+/// [defaultExperimentalFlags] is always returned.
+///
+/// The canonical uri, also known as the import uri, is the absolute uri that
+/// defines the identity of a library, for instance `dart:core`, `package:foo`,
+/// or `file:///path/dir/file.dart`.
+bool isExperimentEnabledInLibrary(ExperimentalFlag flag, Uri canonicalUri,
+    {Map<ExperimentalFlag, bool> experimentalFlags,
+    AllowedExperimentalFlags allowedExperimentalFlags}) {
+  assert(defaultExperimentalFlags.containsKey(flag),
+      "No default value for $flag.");
+  assert(expiredExperimentalFlags.containsKey(flag),
+      "No expired value for $flag.");
+  if (expiredExperimentalFlags[flag]) {
+    return defaultExperimentalFlags[flag];
+  }
+  bool enabled;
+  if (experimentalFlags != null) {
+    enabled = experimentalFlags[flag];
+  }
+  enabled ??= defaultExperimentalFlags[flag];
+  if (!enabled) {
+    allowedExperimentalFlags ??= defaultAllowedExperimentalFlags;
+    Set<ExperimentalFlag> allowedFlags;
+    if (canonicalUri.scheme == 'dart') {
+      allowedFlags = allowedExperimentalFlags.forSdkLibrary(canonicalUri.path);
+    } else if (canonicalUri.scheme == 'package') {
+      int index = canonicalUri.path.indexOf('/');
+      if (index >= 0) {
+        String packageName = canonicalUri.path.substring(0, index);
+        allowedFlags = allowedExperimentalFlags.forPackage(packageName);
+      }
+    }
+    if (allowedFlags != null) {
+      enabled = allowedFlags.contains(flag);
+    }
+  }
+  return enabled;
+}

pkg/front_end/lib/src/api_prototype/experimental_flags_generated.dart

@@ -0,0 +1,85 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// NOTE: THIS FILE IS GENERATED. DO NOT EDIT.
+//
+// Instead modify 'tools/experimental_features.yaml' and run
+// 'dart pkg/front_end/tool/fasta.dart generate-experimental-flags' to update.
+
+part of 'experimental_flags.dart';
+
+enum ExperimentalFlag {
+  constantUpdate2018,
+  controlFlowCollections,
+  extensionMethods,
+  nonNullable,
+  nonfunctionTypeAliases,
+  setLiterals,
+  spreadCollections,
+  tripleShift,
+  variance,
+}
+
+const Version enableConstantUpdate2018Version = const Version(2, 4);
+const Version enableControlFlowCollectionsVersion = const Version(2, 2);
+const Version enableExtensionMethodsVersion = const Version(2, 6);
+const Version enableNonNullableVersion = const Version(2, 9);
+const Version enableNonfunctionTypeAliasesVersion = const Version(2, 9);
+const Version enableSetLiteralsVersion = const Version(2, 2);
+const Version enableSpreadCollectionsVersion = const Version(2, 2);
+const Version enableTripleShiftVersion = const Version(2, 9);
+const Version enableVarianceVersion = const Version(2, 9);
+
+ExperimentalFlag parseExperimentalFlag(String flag) {
+  switch (flag) {
+    case "constant-update-2018":
+      return ExperimentalFlag.constantUpdate2018;
+    case "control-flow-collections":
+      return ExperimentalFlag.controlFlowCollections;
+    case "extension-methods":
+      return ExperimentalFlag.extensionMethods;
+    case "non-nullable":
+      return ExperimentalFlag.nonNullable;
+    case "nonfunction-type-aliases":
+      return ExperimentalFlag.nonfunctionTypeAliases;
+    case "set-literals":
+      return ExperimentalFlag.setLiterals;
+    case "spread-collections":
+      return ExperimentalFlag.spreadCollections;
+    case "triple-shift":
+      return ExperimentalFlag.tripleShift;
+    case "variance":
+      return ExperimentalFlag.variance;
+  }
+  return null;
+}
+
+const Map<ExperimentalFlag, bool> defaultExperimentalFlags = {
+  ExperimentalFlag.constantUpdate2018: true,
+  ExperimentalFlag.controlFlowCollections: true,
+  ExperimentalFlag.extensionMethods: true,
+  ExperimentalFlag.nonNullable: false,
+  ExperimentalFlag.nonfunctionTypeAliases: false,
+  ExperimentalFlag.setLiterals: true,
+  ExperimentalFlag.spreadCollections: true,
+  ExperimentalFlag.tripleShift: false,
+  ExperimentalFlag.variance: false,
+};
+
+const Map<ExperimentalFlag, bool> expiredExperimentalFlags = {
+  ExperimentalFlag.constantUpdate2018: true,
+  ExperimentalFlag.controlFlowCollections: true,
+  ExperimentalFlag.extensionMethods: false,
+  ExperimentalFlag.nonNullable: false,
+  ExperimentalFlag.nonfunctionTypeAliases: false,
+  ExperimentalFlag.setLiterals: true,
+  ExperimentalFlag.spreadCollections: true,
+  ExperimentalFlag.tripleShift: false,
+  ExperimentalFlag.variance: false,
+};
+
+const AllowedExperimentalFlags defaultAllowedExperimentalFlags =
+    const AllowedExperimentalFlags(sdkDefaultExperiments: {
+  ExperimentalFlag.nonNullable,
+}, sdkLibraryExperiments: {}, packageExperiments: {});

pkg/front_end/lib/src/base/processed_options.dart

@@ -22,8 +22,7 @@ import 'package:package_config/package_config.dart';
 import '../api_prototype/compiler_options.dart'
     show CompilerOptions, DiagnosticMessage;
 
-import '../api_prototype/experimental_flags.dart'
-    show defaultExperimentalFlags, ExperimentalFlag, expiredExperimentalFlags;
+import '../api_prototype/experimental_flags.dart' as flags;
 
 import '../api_prototype/file_system.dart'
     show FileSystem, FileSystemEntity, FileSystemException;
@@ -325,15 +324,27 @@ class ProcessedOptions {
   Target get target =>
       _target ??= _raw.target ?? new NoneTarget(new TargetFlags());
 
-  bool isExperimentEnabled(ExperimentalFlag flag) {
-    assert(defaultExperimentalFlags.containsKey(flag),
-        "No default value for $flag.");
-    assert(expiredExperimentalFlags.containsKey(flag),
-        "No expired value for $flag.");
-    if (expiredExperimentalFlags[flag]) {
-      return defaultExperimentalFlags[flag];
-    }
-    return _raw.experimentalFlags[flag] ?? defaultExperimentalFlags[flag];
+  /// Returns `true` if the [flag] is enabled globally.
+  ///
+  /// This is `true` either if the [flag] is passed through an explicit
+  /// `--enable-experiment` option or if the [flag] is expired and on by
+  /// default.
+  bool isExperimentEnabledGlobally(flags.ExperimentalFlag flag) {
+    return flags.isExperimentEnabled(flag,
+        experimentalFlags: _raw.experimentalFlags);
+  }
+
+  /// Returns `true` if the [flag] is enabled in the library with the given
+  /// [importUri].
+  ///
+  /// This is `true` either if the [flag] is enabled globally as defined
+  /// by [isExperimentEnabledGlobally] or is explicitly enabled through
+  /// the 'allowed_experiments.json' file for this library.
+  bool isExperimentEnabledInLibrary(
+      flags.ExperimentalFlag flag, Uri importUri) {
+    return flags.isExperimentEnabledInLibrary(flag, importUri,
+        experimentalFlags: _raw.experimentalFlags,
+        allowedExperimentalFlags: _raw.allowedExperimentalFlags);
   }
 
   /// Get an outline component that summarizes the SDK, if any.

pkg/front_end/lib/src/fasta/kernel/body_builder.dart

@@ -5518,7 +5518,7 @@ class BodyBuilder extends ScopeListener<JumpTarget>
     TypeVariableBuilder variable = typeVariables[index];
     variable.bound = bound?.builder;
     if (variance != null) {
-      if (!libraryBuilder.loader.target.enableVariance) {
+      if (!libraryBuilder.enableVarianceInLibrary) {
         reportVarianceModifierNotEnabled(variance);
       }
       variable.variance = Variance.fromString(variance.lexeme);