Refactor native asset integration into flutter tools (flutter#158932)

Currently the `NativeAsset` target in flutter tools is responsible for
two things:

* performing the dart build (in the app as well as all transitive pub
dependencies)
* taking output (shared libraries) from this build and copying them
around

This intermingling of responsibilities leads to more complex code and
potentially unnecessary work: If the source code changed (e.g. `.c`
files change) we have to run the dart build again. But doing so may
result in the same shared libraries (e.g. adding comments to the `.c`
code). Currently we're going to copy the shared libraries despite them
having not changed, which then may cause upstream things to be dirtied
(if it's based on timestamp of files) and re-built.

Instead this PR splits this `NativeAsset` into the two orthogonal pieces

* `DartBuild` target that is responsible for the dart build
* `InstallCodeAssets` that is responsible for copying shared libraries
to the right place and producing a `native_assets.yaml`.

This decoupling is also preparation for a future where a dart build can
produce other kinds of assets (e.g. data assets) and is used in the web
build as well. (The web build would use `DartBuild` but not
`InstalCodeAssets`).

Author: mkustermann

packages/flutter_tools/lib/src/build_system/targets/common.dart

@@ -289,7 +289,7 @@ class KernelSnapshotNativeAssets extends Target {
 
   @override
   List<Source> get inputs => <Source>[
-    const Source.pattern('{BUILD_DIR}/native_assets.yaml'),
+    const Source.pattern('{BUILD_DIR}/${InstallCodeAssets.nativeAssetsFilename}'),
     ...const KernelSnapshotProgram().inputs,
   ];
 
@@ -302,15 +302,15 @@ class KernelSnapshotNativeAssets extends Target {
   List<String> get depfiles => const <String>[];
 
   @override
-  List<Target> get dependencies => <Target>[
-    const NativeAssets(),
+  List<Target> get dependencies => const <Target>[
+    InstallCodeAssets(),
   ];
 
   static const String dillName = 'native_assets.dill';
 
   @override
   Future<void> build(Environment environment) async {
-    final File nativeAssetsFile = environment.buildDir.childFile('native_assets.yaml');
+    final File nativeAssetsFile = environment.buildDir.childFile(InstallCodeAssets.nativeAssetsFilename);
     final File dillFile = environment.buildDir.childFile(dillName);
 
     final YamlNode nativeAssetContents = loadYamlNode(await nativeAssetsFile.readAsString());

packages/flutter_tools/lib/src/build_system/targets/native_assets.dart

@@ -3,65 +3,43 @@
 // found in the LICENSE file.
 
 import 'package:meta/meta.dart';
-import 'package:native_assets_builder/native_assets_builder.dart';
 import 'package:package_config/package_config_types.dart';
 
 import '../../base/common.dart';
 import '../../base/file_system.dart';
 import '../../build_info.dart';
+import '../../convert.dart';
 import '../../dart/package_map.dart';
 import '../../isolated/native_assets/native_assets.dart';
 import '../build_system.dart';
 import '../depfile.dart';
 import '../exceptions.dart';
 import 'common.dart';
 
-/// Builds the right native assets for a Flutter app.
-///
-/// The build mode and target architecture can be changed from the
-/// native build project (Xcode etc.), so only `flutter assemble` has the
-/// information about build-mode and target architecture.
-/// Invocations of flutter_tools other than `flutter assemble` are dry runs.
-///
-/// This step needs to be consistent with the dry run invocations in `flutter
-/// run`s so that the kernel mapping of asset id to dylib lines up after hot
-/// restart.
-///
-/// [KernelSnapshot] depends on this target. We produce a native_assets.yaml
-/// here, and embed that mapping inside the kernel snapshot.
-///
-/// The build always produces a valid native_assets.yaml and a native_assets.d
-/// even if there are no native assets. This way the caching logic won't try to
-/// rebuild.
-class NativeAssets extends Target {
-  const NativeAssets({
+/// Runs the dart build of the app.
+abstract class DartBuild extends Target {
+  const DartBuild({
     @visibleForTesting FlutterNativeAssetsBuildRunner? buildRunner,
   }) : _buildRunner = buildRunner;
 
   final FlutterNativeAssetsBuildRunner? _buildRunner;
 
   @override
   Future<void> build(Environment environment) async {
-    final String? nativeAssetsEnvironment = environment.defines[kNativeAssets];
     final FileSystem fileSystem = environment.fileSystem;
-    final Uri nativeAssetsFileUri = environment.buildDir.childFile('native_assets.yaml').uri;
+    final String? nativeAssetsEnvironment = environment.defines[kNativeAssets];
 
     final DartBuildResult result;
     if (nativeAssetsEnvironment == 'false') {
       result = const DartBuildResult.empty();
-      await writeNativeAssetsYaml(KernelAssets(), nativeAssetsFileUri, fileSystem);
     } else {
-      final String? targetPlatformEnvironment = environment.defines[kTargetPlatform];
-      if (targetPlatformEnvironment == null) {
-        throw MissingDefineException(kTargetPlatform, name);
-      }
-      final TargetPlatform targetPlatform = getTargetPlatformForName(targetPlatformEnvironment);
-      final Uri projectUri = environment.projectDir.uri;
+      final TargetPlatform targetPlatform = _getTargetPlatformFromEnvironment(environment, name);
 
       final PackageConfig packageConfig = await loadPackageConfigWithLogging(
         fileSystem.file(environment.packageConfigPath),
         logger: environment.logger,
       );
+      final Uri projectUri = environment.projectDir.uri;
       final FlutterNativeAssetsBuildRunner buildRunner = _buildRunner ??
           FlutterNativeAssetsBuildRunnerImpl(
             projectUri,
@@ -70,62 +48,154 @@ class NativeAssets extends Target {
             fileSystem,
             environment.logger,
           );
-
-      (result, _) = await runFlutterSpecificDartBuild(
+      result = await runFlutterSpecificDartBuild(
         environmentDefines: environment.defines,
         buildRunner: buildRunner,
         targetPlatform: targetPlatform,
         projectUri: projectUri,
-        nativeAssetsYamlUri : nativeAssetsFileUri,
         fileSystem: fileSystem,
       );
     }
 
+    final File dartBuildResultJsonFile = environment.buildDir.childFile(dartBuildResultFilename);
+    if (!dartBuildResultJsonFile.parent.existsSync()) {
+      dartBuildResultJsonFile.parent.createSync(recursive: true);
+    }
+    dartBuildResultJsonFile.writeAsStringSync(json.encode(result.toJson()));
+
     final Depfile depfile = Depfile(
       <File>[
         for (final Uri dependency in result.dependencies) fileSystem.file(dependency),
       ],
       <File>[
-        fileSystem.file(nativeAssetsFileUri),
+        fileSystem.file(dartBuildResultJsonFile),
       ],
     );
-    final File outputDepfile = environment.buildDir.childFile('native_assets.d');
+    final File outputDepfile = environment.buildDir.childFile(depFilename);
     if (!outputDepfile.parent.existsSync()) {
       outputDepfile.parent.createSync(recursive: true);
     }
     environment.depFileService.writeToFile(depfile, outputDepfile);
-    if (!await fileSystem.file(nativeAssetsFileUri).exists()) {
-      throwToolExit("${nativeAssetsFileUri.path} doesn't exist.");
-    }
     if (!await outputDepfile.exists()) {
       throwToolExit("${outputDepfile.path} doesn't exist.");
     }
   }
 
   @override
-  List<String> get depfiles => <String>[
-    'native_assets.d',
+  List<String> get depfiles => const <String>[depFilename];
+
+  @override
+  List<Source> get inputs => const <Source>[
+    Source.pattern('{FLUTTER_ROOT}/packages/flutter_tools/lib/src/build_system/targets/native_assets.dart'),
+    // If different packages are resolved, different native assets might need to be built.
+    Source.pattern('{WORKSPACE_DIR}/.dart_tool/package_config_subset'),
+    // TODO(mosuem): Should consume resources.json. https://github.com/flutter/flutter/issues/146263
+  ];
+
+  @override
+  String get name => 'dart_build';
+
+  @override
+  List<Source> get outputs => const <Source>[
+    Source.pattern('{BUILD_DIR}/$dartBuildResultFilename'),
   ];
 
+  /// Dependent build [Target]s can use this to consume the result of the
+  /// [DartBuild] target.
+  static Future<DartBuildResult> loadBuildResult(Environment environment) async {
+    final File dartBuildResultJsonFile = environment.buildDir.childFile(DartBuild.dartBuildResultFilename);
+    return DartBuildResult.fromJson(json.decode(dartBuildResultJsonFile.readAsStringSync()) as Map<String, Object?>);
+  }
+
+  static const String dartBuildResultFilename = 'dart_build_result.json';
+  static const String depFilename = 'dart_build.d';
+}
+
+class DartBuildForNative extends DartBuild {
+  const DartBuildForNative({@visibleForTesting super.buildRunner});
+
   @override
   List<Target> get dependencies => const <Target>[
-    // In AOT, depends on tree-shaking information (resources.json) from compiling dart.
     KernelSnapshotProgram(),
   ];
+}
+
+/// Installs the code assets from a [DartBuild] Flutter app.
+///
+/// The build mode and target architecture can be changed from the
+/// native build project (Xcode etc.), so only `flutter assemble` has the
+/// information about build-mode and target architecture.
+/// Invocations of flutter_tools other than `flutter assemble` are dry runs.
+///
+/// This step needs to be consistent with the dry run invocations in `flutter
+/// run`s so that the kernel mapping of asset id to dylib lines up after hot
+/// restart.
+class InstallCodeAssets extends Target {
+  const InstallCodeAssets();
+
+  @override
+  Future<void> build(Environment environment) async {
+    final Uri projectUri = environment.projectDir.uri;
+    final FileSystem fileSystem = environment.fileSystem;
+    final TargetPlatform targetPlatform = _getTargetPlatformFromEnvironment(environment, name);
+
+    // We fetch the result from the [DartBuild].
+    final DartBuildResult dartBuildResult = await DartBuild.loadBuildResult(environment);
+
+    // And install/copy the code assets to the right place and create a
+    // native_asset.yaml that can be used by the final AOT compilation.
+    final Uri nativeAssetsFileUri = environment.buildDir.childFile(nativeAssetsFilename).uri;
+    await installCodeAssets(dartBuildResult: dartBuildResult, environmentDefines: environment.defines,
+      targetPlatform: targetPlatform, projectUri: projectUri, fileSystem: fileSystem,
+      nativeAssetsFileUri: nativeAssetsFileUri);
+    assert(await fileSystem.file(nativeAssetsFileUri).exists());
+
+    final Depfile depfile = Depfile(
+      <File>[
+        for (final Uri file in dartBuildResult.filesToBeBundled) fileSystem.file(file),
+      ],
+      <File>[
+        fileSystem.file(nativeAssetsFileUri),
+      ],
+    );
+    final File outputDepfile = environment.buildDir.childFile(depFilename);
+    environment.depFileService.writeToFile(depfile, outputDepfile);
+    if (!await outputDepfile.exists()) {
+      throwToolExit("${outputDepfile.path} doesn't exist.");
+    }
+  }
+
+  @override
+  List<String> get depfiles => <String>[depFilename];
+
+  @override
+  List<Target> get dependencies => const <Target>[
+    DartBuildForNative(),
+  ];
 
   @override
   List<Source> get inputs => const <Source>[
     Source.pattern('{FLUTTER_ROOT}/packages/flutter_tools/lib/src/build_system/targets/native_assets.dart'),
     // If different packages are resolved, different native assets might need to be built.
     Source.pattern('{WORKSPACE_DIR}/.dart_tool/package_config_subset'),
-    // TODO(mosuem): Should consume resources.json. https://github.com/flutter/flutter/issues/146263
   ];
 
   @override
-  String get name => 'native_assets';
+  String get name => 'install_code_assets';
 
   @override
   List<Source> get outputs => const <Source>[
-    Source.pattern('{BUILD_DIR}/native_assets.yaml'),
+    Source.pattern('{BUILD_DIR}/$nativeAssetsFilename'),
   ];
+
+  static const String nativeAssetsFilename = 'native_assets.yaml';
+  static const String depFilename = 'install_code_assets.d';
+}
+
+TargetPlatform _getTargetPlatformFromEnvironment(Environment environment, String name) {
+  final String? targetPlatformEnvironment = environment.defines[kTargetPlatform];
+  if (targetPlatformEnvironment == null) {
+    throw MissingDefineException(kTargetPlatform, name);
+  }
+  return getTargetPlatformForName(targetPlatformEnvironment);
 }