[flutter_tools] read expression compilation results into memory before starting next compilation (flutter#77867)

Author: Jonah Williams

packages/flutter_tools/lib/src/compile.dart

@@ -5,6 +5,7 @@
 // @dart = 2.8
 
 import 'dart:async';
+import 'dart:typed_data';
 
 import 'package:meta/meta.dart';
 import 'package:package_config/package_config.dart';
@@ -65,24 +66,30 @@ class TargetModel {
 }
 
 class CompilerOutput {
-  const CompilerOutput(this.outputFilename, this.errorCount, this.sources);
+  const CompilerOutput(this.outputFilename, this.errorCount, this.sources, {this.expressionData});
 
   final String outputFilename;
   final int errorCount;
   final List<Uri> sources;
+
+  /// This field is only non-null for expression compilation requests.
+  final Uint8List expressionData;
 }
 
 enum StdoutState { CollectDiagnostic, CollectDependencies }
 
 /// Handles stdin/stdout communication with the frontend server.
 class StdoutHandler {
   StdoutHandler({
-    @required Logger logger
-  }) : _logger = logger {
+    @required Logger logger,
+    @required FileSystem fileSystem,
+  }) : _logger = logger,
+       _fileSystem = fileSystem {
     reset();
   }
 
   final Logger _logger;
+  final FileSystem _fileSystem;
 
   String boundaryKey;
   StdoutState state = StdoutState.CollectDiagnostic;
@@ -91,6 +98,7 @@ class StdoutHandler {
 
   bool _suppressCompilerMessages;
   bool _expectSources;
+  bool _readFile;
 
   void handler(String message) {
     const String kResultPrefix = 'result ';
@@ -110,11 +118,19 @@ class StdoutHandler {
         return;
       }
       final int spaceDelimiter = message.lastIndexOf(' ');
-      compilerOutput.complete(
-          CompilerOutput(
-              message.substring(boundaryKey.length + 1, spaceDelimiter),
-              int.parse(message.substring(spaceDelimiter + 1).trim()),
-              sources));
+      final String fileName = message.substring(boundaryKey.length + 1, spaceDelimiter);
+      final int errorCount = int.parse(message.substring(spaceDelimiter + 1).trim());
+      Uint8List expressionData;
+      if (_readFile) {
+        expressionData = _fileSystem.file(fileName).readAsBytesSync();
+      }
+      final CompilerOutput output = CompilerOutput(
+        fileName,
+        errorCount,
+        sources,
+        expressionData: expressionData,
+      );
+      compilerOutput.complete(output);
       return;
     }
     if (state == StdoutState.CollectDiagnostic) {
@@ -140,11 +156,12 @@ class StdoutHandler {
 
   // This is needed to get ready to process next compilation result output,
   // with its own boundary key and new completer.
-  void reset({ bool suppressCompilerMessages = false, bool expectSources = true }) {
+  void reset({ bool suppressCompilerMessages = false, bool expectSources = true, bool readFile = false }) {
     boundaryKey = null;
     compilerOutput = Completer<CompilerOutput>();
     _suppressCompilerMessages = suppressCompilerMessages;
     _expectSources = expectSources;
+    _readFile = readFile;
     state = StdoutState.CollectDiagnostic;
   }
 }
@@ -191,7 +208,7 @@ class KernelCompiler {
        _processManager = processManager,
        _fileSystemScheme = fileSystemScheme,
        _fileSystemRoots = fileSystemRoots,
-       _stdoutHandler = stdoutHandler ?? StdoutHandler(logger: logger);
+       _stdoutHandler = stdoutHandler ?? StdoutHandler(logger: logger, fileSystem: fileSystem);
 
   final FileSystem _fileSystem;
   final Artifacts _artifacts;
@@ -430,6 +447,7 @@ abstract class ResidentCompiler {
     @required ProcessManager processManager,
     @required Artifacts artifacts,
     @required Platform platform,
+    @required FileSystem fileSystem,
     bool testCompilation,
     bool trackWidgetCreation,
     String packagesPath,
@@ -530,6 +548,7 @@ class DefaultResidentCompiler implements ResidentCompiler {
     @required ProcessManager processManager,
     @required Artifacts artifacts,
     @required Platform platform,
+    @required FileSystem fileSystem,
     this.testCompilation = false,
     this.trackWidgetCreation = true,
     this.packagesPath,
@@ -547,7 +566,7 @@ class DefaultResidentCompiler implements ResidentCompiler {
        _logger = logger,
        _processManager = processManager,
        _artifacts = artifacts,
-       _stdoutHandler = stdoutHandler ?? StdoutHandler(logger: logger),
+       _stdoutHandler = stdoutHandler ?? StdoutHandler(logger: logger, fileSystem: fileSystem),
        _platform = platform,
        dartDefines = dartDefines ?? const <String>[],
        // This is a URI, not a file path, so the forward slash is correct even on Windows.
@@ -762,21 +781,20 @@ class DefaultResidentCompiler implements ResidentCompiler {
     String libraryUri,
     String klass,
     bool isStatic,
-  ) {
+  ) async {
     if (!_controller.hasListener) {
       _controller.stream.listen(_handleCompilationRequest);
     }
 
     final Completer<CompilerOutput> completer = Completer<CompilerOutput>();
-    _controller.add(
-        _CompileExpressionRequest(
-            completer, expression, definitions, typeDefinitions, libraryUri, klass, isStatic)
-    );
+    final _CompileExpressionRequest request =  _CompileExpressionRequest(
+        completer, expression, definitions, typeDefinitions, libraryUri, klass, isStatic);
+    _controller.add(request);
     return completer.future;
   }
 
   Future<CompilerOutput> _compileExpression(_CompileExpressionRequest request) async {
-    _stdoutHandler.reset(suppressCompilerMessages: true, expectSources: false);
+    _stdoutHandler.reset(suppressCompilerMessages: true, expectSources: false, readFile: true);
 
     // 'compile-expression' should be invoked after compiler has been started,
     // program was compiled.

packages/flutter_tools/lib/src/resident_runner.dart

@@ -69,6 +69,7 @@ class FlutterDevice {
          processManager: globals.processManager,
          logger: globals.logger,
          platform: globals.platform,
+         fileSystem: globals.fs,
        );
 
   /// Create a [FlutterDevice] with optional code generation enabled.
@@ -138,6 +139,7 @@ class FlutterDevice {
         artifacts: globals.artifacts,
         processManager: globals.processManager,
         logger: globals.logger,
+        fileSystem: globals.fs,
         platform: platform,
       );
     } else {
@@ -173,6 +175,7 @@ class FlutterDevice {
         processManager: globals.processManager,
         logger: globals.logger,
         platform: platform,
+        fileSystem: globals.fs,
       );
     }
 

packages/flutter_tools/lib/src/run_hot.dart

@@ -165,8 +165,8 @@ class HotRunner extends ResidentRunner {
         final CompilerOutput compilerOutput =
             await device.generator.compileExpression(expression, definitions,
                 typeDefinitions, libraryUri, klass, isStatic);
-        if (compilerOutput != null && compilerOutput.outputFilename != null) {
-          return base64.encode(globals.fs.file(compilerOutput.outputFilename).readAsBytesSync());
+        if (compilerOutput != null && compilerOutput.expressionData != null) {
+          return base64.encode(compilerOutput.expressionData);
         }
       }
     }

packages/flutter_tools/lib/src/test/flutter_platform.dart

@@ -328,8 +328,8 @@ class FlutterPlatform extends PlatformPlugin {
     final CompilerOutput compilerOutput =
       await compiler.compiler.compileExpression(expression, definitions,
         typeDefinitions, libraryUri, klass, isStatic);
-    if (compilerOutput != null && compilerOutput.outputFilename != null) {
-      return base64.encode(globals.fs.file(compilerOutput.outputFilename).readAsBytesSync());
+    if (compilerOutput != null && compilerOutput.expressionData != null) {
+      return base64.encode(compilerOutput.expressionData);
     }
     throw 'Failed to compile $expression';
   }

packages/flutter_tools/lib/src/test/test_compiler.dart

@@ -111,6 +111,7 @@ class TestCompiler {
       extraFrontEndOptions: buildInfo.extraFrontEndOptions,
       platform: globals.platform,
       testCompilation: true,
+      fileSystem: globals.fs,
     );
     return residentCompiler;
   }

packages/flutter_tools/lib/src/test/web_test_compiler.dart

@@ -130,6 +130,7 @@ class WebTestCompiler {
       processManager: _processManager,
       logger: _logger,
       platform: _platform,
+      fileSystem: _fileSystem,
     );
 
     final CompilerOutput output = await residentCompiler.recompile(

packages/flutter_tools/test/general.shard/compile_batch_test.dart

@@ -19,7 +19,7 @@ import '../src/context.dart';
 void main() {
   testWithoutContext('StdoutHandler can parse output for successful batch compilation', () async {
     final BufferLogger logger = BufferLogger.test();
-    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger);
+    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger, fileSystem: MemoryFileSystem.test());
 
     stdoutHandler.reset();
     'result abc\nline1\nline2\nabc\nabc /path/to/main.dart.dill 0'.split('\n').forEach(stdoutHandler.handler);
@@ -31,7 +31,7 @@ void main() {
 
   testWithoutContext('StdoutHandler can parse output for failed batch compilation', () async {
     final BufferLogger logger = BufferLogger.test();
-    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger);
+    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger, fileSystem: MemoryFileSystem.test());
 
     stdoutHandler.reset();
     'result abc\nline1\nline2\nabc\nabc'.split('\n').forEach(stdoutHandler.handler);
@@ -43,7 +43,7 @@ void main() {
 
   testWithoutContext('KernelCompiler passes correct configuration to frontend server process', () async {
     final BufferLogger logger = BufferLogger.test();
-    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger);
+    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger, fileSystem: MemoryFileSystem.test());
     final Completer<void> completer = Completer<void>();
 
     final KernelCompiler kernelCompiler = KernelCompiler(
@@ -88,7 +88,7 @@ void main() {
 
   testWithoutContext('KernelCompiler returns null if StdoutHandler returns null', () async {
     final BufferLogger logger = BufferLogger.test();
-    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger);
+    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger, fileSystem: MemoryFileSystem.test());
     final Completer<void> completer = Completer<void>();
 
     final KernelCompiler kernelCompiler = KernelCompiler(
@@ -133,7 +133,7 @@ void main() {
 
   testWithoutContext('KernelCompiler returns null if frontend_server process exits with non-zero code', () async {
     final BufferLogger logger = BufferLogger.test();
-    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger);
+    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger, fileSystem: MemoryFileSystem.test());
     final Completer<void> completer = Completer<void>();
 
     final KernelCompiler kernelCompiler = KernelCompiler(
@@ -178,7 +178,7 @@ void main() {
 
   testWithoutContext('KernelCompiler passes correct AOT config to frontend_server in aot/profile mode', () async {
     final BufferLogger logger = BufferLogger.test();
-    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger);
+    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger, fileSystem: MemoryFileSystem.test());
     final Completer<void> completer = Completer<void>();
 
     final KernelCompiler kernelCompiler = KernelCompiler(
@@ -225,7 +225,7 @@ void main() {
 
   testWithoutContext('passes correct AOT config to kernel compiler in aot/release mode', () async {
     final BufferLogger logger = BufferLogger.test();
-    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger);
+    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger, fileSystem: MemoryFileSystem.test());
     final Completer<void> completer = Completer<void>();
 
     final KernelCompiler kernelCompiler = KernelCompiler(
@@ -272,7 +272,7 @@ void main() {
 
   testWithoutContext('KernelCompiler passes dartDefines to the frontend_server', () async {
     final BufferLogger logger = BufferLogger.test();
-    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger);
+    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger, fileSystem: MemoryFileSystem.test());
     final Completer<void> completer = Completer<void>();
 
     final KernelCompiler kernelCompiler = KernelCompiler(
@@ -321,7 +321,7 @@ void main() {
 
   testWithoutContext('KernelCompiler maps a file to a multi-root scheme if provided', () async {
     final BufferLogger logger = BufferLogger.test();
-    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger);
+    final StdoutHandler stdoutHandler = StdoutHandler(logger: logger, fileSystem: MemoryFileSystem.test());
     final Completer<void> completer = Completer<void>();
 
     final KernelCompiler kernelCompiler = KernelCompiler(

packages/flutter_tools/test/general.shard/compile_expression_test.dart

@@ -6,6 +6,7 @@
 
 import 'dart:async';
 
+import 'package:file/memory.dart';
 import 'package:flutter_tools/src/artifacts.dart';
 import 'package:flutter_tools/src/base/common.dart';
 import 'package:flutter_tools/src/base/io.dart';
@@ -30,20 +31,23 @@ void main() {
   MockStream mockFrontendServerStdErr;
   StreamController<String> stdErrStreamController;
   BufferLogger testLogger;
+  MemoryFileSystem fileSystem;
 
   setUp(() {
     testLogger = BufferLogger.test();
     mockProcessManager = MockProcessManager();
     mockFrontendServer = MockProcess();
     mockFrontendServerStdIn = MockStdIn();
     mockFrontendServerStdErr = MockStream();
+    fileSystem = MemoryFileSystem.test();
     generator = ResidentCompiler(
       'sdkroot',
       buildMode: BuildMode.debug,
       artifacts: Artifacts.test(),
       processManager: mockProcessManager,
       logger: testLogger,
       platform: FakePlatform(operatingSystem: 'linux'),
+      fileSystem: fileSystem,
     );
 
     when(mockFrontendServer.stdin).thenReturn(mockFrontendServerStdIn);
@@ -75,6 +79,9 @@ void main() {
         Completer<List<int>>();
     final Completer<List<int>> compileExpressionResponseCompleter =
         Completer<List<int>>();
+    fileSystem.file('/path/to/main.dart.dill')
+      ..createSync(recursive: true)
+      ..writeAsBytesSync(<int>[1, 2, 3, 4]);
 
     when(mockFrontendServer.stdout)
         .thenAnswer((Invocation invocation) =>
@@ -108,8 +115,7 @@ void main() {
           '2+2', null, null, null, null, false).then(
               (CompilerOutput outputExpression) {
                 expect(outputExpression, isNotNull);
-                expect(outputExpression.outputFilename, equals('/path/to/main.dart.dill.incremental'));
-                expect(outputExpression.errorCount, 0);
+                expect(outputExpression.expressionData, <int>[1, 2, 3, 4]);
               }
       );
     });
@@ -141,6 +147,9 @@ void main() {
             equals('line1\nline2\n'));
         expect(outputCompile.outputFilename, equals('/path/to/main.dart.dill'));
 
+        fileSystem.file('/path/to/main.dart.dill.incremental')
+          ..createSync(recursive: true)
+          ..writeAsBytesSync(<int>[0, 1, 2, 3]);
         compileExpressionResponseCompleter1.complete(Future<List<int>>.value(utf8.encode(
             'result def\nline1\nline2\ndef /path/to/main.dart.dill.incremental 0\n'
         )));
@@ -153,9 +162,11 @@ void main() {
       generator.compileExpression('0+1', null, null, null, null, false).then(
         (CompilerOutput outputExpression) {
           expect(outputExpression, isNotNull);
-          expect(outputExpression.outputFilename,
-              equals('/path/to/main.dart.dill.incremental'));
-          expect(outputExpression.errorCount, 0);
+          expect(outputExpression.expressionData, <int>[0, 1, 2, 3]);
+
+          fileSystem.file('/path/to/main.dart.dill.incremental')
+            ..createSync(recursive: true)
+            ..writeAsBytesSync(<int>[4, 5, 6, 7]);
           compileExpressionResponseCompleter2.complete(Future<List<int>>.value(utf8.encode(
               'result def\nline1\nline2\ndef /path/to/main.dart.dill.incremental 0\n'
           )));
@@ -168,9 +179,7 @@ void main() {
       generator.compileExpression('1+1', null, null, null, null, false).then(
         (CompilerOutput outputExpression) {
           expect(outputExpression, isNotNull);
-          expect(outputExpression.outputFilename,
-              equals('/path/to/main.dart.dill.incremental'));
-          expect(outputExpression.errorCount, 0);
+          expect(outputExpression.expressionData, <int>[4, 5, 6, 7]);
           lastExpressionCompleted.complete(true);
         },
       ),