diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md index f5743f848..3aea19f90 100644 --- a/pkgs/test/CHANGELOG.md +++ b/pkgs/test/CHANGELOG.md @@ -8,6 +8,8 @@ of compiling to kernel first. * If no given compiler is compatible for a platform, it will use its default compiler instead. +* Add support for running tests as native executables (vm platform only). + * You can run tests this way with `--compiler exe`. * Support compiler identifiers in platform selectors. * List the supported compilers for each platform in the usage text. * Update all reporters to print the compiler along with the platform name diff --git a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart new file mode 100644 index 000000000..0442d74ce --- /dev/null +++ b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart @@ -0,0 +1,167 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +@TestOn('vm') +import 'dart:io'; + +import 'package:test/test.dart'; +import 'package:test_api/backend.dart'; // ignore: deprecated_member_use +import 'package:test_descriptor/test_descriptor.dart' as d; + +import '../io.dart'; + +void main() { + setUpAll(() async { + await precompileTestExecutable(); + }); + + for (var runtime in Runtime.builtIn) { + for (var compiler in runtime.supportedCompilers) { + // Ignore the platforms we can't run on this OS. + if (runtime == Runtime.internetExplorer && !Platform.isWindows || + runtime == Runtime.safari && !Platform.isMacOS) { + continue; + } + group('--runtime ${runtime.identifier} --compiler ${compiler.identifier}', + () { + final testArgs = [ + 'test.dart', + '-p', + runtime.identifier, + '-c', + compiler.identifier + ]; + + test('can run passing tests', () async { + await d.file('test.dart', _goodTest).create(); + var test = await runTest(testArgs); + + expect(test.stdout, emitsThrough(contains('+1: All tests passed!'))); + await test.shouldExit(0); + }); + + test('fails gracefully for invalid code', () async { + await d.file('test.dart', _compileErrorTest).create(); + var test = await runTest(testArgs); + + expect( + test.stdout, + containsInOrder([ + "Error: A value of type 'String' can't be assigned to a variable of type 'int'.", + "int x = 'hello';", + ])); + + await test.shouldExit(1); + }); + + test('fails gracefully for test failures', () async { + await d.file('test.dart', _failingTest).create(); + var test = await runTest(testArgs); + + expect( + test.stdout, + containsInOrder([ + 'Expected: <2>', + 'Actual: <1>', + 'test.dart 5', + '+0 -1: Some tests failed.', + ])); + + await test.shouldExit(1); + }); + + test('fails gracefully if a test file throws in main', () async { + await d.file('test.dart', _throwingTest).create(); + var test = await runTest(testArgs); + var compileOrLoadMessage = + compiler == Compiler.dart2js ? 'compiling' : 'loading'; + + expect( + test.stdout, + containsInOrder([ + '-1: [${runtime.name}, ${compiler.name}] $compileOrLoadMessage ' + 'test.dart [E]', + 'Failed to load "test.dart": oh no' + ])); + await test.shouldExit(1); + }); + + test('captures prints', () async { + await d.file('test.dart', _testWithPrints).create(); + var test = await runTest([...testArgs, '-r', 'json']); + + expect( + test.stdout, + containsInOrder([ + '"messageType":"print","message":"hello","type":"print"', + ])); + + await test.shouldExit(0); + }); + + if (runtime.isDartVM) { + test('forwards stdout/stderr', () async { + await d.file('test.dart', _testWithStdOutAndErr).create(); + var test = await runTest(testArgs); + + expect(test.stdout, emitsThrough('hello')); + expect(test.stderr, emits('world')); + await test.shouldExit(0); + }); + } + }, + skip: compiler == Compiler.dart2wasm + ? 'Wasm tests are experimental and require special setup' + : [Runtime.firefox, Runtime.nodeJS, Runtime.internetExplorer] + .contains(runtime) && + Platform.isWindows + ? 'https://github.com/dart-lang/test/issues/1942' + : null); + } + } +} + +final _goodTest = ''' + import 'package:test/test.dart'; + + void main() { + test("success", () {}); + } +'''; + +final _failingTest = ''' + import 'package:test/test.dart'; + + void main() { + test("failure", () { + expect(1, 2); + }); + } +'''; + +final _compileErrorTest = ''' +int x = 'hello'; + +void main() {} +'''; + +final _throwingTest = "void main() => throw 'oh no';"; + +final _testWithPrints = ''' +import 'package:test/test.dart'; + +void main() { + print('hello'); + test('success', () {}); +}'''; + +final _testWithStdOutAndErr = ''' +import 'dart:io'; +import 'package:test/test.dart'; + +void main() { + stdout.writeln('hello'); + stderr.writeln('world'); + test('success', () {}); +}'''; diff --git a/pkgs/test/test/runner/runner_test.dart b/pkgs/test/test/runner/runner_test.dart index 4f5d7f467..6adf98200 100644 --- a/pkgs/test/test/runner/runner_test.dart +++ b/pkgs/test/test/runner/runner_test.dart @@ -73,7 +73,7 @@ Running Tests: $_runtimes. Each platform supports the following compilers: $_runtimeCompilers --c, --compiler The compiler(s) to use to run tests, supported compilers are [dart2js, dart2wasm, kernel, source]. +-c, --compiler The compiler(s) to use to run tests, supported compilers are [dart2js, dart2wasm, exe, kernel, source]. Each platform has a default compiler but may support other compilers. You can target a compiler to a specific platform using arguments of the following form [:]. If a platform is specified but no given compiler is supported for that platform, then it will use its default compiler. @@ -124,7 +124,7 @@ final _runtimes = '[vm (default), chrome, firefox' 'experimental-chrome-wasm]'; final _runtimeCompilers = [ - '[vm]: kernel (default), source', + '[vm]: kernel (default), source, exe', '[chrome]: dart2js (default)', '[firefox]: dart2js (default)', if (Platform.isMacOS) '[safari]: dart2js (default)', diff --git a/pkgs/test_api/lib/src/backend/compiler.dart b/pkgs/test_api/lib/src/backend/compiler.dart index da3c59a33..77a637619 100644 --- a/pkgs/test_api/lib/src/backend/compiler.dart +++ b/pkgs/test_api/lib/src/backend/compiler.dart @@ -10,6 +10,9 @@ class Compiler { /// Experimental Dart to Wasm compiler. static const Compiler dart2wasm = Compiler._('Dart2WASM', 'dart2wasm'); + /// Compiles dart code to a native executable. + static const Compiler exe = Compiler._('Exe', 'exe'); + /// The standard compiler for vm tests, compiles tests to kernel before /// running them on the VM. static const Compiler kernel = Compiler._('Kernel', 'kernel'); @@ -21,6 +24,7 @@ class Compiler { static const List builtIn = [ Compiler.dart2js, Compiler.dart2wasm, + Compiler.exe, Compiler.kernel, Compiler.source, ]; diff --git a/pkgs/test_api/lib/src/backend/runtime.dart b/pkgs/test_api/lib/src/backend/runtime.dart index 72c5a8dfd..2572402e7 100644 --- a/pkgs/test_api/lib/src/backend/runtime.dart +++ b/pkgs/test_api/lib/src/backend/runtime.dart @@ -10,8 +10,8 @@ class Runtime { // variable tests in test/backend/platform_selector/evaluate_test. /// The command-line Dart VM. - static const Runtime vm = Runtime( - 'VM', 'vm', Compiler.kernel, [Compiler.kernel, Compiler.source], + static const Runtime vm = Runtime('VM', 'vm', Compiler.kernel, + [Compiler.kernel, Compiler.source, Compiler.exe], isDartVM: true); /// Google Chrome. diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md index 7a8d8871d..376afec8b 100644 --- a/pkgs/test_core/CHANGELOG.md +++ b/pkgs/test_core/CHANGELOG.md @@ -8,6 +8,7 @@ of compiling to kernel first. * If no given compiler is compatible for a platform, it will use its default compiler instead. +* Add support for `-c exe` (the native executable compiler) to the vm platform. * Add `Compiler` class, exposed through `backend.dart`. * Support compiler identifiers in platform selectors. * List the supported compilers for each platform in the usage text. diff --git a/pkgs/test_core/lib/src/bootstrap/vm.dart b/pkgs/test_core/lib/src/bootstrap/vm.dart index d1eb59ada..2166e7d8d 100644 --- a/pkgs/test_core/lib/src/bootstrap/vm.dart +++ b/pkgs/test_core/lib/src/bootstrap/vm.dart @@ -3,14 +3,16 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:developer'; +import 'dart:io'; import 'dart:isolate'; import 'package:stream_channel/isolate_channel.dart'; import 'package:stream_channel/stream_channel.dart'; import 'package:test_core/src/runner/plugin/remote_platform_helpers.dart'; +import 'package:test_core/src/runner/plugin/shared_platform_helpers.dart'; -/// Bootstraps a vm test to communicate with the test runner. +/// Bootstraps a vm test to communicate with the test runner over an isolate. void internalBootstrapVmTest(Function Function() getMain, SendPort sendPort) { var platformChannel = MultiChannel(IsolateChannel.connectSend(sendPort)); @@ -24,3 +26,24 @@ void internalBootstrapVmTest(Function Function() getMain, SendPort sendPort) { platformChannel.sink.add('done'); }); } + +/// Bootstraps a native executable test to communicate with the test runner over +/// a socket. +void internalBootstrapNativeTest( + Function Function() getMain, List args) async { + if (args.length != 2) { + throw StateError( + 'Expected exactly two args, a host and a port, but got $args'); + } + var socket = await Socket.connect(args[0], int.parse(args[1])); + var platformChannel = MultiChannel(jsonSocketStreamChannel(socket)); + var testControlChannel = platformChannel.virtualChannel() + ..pipe(serializeSuite(getMain)); + platformChannel.sink.add(testControlChannel.id); + + platformChannel.stream.forEach((message) { + assert(message == 'debug'); + debugger(message: 'Paused by test runner'); + platformChannel.sink.add('done'); + }); +} diff --git a/pkgs/test_core/lib/src/runner/plugin/shared_platform_helpers.dart b/pkgs/test_core/lib/src/runner/plugin/shared_platform_helpers.dart new file mode 100644 index 000000000..4fdcf23d1 --- /dev/null +++ b/pkgs/test_core/lib/src/runner/plugin/shared_platform_helpers.dart @@ -0,0 +1,21 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:async/async.dart'; +import 'package:stream_channel/stream_channel.dart'; + +/// Converts a raw [Socket] into a [StreamChannel] of JSON objects. +/// +/// JSON messages are separated by newlines. +StreamChannel jsonSocketStreamChannel(Socket socket) => + StreamChannel.withGuarantees(socket, socket) + .cast>() + .transform(StreamChannelTransformer.fromCodec(utf8)) + .transformStream(const LineSplitter()) + .transformSink(StreamSinkTransformer.fromHandlers( + handleData: (original, sink) => sink.add('$original\n'))) + .transform(jsonDocument); diff --git a/pkgs/test_core/lib/src/runner/vm/platform.dart b/pkgs/test_core/lib/src/runner/vm/platform.dart index 9c7b4eaa4..75e56dc5b 100644 --- a/pkgs/test_core/lib/src/runner/vm/platform.dart +++ b/pkgs/test_core/lib/src/runner/vm/platform.dart @@ -23,6 +23,7 @@ import '../../runner/environment.dart'; import '../../runner/load_exception.dart'; import '../../runner/platform.dart'; import '../../runner/plugin/platform_helpers.dart'; +import '../../runner/plugin/shared_platform_helpers.dart'; import '../../runner/runner_suite.dart'; import '../../runner/suite.dart'; import '../../util/package_config.dart'; @@ -39,7 +40,7 @@ class VMPlatform extends PlatformPlugin { p.join(p.current, '.dart_tool', 'test', 'incremental_kernel')); final _closeMemo = AsyncMemoizer(); final _workingDirectory = Directory.current.uri; - final _tempDir = Directory.systemTemp.createTempSync('dart_test.kernel.'); + final _tempDir = Directory.systemTemp.createTempSync('dart_test.vm.'); @override Future load(String path, SuitePlatform platform, @@ -48,23 +49,48 @@ class VMPlatform extends PlatformPlugin { _setupPauseAfterTests(); - var receivePort = ReceivePort(); + MultiChannel outerChannel; + var cleanupCallbacks = []; Isolate? isolate; - try { - isolate = await _spawnIsolate( - path, receivePort.sendPort, suiteConfig.metadata, platform.compiler); - if (isolate == null) return null; - } catch (error) { - receivePort.close(); - rethrow; + if (platform.compiler == Compiler.exe) { + var serverSocket = await ServerSocket.bind('localhost', 0); + Process process; + try { + process = + await _spawnExecutable(path, suiteConfig.metadata, serverSocket); + } catch (error) { + serverSocket.close(); + rethrow; + } + process.stdout.listen(stdout.add); + process.stderr.listen(stderr.add); + var socket = await serverSocket.first; + outerChannel = MultiChannel(jsonSocketStreamChannel(socket)); + cleanupCallbacks + ..add(serverSocket.close) + ..add(process.kill); + } else { + var receivePort = ReceivePort(); + try { + isolate = await _spawnIsolate(path, receivePort.sendPort, + suiteConfig.metadata, platform.compiler); + if (isolate == null) return null; + } catch (error) { + receivePort.close(); + rethrow; + } + outerChannel = MultiChannel(IsolateChannel.connectReceive(receivePort)); + cleanupCallbacks + ..add(receivePort.close) + ..add(isolate.kill); } + cleanupCallbacks.add(outerChannel.sink.close); VmService? client; StreamSubscription? eventSub; // Typical test interaction will go across `channel`, `outerChannel` adds // additional communication directly between the test bootstrapping and this // platform to enable pausing after tests for debugging. - var outerChannel = MultiChannel(IsolateChannel.connectReceive(receivePort)); var outerQueue = StreamQueue(outerChannel.stream); var channelId = (await outerQueue.next) as int; var channel = outerChannel.virtualChannel(channelId).transformStream( @@ -73,8 +99,9 @@ class VMPlatform extends PlatformPlugin { outerChannel.sink.add('debug'); await outerQueue.next; } - receivePort.close(); - isolate!.kill(); + for (var fn in cleanupCallbacks) { + fn(); + } eventSub?.cancel(); client?.dispose(); sink.close(); @@ -83,9 +110,14 @@ class VMPlatform extends PlatformPlugin { Environment? environment; IsolateRef? isolateRef; if (_config.debug) { + if (platform.compiler == Compiler.exe) { + throw UnsupportedError( + 'Unable to debug tests compiled to `exe` (tried to debug $path with ' + 'the `exe` compiler).'); + } var info = await Service.controlWebServer(enable: true, silenceOutput: true); - var isolateID = Service.getIsolateID(isolate)!; + var isolateID = Service.getIsolateID(isolate!)!; var libraryPath = _absolute(path).toString(); var serverUri = info.serverUri!; @@ -134,6 +166,46 @@ class VMPlatform extends PlatformPlugin { return _workingDirectory.resolveUri(uri); } + /// Compiles [path] to a native executable and spawns it as a process. + /// + /// Sets up a communication channel as well by passing command line arguments + /// for the host and port of [socket]. + Future _spawnExecutable( + String path, Metadata suiteMetadata, ServerSocket socket) async { + if (_config.suiteDefaults.precompiledPath != null) { + throw UnsupportedError( + 'Precompiled native executable tests are not supported at this time'); + } + var executable = await _compileToNative(path, suiteMetadata); + return await Process.start( + executable, [socket.address.host, socket.port.toString()]); + } + + /// Compiles [path] to a native executable using `dart compile exe`. + Future _compileToNative(String path, Metadata suiteMetadata) async { + var bootstrapPath = _bootstrapNativeTestFile( + path, + suiteMetadata.languageVersionComment ?? + await rootPackageLanguageVersionComment); + var output = File(p.setExtension(bootstrapPath, '.exe')); + var processResult = await Process.run(Platform.resolvedExecutable, [ + 'compile', + 'exe', + bootstrapPath, + '--output', + output.path, + '--packages', + (await packageConfigUri).toFilePath(), + ]); + if (processResult.exitCode != 0 || !(await output.exists())) { + throw LoadException(path, ''' +exitCode: ${processResult.exitCode} +stdout: ${processResult.stdout} +stderr: ${processResult.stderr}'''); + } + return output.path; + } + /// Spawns an isolate with the current configuration and passes it [message]. /// /// This isolate connects an [IsolateChannel] to [message] and sends the @@ -157,7 +229,7 @@ class VMPlatform extends PlatformPlugin { await _compileToKernel(path, suiteMetadata), message); case Compiler.source: return _spawnIsolateWithUri( - _bootstrapTestFile( + _bootstrapIsolateTestFile( path, suiteMetadata.languageVersionComment ?? await rootPackageLanguageVersionComment), @@ -226,21 +298,41 @@ class VMPlatform extends PlatformPlugin { /// file. /// /// Returns the [Uri] to the created file. - Uri _bootstrapTestFile(String testPath, String languageVersionComment) { - var file = File( - p.join(_tempDir.path, p.setExtension(testPath, '.bootstrap.dart'))); + Uri _bootstrapIsolateTestFile( + String testPath, String languageVersionComment) { + var file = File(p.join( + _tempDir.path, p.setExtension(testPath, '.bootstrap.isolate.dart'))); if (!file.existsSync()) { file ..createSync(recursive: true) - ..writeAsStringSync(_bootstrapTestContents( + ..writeAsStringSync(_bootstrapIsolateTestContents( _absolute(testPath), languageVersionComment)); } return file.uri; } + + /// Bootstraps the test at [testPath] for native execution and writes its + /// contents to a temporary file. + /// + /// Returns the path to the created file. + String _bootstrapNativeTestFile( + String testPath, String languageVersionComment) { + var file = File(p.join( + _tempDir.path, p.setExtension(testPath, '.bootstrap.native.dart'))); + if (!file.existsSync()) { + file + ..createSync(recursive: true) + ..writeAsStringSync(_bootstrapNativeTestContents( + _absolute(testPath), languageVersionComment)); + } + return file.path; + } } -/// Creates bootstrap file contents for running [testUri] in the VM. -String _bootstrapTestContents(Uri testUri, String languageVersionComment) => ''' +/// Creates bootstrap file contents for running [testUri] in a VM isolate. +String _bootstrapIsolateTestContents( + Uri testUri, String languageVersionComment) => + ''' $languageVersionComment import "dart:isolate"; import "package:test_core/src/bootstrap/vm.dart"; @@ -250,6 +342,20 @@ String _bootstrapTestContents(Uri testUri, String languageVersionComment) => ''' } '''; +/// Creates bootstrap file contents for running [testUri] as a native +/// executable. +String _bootstrapNativeTestContents( + Uri testUri, String languageVersionComment) => + ''' + $languageVersionComment + import "dart:isolate"; + import "package:test_core/src/bootstrap/vm.dart"; + import "$testUri" as test; + void main(List args) { + internalBootstrapNativeTest(() => test.main, args); + } + '''; + Future> _gatherCoverage(Environment environment) async { final isolateId = Uri.parse(environment.observatoryUrl!.fragment) .queryParameters['isolateId'];