From 3f3305c9fc8110aa39a1da96f757053a8b3b2682 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Thu, 27 Jun 2024 17:47:10 -0700 Subject: [PATCH 1/5] Migrate run_ios_tests.sh to dart. --- testing/scenario_app/bin/run_ios_tests.dart | 365 ++++++++++++++++++++ testing/scenario_app/ios/README.md | 4 +- testing/scenario_app/run_ios_tests.sh | 80 +++-- 3 files changed, 410 insertions(+), 39 deletions(-) create mode 100644 testing/scenario_app/bin/run_ios_tests.dart diff --git a/testing/scenario_app/bin/run_ios_tests.dart b/testing/scenario_app/bin/run_ios_tests.dart new file mode 100644 index 0000000000000..f90bd9d5b9cea --- /dev/null +++ b/testing/scenario_app/bin/run_ios_tests.dart @@ -0,0 +1,365 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:ffi' as ffi; +import 'dart:io' as io; + +import 'package:args/args.dart'; +import 'package:engine_repo_tools/engine_repo_tools.dart'; +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as path; + +void main(List args) async { + if (!io.Platform.isMacOS) { + io.stderr.writeln('This script is only supported on macOS.'); + io.exitCode = 1; + return; + } + + final engine = Engine.tryFindWithin(); + if (engine == null) { + io.stderr.writeln('Must be run from within the engine repository.'); + io.exitCode = 1; + return; + } + + if (args.length > 1 || args.contains('-h') || args.contains('--help')) { + io.stderr.writeln('Usage: run_ios_tests.dart [ios_engine_variant]'); + io.stderr.writeln(_args.usage); + io.exitCode = 1; + return; + } + + final cleanup = Function()>{}; + final results = _args.parse(args); + try { + // Terminate early on SIGINT. + late final StreamSubscription sigint; + sigint = io.ProcessSignal.sigint.watch().listen((_) { + for (final cleanupTask in cleanup) { + cleanupTask(); + } + throw Exception('Received SIGINT'); + }); + cleanup.add(sigint.cancel); + + _ensureSimulatorsRotateAutomaticallyForPlatformViewRotationTest(); + + final deviceName = results.option('device-name')!; + _deleteAnyExistingDevices(deviceName: deviceName); + io.stderr.writeln(); + + final deviceIdentifier = results.option('device-identifier')!; + final osRuntime = results.option('os-runtime')!; + _createDevice( + deviceName: deviceName, + deviceIdentifier: deviceIdentifier, + osRuntime: osRuntime, + ); + io.stderr.writeln(); + + final String iosEngineVariant; + if (args.length == 1) { + iosEngineVariant = args[0]; + } else if (ffi.Abi.current() == ffi.Abi.macosArm64) { + iosEngineVariant = 'ios_debug_sim_unopt_arm64'; + } else { + iosEngineVariant = 'ios_debug_sim_unopt'; + } + io.stderr.writeln('Using engine variant: $iosEngineVariant'); + io.stderr.writeln(); + + final (scenarioPath, resultBundle) = _buildResultBundlePath( + engine: engine, + iosEngineVariant: iosEngineVariant, + ); + cleanup.add(() => resultBundle.deleteSync(recursive: true)); + + final String dumpXcresultOnFailurePath; + if (results.option('dump-xcresult-on-failure') case final String path) { + dumpXcresultOnFailurePath = path; + } else { + dumpXcresultOnFailurePath = io.Directory.systemTemp.createTempSync().path; + } + + final osVersion = results.option('os-version')!; + if (results.flag('with-skia')) { + io.stderr.writeln('Running simulator tests with Skia'); + io.stderr.writeln(); + final process = await _runTests( + outScenariosPath: scenarioPath, + resultBundlePath: resultBundle.path, + osVersion: osVersion, + deviceName: deviceName, + iosEngineVariant: iosEngineVariant, + ); + cleanup.add(process.kill); + + if (await process.exitCode != 0) { + io.stderr.writeln('test failed.'); + io.exitCode = 1; + final String outputPath = _zipAndStoreFailedTestResults( + iosEngineVariant: iosEngineVariant, + resultBundlePath: resultBundle.path, + storePath: dumpXcresultOnFailurePath, + ); + io.stderr.writeln('Failed test results are stored at $outputPath'); + return; + } else { + io.stderr.writeln('test succcess.'); + } + } + + if (results.flag('with-impeller')) { + final process = await _runTests( + outScenariosPath: scenarioPath, + resultBundlePath: resultBundle.path, + osVersion: osVersion, + deviceName: deviceName, + iosEngineVariant: iosEngineVariant, + xcodeBuildExtraArgs: [ + ..._skipTestsForImpeller, + _infplistFPathForImpeller, + ], + ); + cleanup.add(process.kill); + + if (await process.exitCode != 0) { + io.stderr.writeln('test failed.'); + final String outputPath = _zipAndStoreFailedTestResults( + iosEngineVariant: iosEngineVariant, + resultBundlePath: resultBundle.path, + storePath: dumpXcresultOnFailurePath, + ); + io.stderr.writeln('Failed test results are stored at $outputPath'); + io.exitCode = 1; + return; + } else { + io.stderr.writeln('test succcess.'); + } + } + } on Object catch (anyError) { + io.stderr.writeln('Unexpected error: $anyError'); + io.exitCode = 1; + } finally { + for (final cleanupTask in cleanup) { + await cleanupTask(); + } + } +} + +final _args = ArgParser() + ..addFlag( + 'help', + abbr: 'h', + help: 'Prints usage information.', + negatable: false, + ) + ..addOption( + 'device-name', + help: 'The name of the iOS simulator device to use.', + defaultsTo: 'iPhone SE (3rd generation)', + ) + ..addOption( + 'device-identifier', + help: 'The identifier of the iOS simulator device to use.', + defaultsTo: + 'com.apple.CoreSimulator.SimDeviceType.iPhone-SE-3rd-generation', + ) + ..addOption( + 'os-runtime', + help: 'The OS runtime of the iOS simulator device to use.', + defaultsTo: 'com.apple.CoreSimulator.SimRuntime.iOS-17-0', + ) + ..addOption( + 'os-version', + help: 'The OS version of the iOS simulator device to use.', + defaultsTo: '17.0', + ) + ..addFlag( + 'with-impeller', + help: 'Whether to use the Impeller backend to run the tests.\n\nCan be ' + 'combined with --with-skia to run the test suite with both backends.', + defaultsTo: true, + ) + ..addFlag( + 'with-skia', + help: + 'Whether to use the Skia backend to run the tests.\n\nCan be combined ' + 'with --with-impeller to run the test suite with both backends.', + defaultsTo: true, + ) + ..addOption( + 'dump-xcresult-on-failure', + help: 'The path to dump the xcresult bundle to if the test fails.\n\n' + 'Defaults to the environment variable FLUTTER_TEST_OUTPUTS_DIR, ' + 'otherwise to a randomly generated temporary directory.', + defaultsTo: io.Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'], + ); + +void _ensureSimulatorsRotateAutomaticallyForPlatformViewRotationTest() { + // Can also be set via Simulator Device > Rotate Device Automatically. + final result = io.Process.runSync( + 'defaults', + const [ + 'write', + 'com.apple.iphonesimulator', + 'RotateWindowWhenSignaledByGuest', + '-int 1', + ], + ); + if (result.exitCode != 0) { + throw Exception( + 'Failed to enable automatic rotation for iOS simulator: ${result.stderr}', + ); + } +} + +void _deleteAnyExistingDevices({required String deviceName}) { + io.stderr + .writeln('Deleting any existing simulator devices named $deviceName...'); + + bool deleteSimulator() { + final result = io.Process.runSync( + 'xcrun', + ['simctl', 'delete', deviceName], + ); + if (result.exitCode == 0) { + io.stderr.writeln('Deleted $deviceName'); + return true; + } else { + return false; + } + } + + while (deleteSimulator()) {} +} + +void _createDevice({ + required String deviceName, + required String deviceIdentifier, + required String osRuntime, +}) { + io.stderr.writeln('Creating $deviceName $deviceIdentifier $osRuntime...'); + final result = io.Process.runSync( + 'xcrun', + [ + 'simctl', + 'create', + deviceName, + deviceIdentifier, + osRuntime, + ], + ); + if (result.exitCode != 0) { + throw Exception('Failed to create simulator device: ${result.stderr}'); + } +} + +@useResult +(String scenarios, io.Directory resultBundle) _buildResultBundlePath({ + required Engine engine, + required String iosEngineVariant, +}) { + final scenarioPath = path.normalize(path.join( + engine.outDir.path, + iosEngineVariant, + 'scenario_app', + 'Scenarios', + )); + + // Create a temporary directory to store the test results. + final result = io.Directory(scenarioPath).createTempSync('ios_scenario_xcresult'); + return (scenarioPath, result); +} + +@useResult +Future _runTests({ + required String resultBundlePath, + required String outScenariosPath, + required String osVersion, + required String deviceName, + required String iosEngineVariant, + List xcodeBuildExtraArgs = const [], +}) async { + return io.Process.start( + 'xcodebuild', + [ + '-project', + path.join(outScenariosPath, 'Scenarios.xcodeproj'), + '-sdk', + 'iphonesimulator', + '-scheme', + 'Scenarios', + '-resultBundlePath', + path.join(resultBundlePath, 'ios_scenario.xcresult'), + '-destination', + 'platform=iOS Simulator,OS=$osVersion,name=$deviceName', + 'clean', + 'test', + 'FLUTTER_ENGINE=$iosEngineVariant', + ...xcodeBuildExtraArgs, + ], + mode: io.ProcessStartMode.inheritStdio, + ); +} + +/// -skip-testing {$name} args required to pass the Impeller tests. +/// +/// - Skip testFontRenderingWhenSuppliedWithBogusFont: https://github.com/flutter/flutter/issues/113250 +/// - Skip golden tests that use software rendering: https://github.com/flutter/flutter/issues/131888 +final _skipTestsForImpeller = [ + 'ScenariosUITests/MultiplePlatformViewsBackgroundForegroundTest/testPlatformView', + 'ScenariosUITests/MultiplePlatformViewsTest/testPlatformView', + 'ScenariosUITests/NonFullScreenFlutterViewPlatformViewUITests/testPlatformView', + 'ScenariosUITests/PlatformViewMutationClipPathTests/testPlatformView', + 'ScenariosUITests/PlatformViewMutationClipPathWithTransformTests/testPlatformView', + 'ScenariosUITests/PlatformViewMutationClipRectAfterMovedTests/testPlatformView', + 'ScenariosUITests/PlatformViewMutationClipRectTests/testPlatformView', + 'ScenariosUITests/PlatformViewMutationClipRectWithTransformTests/testPlatformView', + 'ScenariosUITests/PlatformViewMutationClipRRectTests/testPlatformView', + 'ScenariosUITests/PlatformViewMutationClipRRectWithTransformTests/testPlatformView', + 'ScenariosUITests/PlatformViewMutationLargeClipRRectTests/testPlatformView', + 'ScenariosUITests/PlatformViewMutationLargeClipRRectWithTransformTests/testPlatformView', + 'ScenariosUITests/PlatformViewMutationOpacityTests/testPlatformView', + 'ScenariosUITests/PlatformViewMutationTransformTests/testPlatformView', + 'ScenariosUITests/PlatformViewRotation/testPlatformView', + 'ScenariosUITests/PlatformViewUITests/testPlatformView', + 'ScenariosUITests/PlatformViewWithNegativeOtherBackDropFilterTests/testPlatformView', + 'ScenariosUITests/PlatformViewWithOtherBackdropFilterTests/testPlatformView', + 'ScenariosUITests/RenderingSelectionTest/testSoftwareRendering', + 'ScenariosUITests/TwoPlatformViewClipPathTests/testPlatformView', + 'ScenariosUITests/TwoPlatformViewClipRectTests/testPlatformView', + 'ScenariosUITests/TwoPlatformViewClipRRectTests/testPlatformView', + 'ScenariosUITests/TwoPlatformViewsWithOtherBackDropFilterTests/testPlatformView', + 'ScenariosUITests/UnobstructedPlatformViewTests/testMultiplePlatformViewsWithOverlays', +].map((name) => '-skip-testing $name').toList(); + +/// Plist with `FTEEnableImpeller=YES`; all projects in the workspace require this file. +/// +/// For example, `FlutterAppExtensionTestHost` has a dummy file under the below directory. +final _infplistFPathForImpeller = path.join('Scenarios', 'Info_Impeller.plist'); + +@useResult +String _zipAndStoreFailedTestResults({ + required String iosEngineVariant, + required String resultBundlePath, + required String storePath, +}) { + final outputPath = path.join(storePath, '$iosEngineVariant.zip'); + final result = io.Process.runSync( + 'zip', + [ + '-q', + '-r', + outputPath, + resultBundlePath, + ], + ); + if (result.exitCode != 0) { + throw Exception('Failed to zip the test results: ${result.stderr}'); + } + return outputPath; +} diff --git a/testing/scenario_app/ios/README.md b/testing/scenario_app/ios/README.md index 972625056d403..62e3e538f3df6 100644 --- a/testing/scenario_app/ios/README.md +++ b/testing/scenario_app/ios/README.md @@ -11,7 +11,9 @@ run: # From the root of the engine repository $ ./testing/run_ios_tests.sh ios_debug_sim_unopt ``` -or + +or: + ```sh # From the root of the engine repository $ ./testing/run_ios_tests.sh ios_debug_sim_unopt_arm64 diff --git a/testing/scenario_app/run_ios_tests.sh b/testing/scenario_app/run_ios_tests.sh index ed040b16d1b47..be88166c692ab 100755 --- a/testing/scenario_app/run_ios_tests.sh +++ b/testing/scenario_app/run_ios_tests.sh @@ -2,7 +2,6 @@ set -e - # Needed because if it is set, cd may print the path it changed to. unset CDPATH @@ -25,7 +24,10 @@ function follow_links() ( ) SCRIPT_DIR=$(follow_links "$(dirname -- "${BASH_SOURCE[0]}")") -SRC_DIR="$(cd "$SCRIPT_DIR/../../.."; pwd -P)" +SRC_DIR="$( + cd "$SCRIPT_DIR/../../.." + pwd -P +)" if uname -m | grep "arm64"; then FLUTTER_ENGINE="ios_debug_sim_unopt_arm64" @@ -50,7 +52,7 @@ RESULT_BUNDLE_PATH="${SCENARIO_PATH}/${RESULT_BUNDLE_FOLDER}" # Zip and upload xcresult to luci. # First parameter ($1) is the zip output name. -zip_and_upload_xcresult_to_luci () { +zip_and_upload_xcresult_to_luci() { # We don't want the zip to contain the abusolute path, # so use relative path (./$RESULT_BUNDLE_FOLDER) instead. zip -q -r $1 "./$RESULT_BUNDLE_FOLDER" @@ -68,10 +70,10 @@ readonly OS="17.0" echo "Deleting any existing devices names $DEVICE_NAME..." RESULT=0 while [[ $RESULT == 0 ]]; do - xcrun simctl delete "$DEVICE_NAME" || RESULT=1 - if [ $RESULT == 0 ]; then - echo "Deleted $DEVICE_NAME" - fi + xcrun simctl delete "$DEVICE_NAME" || RESULT=1 + if [ $RESULT == 0 ]; then + echo "Deleted $DEVICE_NAME" + fi done echo "" @@ -100,39 +102,41 @@ echo "" # Skip testFontRenderingWhenSuppliedWithBogusFont: https://github.com/flutter/flutter/issues/113250 # Skip golden tests that use software rendering: https://github.com/flutter/flutter/issues/131888 -if set -o pipefail && xcodebuild -sdk iphonesimulator \ - -scheme Scenarios \ - -resultBundlePath "$RESULT_BUNDLE_PATH/ios_scenario.xcresult" \ - -destination "platform=iOS Simulator,OS=$OS,name=$DEVICE_NAME" \ - clean test \ - FLUTTER_ENGINE="$FLUTTER_ENGINE" \ - -skip-testing ScenariosUITests/MultiplePlatformViewsBackgroundForegroundTest/testPlatformView \ - -skip-testing ScenariosUITests/MultiplePlatformViewsTest/testPlatformView \ - -skip-testing ScenariosUITests/NonFullScreenFlutterViewPlatformViewUITests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipPathTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipPathWithTransformTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipRectAfterMovedTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipRectTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipRectWithTransformTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipRRectTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipRRectWithTransformTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationLargeClipRRectTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationLargeClipRRectWithTransformTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationOpacityTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationTransformTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewRotation/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewUITests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewWithNegativeOtherBackDropFilterTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewWithOtherBackdropFilterTests/testPlatformView \ - -skip-testing ScenariosUITests/RenderingSelectionTest/testSoftwareRendering \ - -skip-testing ScenariosUITests/TwoPlatformViewClipPathTests/testPlatformView \ - -skip-testing ScenariosUITests/TwoPlatformViewClipRectTests/testPlatformView \ - -skip-testing ScenariosUITests/TwoPlatformViewClipRRectTests/testPlatformView \ - -skip-testing ScenariosUITests/TwoPlatformViewsWithOtherBackDropFilterTests/testPlatformView \ - -skip-testing ScenariosUITests/UnobstructedPlatformViewTests/testMultiplePlatformViewsWithOverlays \ +if + set -o pipefail && xcodebuild -sdk iphonesimulator \ + -scheme Scenarios \ + -resultBundlePath "$RESULT_BUNDLE_PATH/ios_scenario.xcresult" \ + -destination "platform=iOS Simulator,OS=$OS,name=$DEVICE_NAME" \ + clean test \ + FLUTTER_ENGINE="$FLUTTER_ENGINE" \ + -skip-testing ScenariosUITests/MultiplePlatformViewsBackgroundForegroundTest/testPlatformView \ + -skip-testing ScenariosUITests/MultiplePlatformViewsTest/testPlatformView \ + -skip-testing ScenariosUITests/NonFullScreenFlutterViewPlatformViewUITests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipPathTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipPathWithTransformTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipRectAfterMovedTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipRectTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipRectWithTransformTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipRRectTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipRRectWithTransformTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationLargeClipRRectTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationLargeClipRRectWithTransformTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationOpacityTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationTransformTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewRotation/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewUITests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewWithNegativeOtherBackDropFilterTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewWithOtherBackdropFilterTests/testPlatformView \ + -skip-testing ScenariosUITests/RenderingSelectionTest/testSoftwareRendering \ + -skip-testing ScenariosUITests/TwoPlatformViewClipPathTests/testPlatformView \ + -skip-testing ScenariosUITests/TwoPlatformViewClipRectTests/testPlatformView \ + -skip-testing ScenariosUITests/TwoPlatformViewClipRRectTests/testPlatformView \ + -skip-testing ScenariosUITests/TwoPlatformViewsWithOtherBackDropFilterTests/testPlatformView \ + -skip-testing ScenariosUITests/UnobstructedPlatformViewTests/testMultiplePlatformViewsWithOverlays # Plist with FLTEnableImpeller=YES, all projects in the workspace requires this file. # For example, FlutterAppExtensionTestHost has a dummy file under the below directory. - INFOPLIST_FILE="Scenarios/Info_Impeller.plist"; then + INFOPLIST_FILE="Scenarios/Info_Impeller.plist" +then echo "test success." else echo "test failed." From 3e3949e17dff992f0908124c88c7328637c4cdda Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Fri, 28 Jun 2024 16:19:10 -0700 Subject: [PATCH 2/5] Now working. --- testing/scenario_app/bin/run_ios_tests.dart | 245 ++++++++++++-------- 1 file changed, 144 insertions(+), 101 deletions(-) diff --git a/testing/scenario_app/bin/run_ios_tests.dart b/testing/scenario_app/bin/run_ios_tests.dart index f90bd9d5b9cea..223b7cff2d886 100644 --- a/testing/scenario_app/bin/run_ios_tests.dart +++ b/testing/scenario_app/bin/run_ios_tests.dart @@ -32,124 +32,166 @@ void main(List args) async { return; } + // Collect cleanup tasks to run when the script terminates. final cleanup = Function()>{}; + + // Parse the command-line arguments. final results = _args.parse(args); - try { - // Terminate early on SIGINT. - late final StreamSubscription sigint; - sigint = io.ProcessSignal.sigint.watch().listen((_) { - for (final cleanupTask in cleanup) { - cleanupTask(); - } - throw Exception('Received SIGINT'); - }); - cleanup.add(sigint.cancel); - - _ensureSimulatorsRotateAutomaticallyForPlatformViewRotationTest(); - - final deviceName = results.option('device-name')!; - _deleteAnyExistingDevices(deviceName: deviceName); - io.stderr.writeln(); + final String iosEngineVariant; + if (args.length == 1) { + iosEngineVariant = args[0]; + } else if (ffi.Abi.current() == ffi.Abi.macosArm64) { + iosEngineVariant = 'ios_debug_sim_unopt_arm64'; + } else { + iosEngineVariant = 'ios_debug_sim_unopt'; + } + final String dumpXcresultOnFailurePath; + if (results.option('dump-xcresult-on-failure') case final String path) { + dumpXcresultOnFailurePath = path; + } else { + dumpXcresultOnFailurePath = io.Directory.systemTemp.createTempSync().path; + } - final deviceIdentifier = results.option('device-identifier')!; - final osRuntime = results.option('os-runtime')!; - _createDevice( - deviceName: deviceName, - deviceIdentifier: deviceIdentifier, - osRuntime: osRuntime, + // Run the actual script. + final completer = Completer(); + runZonedGuarded(() async { + await _run( + cleanup, + engine, + iosEngineVariant: iosEngineVariant, + deviceName: results.option('device-name')!, + deviceIdentifier: results.option('device-identifier')!, + osRuntime: results.option('os-runtime')!, + osVersion: results.option('os-version')!, + withImpeller: results.flag('with-impeller'), + withSkia: results.flag('with-skia'), + dumpXcresultOnFailure: dumpXcresultOnFailurePath, ); - io.stderr.writeln(); - - final String iosEngineVariant; - if (args.length == 1) { - iosEngineVariant = args[0]; - } else if (ffi.Abi.current() == ffi.Abi.macosArm64) { - iosEngineVariant = 'ios_debug_sim_unopt_arm64'; + completer.complete(); + }, (e, s) { + if (e is _ToolFailure) { + io.stderr.writeln(e); + io.exitCode = 1; } else { - iosEngineVariant = 'ios_debug_sim_unopt'; + io.stderr.writeln('Uncaught exception: $e\n$s'); + io.exitCode = 255; } - io.stderr.writeln('Using engine variant: $iosEngineVariant'); - io.stderr.writeln(); + completer.complete(); + }); + + // We can't await the result of runZonedGuarded (read the docs on it). + await completer.future; + + // Run cleanup tasks. + for (final task in cleanup) { + await task(); + } +} - final (scenarioPath, resultBundle) = _buildResultBundlePath( - engine: engine, +/// Runs the script. +/// +/// The [cleanup] set contains cleanup tasks to run when the script is either +/// completed normally or terminated early. For example, deleting a temporary +/// directory or killing a process. +/// +/// Each named argument cooresponds to a flag or option in the `ArgParser`. +Future _run( + Set Function()> cleanup, + Engine engine, { + required String iosEngineVariant, + required String deviceName, + required String deviceIdentifier, + required String osRuntime, + required String osVersion, + required bool withImpeller, + required bool withSkia, + required String dumpXcresultOnFailure, +}) async { + // Terminate early on SIGINT. + late final StreamSubscription sigint; + sigint = io.ProcessSignal.sigint.watch().listen((_) { + throw _ToolFailure('Received SIGINT'); + }); + cleanup.add(sigint.cancel); + + _ensureSimulatorsRotateAutomaticallyForPlatformViewRotationTest(); + _deleteAnyExistingDevices(deviceName: deviceName); + _createDevice( + deviceName: deviceName, + deviceIdentifier: deviceIdentifier, + osRuntime: osRuntime, + ); + + final (scenarioPath, resultBundle) = _buildResultBundlePath( + engine: engine, + iosEngineVariant: iosEngineVariant, + ); + + cleanup.add(() => resultBundle.deleteSync(recursive: true)); + + if (withSkia) { + io.stderr.writeln('Running simulator tests with Skia'); + io.stderr.writeln(); + final process = await _runTests( + outScenariosPath: scenarioPath, + resultBundlePath: resultBundle.path, + osVersion: osVersion, + deviceName: deviceName, iosEngineVariant: iosEngineVariant, ); - cleanup.add(() => resultBundle.deleteSync(recursive: true)); - - final String dumpXcresultOnFailurePath; - if (results.option('dump-xcresult-on-failure') case final String path) { - dumpXcresultOnFailurePath = path; - } else { - dumpXcresultOnFailurePath = io.Directory.systemTemp.createTempSync().path; - } + cleanup.add(process.kill); - final osVersion = results.option('os-version')!; - if (results.flag('with-skia')) { - io.stderr.writeln('Running simulator tests with Skia'); - io.stderr.writeln(); - final process = await _runTests( - outScenariosPath: scenarioPath, - resultBundlePath: resultBundle.path, - osVersion: osVersion, - deviceName: deviceName, + if (await process.exitCode != 0) { + final String outputPath = _zipAndStoreFailedTestResults( iosEngineVariant: iosEngineVariant, + resultBundlePath: resultBundle.path, + storePath: dumpXcresultOnFailure, ); - cleanup.add(process.kill); - - if (await process.exitCode != 0) { - io.stderr.writeln('test failed.'); - io.exitCode = 1; - final String outputPath = _zipAndStoreFailedTestResults( - iosEngineVariant: iosEngineVariant, - resultBundlePath: resultBundle.path, - storePath: dumpXcresultOnFailurePath, - ); - io.stderr.writeln('Failed test results are stored at $outputPath'); - return; - } else { - io.stderr.writeln('test succcess.'); - } + io.stderr.writeln('Failed test results are stored at $outputPath'); + throw _ToolFailure('test failed.'); + } else { + io.stderr.writeln('test succcess.'); } + } - if (results.flag('with-impeller')) { - final process = await _runTests( - outScenariosPath: scenarioPath, - resultBundlePath: resultBundle.path, - osVersion: osVersion, - deviceName: deviceName, + if (withImpeller) { + final process = await _runTests( + outScenariosPath: scenarioPath, + resultBundlePath: resultBundle.path, + osVersion: osVersion, + deviceName: deviceName, + iosEngineVariant: iosEngineVariant, + xcodeBuildExtraArgs: [ + ..._skipTestsForImpeller, + _infplistFPathForImpeller, + ], + ); + cleanup.add(process.kill); + + if (await process.exitCode != 0) { + final String outputPath = _zipAndStoreFailedTestResults( iosEngineVariant: iosEngineVariant, - xcodeBuildExtraArgs: [ - ..._skipTestsForImpeller, - _infplistFPathForImpeller, - ], + resultBundlePath: resultBundle.path, + storePath: dumpXcresultOnFailure, ); - cleanup.add(process.kill); - - if (await process.exitCode != 0) { - io.stderr.writeln('test failed.'); - final String outputPath = _zipAndStoreFailedTestResults( - iosEngineVariant: iosEngineVariant, - resultBundlePath: resultBundle.path, - storePath: dumpXcresultOnFailurePath, - ); - io.stderr.writeln('Failed test results are stored at $outputPath'); - io.exitCode = 1; - return; - } else { - io.stderr.writeln('test succcess.'); - } - } - } on Object catch (anyError) { - io.stderr.writeln('Unexpected error: $anyError'); - io.exitCode = 1; - } finally { - for (final cleanupTask in cleanup) { - await cleanupTask(); + io.stderr.writeln('Failed test results are stored at $outputPath'); + throw _ToolFailure('test failed.'); + } else { + io.stderr.writeln('test succcess.'); } } } +/// Exception thrown when the tool should halt execution intentionally. +final class _ToolFailure implements Exception { + _ToolFailure(this.message); + + final String message; + + @override + String toString() => message; +} + final _args = ArgParser() ..addFlag( 'help', @@ -194,8 +236,8 @@ final _args = ArgParser() ..addOption( 'dump-xcresult-on-failure', help: 'The path to dump the xcresult bundle to if the test fails.\n\n' - 'Defaults to the environment variable FLUTTER_TEST_OUTPUTS_DIR, ' - 'otherwise to a randomly generated temporary directory.', + 'Defaults to the environment variable FLUTTER_TEST_OUTPUTS_DIR, ' + 'otherwise to a randomly generated temporary directory.', defaultsTo: io.Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'], ); @@ -271,7 +313,8 @@ void _createDevice({ )); // Create a temporary directory to store the test results. - final result = io.Directory(scenarioPath).createTempSync('ios_scenario_xcresult'); + final result = + io.Directory(scenarioPath).createTempSync('ios_scenario_xcresult'); return (scenarioPath, result); } From 1b447cc9e03f9b18c239325dba7d72cf106c31e8 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Fri, 28 Jun 2024 16:19:40 -0700 Subject: [PATCH 3/5] ++ --- testing/scenario_app/bin/run_ios_tests.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scenario_app/bin/run_ios_tests.dart b/testing/scenario_app/bin/run_ios_tests.dart index 223b7cff2d886..16502797a9612 100644 --- a/testing/scenario_app/bin/run_ios_tests.dart +++ b/testing/scenario_app/bin/run_ios_tests.dart @@ -96,7 +96,7 @@ void main(List args) async { /// /// Each named argument cooresponds to a flag or option in the `ArgParser`. Future _run( - Set Function()> cleanup, + Set Function()> cleanup, Engine engine, { required String iosEngineVariant, required String deviceName, From a4ab3878bade59636b6c873d3b75408ddee813b3 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Fri, 28 Jun 2024 16:30:12 -0700 Subject: [PATCH 4/5] ++ --- testing/scenario_app/bin/run_ios_tests.dart | 21 ++++-- testing/scenario_app/run_ios_tests.sh | 80 ++++++++++----------- 2 files changed, 54 insertions(+), 47 deletions(-) diff --git a/testing/scenario_app/bin/run_ios_tests.dart b/testing/scenario_app/bin/run_ios_tests.dart index 16502797a9612..8a909baae33b7 100644 --- a/testing/scenario_app/bin/run_ios_tests.dart +++ b/testing/scenario_app/bin/run_ios_tests.dart @@ -38,8 +38,8 @@ void main(List args) async { // Parse the command-line arguments. final results = _args.parse(args); final String iosEngineVariant; - if (args.length == 1) { - iosEngineVariant = args[0]; + if (results.rest case [final variant]) { + iosEngineVariant = variant; } else if (ffi.Abi.current() == ffi.Abi.macosArm64) { iosEngineVariant = 'ios_debug_sim_unopt_arm64'; } else { @@ -163,7 +163,7 @@ Future _run( iosEngineVariant: iosEngineVariant, xcodeBuildExtraArgs: [ ..._skipTestsForImpeller, - _infplistFPathForImpeller, + _infoPlistFPathForImpeller(engine), ], ); cleanup.add(process.kill); @@ -378,12 +378,23 @@ final _skipTestsForImpeller = [ 'ScenariosUITests/TwoPlatformViewClipRRectTests/testPlatformView', 'ScenariosUITests/TwoPlatformViewsWithOtherBackDropFilterTests/testPlatformView', 'ScenariosUITests/UnobstructedPlatformViewTests/testMultiplePlatformViewsWithOverlays', -].map((name) => '-skip-testing $name').toList(); +].map((name) => '-skip-testing:$name').toList(); /// Plist with `FTEEnableImpeller=YES`; all projects in the workspace require this file. /// /// For example, `FlutterAppExtensionTestHost` has a dummy file under the below directory. -final _infplistFPathForImpeller = path.join('Scenarios', 'Info_Impeller.plist'); +String _infoPlistFPathForImpeller(Engine engine) { + final infoPath = path.join( + engine.flutterDir.path, + 'testing', + 'scenario_app', + 'ios', + 'Scenarios', + 'Scenarios', + 'Info_Impeller.plist', + ); + return 'INFOPLIST_FILE=$infoPath'; +} @useResult String _zipAndStoreFailedTestResults({ diff --git a/testing/scenario_app/run_ios_tests.sh b/testing/scenario_app/run_ios_tests.sh index be88166c692ab..ed040b16d1b47 100755 --- a/testing/scenario_app/run_ios_tests.sh +++ b/testing/scenario_app/run_ios_tests.sh @@ -2,6 +2,7 @@ set -e + # Needed because if it is set, cd may print the path it changed to. unset CDPATH @@ -24,10 +25,7 @@ function follow_links() ( ) SCRIPT_DIR=$(follow_links "$(dirname -- "${BASH_SOURCE[0]}")") -SRC_DIR="$( - cd "$SCRIPT_DIR/../../.." - pwd -P -)" +SRC_DIR="$(cd "$SCRIPT_DIR/../../.."; pwd -P)" if uname -m | grep "arm64"; then FLUTTER_ENGINE="ios_debug_sim_unopt_arm64" @@ -52,7 +50,7 @@ RESULT_BUNDLE_PATH="${SCENARIO_PATH}/${RESULT_BUNDLE_FOLDER}" # Zip and upload xcresult to luci. # First parameter ($1) is the zip output name. -zip_and_upload_xcresult_to_luci() { +zip_and_upload_xcresult_to_luci () { # We don't want the zip to contain the abusolute path, # so use relative path (./$RESULT_BUNDLE_FOLDER) instead. zip -q -r $1 "./$RESULT_BUNDLE_FOLDER" @@ -70,10 +68,10 @@ readonly OS="17.0" echo "Deleting any existing devices names $DEVICE_NAME..." RESULT=0 while [[ $RESULT == 0 ]]; do - xcrun simctl delete "$DEVICE_NAME" || RESULT=1 - if [ $RESULT == 0 ]; then - echo "Deleted $DEVICE_NAME" - fi + xcrun simctl delete "$DEVICE_NAME" || RESULT=1 + if [ $RESULT == 0 ]; then + echo "Deleted $DEVICE_NAME" + fi done echo "" @@ -102,41 +100,39 @@ echo "" # Skip testFontRenderingWhenSuppliedWithBogusFont: https://github.com/flutter/flutter/issues/113250 # Skip golden tests that use software rendering: https://github.com/flutter/flutter/issues/131888 -if - set -o pipefail && xcodebuild -sdk iphonesimulator \ - -scheme Scenarios \ - -resultBundlePath "$RESULT_BUNDLE_PATH/ios_scenario.xcresult" \ - -destination "platform=iOS Simulator,OS=$OS,name=$DEVICE_NAME" \ - clean test \ - FLUTTER_ENGINE="$FLUTTER_ENGINE" \ - -skip-testing ScenariosUITests/MultiplePlatformViewsBackgroundForegroundTest/testPlatformView \ - -skip-testing ScenariosUITests/MultiplePlatformViewsTest/testPlatformView \ - -skip-testing ScenariosUITests/NonFullScreenFlutterViewPlatformViewUITests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipPathTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipPathWithTransformTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipRectAfterMovedTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipRectTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipRectWithTransformTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipRRectTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationClipRRectWithTransformTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationLargeClipRRectTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationLargeClipRRectWithTransformTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationOpacityTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewMutationTransformTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewRotation/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewUITests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewWithNegativeOtherBackDropFilterTests/testPlatformView \ - -skip-testing ScenariosUITests/PlatformViewWithOtherBackdropFilterTests/testPlatformView \ - -skip-testing ScenariosUITests/RenderingSelectionTest/testSoftwareRendering \ - -skip-testing ScenariosUITests/TwoPlatformViewClipPathTests/testPlatformView \ - -skip-testing ScenariosUITests/TwoPlatformViewClipRectTests/testPlatformView \ - -skip-testing ScenariosUITests/TwoPlatformViewClipRRectTests/testPlatformView \ - -skip-testing ScenariosUITests/TwoPlatformViewsWithOtherBackDropFilterTests/testPlatformView \ - -skip-testing ScenariosUITests/UnobstructedPlatformViewTests/testMultiplePlatformViewsWithOverlays +if set -o pipefail && xcodebuild -sdk iphonesimulator \ + -scheme Scenarios \ + -resultBundlePath "$RESULT_BUNDLE_PATH/ios_scenario.xcresult" \ + -destination "platform=iOS Simulator,OS=$OS,name=$DEVICE_NAME" \ + clean test \ + FLUTTER_ENGINE="$FLUTTER_ENGINE" \ + -skip-testing ScenariosUITests/MultiplePlatformViewsBackgroundForegroundTest/testPlatformView \ + -skip-testing ScenariosUITests/MultiplePlatformViewsTest/testPlatformView \ + -skip-testing ScenariosUITests/NonFullScreenFlutterViewPlatformViewUITests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipPathTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipPathWithTransformTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipRectAfterMovedTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipRectTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipRectWithTransformTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipRRectTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationClipRRectWithTransformTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationLargeClipRRectTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationLargeClipRRectWithTransformTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationOpacityTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewMutationTransformTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewRotation/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewUITests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewWithNegativeOtherBackDropFilterTests/testPlatformView \ + -skip-testing ScenariosUITests/PlatformViewWithOtherBackdropFilterTests/testPlatformView \ + -skip-testing ScenariosUITests/RenderingSelectionTest/testSoftwareRendering \ + -skip-testing ScenariosUITests/TwoPlatformViewClipPathTests/testPlatformView \ + -skip-testing ScenariosUITests/TwoPlatformViewClipRectTests/testPlatformView \ + -skip-testing ScenariosUITests/TwoPlatformViewClipRRectTests/testPlatformView \ + -skip-testing ScenariosUITests/TwoPlatformViewsWithOtherBackDropFilterTests/testPlatformView \ + -skip-testing ScenariosUITests/UnobstructedPlatformViewTests/testMultiplePlatformViewsWithOverlays \ # Plist with FLTEnableImpeller=YES, all projects in the workspace requires this file. # For example, FlutterAppExtensionTestHost has a dummy file under the below directory. - INFOPLIST_FILE="Scenarios/Info_Impeller.plist" -then + INFOPLIST_FILE="Scenarios/Info_Impeller.plist"; then echo "test success." else echo "test failed." From 70138e539d0853bf42c241e58555a2f7d03a40a8 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Mon, 1 Jul 2024 16:31:01 -0700 Subject: [PATCH 5/5] Update testing/scenario_app/bin/run_ios_tests.dart Co-authored-by: John McDole --- testing/scenario_app/bin/run_ios_tests.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scenario_app/bin/run_ios_tests.dart b/testing/scenario_app/bin/run_ios_tests.dart index 8a909baae33b7..5b97f971d8dfb 100644 --- a/testing/scenario_app/bin/run_ios_tests.dart +++ b/testing/scenario_app/bin/run_ios_tests.dart @@ -79,7 +79,7 @@ void main(List args) async { completer.complete(); }); - // We can't await the result of runZonedGuarded (read the docs on it). + // We can't await the result of runZonedGuarded becauase async errors in futures never cross different errorZone boundaries. await completer.future; // Run cleanup tasks.