From 55059edd60914043017235b36a4c766753d61bf6 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Mon, 26 Feb 2024 13:10:28 -0800 Subject: [PATCH 1/7] Refactor args parsing and validation. --- .../scenario_app/bin/utils/environment.dart | 45 +++ testing/scenario_app/bin/utils/options.dart | 280 ++++++++++++++++++ testing/scenario_app/pubspec.yaml | 1 + 3 files changed, 326 insertions(+) create mode 100644 testing/scenario_app/bin/utils/environment.dart create mode 100644 testing/scenario_app/bin/utils/options.dart diff --git a/testing/scenario_app/bin/utils/environment.dart b/testing/scenario_app/bin/utils/environment.dart new file mode 100644 index 0000000000000..2ec16124052e8 --- /dev/null +++ b/testing/scenario_app/bin/utils/environment.dart @@ -0,0 +1,45 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:meta/meta.dart'; + +/// An overridable collection of values provided by the environment. +@immutable +final class Environment { + /// Creates a new environment from the given values. + const Environment({ + required this.isCi, + required this.showVerbose, + required this.logsDir, + }); + + /// Whether the current program is running on a CI environment. + /// + /// Useful for determining if certain features should be enabled or disabled + /// based on the environment, or to add safety checks (for example, using + /// confusing or ambiguous flags). + final bool isCi; + + /// Whether the user has requested verbose logging and program output. + final bool showVerbose; + + /// What directory to store logs and screenshots in. + final String logsDir; + + @override + bool operator ==(Object o) { + return o is Environment && + o.isCi == isCi && + o.showVerbose == showVerbose && + o.logsDir == logsDir; + } + + @override + int get hashCode => Object.hash(isCi, showVerbose, logsDir); + + @override + String toString() { + return 'Environment(isCi: $isCi, showVerbose: $showVerbose, logsDir: $logsDir)'; + } +} diff --git a/testing/scenario_app/bin/utils/options.dart b/testing/scenario_app/bin/utils/options.dart new file mode 100644 index 0000000000000..8052f20d5b5de --- /dev/null +++ b/testing/scenario_app/bin/utils/options.dart @@ -0,0 +1,280 @@ +import 'dart:io' as io; + +import 'package:args/args.dart'; +import 'package:engine_repo_tools/engine_repo_tools.dart'; +import 'package:path/path.dart' as p; + +import 'environment.dart'; + +/// Command line options and parser for the Android `scenario_app` test runner. +extension type const Options._(ArgResults _args) { + /// Parses the command line [args] into a set of options. + factory Options.parse( + List args, { + required Environment environment, + required Engine? localEngine, + }) { + final ArgResults results = _parser(environment, localEngine).parse(args); + final Options options = Options._(results); + + // The 'adb' tool must exist. + if (results['adb'] == null) { + throw ArgumentError('The --adb option must be set.'); + } else if (!io.File(options.adb).existsSync()) { + throw ArgumentError('The adb tool does not exist at ${options.adb}.'); + } + + // The 'ndk-stack' tool must exist. + if (results['ndk-stack'] == null) { + throw ArgumentError('The --ndk-stack option must be set.'); + } else if (!io.File(options.ndkStack).existsSync()) { + throw ArgumentError('The ndk-stack tool does not exist at ${options.ndkStack}.'); + } + + // The 'out-dir' must exist. + if (results['out-dir'] == null) { + throw ArgumentError('The --out-dir option must be set.'); + } else if (!io.Directory(options.outDir).existsSync()) { + throw ArgumentError('The out directory does not exist at ${options.outDir}.'); + } + + return options; + } + + /// A subset of command line options used by the `scenario_app` runner. + /// + /// These options are expected to be parsed before the full set of options. + static final ArgParser _miniParser = ArgParser() + ..addFlag( + 'verbose', + abbr: 'v', + help: 'Enable verbose logging', + ) + ..addFlag( + 'help', + abbr: 'h', + help: 'Print usage information', + negatable: false, + ); + + /// Whether usage information should be shown based on command line [args]. + /// + /// This is a shortcut that can be used to determine if the usage information + /// before parsing the remaining command line arguments. For example: + /// + /// ``` + /// void main(List args) { + /// if (Options.showUsage(args)) { + /// stdout.writeln(Options.usage); + /// return; + /// } + /// final options = Options.parse(args); + /// // ... + /// } + /// ``` + static bool showUsage(List args) { + final ArgResults results = _miniParser.parse(args); + return results['help'] as bool; + } + + /// Whether verbose logging should be enabled based on command line [args]. + /// + /// This is a shortcut that can be used to determine if verbose logging should + /// be enabled before parsing the remaining command line arguments. For + /// example: + /// + /// ``` + /// void main(List args) { + /// final bool verbose = Options.showVerbose(args); + /// // ... + /// } + /// ``` + static bool showVerbose(List args) { + final ArgResults results = _miniParser.parse(args); + return results['verbose'] as bool; + } + + /// Returns usage information for the `scenario_app` test runner. + /// + /// If [verbose] is `true`, then additional options are shown. + static String usage({ + required Environment environment, + required Engine? localEngineDir, + }) { + return _parser(environment, localEngineDir).usage; + } + + /// Parses the command line [args] into a set of options. + /// + /// Unlike [_miniParser], this parser includes all options. + static ArgParser _parser(Environment environment, Engine? localEngine) { + final bool hideUnusualOptions = !environment.showVerbose; + return ArgParser() + ..addFlag( + 'verbose', + abbr: 'v', + help: 'Enable verbose logging', + ) + ..addFlag( + 'help', + abbr: 'h', + help: 'Print usage information', + ) + ..addFlag( + 'use-skia-gold', + help: + 'Whether to use Skia Gold to compare screenshots. Defaults to true ' + 'on CI and false otherwise.', + defaultsTo: environment.isCi, + hide: hideUnusualOptions, + ) + ..addFlag( + 'enable-impeller', + help: + 'Whether to enable Impeller as the graphics backend. If true, the ' + 'test runner will use --impeller-backend if set, otherwise the ' + 'default backend will be used. To explicitly run with the Skia ' + 'backend, set this to false (--no-enable-impeller).', + ) + ..addOption( + 'impeller-backend', + help: 'The graphics backend to use when --enable-impeller is true. ' + 'Unlike the similar option when launching an app, there is no ' + 'fallback; that is, either Vulkan or OpenGLES must be specified. ', + allowed: ['vulkan', 'opengles'], + defaultsTo: 'vulkan', + ) + ..addOption( + 'logs-dir', + help: 'Path to a directory where logs and screenshots are stored.', + defaultsTo: environment.logsDir, + ) + ..addOption( + 'adb', + help: 'Path to the Android Debug Bridge (adb) executable. ' + 'If the current working directory is within the engine repository, ' + 'defaults to ./third_party/android_tools/sdk/platform-tools/adb.', + defaultsTo: localEngine != null + ? p.join( + localEngine.srcDir.path, + 'third_party', + 'android_tools', + 'sdk', + 'platform-tools', + 'adb', + ) + : null, + valueHelp: 'path/to/adb', + hide: hideUnusualOptions, + ) + ..addOption( + 'ndk-stack', + help: + 'Path to the NDK stack tool. Defaults to the checked-in version in ' + 'third_party/android_tools if the current working directory is ' + 'within the engine repository on a supported platform.', + defaultsTo: localEngine != null && + (io.Platform.isLinux || + io.Platform.isMacOS || + io.Platform.isWindows) + ? p.join( + localEngine.srcDir.path, + 'third_party', + 'android_tools', + 'ndk', + 'prebuilt', + () { + if (io.Platform.isLinux) { + return 'linux-x86_64'; + } else if (io.Platform.isMacOS) { + return 'darwin-x86_64'; + } else if (io.Platform.isWindows) { + return 'windows-x86_64'; + } else { + // Unreachable. + throw UnsupportedError( + 'Unsupported platform: ${io.Platform.operatingSystem}', + ); + } + }(), + 'bin', + 'ndk-stack', + ) + : null, + valueHelp: 'path/to/ndk-stack', + hide: hideUnusualOptions, + ) + ..addOption( + 'out-dir', + help: 'Path to a out/{variant} directory where the APKs are built. ' + 'Defaults to the latest updated out/ directory that starts with ' + '"android_" if the current working directory is within the engine ' + 'repository.', + defaultsTo: localEngine + ?.outputs() + .where((Output o) => p.basename(o.path.path).startsWith('android_')) + .firstOrNull + ?.path + .path, + mandatory: environment.isCi, + valueHelp: 'path/to/out/android_variant', + ) + ..addOption( + 'smoke-test', + help: 'Fully qualified class name of a single test to run. For example ' + 'try "dev.flutter.scenarios.EngineLaunchE2ETest" or ' + '"dev.flutter.scenariosui.ExternalTextureTests".', + valueHelp: 'package.ClassName', + ) + ..addOption( + 'output-contents-golden', + help: 'Path to a file that contains the expected filenames of golden ' + 'files. If the current working directory is within the engine ' + 'repository, defaults to ./testing/scenario_app/android/' + 'expected_golden_output.txt.', + defaultsTo: localEngine != null + ? p.join( + localEngine.srcDir.path, + 'testing', + 'scenario_app', + 'android', + 'expected_golden_output.txt', + ) + : null, + valueHelp: 'path/to/golden.txt', + ); + } + + /// Whether verbose logging should be enabled. + bool get verbose => _args['verbose'] as bool; + + /// Whether usage information should be shown. + bool get help => _args['help'] as bool; + + /// Whether to use Skia Gold to compare screenshots. + bool get useSkiaGold => _args['use-skia-gold'] as bool; + + /// Whether to enable Impeller as the graphics backend. + bool get enableImpeller => _args['enable-impeller'] as bool; + + /// The graphics backend to use when --enable-impeller is true. + String get impellerBackend => _args['impeller-backend'] as String; + + /// Path to a directory where logs and screenshots are stored. + String get logsDir => _args['logs-dir'] as String; + + /// Path to the Android Debug Bridge (adb) executable. + String get adb => _args['adb'] as String; + + /// Path to the NDK stack tool. + String get ndkStack => _args['ndk-stack'] as String; + + /// Path to a out/{variant} directory where the APKs are built. + String get outDir => _args['out-dir'] as String; + + /// Fully qualified class name of a single test to run. + String? get smokeTest => _args['smoke-test'] as String?; + + /// Path to a file that contains the expected filenames of golden files. + String? get outputContentsGolden => _args['output-contents-golden'] as String; +} diff --git a/testing/scenario_app/pubspec.yaml b/testing/scenario_app/pubspec.yaml index 039ce83e22854..f42e15868a9b2 100644 --- a/testing/scenario_app/pubspec.yaml +++ b/testing/scenario_app/pubspec.yaml @@ -18,6 +18,7 @@ dependencies: args: any dir_contents_diff: any engine_repo_tools: any + meta: any path: any process: any sky_engine: any From 7f2c61081acdef8bcc19a5e63b11b3ef29189c50 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Mon, 26 Feb 2024 13:30:54 -0800 Subject: [PATCH 2/7] ++ --- .../scenario_app/bin/run_android_tests.dart | 227 +++++------------- .../scenario_app/bin/utils/environment.dart | 2 +- testing/scenario_app/bin/utils/options.dart | 55 +++-- 3 files changed, 87 insertions(+), 197 deletions(-) diff --git a/testing/scenario_app/bin/run_android_tests.dart b/testing/scenario_app/bin/run_android_tests.dart index 436cb421424ee..3530650099214 100644 --- a/testing/scenario_app/bin/run_android_tests.dart +++ b/testing/scenario_app/bin/run_android_tests.dart @@ -7,7 +7,6 @@ import 'dart:convert'; import 'dart:io'; import 'dart:typed_data'; -import 'package:args/args.dart'; import 'package:dir_contents_diff/dir_contents_diff.dart' show dirContentsDiff; import 'package:engine_repo_tools/engine_repo_tools.dart'; import 'package:path/path.dart'; @@ -15,180 +14,73 @@ import 'package:process/process.dart'; import 'package:skia_gold_client/skia_gold_client.dart'; import 'utils/adb_logcat_filtering.dart'; +import 'utils/environment.dart'; import 'utils/logs.dart'; +import 'utils/options.dart'; import 'utils/process_manager_extension.dart'; import 'utils/screenshot_transformer.dart'; -void _withTemporaryCwd(String path, void Function() callback) { - final String originalCwd = Directory.current.path; - Directory.current = Directory(path).parent.path; +// If you update the arguments, update the documentation in the README.md file. +void main(List args) async { + // Get some basic environment information to guide the rest of the program. + final Environment environment = Environment( + isCi: Platform.environment['LUCI_CONTEXT'] != null, + showVerbose: Options.showVerbose(args), + logsDir: Platform.environment['FLUTTER_LOGS_DIR'], + ); - try { - callback(); - } finally { - Directory.current = originalCwd; + // Determine if the CWD is within an engine checkout. + final Engine? localEngineDir = Engine.tryFindWithin(); + + // Show usage if requested. + if (Options.showUsage(args)) { + stdout.writeln(Options.usage( + environment: environment, + localEngineDir: localEngineDir, + )); + return; } -} -// If you update the arguments, update the documentation in the README.md file. -void main(List args) async { - final Engine? engine = Engine.tryFindWithin(); - final ArgParser parser = ArgParser() - ..addFlag( - 'help', - help: 'Prints usage information', - negatable: false, - ) - ..addFlag( - 'verbose', - help: 'Prints verbose output', - negatable: false, - ) - ..addOption( - 'adb', - help: 'Path to the adb tool', - defaultsTo: engine != null - ? join( - engine.srcDir.path, - 'third_party', - 'android_tools', - 'sdk', - 'platform-tools', - 'adb', - ) - : null, - ) - ..addOption( - 'ndk-stack', - help: 'Path to the ndk-stack tool', - defaultsTo: engine != null - ? join( - engine.srcDir.path, - 'third_party', - 'android_tools', - 'ndk', - 'prebuilt', - () { - if (Platform.isLinux) { - return 'linux-x86_64'; - } else if (Platform.isMacOS) { - return 'darwin-x86_64'; - } else if (Platform.isWindows) { - return 'windows-x86_64'; - } else { - throw UnsupportedError('Unsupported platform: ${Platform.operatingSystem}'); - } - }(), - 'bin', - 'ndk-stack', - ) - : null, - ) - ..addOption( - 'out-dir', - help: 'Out directory', - defaultsTo: engine - ?.outputs() - .where((Output o) => basename(o.path.path).startsWith('android_')) - .firstOrNull - ?.path - .path, - ) - ..addOption( - 'smoke-test', - help: 'Runs a single test to verify the setup', - ) - ..addFlag( - 'use-skia-gold', - help: 'Use Skia Gold to compare screenshots.', - defaultsTo: isLuciEnv, - ) - ..addFlag( - 'enable-impeller', - help: 'Enable Impeller for the Android app.', - ) - ..addOption( - 'output-contents-golden', - help: 'Path to a file that contains the expected filenames of golden files.', - defaultsTo: engine != null - ? join( - engine.srcDir.path, - 'flutter', - 'testing', - 'scenario_app', - 'android', - 'expected_golden_output.txt', - ) - : null, - ) - ..addOption( - 'impeller-backend', - help: 'The Impeller backend to use for the Android app.', - allowed: ['vulkan', 'opengles'], - defaultsTo: 'vulkan', - ) - ..addOption( - 'logs-dir', - help: 'The directory to store the logs and screenshots. Defaults to ' - 'the value of the FLUTTER_LOGS_DIR environment variable, if set, ' - 'otherwise it defaults to a path within out-dir.', - defaultsTo: Platform.environment['FLUTTER_LOGS_DIR'], + // Parse the command line arguments. + final Options options; + try { + options = Options.parse( + args, + environment: environment, + localEngine: localEngineDir, ); + } on FormatException catch (error) { + stderr.writeln(error); + stderr.writeln(Options.usage( + environment: environment, + localEngineDir: localEngineDir, + )); + exitCode = 1; + return; + } runZonedGuarded( () async { - final ArgResults results = parser.parse(args); - if (results['help'] as bool) { - stdout.writeln(parser.usage); - return; - } - - if (results['out-dir'] == null) { - panic(['--out-dir is required']); - } - if (results['adb'] == null) { - panic(['--adb is required']); - } - - final bool verbose = results['verbose'] as bool; - final Directory outDir = Directory(results['out-dir'] as String); - final File adb = File(results['adb'] as String); - final bool useSkiaGold = results['use-skia-gold'] as bool; - final String? smokeTest = results['smoke-test'] as String?; - final bool enableImpeller = results['enable-impeller'] as bool; - final String? contentsGolden = results['output-contents-golden'] as String?; - final _ImpellerBackend? impellerBackend = _ImpellerBackend.tryParse(results['impeller-backend'] as String?); - if (enableImpeller && impellerBackend == null) { - panic([ - 'invalid graphics-backend', - results['impeller-backend'] as String? ?? '' - ]); - } - final Directory logsDir = Directory(results['logs-dir'] as String? ?? join(outDir.path, 'scenario_app', 'logs')); - final String? ndkStack = results['ndk-stack'] as String?; - if (ndkStack == null) { - panic(['--ndk-stack is required']); - } await _run( - verbose: verbose, - outDir: outDir, - adb: adb, - smokeTestFullPath: smokeTest, - useSkiaGold: useSkiaGold, - enableImpeller: enableImpeller, - impellerBackend: impellerBackend, - logsDir: logsDir, - contentsGolden: contentsGolden, - ndkStack: ndkStack, + verbose: options.verbose, + outDir: Directory(options.outDir), + adb: File(options.adb), + smokeTestFullPath: options.smokeTest, + useSkiaGold: options.useSkiaGold, + enableImpeller: options.enableImpeller, + impellerBackend: _ImpellerBackend.tryParse(options.impellerBackend), + logsDir: Directory(options.logsDir), + contentsGolden: options.outputContentsGolden, + ndkStack: options.ndkStack, ); exit(0); }, (Object error, StackTrace stackTrace) { if (error is! Panic) { - stderr.writeln(error); + stderr.writeln('Unhandled error: $error'); stderr.writeln(stackTrace); } - exit(1); + exitCode = 1; }, ); } @@ -222,18 +114,6 @@ Future _run({ required String ndkStack, }) async { const ProcessManager pm = LocalProcessManager(); - - if (!outDir.existsSync()) { - panic([ - 'out-dir does not exist: $outDir', - 'make sure to build the selected engine variant' - ]); - } - - if (!adb.existsSync()) { - panic(['cannot find adb: $adb', 'make sure to run gclient sync']); - } - final String scenarioAppPath = join(outDir.path, 'scenario_app'); final String logcatPath = join(logsDir.path, 'logcat.txt'); @@ -531,3 +411,14 @@ Future _run({ } } } + +void _withTemporaryCwd(String path, void Function() callback) { + final String originalCwd = Directory.current.path; + Directory.current = Directory(path).path; + + try { + callback(); + } finally { + Directory.current = originalCwd; + } +} diff --git a/testing/scenario_app/bin/utils/environment.dart b/testing/scenario_app/bin/utils/environment.dart index 2ec16124052e8..9ad5561f83e24 100644 --- a/testing/scenario_app/bin/utils/environment.dart +++ b/testing/scenario_app/bin/utils/environment.dart @@ -25,7 +25,7 @@ final class Environment { final bool showVerbose; /// What directory to store logs and screenshots in. - final String logsDir; + final String? logsDir; @override bool operator ==(Object o) { diff --git a/testing/scenario_app/bin/utils/options.dart b/testing/scenario_app/bin/utils/options.dart index 8052f20d5b5de..f508224767de6 100644 --- a/testing/scenario_app/bin/utils/options.dart +++ b/testing/scenario_app/bin/utils/options.dart @@ -9,6 +9,8 @@ import 'environment.dart'; /// Command line options and parser for the Android `scenario_app` test runner. extension type const Options._(ArgResults _args) { /// Parses the command line [args] into a set of options. + /// + /// Throws a [FormatException] if command line arguments are invalid. factory Options.parse( List args, { required Environment environment, @@ -16,47 +18,37 @@ extension type const Options._(ArgResults _args) { }) { final ArgResults results = _parser(environment, localEngine).parse(args); final Options options = Options._(results); - + // The 'adb' tool must exist. if (results['adb'] == null) { - throw ArgumentError('The --adb option must be set.'); + throw const FormatException('The --adb option must be set.'); } else if (!io.File(options.adb).existsSync()) { - throw ArgumentError('The adb tool does not exist at ${options.adb}.'); + throw FormatException( + 'The adb tool does not exist at ${options.adb}.', + ); } // The 'ndk-stack' tool must exist. if (results['ndk-stack'] == null) { - throw ArgumentError('The --ndk-stack option must be set.'); + throw const FormatException('The --ndk-stack option must be set.'); } else if (!io.File(options.ndkStack).existsSync()) { - throw ArgumentError('The ndk-stack tool does not exist at ${options.ndkStack}.'); + throw FormatException( + 'The ndk-stack tool does not exist at ${options.ndkStack}.', + ); } // The 'out-dir' must exist. if (results['out-dir'] == null) { - throw ArgumentError('The --out-dir option must be set.'); + throw const FormatException('The --out-dir option must be set.'); } else if (!io.Directory(options.outDir).existsSync()) { - throw ArgumentError('The out directory does not exist at ${options.outDir}.'); + throw FormatException( + 'The out directory does not exist at ${options.outDir}.', + ); } return options; } - /// A subset of command line options used by the `scenario_app` runner. - /// - /// These options are expected to be parsed before the full set of options. - static final ArgParser _miniParser = ArgParser() - ..addFlag( - 'verbose', - abbr: 'v', - help: 'Enable verbose logging', - ) - ..addFlag( - 'help', - abbr: 'h', - help: 'Print usage information', - negatable: false, - ); - /// Whether usage information should be shown based on command line [args]. /// /// This is a shortcut that can be used to determine if the usage information @@ -73,8 +65,10 @@ extension type const Options._(ArgResults _args) { /// } /// ``` static bool showUsage(List args) { - final ArgResults results = _miniParser.parse(args); - return results['help'] as bool; + // If any of the arguments are '--help' or -'h'. + return args.isNotEmpty && args.any((String arg) { + return arg == '--help' || arg == '-h'; + }); } /// Whether verbose logging should be enabled based on command line [args]. @@ -90,8 +84,10 @@ extension type const Options._(ArgResults _args) { /// } /// ``` static bool showVerbose(List args) { - final ArgResults results = _miniParser.parse(args); - return results['verbose'] as bool; + // If any of the arguments are '--verbose' or -'v'. + return args.isNotEmpty && args.any((String arg) { + return arg == '--verbose' || arg == '-v'; + }); } /// Returns usage information for the `scenario_app` test runner. @@ -261,7 +257,10 @@ extension type const Options._(ArgResults _args) { String get impellerBackend => _args['impeller-backend'] as String; /// Path to a directory where logs and screenshots are stored. - String get logsDir => _args['logs-dir'] as String; + String get logsDir { + final String? logsDir = _args['logs-dir'] as String?; + return logsDir ?? p.join(outDir, 'logs'); + } /// Path to the Android Debug Bridge (adb) executable. String get adb => _args['adb'] as String; From c799edb84a764fb532f96d3976cbe624fbc012da Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Mon, 26 Feb 2024 13:36:34 -0800 Subject: [PATCH 3/7] ++ --- testing/scenario_app/bin/README.md | 55 ++++++++++++++------- testing/scenario_app/bin/utils/options.dart | 4 +- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/testing/scenario_app/bin/README.md b/testing/scenario_app/bin/README.md index 666f83c892d5a..698ea20a63139 100644 --- a/testing/scenario_app/bin/README.md +++ b/testing/scenario_app/bin/README.md @@ -24,23 +24,40 @@ dart bin/android_integration_tests.dart --smoke-test dev.flutter.scenarios.Engin ## Additional arguments -- `--verbose`: Print additional information about the test run. - -- `--adb`: The path to the `adb` tool. Defaults to - `third_party/android_tools/sdk/platform-tools/adb`. - -- `--out-dir`: The directory containing the build artifacts. Defaults to the - last updated build directory in `out/` that starts with `android_`. - -- `--logs-dir`: The directory to store logs and screenshots. Defaults to - `FLUTTER_LOGS_DIR` if set, or `out/.../scenario_app/logs` otherwise. - -- `--use-skia-gold`: Use Skia Gold to compare screenshots. Defaults to true - when running on CI, and false otherwise (i.e. when running locally). If - set to true, `isSkiaGoldClientAvailable` must be true. - -- `--enable-impeller`: Enable Impeller for the Android app. Defaults to - false, which means that the app will use Skia as the graphics backend. +```txt +-v, --verbose Enable verbose logging +-h, --help Print usage information + --[no-]enable-impeller Whether to enable Impeller as the graphics backend. If true, the + test runner will use --impeller-backend if set, otherwise the + default backend will be used. To explicitly run with the Skia + backend, set this to false (--no-enable-impeller). + --impeller-backend The graphics backend to use when --enable-impeller is true. Unlike + the similar option when launching an app, there is no fallback; + that is, either Vulkan or OpenGLES must be specified. + [vulkan (default), opengles] + --logs-dir Path to a directory where logs and screenshots are stored. + --out-dir= Path to a out/{variant} directory where the APKs are built. + Defaults to the latest updated out/ directory that starts with + "android_" if the current working directory is within the engine + repository. + --smoke-test= Fully qualified class name of a single test to run. For example try + "dev.flutter.scenarios.EngineLaunchE2ETest" or + "dev.flutter.scenariosui.ExternalTextureTests". + --output-contents-golden= Path to a file that contains the expected filenames of golden + files. If the current working directory is within the engine + repository, defaults to + ./testing/scenario_app/android/expected_golden_output.txt. +``` -- `--impeller-backend`: The Impeller backend to use for the Android app. - Defaults to 'vulkan'. Only used when `--enable-impeller` is set to true. +## Advanced usage + +```txt + --[no-]use-skia-gold Whether to use Skia Gold to compare screenshots. Defaults to true + on CI and false otherwise. + --adb= Path to the Android Debug Bridge (adb) executable. If the current + working directory is within the engine repository, defaults to + ./third_party/android_tools/sdk/platform-tools/adb. + --ndk-stack= Path to the NDK stack tool. Defaults to the checked-in version in + third_party/android_tools if the current working directory is + within the engine repository on a supported platform. +``` diff --git a/testing/scenario_app/bin/utils/options.dart b/testing/scenario_app/bin/utils/options.dart index f508224767de6..bfcbe2e08811d 100644 --- a/testing/scenario_app/bin/utils/options.dart +++ b/testing/scenario_app/bin/utils/options.dart @@ -105,16 +105,18 @@ extension type const Options._(ArgResults _args) { /// Unlike [_miniParser], this parser includes all options. static ArgParser _parser(Environment environment, Engine? localEngine) { final bool hideUnusualOptions = !environment.showVerbose; - return ArgParser() + return ArgParser(usageLineLength: 120) ..addFlag( 'verbose', abbr: 'v', help: 'Enable verbose logging', + negatable: false, ) ..addFlag( 'help', abbr: 'h', help: 'Print usage information', + negatable: false, ) ..addFlag( 'use-skia-gold', From 3b90ff574ea2464f72cd3989426ab6b2319e2b08 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Mon, 26 Feb 2024 13:37:36 -0800 Subject: [PATCH 4/7] ++ --- testing/scenario_app/bin/run_android_tests.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scenario_app/bin/run_android_tests.dart b/testing/scenario_app/bin/run_android_tests.dart index 3530650099214..1be584cf25da5 100644 --- a/testing/scenario_app/bin/run_android_tests.dart +++ b/testing/scenario_app/bin/run_android_tests.dart @@ -35,7 +35,7 @@ void main(List args) async { // Show usage if requested. if (Options.showUsage(args)) { stdout.writeln(Options.usage( - environment: environment, + environment: environment, localEngineDir: localEngineDir, )); return; From f70b4b71b20ead7b8d81865ceb4e0f86b2b0faf5 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Mon, 26 Feb 2024 13:50:00 -0800 Subject: [PATCH 5/7] ++ --- testing/scenario_app/bin/utils/options.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scenario_app/bin/utils/options.dart b/testing/scenario_app/bin/utils/options.dart index bfcbe2e08811d..4f5c1357838b1 100644 --- a/testing/scenario_app/bin/utils/options.dart +++ b/testing/scenario_app/bin/utils/options.dart @@ -232,7 +232,7 @@ extension type const Options._(ArgResults _args) { 'expected_golden_output.txt.', defaultsTo: localEngine != null ? p.join( - localEngine.srcDir.path, + localEngine.flutterDir.path, 'testing', 'scenario_app', 'android', From 7b8335a7a4ad3759a2636cdaedb28128bad7b7f6 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Mon, 26 Feb 2024 14:08:49 -0800 Subject: [PATCH 6/7] ++ --- testing/scenario_app/bin/utils/options.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scenario_app/bin/utils/options.dart b/testing/scenario_app/bin/utils/options.dart index 4f5c1357838b1..057256afe1cac 100644 --- a/testing/scenario_app/bin/utils/options.dart +++ b/testing/scenario_app/bin/utils/options.dart @@ -208,7 +208,7 @@ extension type const Options._(ArgResults _args) { 'Defaults to the latest updated out/ directory that starts with ' '"android_" if the current working directory is within the engine ' 'repository.', - defaultsTo: localEngine + defaultsTo: environment.isCi ? null : localEngine ?.outputs() .where((Output o) => p.basename(o.path.path).startsWith('android_')) .firstOrNull From 7195e4d948d915d154f707b7f29edbf64999ce31 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Mon, 26 Feb 2024 15:17:43 -0800 Subject: [PATCH 7/7] ++ --- testing/scenario_app/bin/run_android_tests.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scenario_app/bin/run_android_tests.dart b/testing/scenario_app/bin/run_android_tests.dart index 1be584cf25da5..6a6ed8e9bb659 100644 --- a/testing/scenario_app/bin/run_android_tests.dart +++ b/testing/scenario_app/bin/run_android_tests.dart @@ -401,7 +401,7 @@ Future _run({ // TODO(matanlurey): Resolve this in a better way. On CI this file always exists. File(join(screenshotPath, 'noop.txt')).writeAsStringSync(''); // TODO(gaaclarke): We should move this into dir_contents_diff. - _withTemporaryCwd(contentsGolden, () { + _withTemporaryCwd(dirname(contentsGolden), () { final int exitCode = dirContentsDiff(basename(contentsGolden), screenshotPath); if (exitCode != 0) { panic(['Output contents incorrect.']);