diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 771fd9a20ca8a..d987a83d8abf8 100755 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1514,6 +1514,10 @@ FILE: ../../../flutter/vulkan/vulkan_window.cc FILE: ../../../flutter/vulkan/vulkan_window.h FILE: ../../../flutter/web_sdk/libraries.json FILE: ../../../flutter/web_sdk/sdk_rewriter.dart +FILE: ../../../flutter/web_sdk/web_test_utils/lib/environment.dart +FILE: ../../../flutter/web_sdk/web_test_utils/lib/exceptions.dart +FILE: ../../../flutter/web_sdk/web_test_utils/lib/goldens.dart +FILE: ../../../flutter/web_sdk/web_test_utils/lib/image_compare.dart ---------------------------------------------------------------------------------------------------- Copyright 2013 The Flutter Authors. All rights reserved. diff --git a/e2etests/web/regular_integration_tests/README.md b/e2etests/web/regular_integration_tests/README.md index ba7f0c6c735d2..2e0aa9616c692 100644 --- a/e2etests/web/regular_integration_tests/README.md +++ b/e2etests/web/regular_integration_tests/README.md @@ -37,3 +37,34 @@ flutter drive -v --target=test_driver/text_editing_integration.dart -d web-serve ``` More details for "Running Flutter Driver tests with Web" can be found in [wiki](https://github.com/flutter/flutter/wiki/Running-Flutter-Driver-tests-with-Web). + +## Adding screenshot tests + +In order to test screenshot tests the tests on the driver side needs to call the `integration_test` package with an `onScreenshot` callback which can do a comparison between the `screenshotBytes` taken during the test and a golden file. We added a utility method that can do this comparison by using a golden in `flutter/goldens` repository. + +In order to use screenshot testing first, import `screenshot_support.dart` from the driver side test (example: `text_editing_integration_test.dart`). Default value for `diffRateFailure` is 0.5. + +``` +import 'package:regular_integration_tests/screenshot_support.dart' as test; + +Future main() async { + final double kMaxDiffRateFailure = 0.1; + await test.runTestWithScreenshots(diffRateFailure = kMaxDiffRateFailure); +} +``` + +In order to run the tests follow these steps: + +1. You can use two different approaches, using [felt](https://github.com/flutter/engine/blob/master/lib/web_ui/dev/README.md) tool will run all the tests, an update all the goldens. For running individual tests, we need to set UPDATE_GOLDENS environment variable. + +``` +felt test --integration-tests-only --update-screenshot-goldens +``` + +``` +UPDATE_GOLDENS=true flutter drive -v --target=test_driver/text_editing_integration.dart -d web-server --release --local-engine=host_debug_unopt +``` + +2. The golden will be under `engine/src/flutter/lib/web_ui/.dart_tool/goldens/engine/web/` directory, you should create a PR for that file and merge it to `flutter/goldens`. + +3. Get the commit SHA and replace the `revision` in this file: `engine/src/flutter/lib/web_ui/dev/goldens_lock.yaml` diff --git a/e2etests/web/regular_integration_tests/fonts/RobotoMono-Bold.ttf b/e2etests/web/regular_integration_tests/fonts/RobotoMono-Bold.ttf new file mode 100644 index 0000000000000..900fce6848210 Binary files /dev/null and b/e2etests/web/regular_integration_tests/fonts/RobotoMono-Bold.ttf differ diff --git a/e2etests/web/regular_integration_tests/fonts/RobotoMono-Regular.ttf b/e2etests/web/regular_integration_tests/fonts/RobotoMono-Regular.ttf new file mode 100644 index 0000000000000..7c4ce36a442d1 Binary files /dev/null and b/e2etests/web/regular_integration_tests/fonts/RobotoMono-Regular.ttf differ diff --git a/e2etests/web/regular_integration_tests/lib/screenshot_support.dart b/e2etests/web/regular_integration_tests/lib/screenshot_support.dart new file mode 100644 index 0000000000000..01d97067d0082 --- /dev/null +++ b/e2etests/web/regular_integration_tests/lib/screenshot_support.dart @@ -0,0 +1,96 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io' as io; +import 'dart:math'; + +import 'package:flutter_driver/flutter_driver.dart'; +import 'package:integration_test/integration_test_driver_extended.dart' as test; + +import 'package:web_test_utils/goldens.dart'; +import 'package:web_test_utils/image_compare.dart'; +import 'package:webdriver/src/async/window.dart'; + +import 'package:image/image.dart'; + +/// Tolerable pixel difference ratio between the goldens and the screenshots. +/// +/// We are allowing a higher difference rate compared to the unit tests (where +/// this rate is set to 0.28), since during the end to end tests there are +/// more components on the screen which are not related to the functionality +/// under test ex: a blinking cursor. +const double kMaxDiffRateFailure = 0.5 / 100; // 0.5% + +/// SBrowser screen dimensions for the Flutter Driver test. +const int _kScreenshotWidth = 1024; +const int _kScreenshotHeight = 1024; + +/// Used for calling `integration_test` package. +/// +/// Compared to other similar classes which only included the following call: +/// ``` +/// Future main() async => test.integrationDriver(); +/// ``` +/// +/// this method is able to take screenshot. +/// +/// It provides an `onScreenshot` callback to the `integrationDriver` method. +/// It also includes options for updating the golden files. +Future runTestWithScreenshots( + {double diffRateFailure = kMaxDiffRateFailure, + int browserWidth = _kScreenshotWidth, + int browserHeight = _kScreenshotHeight}) async { + final WebFlutterDriver driver = + await FlutterDriver.connect() as WebFlutterDriver; + + // Learn the browser in use from the webDriver. + final String browser = driver.webDriver.capabilities['browserName'] as String; + + final Window window = await driver.webDriver.window; + window.setSize(Rectangle(0, 0, browserWidth, browserHeight)); + + bool updateGoldens = false; + // We are using an environment variable instead of an argument, since + // this code is not invoked from the shell but from the `flutter drive` + // tool itself. Therefore we do not have control on the command line + // arguments. + // Please read the README, further info on how to update the goldens. + final String updateGoldensFlag = io.Platform.environment['UPDATE_GOLDENS']; + // Validate if the environment variable is set correctly. + if (updateGoldensFlag != null && + !(updateGoldensFlag.toLowerCase() == 'true' || + updateGoldensFlag.toLowerCase() == 'false')) { + throw StateError( + 'UPDATE_GOLDENS environment variable is not set correctly'); + } + if (updateGoldensFlag != null && updateGoldensFlag.toLowerCase() == 'true') { + updateGoldens = true; + } + + test.integrationDriver( + driver: driver, + onScreenshot: (String screenshotName, List screenshotBytes) async { + if (browser == 'chrome') { + final Image screenshot = decodePng(screenshotBytes); + final String result = compareImage( + screenshot, + updateGoldens, + '$screenshotName-$browser.png', + PixelComparison.fuzzy, + diffRateFailure, + forIntegrationTests: true, + write: updateGoldens, + ); + if (result == 'OK') { + return true; + } else { + io.stderr.writeln('ERROR: $result'); + return false; + } + } else { + return true; + } + }, + ); +} diff --git a/e2etests/web/regular_integration_tests/lib/text_editing_main.dart b/e2etests/web/regular_integration_tests/lib/text_editing_main.dart index f1aefa7ef1967..d36c0c7ae9f6c 100644 --- a/e2etests/web/regular_integration_tests/lib/text_editing_main.dart +++ b/e2etests/web/regular_integration_tests/lib/text_editing_main.dart @@ -11,6 +11,7 @@ class MyApp extends StatelessWidget { Widget build(BuildContext context) { return MaterialApp( key: const Key('mainapp'), + theme: ThemeData(fontFamily: 'RobotoMono'), title: 'Integration Test App', home: MyHomePage(title: 'Integration Test App'), ); @@ -56,6 +57,7 @@ class _MyHomePageState extends State { enabled: true, controller: _emptyController, decoration: const InputDecoration( + contentPadding: EdgeInsets.all(10.0), labelText: 'Empty Input Field:', ), ), @@ -67,6 +69,7 @@ class _MyHomePageState extends State { enabled: true, controller: _controller, decoration: const InputDecoration( + contentPadding: EdgeInsets.all(10.0), labelText: 'Text Input Field:', ), ), @@ -78,6 +81,7 @@ class _MyHomePageState extends State { enabled: true, controller: _controller2, decoration: const InputDecoration( + contentPadding: EdgeInsets.all(10.0), labelText: 'Text Input Field 2:', ), onFieldSubmitted: (String str) { @@ -94,7 +98,7 @@ class _MyHomePageState extends State { child: SelectableText( 'Lorem ipsum dolor sit amet', key: Key('selectable'), - style: TextStyle(fontFamily: 'Roboto', fontSize: 20.0), + style: TextStyle(fontFamily: 'RobotoMono', fontSize: 20.0), ), ), ], diff --git a/e2etests/web/regular_integration_tests/pubspec.yaml b/e2etests/web/regular_integration_tests/pubspec.yaml index 26ee22f810be2..407980bc02496 100644 --- a/e2etests/web/regular_integration_tests/pubspec.yaml +++ b/e2etests/web/regular_integration_tests/pubspec.yaml @@ -15,8 +15,14 @@ dev_dependencies: sdk: flutter integration_test: 0.9.0 http: 0.12.0+2 - test: any + web_test_utils: + path: ../../../web_sdk/web_test_utils flutter: assets: - assets/images/ + fonts: + - family: RobotoMono + fonts: + - asset: fonts/RobotoMono-Bold.ttf + - asset: fonts/RobotoMono-Regular.ttf diff --git a/e2etests/web/regular_integration_tests/test_driver/text_editing_integration.dart b/e2etests/web/regular_integration_tests/test_driver/text_editing_integration.dart index af1b8d6b00a0f..30acf5a3efb2c 100644 --- a/e2etests/web/regular_integration_tests/test_driver/text_editing_integration.dart +++ b/e2etests/web/regular_integration_tests/test_driver/text_editing_integration.dart @@ -13,7 +13,7 @@ import 'package:flutter/material.dart'; import 'package:integration_test/integration_test.dart'; void main() { - IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding; testWidgets('Focused text field creates a native input element', (WidgetTester tester) async { @@ -41,6 +41,8 @@ void main() { textFormField.controller.text = 'New Value'; // DOM element's value also changes. expect(input.value, 'New Value'); + + await binding.takeScreenshot('focused_text_field'); }); testWidgets('Input field with no initial value works', diff --git a/e2etests/web/regular_integration_tests/test_driver/text_editing_integration_test.dart b/e2etests/web/regular_integration_tests/test_driver/text_editing_integration_test.dart index 96b5ad0bf52a4..9c2c0fdcadc77 100644 --- a/e2etests/web/regular_integration_tests/test_driver/text_editing_integration_test.dart +++ b/e2etests/web/regular_integration_tests/test_driver/text_editing_integration_test.dart @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'package:integration_test/integration_test_driver.dart' as test; +import 'package:regular_integration_tests/screenshot_support.dart' as test; -Future main() async => test.integrationDriver(); +Future main() async { + await test.runTestWithScreenshots(); +} diff --git a/lib/web_ui/dev/goldens_lock.yaml b/lib/web_ui/dev/goldens_lock.yaml index 067577a642f6f..579e6c5086add 100644 --- a/lib/web_ui/dev/goldens_lock.yaml +++ b/lib/web_ui/dev/goldens_lock.yaml @@ -1,2 +1,2 @@ repository: https://github.com/flutter/goldens.git -revision: 1a4722227af42c3f51450266016b1a07ae459e73 +revision: da3fef0c0eb849dfbb14b09a088c5f7916677482 diff --git a/lib/web_ui/dev/integration_tests_manager.dart b/lib/web_ui/dev/integration_tests_manager.dart index e01a632f219cd..ac6c2b50a6183 100644 --- a/lib/web_ui/dev/integration_tests_manager.dart +++ b/lib/web_ui/dev/integration_tests_manager.dart @@ -24,7 +24,10 @@ class IntegrationTestsManager { final DriverManager _driverManager; - IntegrationTestsManager(this._browser, this._useSystemFlutter) + final bool _doUpdateScreenshotGoldens; + + IntegrationTestsManager( + this._browser, this._useSystemFlutter, this._doUpdateScreenshotGoldens) : _driverManager = DriverManager.chooseDriver(_browser); Future runTests() async { @@ -159,14 +162,19 @@ class IntegrationTestsManager { Future _runTestsInProfileMode( io.Directory directory, String testName) async { - final String executable = + String executable = _useSystemFlutter ? 'flutter' : environment.flutterCommand.path; + Map enviroment = Map(); + if (_doUpdateScreenshotGoldens) { + enviroment['UPDATE_GOLDENS'] = 'true'; + } final IntegrationArguments arguments = IntegrationArguments.fromBrowser(_browser); final int exitCode = await runProcess( executable, arguments.getTestArguments(testName, 'profile'), workingDirectory: directory.path, + environment: enviroment, ); if (exitCode != 0) { @@ -334,7 +342,7 @@ class ChromeIntegrationArguments extends IntegrationArguments { '--$mode', '--browser-name=chrome', if (isLuci) '--chrome-binary=${preinstalledChromeExecutable()}', - if (isLuci) '--headless', + '--headless', '--local-engine=host_debug_unopt', ]; } diff --git a/lib/web_ui/dev/test_platform.dart b/lib/web_ui/dev/test_platform.dart index 18dd92cf1c45e..330457c23b403 100644 --- a/lib/web_ui/dev/test_platform.dart +++ b/lib/web_ui/dev/test_platform.dart @@ -22,6 +22,8 @@ import 'package:shelf_web_socket/shelf_web_socket.dart'; import 'package:shelf_packages_handler/shelf_packages_handler.dart'; import 'package:stream_channel/stream_channel.dart'; import 'package:web_socket_channel/web_socket_channel.dart'; +import 'package:web_test_utils/goldens.dart'; +import 'package:web_test_utils/image_compare.dart'; import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports @@ -39,7 +41,6 @@ import 'package:test_core/src/runner/configuration.dart'; // ignore: implementat import 'browser.dart'; import 'common.dart'; import 'environment.dart' as env; -import 'goldens.dart'; import 'screenshot_manager.dart'; import 'supported_browsers.dart'; @@ -197,17 +198,6 @@ class BrowserPlatform extends PlatformPlugin { 'golden_files', ); } else { - // On LUCI MacOS bots the goldens are fetched by the recipe code. - // Fetch the goldens if: - // - Tests are running on a local machine. - // - Tests are running on an OS other than macOS. - if (!isLuci || !Platform.isMacOS) { - await fetchGoldens(); - } else { - if (!env.environment.webUiGoldensRepositoryDirectory.existsSync()) { - throw Exception('The goldens directory must have been copied'); - } - } goldensDirectory = p.join( env.environment.webUiGoldensRepositoryDirectory.path, 'engine', @@ -215,19 +205,6 @@ class BrowserPlatform extends PlatformPlugin { ); } - // Bail out fast if golden doesn't exist, and user doesn't want to create it. - final File file = File(p.join( - goldensDirectory, - filename, - )); - if (!file.existsSync() && !write) { - return ''' -Golden file $filename does not exist on path ${file.absolute.path} - -To automatically create this file call matchGoldenFile('$filename', write: true). -'''; - } - final Rectangle regionAsRectange = Rectangle( region['x'] as num, region['y'] as num, @@ -238,115 +215,14 @@ To automatically create this file call matchGoldenFile('$filename', write: true) // Take screenshot. final Image screenshot = await _screenshotManager.capture(regionAsRectange); - if (write) { - // Don't even bother with the comparison, just write and return - print('Updating screenshot golden: $file'); - file.writeAsBytesSync(encodePng(screenshot), flush: true); - if (doUpdateScreenshotGoldens) { - // Do not fail tests when bulk-updating screenshot goldens. - return 'OK'; - } else { - return 'Golden file $filename was updated. You can remove "write: true" in the call to matchGoldenFile.'; - } - } - - // Compare screenshots. - ImageDiff diff = ImageDiff( - golden: decodeNamedImage(file.readAsBytesSync(), filename), - other: screenshot, - pixelComparison: pixelComparison, - ); - - if (diff.rate > 0) { - final String testResultsPath = - env.environment.webUiTestResultsDirectory.path; - final String basename = p.basenameWithoutExtension(file.path); - - final File actualFile = - File(p.join(testResultsPath, '$basename.actual.png')); - actualFile.writeAsBytesSync(encodePng(screenshot), flush: true); - - final File diffFile = File(p.join(testResultsPath, '$basename.diff.png')); - diffFile.writeAsBytesSync(encodePng(diff.diff), flush: true); - - final File expectedFile = - File(p.join(testResultsPath, '$basename.expected.png')); - file.copySync(expectedFile.path); - - final File reportFile = - File(p.join(testResultsPath, '$basename.report.html')); - reportFile.writeAsStringSync(''' -Golden file $filename did not match the image generated by the test. - - - - - - - - - - - - -
ExpectedDiffActual
- - - - - -
-'''); - - final StringBuffer message = StringBuffer(); - message.writeln( - 'Golden file $filename did not match the image generated by the test.'); - message.writeln(getPrintableDiffFilesInfo(diff.rate, maxDiffRateFailure)); - message - .writeln('You can view the test report in your browser by opening:'); - - // Cirrus cannot serve HTML pages generated by build jobs, so we - // archive all the files so that they can be downloaded and inspected - // locally. - if (isCirrus) { - final String taskId = Platform.environment['CIRRUS_TASK_ID']; - final String baseArtifactsUrl = - 'https://api.cirrus-ci.com/v1/artifact/task/$taskId/web_engine_test/test_results'; - final String cirrusReportUrl = '$baseArtifactsUrl/$basename.report.zip'; - message.writeln(cirrusReportUrl); - - await Process.run( - 'zip', - [ - '$basename.report.zip', - '$basename.report.html', - '$basename.expected.png', - '$basename.diff.png', - '$basename.actual.png', - ], - workingDirectory: testResultsPath, - ); - } else { - final String localReportPath = '$testResultsPath/$basename.report.html'; - message.writeln(localReportPath); - } - - message.writeln( - 'To update the golden file call matchGoldenFile(\'$filename\', write: true).'); - message.writeln('Golden file: ${expectedFile.path}'); - message.writeln('Actual file: ${actualFile.path}'); - - if (diff.rate < maxDiffRateFailure) { - // Issue a warning but do not fail the test. - print('WARNING:'); - print(message); - return 'OK'; - } else { - // Fail test - return '$message'; - } - } - return 'OK'; + return compareImage( + screenshot, + doUpdateScreenshotGoldens, + filename, + pixelComparison, + maxDiffRateFailure, + goldensDirectory: goldensDirectory, + write: write); } /// A handler that serves wrapper files used to bootstrap tests. diff --git a/lib/web_ui/dev/test_runner.dart b/lib/web_ui/dev/test_runner.dart index 5ad1b3b53dfc3..74c8de6d89248 100644 --- a/lib/web_ui/dev/test_runner.dart +++ b/lib/web_ui/dev/test_runner.dart @@ -15,6 +15,7 @@ import 'package:test_core/src/runner/hack_register_platform.dart' import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports import 'package:test_core/src/executable.dart' as test; // ignore: implementation_imports +import 'package:web_test_utils/goldens.dart'; import 'common.dart'; import 'environment.dart'; @@ -157,7 +158,10 @@ class TestCommand extends Command with ArgUtils { } Future runIntegrationTests() async { - return IntegrationTestsManager(browser, useSystemFlutter).runTests(); + await _prepare(); + return IntegrationTestsManager( + browser, useSystemFlutter, doUpdateScreenshotGoldens) + .runTests(); } Future runUnitTests() async { @@ -189,6 +193,15 @@ class TestCommand extends Command with ArgUtils { } environment.webUiTestResultsDirectory.createSync(recursive: true); + // If screenshot tests are available, fetch the screenshot goldens. + if (isScreenshotTestsAvailable) { + print('screenshot tests available'); + final GoldensRepoFetcher goldensRepoFetcher = GoldensRepoFetcher( + environment.webUiGoldensRepositoryDirectory, + path.join(environment.webUiDevDir.path, 'goldens_lock.yaml')); + await goldensRepoFetcher.fetch(); + } + // In order to run iOS Safari unit tests we need to make sure iOS Simulator // is booted. if (isSafariIOS) { @@ -371,6 +384,15 @@ class TestCommand extends Command with ArgUtils { isFirefoxIntegrationTestAvailable || isSafariIntegrationTestAvailable; + // Whether the tests will do screenshot testing. + bool get isScreenshotTestsAvailable => + isIntegrationTestsAvailable || isUnitTestsScreenshotsAvailable; + + // For unit tests screenshot tests and smoke tests only run on: + // "Chrome/iOS" for LUCI/local. + bool get isUnitTestsScreenshotsAvailable => + isChrome && (io.Platform.isLinux || !isLuci) || isSafariIOS; + /// Use system flutter instead of cloning the repository. /// /// Read the flag help for more details. Uses PATH to locate flutter. @@ -397,13 +419,7 @@ class TestCommand extends Command with ArgUtils { 'test', )); - // Screenshot tests and smoke tests only run on: "Chrome/iOS Safari" - // locally and on LUCI. They are not available on Windows bots: - // TODO: https://github.com/flutter/flutter/issues/63710 - if ((isChrome && isLuci && io.Platform.isLinux) || - ((isChrome || isSafariIOS) && !isLuci) || - (isSafariIOS && isLuci)) { - print('INFO: Also running the screenshot tests.'); + if (isUnitTestsScreenshotsAvailable) { // Separate screenshot tests from unit-tests. Screenshot tests must run // one at a time. Otherwise, they will end up screenshotting each other. // This is not an issue for unit-tests. @@ -621,7 +637,8 @@ class TestCommand extends Command with ArgUtils { /// Runs a batch of tests. /// - /// Unless [expectFailure] is set to false, sets [io.exitCode] to a non-zero value if any tests fail. + /// Unless [expectFailure] is set to false, sets [io.exitCode] to a non-zero + /// value if any tests fail. Future _runTestBatch( List testFiles, { @required int concurrency, @@ -644,7 +661,8 @@ class TestCommand extends Command with ArgUtils { return BrowserPlatform.start( browser, root: io.Directory.current.path, - // It doesn't make sense to update a screenshot for a test that is expected to fail. + // It doesn't make sense to update a screenshot for a test that is + // expected to fail. doUpdateScreenshotGoldens: !expectFailure && doUpdateScreenshotGoldens, ); }); diff --git a/lib/web_ui/pubspec.yaml b/lib/web_ui/pubspec.yaml index 35f5af4d3fea5..c761816991502 100644 --- a/lib/web_ui/pubspec.yaml +++ b/lib/web_ui/pubspec.yaml @@ -14,7 +14,7 @@ dev_dependencies: image: 2.1.13 js: 0.6.1+1 mockito: 4.1.1 - path: 1.7.0 + path: 1.8.0-nullsafety.1 test: 1.14.3 quiver: 2.1.3 build_resolvers: 1.3.10 @@ -23,6 +23,8 @@ dev_dependencies: build_web_compilers: 2.11.0 yaml: 2.2.1 watcher: 0.9.7+15 + web_test_utils: + path: ../../web_sdk/web_test_utils web_engine_tester: path: ../../web_sdk/web_engine_tester simulators: diff --git a/web_sdk/web_test_utils/lib/environment.dart b/web_sdk/web_test_utils/lib/environment.dart new file mode 100644 index 0000000000000..b56fd552713de --- /dev/null +++ b/web_sdk/web_test_utils/lib/environment.dart @@ -0,0 +1,214 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// @dart = 2.6 +import 'dart:io' as io; +import 'package:path/path.dart' as pathlib; + +import 'exceptions.dart'; + +/// Contains various environment variables, such as common file paths and command-line options. +Environment get environment { + _environment ??= Environment(); + return _environment; +} + +Environment _environment; + +/// Contains various environment variables, such as common file paths and command-line options. +class Environment { + factory Environment() { + final io.File self = io.File.fromUri(io.Platform.script); + final io.Directory engineSrcDir = self.parent.parent.parent.parent.parent; + return _prepareEnvironmentFromEngineDir(self, engineSrcDir); + } + + factory Environment.forIntegrationTests() { + final io.File self = io.File.fromUri(io.Platform.script); + final io.Directory engineSrcDir = + self.parent.parent.parent.parent.parent.parent; + return _prepareEnvironmentFromEngineDir(self, engineSrcDir); + } + + static Environment _prepareEnvironmentFromEngineDir( + io.File self, io.Directory engineSrcDir) { + final io.Directory engineToolsDir = + io.Directory(pathlib.join(engineSrcDir.path, 'flutter', 'tools')); + final io.Directory outDir = + io.Directory(pathlib.join(engineSrcDir.path, 'out')); + final io.Directory hostDebugUnoptDir = + io.Directory(pathlib.join(outDir.path, 'host_debug_unopt')); + final io.Directory dartSdkDir = + io.Directory(pathlib.join(hostDebugUnoptDir.path, 'dart-sdk')); + final io.Directory webUiRootDir = io.Directory( + pathlib.join(engineSrcDir.path, 'flutter', 'lib', 'web_ui')); + final io.Directory integrationTestsDir = io.Directory( + pathlib.join(engineSrcDir.path, 'flutter', 'e2etests', 'web')); + + for (io.Directory expectedDirectory in [ + engineSrcDir, + outDir, + hostDebugUnoptDir, + dartSdkDir, + webUiRootDir + ]) { + if (!expectedDirectory.existsSync()) { + throw ToolException('$expectedDirectory does not exist.'); + } + } + + return Environment._( + self: self, + webUiRootDir: webUiRootDir, + engineSrcDir: engineSrcDir, + engineToolsDir: engineToolsDir, + integrationTestsDir: integrationTestsDir, + outDir: outDir, + hostDebugUnoptDir: hostDebugUnoptDir, + dartSdkDir: dartSdkDir, + ); + } + + Environment._({ + this.self, + this.webUiRootDir, + this.engineSrcDir, + this.engineToolsDir, + this.integrationTestsDir, + this.outDir, + this.hostDebugUnoptDir, + this.dartSdkDir, + }); + + /// The Dart script that's currently running. + final io.File self; + + /// Path to the "web_ui" package sources. + final io.Directory webUiRootDir; + + /// Path to the engine's "src" directory. + final io.Directory engineSrcDir; + + /// Path to the engine's "tools" directory. + final io.Directory engineToolsDir; + + /// Path to the web integration tests. + final io.Directory integrationTestsDir; + + /// Path to the engine's "out" directory. + /// + /// This is where you'll find the ninja output, such as the Dart SDK. + final io.Directory outDir; + + /// The "host_debug_unopt" build of the Dart SDK. + final io.Directory hostDebugUnoptDir; + + /// The root of the Dart SDK. + final io.Directory dartSdkDir; + + /// The "dart" executable file. + String get dartExecutable => pathlib.join(dartSdkDir.path, 'bin', 'dart'); + + /// The "pub" executable file. + String get pubExecutable => pathlib.join(dartSdkDir.path, 'bin', 'pub'); + + /// The "dart2js" executable file. + String get dart2jsExecutable => + pathlib.join(dartSdkDir.path, 'bin', 'dart2js'); + + /// Path to where github.com/flutter/engine is checked out inside the engine workspace. + io.Directory get flutterDirectory => + io.Directory(pathlib.join(engineSrcDir.path, 'flutter')); + io.Directory get webSdkRootDir => io.Directory(pathlib.join( + flutterDirectory.path, + 'web_sdk', + )); + + /// Path to the "web_engine_tester" package. + io.Directory get webEngineTesterRootDir => io.Directory(pathlib.join( + webSdkRootDir.path, + 'web_engine_tester', + )); + + /// Path to the "build" directory, generated by "package:build_runner". + /// + /// This is where compiled output goes. + io.Directory get webUiBuildDir => io.Directory(pathlib.join( + webUiRootDir.path, + 'build', + )); + + /// Path to the ".dart_tool" directory, generated by various Dart tools. + io.Directory get webUiDartToolDir => io.Directory(pathlib.join( + webUiRootDir.path, + '.dart_tool', + )); + + /// Path to the ".dart_tool" directory living under `engine/src/flutter`. + /// + /// This is a designated area for tool downloads which can be used by + /// multiple platforms. For exampe: Flutter repo for e2e tests. + io.Directory get engineDartToolDir => io.Directory(pathlib.join( + engineSrcDir.path, + 'flutter', + '.dart_tool', + )); + + /// Path to the "dev" directory containing engine developer tools and + /// configuration files. + io.Directory get webUiDevDir => io.Directory(pathlib.join( + webUiRootDir.path, + 'dev', + )); + + /// Path to the "test" directory containing web engine tests. + io.Directory get webUiTestDir => io.Directory(pathlib.join( + webUiRootDir.path, + 'test', + )); + + /// Path to the "lib" directory containing web engine code. + io.Directory get webUiLibDir => io.Directory(pathlib.join( + webUiRootDir.path, + 'lib', + )); + + /// Path to the clone of the flutter/goldens repository. + io.Directory get webUiGoldensRepositoryDirectory => io.Directory(pathlib.join( + webUiDartToolDir.path, + 'goldens', + )); + + /// Directory to add test results which would later be uploaded to a gcs + /// bucket by LUCI. + io.Directory get webUiTestResultsDirectory => io.Directory(pathlib.join( + webUiDartToolDir.path, + 'test_results', + )); + + /// Path to the screenshots taken by iOS simulator. + io.Directory get webUiSimulatorScreenshotsDirectory => + io.Directory(pathlib.join( + webUiDartToolDir.path, + 'ios_screenshots', + )); + + /// Path to the script that clones the Flutter repo. + io.File get cloneFlutterScript => io.File(pathlib.join( + engineToolsDir.path, + 'clone_flutter.sh', + )); + + /// Path to flutter. + /// + /// For example, this can be used to run `flutter pub get`. + /// + /// Only use [cloneFlutterScript] to clone flutter to the engine build. + io.File get flutterCommand => io.File(pathlib.join( + engineDartToolDir.path, + 'flutter', + 'bin', + 'flutter', + )); +} diff --git a/web_sdk/web_test_utils/lib/exceptions.dart b/web_sdk/web_test_utils/lib/exceptions.dart new file mode 100644 index 0000000000000..167d1734e655f --- /dev/null +++ b/web_sdk/web_test_utils/lib/exceptions.dart @@ -0,0 +1,30 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +class BrowserInstallerException implements Exception { + BrowserInstallerException(this.message); + + final String message; + + @override + String toString() => message; +} + +class DriverException implements Exception { + DriverException(this.message); + + final String message; + + @override + String toString() => message; +} + +class ToolException implements Exception { + ToolException(this.message); + + final String message; + + @override + String toString() => message; +} diff --git a/lib/web_ui/dev/goldens.dart b/web_sdk/web_test_utils/lib/goldens.dart similarity index 68% rename from lib/web_ui/dev/goldens.dart rename to web_sdk/web_test_utils/lib/goldens.dart index fef621c724385..480575dc0c998 100644 --- a/lib/web_ui/dev/goldens.dart +++ b/web_sdk/web_test_utils/lib/goldens.dart @@ -6,10 +6,8 @@ import 'dart:io' as io; import 'package:image/image.dart'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as path; -import 'package:yaml/yaml.dart'; -import 'environment.dart'; -import 'utils.dart'; +import 'package:yaml/yaml.dart'; /// How to compares pixels within the image. /// @@ -30,14 +28,14 @@ void main(List args) { final io.File fileB = io.File(args[1]); final Image imageA = decodeNamedImage(fileA.readAsBytesSync(), 'a.png'); final Image imageB = decodeNamedImage(fileB.readAsBytesSync(), 'b.png'); - final ImageDiff diff = ImageDiff(golden: imageA, other: imageB, pixelComparison: PixelComparison.fuzzy); + final ImageDiff diff = ImageDiff( + golden: imageA, other: imageB, pixelComparison: PixelComparison.fuzzy); print('Diff: ${(diff.rate * 100).toStringAsFixed(4)}%'); } /// This class encapsulates visually diffing an Image with any other. /// Both images need to be the exact same size. class ImageDiff { - /// The image to match final Image golden; @@ -59,6 +57,8 @@ class ImageDiff { /// This gets set to 1 (100% difference) when golden and other aren't the same size. double get rate => _wrongPixels / _pixelCount; + /// Image diff constructor which requires two [Image]s to compare and + /// [PixelComparison] algorithm. ImageDiff({ @required this.golden, @required this.other, @@ -72,8 +72,8 @@ class ImageDiff { /// That would be the distance between black and white. static final double _maxTheoreticalColorDistance = Color.distance( - [255, 255, 255], // white - [0, 0, 0], // black + [255, 255, 255], // white + [0, 0, 0], // black false, ).toDouble(); @@ -121,11 +121,9 @@ class ImageDiff { _reflectedPixel(image, x - 1, y - 1), _reflectedPixel(image, x - 1, y), _reflectedPixel(image, x - 1, y + 1), - _reflectedPixel(image, x, y - 1), _reflectedPixel(image, x, y), _reflectedPixel(image, x, y + 1), - _reflectedPixel(image, x + 1, y - 1), _reflectedPixel(image, x + 1, y), _reflectedPixel(image, x + 1, y + 1), @@ -148,25 +146,30 @@ class ImageDiff { } void _computeDiff() { - int goldenWidth = golden.width; - int goldenHeight = golden.height; + final int goldenWidth = golden.width; + final int goldenHeight = golden.height; _pixelCount = goldenWidth * goldenHeight; diff = Image(goldenWidth, goldenHeight); if (goldenWidth == other.width && goldenHeight == other.height) { - for(int y = 0; y < goldenHeight; y++) { + for (int y = 0; y < goldenHeight; y++) { for (int x = 0; x < goldenWidth; x++) { - final bool isExactlySame = golden.getPixel(x, y) == other.getPixel(x, y); + final bool isExactlySame = + golden.getPixel(x, y) == other.getPixel(x, y); final List goldenPixel = _getPixelRgbForComparison(golden, x, y); final List otherPixel = _getPixelRgbForComparison(other, x, y); - final double colorDistance = Color.distance(goldenPixel, otherPixel, false) / _maxTheoreticalColorDistance; + final double colorDistance = + Color.distance(goldenPixel, otherPixel, false) / + _maxTheoreticalColorDistance; final bool isFuzzySame = colorDistance < _kColorDistanceThreshold; if (isExactlySame || isFuzzySame) { diff.setPixel(x, y, _colorOk); } else { - final int goldenLuminance = getLuminanceRgb(goldenPixel[0], goldenPixel[1], goldenPixel[2]); - final int otherLuminance = getLuminanceRgb(otherPixel[0], otherPixel[1], otherPixel[2]); + final int goldenLuminance = + getLuminanceRgb(goldenPixel[0], goldenPixel[1], goldenPixel[2]); + final int otherLuminance = + getLuminanceRgb(otherPixel[0], otherPixel[1], otherPixel[2]); if (goldenLuminance < otherLuminance) { diff.setPixel(x, y, _colorExpectedPixel); } else { @@ -183,26 +186,31 @@ class ImageDiff { } } -// Returns text explaining pixel difference rate. +/// Returns text explaining pixel difference rate. String getPrintableDiffFilesInfo(double diffRate, double maxRate) => - '(${((diffRate) * 100).toStringAsFixed(4)}% of pixels were different. ' - 'Maximum allowed rate is: ${(maxRate * 100).toStringAsFixed(4)}%).'; + '(${((diffRate) * 100).toStringAsFixed(4)}% of pixels were different. ' + 'Maximum allowed rate is: ${(maxRate * 100).toStringAsFixed(4)}%).'; -/// Fetches golden files from github.com/flutter/goldens, cloning the repository if necessary. +/// Downloads the repository that stores the golden files. /// -/// The repository is cloned into web_ui/.dart_tool. -Future fetchGoldens() async { - await _GoldensRepoFetcher().fetch(); -} - -class _GoldensRepoFetcher { +/// Reads the url of the repo and `commit no` to sync to, from +/// `goldens_lock.yaml`. +class GoldensRepoFetcher { String _repository; String _revision; + final io.Directory _webUiGoldensRepositoryDirectory; + final String _lockFilePath; + /// Constructor that takes directory to download the repository and + /// file with goldens repo information. + GoldensRepoFetcher(this._webUiGoldensRepositoryDirectory, this._lockFilePath); + + /// Fetches golden files from github.com/flutter/goldens, cloning the + /// repository if necessary. + /// + /// The repository is cloned into web_ui/.dart_tool. Future fetch() async { - final io.File lockFile = io.File( - path.join(environment.webUiDevDir.path, 'goldens_lock.yaml') - ); + final io.File lockFile = io.File(path.join(_lockFilePath)); final YamlMap lock = loadYaml(lockFile.readAsStringSync()) as YamlMap; _repository = lock['repository'] as String; _revision = lock['revision'] as String; @@ -214,40 +222,32 @@ class _GoldensRepoFetcher { print('Fetching $_repository@$_revision'); - if (!environment.webUiGoldensRepositoryDirectory.existsSync()) { - environment.webUiGoldensRepositoryDirectory.createSync(recursive: true); - await runProcess( - 'git', + if (!_webUiGoldensRepositoryDirectory.existsSync()) { + _webUiGoldensRepositoryDirectory.createSync(recursive: true); + await _runGit( ['init'], - workingDirectory: environment.webUiGoldensRepositoryDirectory.path, - mustSucceed: true, + _webUiGoldensRepositoryDirectory.path, ); - await runProcess( - 'git', + + await _runGit( ['remote', 'add', 'origin', _repository], - workingDirectory: environment.webUiGoldensRepositoryDirectory.path, - mustSucceed: true, + _webUiGoldensRepositoryDirectory.path, ); } - await runProcess( - 'git', + await _runGit( ['fetch', 'origin', 'master'], - workingDirectory: environment.webUiGoldensRepositoryDirectory.path, - mustSucceed: true, + _webUiGoldensRepositoryDirectory.path, ); - await runProcess( - 'git', + await _runGit( ['checkout', _revision], - workingDirectory: environment.webUiGoldensRepositoryDirectory.path, - mustSucceed: true, + _webUiGoldensRepositoryDirectory.path, ); } Future _getLocalRevision() async { - final io.File head = io.File(path.join( - environment.webUiGoldensRepositoryDirectory.path, '.git', 'HEAD' - )); + final io.File head = io.File( + path.join(_webUiGoldensRepositoryDirectory.path, '.git', 'HEAD')); if (!head.existsSync()) { return null; @@ -255,4 +255,26 @@ class _GoldensRepoFetcher { return head.readAsStringSync().trim(); } + + /// Runs `git` with given arguments. + Future _runGit( + List arguments, + String workingDirectory, + ) async { + final io.Process process = await io.Process.start( + 'git', + arguments, + workingDirectory: workingDirectory, + // Running the process in a system shell for Windows. Otherwise + // the process is not able to get Dart from path. + runInShell: io.Platform.isWindows, + mode: io.ProcessStartMode.inheritStdio, + ); + final int exitCode = await process.exitCode; + if (exitCode != 0) { + throw Exception('Git command failed with arguments $arguments on ' + 'workingDirectory: $workingDirectory resulting with exitCode: ' + '$exitCode'); + } + } } diff --git a/web_sdk/web_test_utils/lib/image_compare.dart b/web_sdk/web_test_utils/lib/image_compare.dart new file mode 100644 index 0000000000000..137d04701aeda --- /dev/null +++ b/web_sdk/web_test_utils/lib/image_compare.dart @@ -0,0 +1,141 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:image/image.dart'; +import 'package:path/path.dart' as p; + +import 'environment.dart'; +import 'goldens.dart'; + +/// Compares a screenshot taken through a test with it's golden. +/// +/// Used by Flutter Web Engine unit tests and the integration tests. +/// +/// Returns the results of the tests as `String`. When tests passes the result +/// is simply `OK`, however when they fail it contains a detailed explanation +/// on which files are compared, their absolute locations and an HTML page +/// that the developer can see the comparison. +String compareImage( + Image screenshot, + bool doUpdateScreenshotGoldens, + String filename, + PixelComparison pixelComparison, + double maxDiffRateFailure, { + String goldensDirectory = '', + bool forIntegrationTests = false, + bool write = false, +}) { + final Environment environment = + forIntegrationTests ? Environment.forIntegrationTests() : Environment(); + if (goldensDirectory.isEmpty) { + goldensDirectory = p.join( + environment.webUiGoldensRepositoryDirectory.path, + 'engine', + 'web', + ); + } + // Bail out fast if golden doesn't exist, and user doesn't want to create it. + final File file = File(p.join( + goldensDirectory, + filename, + )); + if (!file.existsSync() && !write) { + return ''' +Golden file $filename does not exist. + +To automatically create this file call matchGoldenFile('$filename', write: true). +'''; + } + if (write) { + // Don't even bother with the comparison, just write and return + print('Updating screenshot golden: $file'); + file.writeAsBytesSync(encodePng(screenshot), flush: true); + if (doUpdateScreenshotGoldens) { + // Do not fail tests when bulk-updating screenshot goldens. + return 'OK'; + } else { + return 'Golden file $filename was updated. You can remove "write: true" ' + 'in the call to matchGoldenFile.'; + } + } + + final Image golden = decodeNamedImage(file.readAsBytesSync(), filename); + + // Compare screenshots. + final ImageDiff diff = ImageDiff( + golden: golden, + other: screenshot, + pixelComparison: pixelComparison, + ); + + if (diff.rate > 0) { + final String testResultsPath = environment.webUiTestResultsDirectory.path; + Directory(testResultsPath).createSync(recursive: true); + final String basename = p.basenameWithoutExtension(file.path); + + final File actualFile = + File(p.join(testResultsPath, '$basename.actual.png')); + actualFile.writeAsBytesSync(encodePng(screenshot), flush: true); + + final File diffFile = File(p.join(testResultsPath, '$basename.diff.png')); + diffFile.writeAsBytesSync(encodePng(diff.diff), flush: true); + + final File expectedFile = + File(p.join(testResultsPath, '$basename.expected.png')); + file.copySync(expectedFile.path); + + final File reportFile = + File(p.join(testResultsPath, '$basename.report.html')); + reportFile.writeAsStringSync(''' +Golden file $filename did not match the image generated by the test. + + + + + + + + + + + + +
ExpectedDiffActual
+ + + + + +
+'''); + + final StringBuffer message = StringBuffer(); + message.writeln( + 'Golden file $filename did not match the image generated by the test.'); + message.writeln(getPrintableDiffFilesInfo(diff.rate, maxDiffRateFailure)); + message.writeln('You can view the test report in your browser by opening:'); + + final String localReportPath = '$testResultsPath/$basename.report.html'; + message.writeln(localReportPath); + + message.writeln( + 'To update the golden file call matchGoldenFile(\'$filename\', write: ' + 'true).'); + message.writeln('Golden file: ${expectedFile.path}'); + message.writeln('Actual file: ${actualFile.path}'); + + if (diff.rate < maxDiffRateFailure) { + // Issue a warning but do not fail the test. + print('WARNING:'); + print(message); + return 'OK'; + } else { + // Fail test + return '$message'; + } + } + return 'OK'; +} diff --git a/web_sdk/web_test_utils/pubspec.yaml b/web_sdk/web_test_utils/pubspec.yaml new file mode 100644 index 0000000000000..37de1d9c5d34f --- /dev/null +++ b/web_sdk/web_test_utils/pubspec.yaml @@ -0,0 +1,10 @@ +name: web_test_utils + +environment: + sdk: ">=2.2.0 <3.0.0" + +dependencies: + path: 1.8.0-nullsafety.1 + image: 2.1.13 + js: 0.6.1+1 + yaml: 2.2.1