diff --git a/e2etests/web/regular_integration_tests/README.md b/e2etests/web/regular_integration_tests/README.md index 2e0aa9616c692..5e3501564680e 100644 --- a/e2etests/web/regular_integration_tests/README.md +++ b/e2etests/web/regular_integration_tests/README.md @@ -55,7 +55,7 @@ Future main() async { In order to run the tests follow these steps: -1. You can use two different approaches, using [felt](https://github.com/flutter/engine/blob/master/lib/web_ui/dev/README.md) tool will run all the tests, an update all the goldens. For running individual tests, we need to set UPDATE_GOLDENS environment variable. +1. You can use two different approaches, using [felt](https://github.com/flutter/engine/blob/master/lib/web_ui/dev/README.md) tool will run all the tests, hence update all the goldens. For running individual tests, we need to set UPDATE_GOLDENS environment variable. Screenshots are saved differently per browser, therefore do not forget to also update the screenshots for other browsers. Note that, LUCI is only running screenshot testing for integration tests on Firefox and Chrome. ``` felt test --integration-tests-only --update-screenshot-goldens @@ -65,6 +65,10 @@ felt test --integration-tests-only --update-screenshot-goldens UPDATE_GOLDENS=true flutter drive -v --target=test_driver/text_editing_integration.dart -d web-server --release --local-engine=host_debug_unopt ``` -2. The golden will be under `engine/src/flutter/lib/web_ui/.dart_tool/goldens/engine/web/` directory, you should create a PR for that file and merge it to `flutter/goldens`. +``` +UPDATE_GOLDENS=true flutter drive -v --target=test_driver/text_editing_integration.dart -d web-server --release --local-engine=host_debug_unopt --browser-name=firefox +``` + +2. The golden will be under `engine/src/flutter/lib/web_ui/.dart_tool/goldens/engine/web/` directory, you should create a PR for that file and merge it to `flutter/goldens`. For each browser the browser name would be appended to the end of the golden file such as: `screenshot_name-chrome.png` or `screenshot_name-firefox.png` 3. Get the commit SHA and replace the `revision` in this file: `engine/src/flutter/lib/web_ui/dev/goldens_lock.yaml` diff --git a/e2etests/web/regular_integration_tests/lib/screenshot_support.dart b/e2etests/web/regular_integration_tests/lib/screenshot_support.dart index 01d97067d0082..bfdf34ff0000d 100644 --- a/e2etests/web/regular_integration_tests/lib/screenshot_support.dart +++ b/e2etests/web/regular_integration_tests/lib/screenshot_support.dart @@ -53,8 +53,7 @@ Future runTestWithScreenshots( bool updateGoldens = false; // We are using an environment variable instead of an argument, since // this code is not invoked from the shell but from the `flutter drive` - // tool itself. Therefore we do not have control on the command line - // arguments. + // tool itself, we do not have control on the command line arguments. // Please read the README, further info on how to update the goldens. final String updateGoldensFlag = io.Platform.environment['UPDATE_GOLDENS']; // Validate if the environment variable is set correctly. @@ -71,25 +70,21 @@ Future runTestWithScreenshots( test.integrationDriver( driver: driver, onScreenshot: (String screenshotName, List screenshotBytes) async { - if (browser == 'chrome') { - final Image screenshot = decodePng(screenshotBytes); - final String result = compareImage( - screenshot, - updateGoldens, - '$screenshotName-$browser.png', - PixelComparison.fuzzy, - diffRateFailure, - forIntegrationTests: true, - write: updateGoldens, - ); - if (result == 'OK') { - return true; - } else { - io.stderr.writeln('ERROR: $result'); - return false; - } - } else { + final Image screenshot = decodePng(screenshotBytes); + final String result = compareImage( + screenshot, + updateGoldens, + '$screenshotName-$browser.png', + PixelComparison.fuzzy, + diffRateFailure, + forIntegrationTests: true, + write: updateGoldens, + ); + if (result == 'OK') { return true; + } else { + io.stderr.writeln('ERROR: $result'); + return false; } }, ); diff --git a/lib/web_ui/dev/goldens_lock.yaml b/lib/web_ui/dev/goldens_lock.yaml index 579e6c5086add..66a641aed621b 100644 --- a/lib/web_ui/dev/goldens_lock.yaml +++ b/lib/web_ui/dev/goldens_lock.yaml @@ -1,2 +1,2 @@ repository: https://github.com/flutter/goldens.git -revision: da3fef0c0eb849dfbb14b09a088c5f7916677482 +revision: 672510dc52daa5b059081f6990582bccdb4ea48f diff --git a/lib/web_ui/dev/test_runner.dart b/lib/web_ui/dev/test_runner.dart index 74c8de6d89248..9538a380ec58b 100644 --- a/lib/web_ui/dev/test_runner.dart +++ b/lib/web_ui/dev/test_runner.dart @@ -103,6 +103,13 @@ class TestCommand extends Command with ArgUtils { /// How many dart2js build tasks are running at the same time. final Pool _pool = Pool(8); + /// Checks if test harness preparation (such as fetching the goldens, + /// creating test_results directory or starting ios-simulator) has been done. + /// + /// If unit tests already did these steps, integration tests do not have to + /// repeat them. + bool _testPreparationReady = false; + /// Check the flags to see what type of tests are requested. TestTypesRequested findTestType() { if (boolArg('unit-tests-only') && boolArg('integration-tests-only')) { @@ -158,7 +165,9 @@ class TestCommand extends Command with ArgUtils { } Future runIntegrationTests() async { - await _prepare(); + if(!_testPreparationReady) { + await _prepare(); + } return IntegrationTestsManager( browser, useSystemFlutter, doUpdateScreenshotGoldens) .runTests(); @@ -207,6 +216,7 @@ class TestCommand extends Command with ArgUtils { if (isSafariIOS) { await IosSafariArgParser.instance.initIosSimulator(); } + _testPreparationReady = true; } /// Builds all test targets that will be run.