From a10c69a459b93ae885fb2b9a11ef5b9eaa92f0aa Mon Sep 17 00:00:00 2001 From: nturgut Date: Tue, 13 Oct 2020 08:37:37 -0700 Subject: [PATCH 1/5] enabling firefox screenshot tests. adding to documentation --- .../web/regular_integration_tests/README.md | 8 +++-- .../lib/screenshot_support.dart | 35 ++++++++----------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/e2etests/web/regular_integration_tests/README.md b/e2etests/web/regular_integration_tests/README.md index 2e0aa9616c692..5e3501564680e 100644 --- a/e2etests/web/regular_integration_tests/README.md +++ b/e2etests/web/regular_integration_tests/README.md @@ -55,7 +55,7 @@ Future main() async { In order to run the tests follow these steps: -1. You can use two different approaches, using [felt](https://github.com/flutter/engine/blob/master/lib/web_ui/dev/README.md) tool will run all the tests, an update all the goldens. For running individual tests, we need to set UPDATE_GOLDENS environment variable. +1. You can use two different approaches, using [felt](https://github.com/flutter/engine/blob/master/lib/web_ui/dev/README.md) tool will run all the tests, hence update all the goldens. For running individual tests, we need to set UPDATE_GOLDENS environment variable. Screenshots are saved differently per browser, therefore do not forget to also update the screenshots for other browsers. Note that, LUCI is only running screenshot testing for integration tests on Firefox and Chrome. ``` felt test --integration-tests-only --update-screenshot-goldens @@ -65,6 +65,10 @@ felt test --integration-tests-only --update-screenshot-goldens UPDATE_GOLDENS=true flutter drive -v --target=test_driver/text_editing_integration.dart -d web-server --release --local-engine=host_debug_unopt ``` -2. The golden will be under `engine/src/flutter/lib/web_ui/.dart_tool/goldens/engine/web/` directory, you should create a PR for that file and merge it to `flutter/goldens`. +``` +UPDATE_GOLDENS=true flutter drive -v --target=test_driver/text_editing_integration.dart -d web-server --release --local-engine=host_debug_unopt --browser-name=firefox +``` + +2. The golden will be under `engine/src/flutter/lib/web_ui/.dart_tool/goldens/engine/web/` directory, you should create a PR for that file and merge it to `flutter/goldens`. For each browser the browser name would be appended to the end of the golden file such as: `screenshot_name-chrome.png` or `screenshot_name-firefox.png` 3. Get the commit SHA and replace the `revision` in this file: `engine/src/flutter/lib/web_ui/dev/goldens_lock.yaml` diff --git a/e2etests/web/regular_integration_tests/lib/screenshot_support.dart b/e2etests/web/regular_integration_tests/lib/screenshot_support.dart index 01d97067d0082..bfdf34ff0000d 100644 --- a/e2etests/web/regular_integration_tests/lib/screenshot_support.dart +++ b/e2etests/web/regular_integration_tests/lib/screenshot_support.dart @@ -53,8 +53,7 @@ Future runTestWithScreenshots( bool updateGoldens = false; // We are using an environment variable instead of an argument, since // this code is not invoked from the shell but from the `flutter drive` - // tool itself. Therefore we do not have control on the command line - // arguments. + // tool itself, we do not have control on the command line arguments. // Please read the README, further info on how to update the goldens. final String updateGoldensFlag = io.Platform.environment['UPDATE_GOLDENS']; // Validate if the environment variable is set correctly. @@ -71,25 +70,21 @@ Future runTestWithScreenshots( test.integrationDriver( driver: driver, onScreenshot: (String screenshotName, List screenshotBytes) async { - if (browser == 'chrome') { - final Image screenshot = decodePng(screenshotBytes); - final String result = compareImage( - screenshot, - updateGoldens, - '$screenshotName-$browser.png', - PixelComparison.fuzzy, - diffRateFailure, - forIntegrationTests: true, - write: updateGoldens, - ); - if (result == 'OK') { - return true; - } else { - io.stderr.writeln('ERROR: $result'); - return false; - } - } else { + final Image screenshot = decodePng(screenshotBytes); + final String result = compareImage( + screenshot, + updateGoldens, + '$screenshotName-$browser.png', + PixelComparison.fuzzy, + diffRateFailure, + forIntegrationTests: true, + write: updateGoldens, + ); + if (result == 'OK') { return true; + } else { + io.stderr.writeln('ERROR: $result'); + return false; } }, ); From fb67579b3afe8f98479fd1967f66c54c877189f9 Mon Sep 17 00:00:00 2001 From: Nurhan Turgut Date: Tue, 13 Oct 2020 09:53:37 -0700 Subject: [PATCH 2/5] test with correct goldens --- lib/web_ui/dev/goldens_lock.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/web_ui/dev/goldens_lock.yaml b/lib/web_ui/dev/goldens_lock.yaml index 579e6c5086add..b6b3336cabc3c 100644 --- a/lib/web_ui/dev/goldens_lock.yaml +++ b/lib/web_ui/dev/goldens_lock.yaml @@ -1,2 +1,2 @@ -repository: https://github.com/flutter/goldens.git -revision: da3fef0c0eb849dfbb14b09a088c5f7916677482 +repository: https://github.com/nturgut/goldens.git +revision: 33ca76dd0494594da82e3a5710e57ab1b6c5c2dd From 57eace5826c44577b6f0c95c73dc86e52d0c4709 Mon Sep 17 00:00:00 2001 From: Nurhan Turgut Date: Tue, 13 Oct 2020 13:57:29 -0700 Subject: [PATCH 3/5] update goldens SHA --- lib/web_ui/dev/goldens_lock.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/web_ui/dev/goldens_lock.yaml b/lib/web_ui/dev/goldens_lock.yaml index b6b3336cabc3c..45759afa2bfca 100644 --- a/lib/web_ui/dev/goldens_lock.yaml +++ b/lib/web_ui/dev/goldens_lock.yaml @@ -1,2 +1,2 @@ repository: https://github.com/nturgut/goldens.git -revision: 33ca76dd0494594da82e3a5710e57ab1b6c5c2dd +revision: daf0cdfdb7b7386e98896ef9e0b47823274bb742 From d2575b1b48d96a3baa30c2d323845bf8944ebcfc Mon Sep 17 00:00:00 2001 From: Nurhan Turgut Date: Tue, 13 Oct 2020 14:42:41 -0700 Subject: [PATCH 4/5] change the goldens with flutter/goldens repo --- lib/web_ui/dev/goldens_lock.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/web_ui/dev/goldens_lock.yaml b/lib/web_ui/dev/goldens_lock.yaml index 45759afa2bfca..66a641aed621b 100644 --- a/lib/web_ui/dev/goldens_lock.yaml +++ b/lib/web_ui/dev/goldens_lock.yaml @@ -1,2 +1,2 @@ -repository: https://github.com/nturgut/goldens.git -revision: daf0cdfdb7b7386e98896ef9e0b47823274bb742 +repository: https://github.com/flutter/goldens.git +revision: 672510dc52daa5b059081f6990582bccdb4ea48f From 19d5098b57ff948ea88cbafc4a16e4bd3a83370e Mon Sep 17 00:00:00 2001 From: Nurhan Turgut Date: Tue, 13 Oct 2020 17:53:33 -0700 Subject: [PATCH 5/5] do not run preparation step twice. this will cause test_results to be deleted --- lib/web_ui/dev/test_runner.dart | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/web_ui/dev/test_runner.dart b/lib/web_ui/dev/test_runner.dart index 74c8de6d89248..9538a380ec58b 100644 --- a/lib/web_ui/dev/test_runner.dart +++ b/lib/web_ui/dev/test_runner.dart @@ -103,6 +103,13 @@ class TestCommand extends Command with ArgUtils { /// How many dart2js build tasks are running at the same time. final Pool _pool = Pool(8); + /// Checks if test harness preparation (such as fetching the goldens, + /// creating test_results directory or starting ios-simulator) has been done. + /// + /// If unit tests already did these steps, integration tests do not have to + /// repeat them. + bool _testPreparationReady = false; + /// Check the flags to see what type of tests are requested. TestTypesRequested findTestType() { if (boolArg('unit-tests-only') && boolArg('integration-tests-only')) { @@ -158,7 +165,9 @@ class TestCommand extends Command with ArgUtils { } Future runIntegrationTests() async { - await _prepare(); + if(!_testPreparationReady) { + await _prepare(); + } return IntegrationTestsManager( browser, useSystemFlutter, doUpdateScreenshotGoldens) .runTests(); @@ -207,6 +216,7 @@ class TestCommand extends Command with ArgUtils { if (isSafariIOS) { await IosSafariArgParser.instance.initIosSimulator(); } + _testPreparationReady = true; } /// Builds all test targets that will be run.