From 1432e6db99d61bbb8708c0c875ba0375a183fc8a Mon Sep 17 00:00:00 2001 From: Hzj_jie Date: Thu, 25 Jan 2024 14:28:35 -0800 Subject: [PATCH 1/5] release format typo yaml format safe_load format missing imports vpython3 typo assert format typo unexpect change release test remove todo typo format path format typo targets deps format disable run all logging lint filter cl format typo format --- ci/builders/linux_fuchsia.json | 24 +++- .../fuchsia/dart_runner/embedder/BUILD.gn | 3 +- .../fuchsia/dart_runner/kernel/BUILD.gn | 3 +- .../fuchsia/dart_runner/vmservice/BUILD.gn | 3 +- .../platform/fuchsia/flutter/kernel/BUILD.gn | 3 +- testing/fuchsia/run_tests.py | 115 ++++++++++++++---- testing/fuchsia/test_suites.yaml | 4 + 7 files changed, 126 insertions(+), 29 deletions(-) diff --git a/ci/builders/linux_fuchsia.json b/ci/builders/linux_fuchsia.json index 011a13618aa11..facbae0827b6c 100644 --- a/ci/builders/linux_fuchsia.json +++ b/ci/builders/linux_fuchsia.json @@ -115,10 +115,12 @@ { "drone_dimensions": [ "device_type=none", + "kvm=1", "os=Linux" ], "gclient_variables": { - "download_android_deps": false + "download_android_deps": false, + "run_fuchsia_emu": true }, "gn": [ "--fuchsia", @@ -131,9 +133,22 @@ "ninja": { "config": "fuchsia_release_x64", "targets": [ - "flutter/shell/platform/fuchsia:fuchsia" + "flutter/shell/platform/fuchsia:fuchsia", + "flutter/shell/platform/fuchsia/dart_runner:dart_runner_tests", + "fuchsia_tests" ] - } + }, + "tests": [ + { + "name": "x64 emulator based release tests", + "language": "python3", + "script": "flutter/tools/fuchsia/with_envs.py", + "parameters": [ + "testing/fuchsia/run_tests.py", + "fuchsia_release_x64" + ] + } + ] }, { "drone_dimensions": [ @@ -158,6 +173,7 @@ "config": "fuchsia_debug_x64", "targets": [ "flutter/shell/platform/fuchsia:fuchsia", + "flutter/shell/platform/fuchsia/dart_runner:dart_runner_tests", "fuchsia_tests" ] }, @@ -176,7 +192,7 @@ ] }, { - "name": "x64 emulator based tests", + "name": "x64 emulator based debug tests", "language": "python3", "script": "flutter/tools/fuchsia/with_envs.py", "parameters": [ diff --git a/shell/platform/fuchsia/dart_runner/embedder/BUILD.gn b/shell/platform/fuchsia/dart_runner/embedder/BUILD.gn index afd12c4b51be8..9536b487cb518 100644 --- a/shell/platform/fuchsia/dart_runner/embedder/BUILD.gn +++ b/shell/platform/fuchsia/dart_runner/embedder/BUILD.gn @@ -55,7 +55,8 @@ template("create_aot_snapshot") { # No asserts in release with flutter_profile=true (non-product) # Yes asserts in non-product debug. if (!invoker.product && - (!(flutter_runtime_mode == "profile") || is_debug)) { + (!(flutter_runtime_mode == "profile" || + flutter_runtime_mode == "release") || is_debug)) { args += [ "--enable_asserts" ] } args += [ rebase_path(shim_kernel) ] diff --git a/shell/platform/fuchsia/dart_runner/kernel/BUILD.gn b/shell/platform/fuchsia/dart_runner/kernel/BUILD.gn index af896f65e28b8..891a393a4557e 100644 --- a/shell/platform/fuchsia/dart_runner/kernel/BUILD.gn +++ b/shell/platform/fuchsia/dart_runner/kernel/BUILD.gn @@ -70,7 +70,8 @@ template("create_kernel_core_snapshot") { # No asserts in release with flutter_profile=true (non-product) # Yes asserts in non-product debug. if (!invoker.product && - (is_debug || !(flutter_runtime_mode == "profile"))) { + (is_debug || !(flutter_runtime_mode == "profile" || + flutter_runtime_mode == "release"))) { args += [ "--enable_asserts" ] } args += [ rebase_path(platform_dill) ] diff --git a/shell/platform/fuchsia/dart_runner/vmservice/BUILD.gn b/shell/platform/fuchsia/dart_runner/vmservice/BUILD.gn index 8aa7da550baab..15bf14a7fc5f6 100644 --- a/shell/platform/fuchsia/dart_runner/vmservice/BUILD.gn +++ b/shell/platform/fuchsia/dart_runner/vmservice/BUILD.gn @@ -65,7 +65,8 @@ template("aot_snapshot") { # No asserts in debug or release product. # No asserts in release with flutter_profile=true (non-product) # Yes asserts in non-product debug. - if (!product && (!(flutter_runtime_mode == "profile") || is_debug)) { + if (!product && (!(flutter_runtime_mode == "profile" || + flutter_runtime_mode == "release") || is_debug)) { args += [ "--enable_asserts" ] } diff --git a/shell/platform/fuchsia/flutter/kernel/BUILD.gn b/shell/platform/fuchsia/flutter/kernel/BUILD.gn index 7564a3584e744..3c6f5a207732a 100644 --- a/shell/platform/fuchsia/flutter/kernel/BUILD.gn +++ b/shell/platform/fuchsia/flutter/kernel/BUILD.gn @@ -84,7 +84,8 @@ template("core_snapshot") { # No asserts in release with flutter_profile=true (non-product) # Yes asserts in non-product debug. if (!invoker.product && - (is_debug || !(flutter_runtime_mode == "profile"))) { + (is_debug || !(flutter_runtime_mode == "profile" || + flutter_runtime_mode == "release"))) { args += [ "--enable_asserts" ] } args += [ rebase_path(platform_dill) ] diff --git a/testing/fuchsia/run_tests.py b/testing/fuchsia/run_tests.py index 2a301c828299c..853db5bd9cda6 100755 --- a/testing/fuchsia/run_tests.py +++ b/testing/fuchsia/run_tests.py @@ -1,12 +1,28 @@ -#!/usr/bin/env python3 +#!/usr/bin/env vpython3 + +# [VPYTHON:BEGIN] +# python_version: "3.8" +# wheel < +# name: "infra/python/wheels/pyyaml/${platform}_${py_python}_${py_abi}" +# version: "version:5.4.1.chromium.1" +# > +# [VPYTHON:END] + # Copyright (c) 2013, the Flutter project authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be found # in the LICENSE file. import argparse +import logging import os import sys +from subprocess import CompletedProcess +from typing import List + +# The import is coming from vpython wheel and pylint cannot find it. +import yaml # pylint: disable=import-error + # The imports are coming from fuchsia/test_scripts and pylint cannot find them # without setting a global init-hook which is less favorable. # But this file will be executed as part of the CI, its correctness of importing @@ -25,32 +41,89 @@ from run_executable_test import ExecutableTestRunner from test_runner import TestRunner -# TODO(https://github.com/flutter/flutter/issues/140179): Respect build -# configurations. -OUT_DIR = os.path.join(DIR_SRC_ROOT, 'out/fuchsia_debug_x64') +if len(sys.argv) == 2: + VARIANT = sys.argv[1] + sys.argv.pop() +elif len(sys.argv) == 1: + VARIANT = 'fuchsia_debug_x64' +else: + assert False, 'Expect only one parameter as the compile output directory.' +OUT_DIR = os.path.join(DIR_SRC_ROOT, 'out', VARIANT) -# TODO(https://github.com/flutter/flutter/issues/140179): Execute all the tests -# in -# https://github.com/flutter/engine/blob/main/testing/fuchsia/test_suites.yaml -# and avoid hardcoded paths. -def _get_test_runner(runner_args: argparse.Namespace, *_) -> TestRunner: - return ExecutableTestRunner( - OUT_DIR, [], - 'fuchsia-pkg://fuchsia.com/dart_runner_tests#meta/dart_runner_tests.cm', - runner_args.target_id, None, '/tmp/log', - [os.path.join(OUT_DIR, 'dart_runner_tests.far')], None +class BundledTestRunner(TestRunner): + + # private, use bundled_test_runner_of function instead. + def __init__( + self, target_id: str, package_deps: List[str], tests: List[str], + logs_dir: str + ): + super().__init__(OUT_DIR, [], None, target_id, package_deps) + self.tests = tests + self.logs_dir = logs_dir + + def run_test(self) -> CompletedProcess: + returncode = 0 + for test in self.tests: + # pylint: disable=protected-access + test_runner = ExecutableTestRunner( + OUT_DIR, [], test, self._target_id, None, self.logs_dir, [], None + ) + test_runner._package_deps = self._package_deps + result = test_runner.run_test().returncode + logging.warning('Result of test %s is %s', test, result) + if result != 0: + returncode = result + return CompletedProcess(args='', returncode=returncode) + + +def bundled_test_runner_of(target_id: str) -> BundledTestRunner: + log_dir = os.environ.get('FLUTTER_LOGS_DIR', '/tmp/log') + with open(os.path.join(os.path.dirname(__file__), 'test_suites.yaml'), + 'r') as file: + tests = yaml.safe_load(file) + # Ignore tests needing multiple packages or with extra test arguments for + # now. + tests = list( + filter( + lambda test: test['test_command'].startswith('test run ') and test[ + 'test_command'].endswith('.cm'), tests + ) + ) + tests = list( + filter( + lambda test: 'package' in test and test['package'].endswith('-0.far'), + tests + ) + ) + tests = list( + filter( + lambda test: not 'variant' in test or VARIANT in test['variant'], + tests + ) + ) + for test in tests: + original_package = test['package'] + test['package'] = os.path.join( + OUT_DIR, test['package'].replace('-0.far', '.far') + ) + try: + os.remove(test['package']) + except FileNotFoundError: + pass + os.symlink(original_package, test['package']) + return BundledTestRunner( + target_id, [test['package'] for test in tests], + [test['test_command'][len('test run '):] for test in tests], log_dir ) +def _get_test_runner(runner_args: argparse.Namespace, *_) -> TestRunner: + return bundled_test_runner_of(runner_args.target_id) + + if __name__ == '__main__': - try: - os.remove(os.path.join(OUT_DIR, 'dart_runner_tests.far')) - except FileNotFoundError: - pass - os.symlink( - 'dart_runner_tests-0.far', os.path.join(OUT_DIR, 'dart_runner_tests.far') - ) + logging.warning('Running tests in %s', OUT_DIR) sys.argv.append('--out-dir=' + OUT_DIR) # The 'flutter-test-type' is a place holder and has no specific meaning; the # _get_test_runner is overrided. diff --git a/testing/fuchsia/test_suites.yaml b/testing/fuchsia/test_suites.yaml index c311fe2fc28dc..876a51ad8914f 100644 --- a/testing/fuchsia/test_suites.yaml +++ b/testing/fuchsia/test_suites.yaml @@ -19,16 +19,20 @@ package: flow_tests-0.far - test_command: test run fuchsia-pkg://fuchsia.com/runtime_tests#meta/runtime_tests.cm package: runtime_tests-0.far + variant: fuchsia_debug_x64 - test_command: test run fuchsia-pkg://fuchsia.com/shell_tests#meta/shell_tests.cm package: shell_tests-0.far + variant: fuchsia_debug_x64 - test_command: test run fuchsia-pkg://fuchsia.com/testing_tests#meta/testing_tests.cm package: testing_tests-0.far - test_command: test run fuchsia-pkg://fuchsia.com/txt_tests#meta/txt_tests.cm -- --gtest_filter=-ParagraphTest.* package: txt_tests-0.far - test_command: test run fuchsia-pkg://fuchsia.com/ui_tests#meta/ui_tests.cm package: ui_tests-0.far + variant: fuchsia_debug_x64 - test_command: test run fuchsia-pkg://fuchsia.com/embedder_tests#meta/embedder_tests.cm package: embedder_tests-0.far + variant: fuchsia_debug_x64 - test_command: test run fuchsia-pkg://fuchsia.com/dart_utils_tests#meta/dart_utils_tests.cm package: dart_utils_tests-0.far - test_command: test run fuchsia-pkg://fuchsia.com/dart-jit-runner-integration-test#meta/dart-jit-runner-integration-test.cm From 567955262106fb561ee376a59003ae3b69aa2a39 Mon Sep 17 00:00:00 2001 From: Hzj_jie Date: Mon, 29 Jan 2024 16:19:34 -0800 Subject: [PATCH 2/5] Address review comments --- testing/fuchsia/run_tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testing/fuchsia/run_tests.py b/testing/fuchsia/run_tests.py index 853db5bd9cda6..3660cec799675 100755 --- a/testing/fuchsia/run_tests.py +++ b/testing/fuchsia/run_tests.py @@ -71,7 +71,7 @@ def run_test(self) -> CompletedProcess: ) test_runner._package_deps = self._package_deps result = test_runner.run_test().returncode - logging.warning('Result of test %s is %s', test, result) + logging.info('Result of test %s is %s', test, result) if result != 0: returncode = result return CompletedProcess(args='', returncode=returncode) @@ -82,8 +82,8 @@ def bundled_test_runner_of(target_id: str) -> BundledTestRunner: with open(os.path.join(os.path.dirname(__file__), 'test_suites.yaml'), 'r') as file: tests = yaml.safe_load(file) - # Ignore tests needing multiple packages or with extra test arguments for - # now. + # TODO: 140179 - Run tests with multiple packages or with extra test + # arguments. tests = list( filter( lambda test: test['test_command'].startswith('test run ') and test[ @@ -123,7 +123,7 @@ def _get_test_runner(runner_args: argparse.Namespace, *_) -> TestRunner: if __name__ == '__main__': - logging.warning('Running tests in %s', OUT_DIR) + logging.info('Running tests in %s', OUT_DIR) sys.argv.append('--out-dir=' + OUT_DIR) # The 'flutter-test-type' is a place holder and has no specific meaning; the # _get_test_runner is overrided. From c9c3f0dd80f9849796225d25837716f4a6ca768a Mon Sep 17 00:00:00 2001 From: Hzj_jie Date: Tue, 30 Jan 2024 10:49:47 -0800 Subject: [PATCH 3/5] Address review comments --- shell/platform/fuchsia/dart_runner/embedder/BUILD.gn | 3 +-- shell/platform/fuchsia/dart_runner/kernel/BUILD.gn | 3 +-- shell/platform/fuchsia/dart_runner/vmservice/BUILD.gn | 3 +-- shell/platform/fuchsia/flutter/kernel/BUILD.gn | 3 +-- testing/fuchsia/run_tests.py | 4 ++-- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/shell/platform/fuchsia/dart_runner/embedder/BUILD.gn b/shell/platform/fuchsia/dart_runner/embedder/BUILD.gn index 9536b487cb518..4a164bc93490e 100644 --- a/shell/platform/fuchsia/dart_runner/embedder/BUILD.gn +++ b/shell/platform/fuchsia/dart_runner/embedder/BUILD.gn @@ -55,8 +55,7 @@ template("create_aot_snapshot") { # No asserts in release with flutter_profile=true (non-product) # Yes asserts in non-product debug. if (!invoker.product && - (!(flutter_runtime_mode == "profile" || - flutter_runtime_mode == "release") || is_debug)) { + (flutter_runtime_mode == "debug" || is_debug)) { args += [ "--enable_asserts" ] } args += [ rebase_path(shim_kernel) ] diff --git a/shell/platform/fuchsia/dart_runner/kernel/BUILD.gn b/shell/platform/fuchsia/dart_runner/kernel/BUILD.gn index 891a393a4557e..88a232cefe2f0 100644 --- a/shell/platform/fuchsia/dart_runner/kernel/BUILD.gn +++ b/shell/platform/fuchsia/dart_runner/kernel/BUILD.gn @@ -70,8 +70,7 @@ template("create_kernel_core_snapshot") { # No asserts in release with flutter_profile=true (non-product) # Yes asserts in non-product debug. if (!invoker.product && - (is_debug || !(flutter_runtime_mode == "profile" || - flutter_runtime_mode == "release"))) { + (is_debug || flutter_runtime_mode == "debug")) { args += [ "--enable_asserts" ] } args += [ rebase_path(platform_dill) ] diff --git a/shell/platform/fuchsia/dart_runner/vmservice/BUILD.gn b/shell/platform/fuchsia/dart_runner/vmservice/BUILD.gn index 15bf14a7fc5f6..c299edb5f3f78 100644 --- a/shell/platform/fuchsia/dart_runner/vmservice/BUILD.gn +++ b/shell/platform/fuchsia/dart_runner/vmservice/BUILD.gn @@ -65,8 +65,7 @@ template("aot_snapshot") { # No asserts in debug or release product. # No asserts in release with flutter_profile=true (non-product) # Yes asserts in non-product debug. - if (!product && (!(flutter_runtime_mode == "profile" || - flutter_runtime_mode == "release") || is_debug)) { + if (!product && (flutter_runtime_mode == "debug" || is_debug)) { args += [ "--enable_asserts" ] } diff --git a/shell/platform/fuchsia/flutter/kernel/BUILD.gn b/shell/platform/fuchsia/flutter/kernel/BUILD.gn index 3c6f5a207732a..eb5b8e1c249e7 100644 --- a/shell/platform/fuchsia/flutter/kernel/BUILD.gn +++ b/shell/platform/fuchsia/flutter/kernel/BUILD.gn @@ -84,8 +84,7 @@ template("core_snapshot") { # No asserts in release with flutter_profile=true (non-product) # Yes asserts in non-product debug. if (!invoker.product && - (is_debug || !(flutter_runtime_mode == "profile" || - flutter_runtime_mode == "release"))) { + (is_debug || flutter_runtime_mode == "debug")) { args += [ "--enable_asserts" ] } args += [ rebase_path(platform_dill) ] diff --git a/testing/fuchsia/run_tests.py b/testing/fuchsia/run_tests.py index 3660cec799675..4211758c01a48 100755 --- a/testing/fuchsia/run_tests.py +++ b/testing/fuchsia/run_tests.py @@ -82,8 +82,8 @@ def bundled_test_runner_of(target_id: str) -> BundledTestRunner: with open(os.path.join(os.path.dirname(__file__), 'test_suites.yaml'), 'r') as file: tests = yaml.safe_load(file) - # TODO: 140179 - Run tests with multiple packages or with extra test - # arguments. + # TODO(zijiehe-google-com): Run tests with multiple packages or with extra + # test arguments, https://github.com/flutter/flutter/issues/140179. tests = list( filter( lambda test: test['test_command'].startswith('test run ') and test[ From 82267fc7489a092a49be6c35ec7dd993b189d3c6 Mon Sep 17 00:00:00 2001 From: Hzj_jie Date: Tue, 30 Jan 2024 10:50:48 -0800 Subject: [PATCH 4/5] format --- shell/platform/fuchsia/dart_runner/embedder/BUILD.gn | 3 +-- shell/platform/fuchsia/dart_runner/kernel/BUILD.gn | 3 +-- shell/platform/fuchsia/flutter/kernel/BUILD.gn | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/shell/platform/fuchsia/dart_runner/embedder/BUILD.gn b/shell/platform/fuchsia/dart_runner/embedder/BUILD.gn index 4a164bc93490e..562191cbcf6b0 100644 --- a/shell/platform/fuchsia/dart_runner/embedder/BUILD.gn +++ b/shell/platform/fuchsia/dart_runner/embedder/BUILD.gn @@ -54,8 +54,7 @@ template("create_aot_snapshot") { # No asserts in debug or release product. # No asserts in release with flutter_profile=true (non-product) # Yes asserts in non-product debug. - if (!invoker.product && - (flutter_runtime_mode == "debug" || is_debug)) { + if (!invoker.product && (flutter_runtime_mode == "debug" || is_debug)) { args += [ "--enable_asserts" ] } args += [ rebase_path(shim_kernel) ] diff --git a/shell/platform/fuchsia/dart_runner/kernel/BUILD.gn b/shell/platform/fuchsia/dart_runner/kernel/BUILD.gn index 88a232cefe2f0..c990742404703 100644 --- a/shell/platform/fuchsia/dart_runner/kernel/BUILD.gn +++ b/shell/platform/fuchsia/dart_runner/kernel/BUILD.gn @@ -69,8 +69,7 @@ template("create_kernel_core_snapshot") { # No asserts in debug or release product. # No asserts in release with flutter_profile=true (non-product) # Yes asserts in non-product debug. - if (!invoker.product && - (is_debug || flutter_runtime_mode == "debug")) { + if (!invoker.product && (is_debug || flutter_runtime_mode == "debug")) { args += [ "--enable_asserts" ] } args += [ rebase_path(platform_dill) ] diff --git a/shell/platform/fuchsia/flutter/kernel/BUILD.gn b/shell/platform/fuchsia/flutter/kernel/BUILD.gn index eb5b8e1c249e7..516e7515da766 100644 --- a/shell/platform/fuchsia/flutter/kernel/BUILD.gn +++ b/shell/platform/fuchsia/flutter/kernel/BUILD.gn @@ -83,8 +83,7 @@ template("core_snapshot") { # No asserts in debug or release product. # No asserts in release with flutter_profile=true (non-product) # Yes asserts in non-product debug. - if (!invoker.product && - (is_debug || flutter_runtime_mode == "debug")) { + if (!invoker.product && (is_debug || flutter_runtime_mode == "debug")) { args += [ "--enable_asserts" ] } args += [ rebase_path(platform_dill) ] From 86bfdc239bcd28dcc55caf6aa81b2b9a3e8ff82c Mon Sep 17 00:00:00 2001 From: Hzj_jie Date: Tue, 30 Jan 2024 13:37:01 -0800 Subject: [PATCH 5/5] == --- testing/fuchsia/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/fuchsia/run_tests.py b/testing/fuchsia/run_tests.py index 4211758c01a48..40c66be2b96a1 100755 --- a/testing/fuchsia/run_tests.py +++ b/testing/fuchsia/run_tests.py @@ -98,7 +98,7 @@ def bundled_test_runner_of(target_id: str) -> BundledTestRunner: ) tests = list( filter( - lambda test: not 'variant' in test or VARIANT in test['variant'], + lambda test: not 'variant' in test or VARIANT == test['variant'], tests ) )