[Fuchsia] Run tests with test arguments (#50478)

This change starts executing the suites with extra arguments, namely txt_tests, though the implementation does not limit to the suite.

Bug: flutter/flutter#140179

[C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style

Author: zijiehe-google-com

ci/builders/linux_fuchsia.json

@@ -192,6 +192,10 @@
                         "--upload"
                     ]
                 },
+                {
+                    "name": "run_tests test",
+                    "script": "flutter/testing/fuchsia/run_tests_test.py"
+                },
                 {
                     "name": "x64 emulator based debug tests",
                     "language": "python3",

testing/fuchsia/run_tests.py

@@ -18,7 +18,7 @@
 import sys
 
 from subprocess import CompletedProcess
-from typing import List, Set
+from typing import Any, Iterable, List, Mapping, NamedTuple, Set
 
 # The import is coming from vpython wheel and pylint cannot find it.
 import yaml  # pylint: disable=import-error
@@ -29,10 +29,7 @@
 # is guaranteed.
 
 sys.path.insert(
-    0,
-    os.path.join(
-        os.path.dirname(__file__), '../../tools/fuchsia/test_scripts/test/'
-    )
+    0, os.path.join(os.path.dirname(__file__), '../../tools/fuchsia/test_scripts/test/')
 )
 
 # pylint: disable=import-error, wrong-import-position
@@ -51,24 +48,28 @@
 OUT_DIR = os.path.join(DIR_SRC_ROOT, 'out', VARIANT)
 
 
-class BundledTestRunner(TestRunner):
+# Visible for testing
+class TestCase(NamedTuple):
+  package: str
+  args: str = ''
+
+
+class _BundledTestRunner(TestRunner):
 
   # private, use bundled_test_runner_of function instead.
-  def __init__(
-      self, target_id: str, package_deps: Set[str], tests: List[str],
-      logs_dir: str
-  ):
+  def __init__(self, target_id: str, package_deps: Set[str], tests: List[TestCase], logs_dir: str):
     super().__init__(OUT_DIR, [], None, target_id, list(package_deps))
     self.tests = tests
     self.logs_dir = logs_dir
 
   def run_test(self) -> CompletedProcess:
     returncode = 0
     for test in self.tests:
-      # pylint: disable=protected-access
+      assert test.package.endswith('.cm')
       test_runner = ExecutableTestRunner(
-          OUT_DIR, [], test, self._target_id, None, self.logs_dir, [], None
+          OUT_DIR, test.args.split(), test.package, self._target_id, None, self.logs_dir, [], None
       )
+      # pylint: disable=protected-access
       test_runner._package_deps = self._package_deps
       result = test_runner.run_test().returncode
       logging.info('Result of test %s is %s', test, result)
@@ -77,33 +78,8 @@ def run_test(self) -> CompletedProcess:
     return CompletedProcess(args='', returncode=returncode)
 
 
-def bundled_test_runner_of(target_id: str) -> BundledTestRunner:
-  log_dir = os.environ.get('FLUTTER_LOGS_DIR', '/tmp/log')
-  with open(os.path.join(os.path.dirname(__file__), 'test_suites.yaml'),
-            'r') as file:
-    tests = yaml.safe_load(file)
-  # TODO(zijiehe-google-com): Run tests with extra test arguments,
-  # https://github.com/flutter/flutter/issues/140179.
-  tests = list(
-      filter(
-          lambda test: test['test_command'].startswith('test run ') and test[
-              'test_command'].endswith('.cm'), tests
-      )
-  )
-  # TODO(zijiehe-google-com): Run tests with dart aot,
-  # https://github.com/flutter/flutter/issues/140179.
-  tests = list(
-      filter(
-          lambda test: 'run_with_dart_aot' not in test or test[
-              'run_with_dart_aot'] != 'true', tests
-      )
-  )
-  tests = list(
-      filter(
-          lambda test: 'variant' not in test or VARIANT == test['variant'],
-          tests
-      )
-  )
+# Visible for testing
+def resolve_packages(tests: Iterable[Mapping[str, Any]]) -> Set[str]:
   packages = set()
   for test in tests:
     if 'package' in test:
@@ -128,14 +104,43 @@ def bundled_test_runner_of(target_id: str) -> BundledTestRunner:
       resolved_packages.add(new_package)
     else:
       resolved_packages.add(os.path.join(OUT_DIR, package))
-  return BundledTestRunner(
-      target_id, resolved_packages,
-      [test['test_command'][len('test run '):] for test in tests], log_dir
-  )
+  return resolved_packages
+
+
+# Visible for testing
+def build_test_cases(tests: Iterable[Mapping[str, Any]]) -> List[TestCase]:
+  test_cases = []
+  for test in [t['test_command'] for t in tests]:
+    assert test.startswith('test run ')
+    test = test[len('test run '):]
+    if ' -- ' in test:
+      package, args = test.split(' -- ', 1)
+      test_cases.append(TestCase(package=package, args=args))
+    else:
+      test_cases.append(TestCase(package=test))
+  return test_cases
+
+
+def _bundled_test_runner_of(target_id: str) -> _BundledTestRunner:
+  log_dir = os.environ.get('FLUTTER_LOGS_DIR', '/tmp/log')
+  with open(os.path.join(os.path.dirname(__file__), 'test_suites.yaml'), 'r') as file:
+    tests = yaml.safe_load(file)
+  # TODO(zijiehe-google-com): Run tests with dart aot,
+  # https://github.com/flutter/flutter/issues/140179.
+  def dart_jit(test) -> bool:
+    return 'run_with_dart_aot' not in test or test['run_with_dart_aot'] != 'true'
+
+  # TODO(zijiehe-google-com): Run all tests in release build,
+  # https://github.com/flutter/flutter/issues/140179.
+  def variant(test) -> bool:
+    return 'variant' not in test or test['variant'] == VARIANT
+
+  tests = [t for t in tests if dart_jit(t) and variant(t)]
+  return _BundledTestRunner(target_id, resolve_packages(tests), build_test_cases(tests), log_dir)
 
 
 def _get_test_runner(runner_args: argparse.Namespace, *_) -> TestRunner:
-  return bundled_test_runner_of(runner_args.target_id)
+  return _bundled_test_runner_of(runner_args.target_id)
 
 
 if __name__ == '__main__':

testing/fuchsia/run_tests_test.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env vpython3
+
+# [VPYTHON:BEGIN]
+# python_version: "3.8"
+# wheel <
+#   name: "infra/python/wheels/pyyaml/${platform}_${py_python}_${py_abi}"
+#   version: "version:5.4.1.chromium.1"
+# >
+# [VPYTHON:END]
+
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import shutil
+import unittest
+
+from pathlib import Path
+
+# pylint is looking for a wrong //testing/run_tests.py.
+# pylint: disable=no-member
+import run_tests
+
+run_tests.OUT_DIR = '/tmp/out/fuchsia_debug_x64'
+os.makedirs(run_tests.OUT_DIR, exist_ok=True)
+
+
+class RunTestsTest(unittest.TestCase):
+
+  def test_resolve_both_package_and_packages(self):
+    packages = run_tests.resolve_packages([{'package': 'abc'}, {'packages': ['abc', 'def']}])
+    self.assertEqual(
+        packages, {os.path.join(run_tests.OUT_DIR, 'abc'),
+                   os.path.join(run_tests.OUT_DIR, 'def')}
+    )
+
+  def test_resolve_package_make_symbolic_link(self):
+    Path(os.path.join(run_tests.OUT_DIR, 'abc-0.far')).touch()
+    packages = run_tests.resolve_packages([
+        {'package': 'abc-0.far'},
+    ])
+    self.assertEqual(packages, {os.path.join(run_tests.OUT_DIR, 'abc.far')})
+    self.assertTrue(os.path.islink(os.path.join(run_tests.OUT_DIR, 'abc.far')))
+
+  def test_build_test_cases_with_arguments(self):
+    test_cases = run_tests.build_test_cases([
+        {'test_command': 'test run abc'},
+        {'test_command': 'test run def -- --args'},
+    ])
+    self.assertEqual(
+        test_cases,
+        [run_tests.TestCase(package='abc'),
+         run_tests.TestCase(package='def', args='--args')]
+    )
+
+
+if __name__ == '__main__':
+  try:
+    unittest.main()
+  finally:
+    # Clean up the temporary files. Since it's in /tmp/, ignore the errors.
+    shutil.rmtree(run_tests.OUT_DIR, ignore_errors=True)

testing/fuchsia/test_suites.yaml

@@ -27,6 +27,7 @@
   package: testing_tests-0.far
 - test_command: test run fuchsia-pkg://fuchsia.com/txt_tests#meta/txt_tests.cm -- --gtest_filter=-ParagraphTest.*
   package: txt_tests-0.far
+  variant: fuchsia_debug_x64
 - test_command: test run fuchsia-pkg://fuchsia.com/ui_tests#meta/ui_tests.cm
   package: ui_tests-0.far
   variant: fuchsia_debug_x64