diff --git a/ci/firebase_testlab.py b/ci/firebase_testlab.py new file mode 100644 index 0000000000000..c66f58ed4025c --- /dev/null +++ b/ci/firebase_testlab.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import glob +import re +import os +import subprocess +import sys + +script_dir = os.path.dirname(os.path.realpath(__file__)) +buildroot_dir = os.path.abspath(os.path.join(script_dir, '..', '..')) +out_dir = os.path.join(buildroot_dir, 'out') +bucket = 'gs://flutter_firebase_testlab' +error_re = re.compile('[EF]/flutter') + + +def RunFirebaseTest(apk, results_dir): + try: + # game-loop tests are meant for OpenGL apps. + # This type of test will give the application a handle to a file, and + # we'll write the timeline JSON to that file. + # See https://firebase.google.com/docs/test-lab/android/game-loop + # Pixel 4. As of this commit, this is a highly available device in FTL. + subprocess.check_output([ + 'gcloud', + '--project', 'flutter-infra', + 'firebase', 'test', 'android', 'run', + '--type', 'game-loop', + '--app', apk, + '--timeout', '2m', + '--results-bucket', bucket, + '--results-dir', results_dir, + '--device', 'model=flame,version=29', + ]) + except subprocess.CalledProcessError as ex: + print(ex.output) + # Recipe will retry return codes from firebase that indicate an infra + # failure. + sys.exit(ex.returncode) + + +def CheckLogcat(results_dir): + logcat = subprocess.check_output([ + 'gsutil', 'cat', '%s/%s/*/logcat' % (bucket, results_dir) + ]) + + logcat_match = error_re.match(logcat) + if logcat_match: + print('Errors in logcat:') + print(logcat_match) + sys.exit(1) + + +def CheckTimeline(results_dir): + du = subprocess.check_output([ + 'gsutil', 'du', + '%s/%s/*/game_loop_results/results_scenario_0.json' % (bucket, results_dir) + ]).strip() + if du == '0': + print('Failed to produce a timeline.') + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--variant', dest='variant', action='store', + default='android_profile_arm64', help='The engine variant to run tests for.') + parser.add_argument('--build-id', + default=os.environ.get('SWARMING_TASK_ID', 'local_test'), + help='A unique build identifier for this test. Used to sort results in the GCS bucket.') + + args = parser.parse_args() + + apks_dir = os.path.join(out_dir, args.variant, 'firebase_apks') + apks = glob.glob('%s/*.apk' % apks_dir) + + if not apks: + print('No APKs found at %s' % apks_dir) + return 1 + + git_revision = subprocess.check_output( + ['git', 'rev-parse', 'HEAD'], cwd=script_dir).strip() + + for apk in apks: + results_dir = '%s/%s/%s' % (os.path.basename(apk), git_revision, args.build_id) + + RunFirebaseTest(apk, results_dir) + CheckLogcat(results_dir) + # scenario_app produces a timeline, but the android image test does not. + if 'scenario' in apk: + CheckTimeline(results_dir) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ci/firebase_testlab.sh b/ci/firebase_testlab.sh index 9bd8f10074a1f..38bb393626d22 100755 --- a/ci/firebase_testlab.sh +++ b/ci/firebase_testlab.sh @@ -4,45 +4,10 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -set -e - -APP="$1" -if [[ -z "$APP" ]]; then - echo "Application must be specified as the first argument to the script." - exit 255 -fi - -if [[ ! -f "$APP" ]]; then - echo "File '$APP' not found." - exit 255 -fi +# TODO(dnfield): delete this script once recipes point to the python version. -GIT_REVISION="${2:-$(git rev-parse HEAD)}" -BUILD_ID="${3:-$SWARMING_TASK_ID}" - -# Run the test. -# game-loop tests are meant for OpenGL apps. -# This type of test will give the application a handle to a file, and -# we'll write the timeline JSON to that file. -# See https://firebase.google.com/docs/test-lab/android/game-loop -# Pixel 4. As of this commit, this is a highly available device in FTL. -gcloud --project flutter-infra firebase test android run \ - --type game-loop \ - --app "$APP" \ - --timeout 2m \ - --results-bucket=gs://flutter_firebase_testlab \ - --results-dir="engine_scenario_test/$GIT_REVISION/$BUILD_ID" \ - --device model=flame,version=29 +set -e -errors=$(gsutil cat gs://flutter_firebase_testlab/engine_scenario_test/$GIT_REVISION/$BUILD_ID/\*/logcat | grep "[FE]/flutter" | true) -if [[ ! -z $errors ]]; then - echo "Errors detected in logcat:" - echo "$errors" - exit 1 -fi +CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -result_size=$(gsutil du gs://flutter_firebase_testlab/engine_scenario_test/$GIT_REVISION/$BUILD_ID/\*/game_loop_results/results_scenario_0.json | cut -d " " -f1) -if [[ $result_size == "0" ]]; then - echo "Failed to produce a timeline." - exit 1 -fi +python $CURRENT_DIR/firebase_testlab.py diff --git a/testing/scenario_app/android/BUILD.gn b/testing/scenario_app/android/BUILD.gn index 0cf5ae1fee033..e2acd01b58310 100644 --- a/testing/scenario_app/android/BUILD.gn +++ b/testing/scenario_app/android/BUILD.gn @@ -4,7 +4,7 @@ import("//flutter/testing/scenario_app/runtime_mode.gni") -action("android") { +action("build_apk") { script = "run_gradle.py" inputs = [ "$root_out_dir/flutter.jar" ] @@ -48,3 +48,9 @@ action("android") { "//flutter/testing/scenario_app:scenario_app_snapshot", ] } + +copy("android") { + sources = get_target_outputs(":build_apk") + outputs = [ "$root_out_dir/firebase_apks/scenario_app.apk" ] + deps = [ ":build_apk" ] +}