diff --git a/.cirrus.yml b/.cirrus.yml index 3c1704595c788..53f29cd4d5190 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -19,6 +19,9 @@ task: FRAMEWORK_PATH: "/tmp/master_framework" PATH: "$FLUTTER_ENGINE/third_party/dart/tools/sdks/dart-sdk/bin:$DEPOT_TOOLS:$PATH" USE_ANDROID: "False" + # TODO(liyuqian): currently we're using flutter-cirrus GCP project. Migrate + # to flutter-infra project once the metrics_center service is stabilized, + BENCHMARK_GCP_CREDENTIALS: ENCRYPTED[da76d2b7b39894de70fae1fc9182c97cc41400adc93f0f1c49bc7442f15fb933da8d756ed88523810a9a77c34f51a693] setup_script: | git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git $DEPOT_TOOLS mkdir -p $ENGINE_PATH/src @@ -29,6 +32,18 @@ task: mv $CIRRUS_WORKING_DIR flutter gclient sync matrix: + - name: build_and_benchmark_linux_release + only_if: $CIRRUS_BRANCH == 'master' # Only run for post-submit commits. + compile_host_script: | + cd $ENGINE_PATH/src + ./flutter/tools/gn --runtime-mode=release + ninja -C out/host_release + benchmark_host_script: | + cd $ENGINE_PATH/src/out/host_release/ + ./txt_benchmarks --benchmark_format=json > txt_benchmarks.json + cd $ENGINE_PATH/src/flutter/testing/benchmark + pub get + dart bin/parse_and_send.dart ../../../out/host_release/txt_benchmarks.json - name: build_and_test_linux_unopt_debug compile_host_script: | cd $ENGINE_PATH/src diff --git a/testing/benchmark/.gitignore b/testing/benchmark/.gitignore new file mode 100644 index 0000000000000..d97c5eada5d8c --- /dev/null +++ b/testing/benchmark/.gitignore @@ -0,0 +1 @@ +secret diff --git a/testing/benchmark/README.md b/testing/benchmark/README.md new file mode 100644 index 0000000000000..b01f37ab186ef --- /dev/null +++ b/testing/benchmark/README.md @@ -0,0 +1,2 @@ +This is a Dart project that runs the engine benchmarks, and send the metrics to +the cloud for storage and analysis. diff --git a/testing/benchmark/bin/parse_and_send.dart b/testing/benchmark/bin/parse_and_send.dart new file mode 100644 index 0000000000000..f6222b1ca6d8d --- /dev/null +++ b/testing/benchmark/bin/parse_and_send.dart @@ -0,0 +1,54 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:git/git.dart'; +import 'package:metrics_center/flutter.dart'; +import 'package:metrics_center/google_benchmark.dart'; + +Future _getGitRevision() async { + final GitDir gitDir = await GitDir.fromExisting('../../'); + // Somehow gitDir.currentBranch() doesn't work in Cirrus with "fatal: 'HEAD' - + // not a valid ref". Therefore, we use "git log" to get the revision manually. + final ProcessResult logResult = + await gitDir.runCommand(['log', '--pretty=format:%H', '-n', '1']); + if (logResult.exitCode != 0) { + throw 'Unexpected exit code ${logResult.exitCode}'; + } + return logResult.stdout.toString(); +} + +Future> _parse(String jsonFileName) async { + final String gitRevision = await _getGitRevision(); + final List rawPoints = + await GoogleBenchmarkParser.parse(jsonFileName); + final List points = []; + for (MetricPoint rawPoint in rawPoints) { + points.add(FlutterEngineMetricPoint( + rawPoint.tags[kNameKey], + rawPoint.value, + gitRevision, + moreTags: rawPoint.tags, + )); + } + return points; +} + +Future main(List args) async { + if (args.length != 1) { + throw 'Must have one argument: '; + } + final List points = await _parse(args[0]); + // The data will be sent to the Datastore of the GCP project specified through + // environment variable BENCHMARK_GCP_CREDENTIALS. The engine Cirrus job has + // currently configured the GCP project to flutter-cirrus for test. We'll + // eventually migrate to flutter-infra project once the test is done. + final FlutterDestination destination = + await FlutterDestination.makeFromCredentialsJson( + jsonDecode(Platform.environment['BENCHMARK_GCP_CREDENTIALS']), + ); + await destination.update(points); +} diff --git a/testing/benchmark/example/txt_benchmarks.json b/testing/benchmark/example/txt_benchmarks.json new file mode 100644 index 0000000000000..5db0e3d33cea2 --- /dev/null +++ b/testing/benchmark/example/txt_benchmarks.json @@ -0,0 +1,32 @@ +{ + "context": { + "date": "2019-12-17 15:14:14", + "num_cpus": 56, + "mhz_per_cpu": 2594, + "cpu_scaling_enabled": true, + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "BM_PaintRecordInit", + "iterations": 6749079, + "real_time": 101, + "cpu_time": 101, + "time_unit": "ns" + }, + { + "name": "BM_ParagraphShortLayout", + "iterations": 151761, + "real_time": 4460, + "cpu_time": 4460, + "time_unit": "ns" + }, + { + "name": "BM_ParagraphStylesBigO_BigO", + "cpu_coefficient": 6548, + "real_coefficient": 6548, + "big_o": "N", + "time_unit": "ns" + } + ] +} diff --git a/testing/benchmark/pubspec.yaml b/testing/benchmark/pubspec.yaml new file mode 100644 index 0000000000000..078c7751aa1e1 --- /dev/null +++ b/testing/benchmark/pubspec.yaml @@ -0,0 +1,15 @@ +name: flutter_engine_benchmark + +dependencies: + git: any + metrics_center: + # TODO(liyuqian): once metrics_center is properly reviewed, add it to + # flutter/packages, publish on pub.dev, and use the published package here. + git: https://github.com/liyuqian/metrics_center.git + +dev_dependencies: + test: any + pedantic: ^1.8.0 + +environment: + sdk: ">=2.2.2 <3.0.0" diff --git a/testing/benchmark/test/parse_and_send_test.dart b/testing/benchmark/test/parse_and_send_test.dart new file mode 100644 index 0000000000000..c312d6381c129 --- /dev/null +++ b/testing/benchmark/test/parse_and_send_test.dart @@ -0,0 +1,24 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:test/test.dart'; + +void main() { + // In order to run this test, one should download a service account + // credentials json from a test GCP project, and put that json as + // `secret/test_gcp_credentials.json`. There's a `flutter-test` project for + // Flutter team members. + test('parse_and_send with example json does not crash.', () async { + final String testCred = + File('secret/test_gcp_credentials.json').readAsStringSync(); + Process.runSync('dart', [ + 'bin/parse_and_send.dart', + 'example/txt_benchmarks.json', + ], environment: { + 'BENCHMARK_GCP_CREDENTIALS': testCred, + }); + }); +}