Run and collect benchmarks

Author: liyuqian

.cirrus.yml

@@ -19,6 +19,7 @@ task:
     FRAMEWORK_PATH: "/tmp/master_framework"
     PATH: "$FLUTTER_ENGINE/third_party/dart/tools/sdks/dart-sdk/bin:$DEPOT_TOOLS:$PATH"
     USE_ANDROID: "False"
+    BENCHMARK_GCP_CREDENTIALS: ENCRYPTED[da76d2b7b39894de70fae1fc9182c97cc41400adc93f0f1c49bc7442f15fb933da8d756ed88523810a9a77c34f51a693]
   setup_script: |
     git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git $DEPOT_TOOLS
     mkdir -p $ENGINE_PATH/src
@@ -29,6 +30,18 @@ task:
     mv $CIRRUS_WORKING_DIR flutter
     gclient sync
   matrix:
+    - name: build_and_benchmark_linux_release
+      only_if: $CIRRUS_BRANCH == 'master' # Only run for post-submit commits.
+      compile_host_script: |
+        cd $ENGINE_PATH/src
+        ./flutter/tools/gn --runtime-mode=release
+        ninja -C out/host_release
+      benchmark_host_script: |
+        cd $ENGINE_PATH/src/out/host_release/
+        ./txt_benchmarks --benchmark_format=json > txt_benchmarks.json
+        cd $ENGINE_PATH/src/flutter/testing/benchmark
+        pub get
+        dart bin/parse_and_send.dart ../../../out/host_release/txt_benchmarks.json
     - name: build_and_test_linux_unopt_debug
       compile_host_script: |
         cd $ENGINE_PATH/src

testing/benchmark/.gitignore

@@ -0,0 +1 @@
+secret

testing/benchmark/README.md

@@ -0,0 +1,3 @@
+This is a Dart project that runs the engine benchmarks, and send the metrics to
+the cloud for storage and analysis.
+

testing/benchmark/bin/parse_and_send.dart

@@ -0,0 +1,80 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:git/git.dart';
+import 'package:metrics_center/flutter.dart';
+
+const String kNameKey = 'name';
+const String kTimeUnitKey = 'time_unit';
+const String kSubResultKey = 'sub_result';
+
+const List<String> kSkippedSubResults = <String>[
+  kNameKey,
+  kTimeUnitKey,
+  'iterations',
+  'big_o',
+];
+
+Future<List<FlutterEngineMetricsPoint>> _parse(String jsonFileName) async {
+  final GitDir gitDir = await GitDir.fromExisting('../../');
+  // Somehow gitDir.currentBranch() doesn't work in Cirrus with "fatal: 'HEAD' -
+  // not a valid ref". Therefore, we use "git log" to get the revision manually.
+  final ProcessResult logResult = await gitDir
+      .runCommand(<String>['log', '--pretty=format:%H]', '-n', '1']);
+  if (logResult.exitCode != 0) {
+    throw 'Unexpected exit code ${logResult.exitCode}';
+  }
+  final String gitRevision = logResult.stdout.toString();
+
+  final Map<String, dynamic> jsonResult =
+      jsonDecode(File(jsonFileName).readAsStringSync());
+
+  final Map<String, dynamic> rawContext = jsonResult['context'];
+  final Map<String, String> context = rawContext.map<String, String>(
+    (String k, dynamic v) => MapEntry<String, String>(k, v.toString()),
+  );
+
+  final List<FlutterEngineMetricsPoint> points = <FlutterEngineMetricsPoint>[];
+  for (Map<String, dynamic> item in jsonResult['benchmarks']) {
+    final String name = item[kNameKey];
+    final Map<String, String> timeUnitMap = <String, String>{
+      kUnitKey: item[kTimeUnitKey]
+    };
+    for (String subResult in item.keys) {
+      if (!kSkippedSubResults.contains(subResult)) {
+        num rawValue;
+        try {
+          rawValue = item[subResult];
+        } catch (e) {
+          print(
+              '$subResult: ${item[subResult]} (${item[subResult].runtimeType}) is not a number');
+          continue;
+        }
+
+        final double value = rawValue is int ? rawValue.toDouble() : rawValue;
+        points.add(
+          FlutterEngineMetricsPoint(name, value, gitRevision,
+              moreTags: <String, String>{kSubResultKey: subResult}
+                ..addAll(context)
+                ..addAll(subResult.endsWith('time')
+                    ? timeUnitMap
+                    : <String, String>{})),
+        );
+      }
+    }
+  }
+
+  return points;
+}
+
+Future<void> main(List<String> args) async {
+  if (args.length != 1) {
+    throw 'Must have one argument: <benchmark_json_file>';
+  }
+  final List<FlutterEngineMetricsPoint> points = await _parse(args[0]);
+  final FlutterDestination destination =
+      await FlutterDestination.makeFromCredentialsJson(
+    jsonDecode(Platform.environment['BENCHMARK_GCP_CREDENTIALS']),
+  );
+  await destination.update(points);
+}

testing/benchmark/example/txt_benchmarks.json

@@ -0,0 +1,32 @@
+{
+  "context": {
+    "date": "2019-12-17 15:14:14",
+    "num_cpus": 56,
+    "mhz_per_cpu": 2594,
+    "cpu_scaling_enabled": true,
+    "library_build_type": "release"
+  },
+  "benchmarks": [
+    {
+      "name": "BM_PaintRecordInit",
+      "iterations": 6749079,
+      "real_time": 101,
+      "cpu_time": 101,
+      "time_unit": "ns"
+    },
+    {
+      "name": "BM_ParagraphShortLayout",
+      "iterations": 151761,
+      "real_time": 4460,
+      "cpu_time": 4460,
+      "time_unit": "ns"
+    },
+    {
+      "name": "BM_ParagraphStylesBigO_BigO",
+      "cpu_coefficient": 6548,
+      "real_coefficient": 6548,
+      "big_o": "N",
+      "time_unit": "ns"
+    }
+  ]
+}

testing/benchmark/pubspec.yaml

@@ -0,0 +1,13 @@
+name: flutter_engine_benchmark
+
+dependencies:
+  git: any
+  metrics_center:
+    git: https://github.com/liyuqian/metrics_center.git
+
+dev_dependencies:
+  test: any
+  pedantic: ^1.8.0
+
+environment:
+  sdk: ">=2.2.2 <3.0.0"

testing/benchmark/test/parse_and_send_test.dart

@@ -0,0 +1,16 @@
+import 'dart:io';
+
+import 'package:test/test.dart';
+
+void main() {
+  test('parse_and_send with example json does not crash.', () async {
+    final String testCred =
+        File('secret/test_gcp_credentials.json').readAsStringSync();
+    Process.runSync('dart', <String>[
+      'bin/parse_and_send.dart',
+      'example/txt_benchmarks.json',
+    ], environment: <String, String>{
+      'BENCHMARK_GCP_CREDENTIALS': testCred,
+    });
+  });
+}