Merge protobuf_benchmarks and query_benchmark

.github/workflows/dart.yml

@@ -68,16 +68,16 @@ jobs:
         working-directory: protobuf
         run: dart analyze test
   job_003:
-    name: "format_analyze; linux; Dart dev; PKGS: api_benchmark, query_benchmark; `dart format --output=none --set-exit-if-changed .`, `./../tool/setup.sh`, `./compile_protos.sh`, `dart analyze --fatal-infos`"
+    name: "format_analyze; linux; Dart dev; PKG: api_benchmark; `dart format --output=none --set-exit-if-changed .`, `./../tool/setup.sh`, `./compile_protos.sh`, `dart analyze --fatal-infos`"
     runs-on: ubuntu-latest
     steps:
       - name: Cache Pub hosted dependencies
         uses: actions/cache@v3
         with:
           path: "~/.pub-cache/hosted"
-          key: "os:ubuntu-latest;pub-cache-hosted;sdk:dev;packages:api_benchmark-query_benchmark;commands:format-command_0-command_1-analyze_0"
+          key: "os:ubuntu-latest;pub-cache-hosted;sdk:dev;packages:api_benchmark;commands:format-command_0-command_1-analyze_0"
           restore-keys: |
-            os:ubuntu-latest;pub-cache-hosted;sdk:dev;packages:api_benchmark-query_benchmark
+            os:ubuntu-latest;pub-cache-hosted;sdk:dev;packages:api_benchmark
             os:ubuntu-latest;pub-cache-hosted;sdk:dev
             os:ubuntu-latest;pub-cache-hosted
             os:ubuntu-latest
@@ -107,28 +107,47 @@ jobs:
         if: "always() && steps.api_benchmark_pub_upgrade.conclusion == 'success'"
         working-directory: api_benchmark
         run: dart analyze --fatal-infos
-      - id: query_benchmark_pub_upgrade
-        name: query_benchmark; dart pub upgrade
+  job_004:
+    name: "format_analyze; linux; Dart dev; PKG: benchmarks; `dart format --output=none --set-exit-if-changed .`, `./../tool/setup.sh`, `./tool/compile_protos.sh`, `dart analyze --fatal-infos`"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cache Pub hosted dependencies
+        uses: actions/cache@v3
+        with:
+          path: "~/.pub-cache/hosted"
+          key: "os:ubuntu-latest;pub-cache-hosted;sdk:dev;packages:benchmarks;commands:format-command_0-command_2-analyze_0"
+          restore-keys: |
+            os:ubuntu-latest;pub-cache-hosted;sdk:dev;packages:benchmarks
+            os:ubuntu-latest;pub-cache-hosted;sdk:dev
+            os:ubuntu-latest;pub-cache-hosted
+            os:ubuntu-latest
+      - uses: dart-lang/[email protected]
+        with:
+          sdk: dev
+      - id: checkout
+        uses: actions/[email protected]
+      - id: benchmarks_pub_upgrade
+        name: benchmarks; dart pub upgrade
         if: "always() && steps.checkout.conclusion == 'success'"
-        working-directory: query_benchmark
+        working-directory: benchmarks
         run: dart pub upgrade
-      - name: "query_benchmark; dart format --output=none --set-exit-if-changed ."
-        if: "always() && steps.query_benchmark_pub_upgrade.conclusion == 'success'"
-        working-directory: query_benchmark
+      - name: "benchmarks; dart format --output=none --set-exit-if-changed ."
+        if: "always() && steps.benchmarks_pub_upgrade.conclusion == 'success'"
+        working-directory: benchmarks
         run: "dart format --output=none --set-exit-if-changed ."
-      - name: query_benchmark; ./../tool/setup.sh
-        if: "always() && steps.query_benchmark_pub_upgrade.conclusion == 'success'"
-        working-directory: query_benchmark
+      - name: benchmarks; ./../tool/setup.sh
+        if: "always() && steps.benchmarks_pub_upgrade.conclusion == 'success'"
+        working-directory: benchmarks
         run: ./../tool/setup.sh
-      - name: query_benchmark; ./compile_protos.sh
-        if: "always() && steps.query_benchmark_pub_upgrade.conclusion == 'success'"
-        working-directory: query_benchmark
-        run: ./compile_protos.sh
-      - name: "query_benchmark; dart analyze --fatal-infos"
-        if: "always() && steps.query_benchmark_pub_upgrade.conclusion == 'success'"
-        working-directory: query_benchmark
+      - name: benchmarks; ./tool/compile_protos.sh
+        if: "always() && steps.benchmarks_pub_upgrade.conclusion == 'success'"
+        working-directory: benchmarks
+        run: ./tool/compile_protos.sh
+      - name: "benchmarks; dart analyze --fatal-infos"
+        if: "always() && steps.benchmarks_pub_upgrade.conclusion == 'success'"
+        working-directory: benchmarks
         run: dart analyze --fatal-infos
-  job_004:
+  job_005:
     name: "format_analyze; linux; Dart dev; PKG: protobuf; `dart format --output=none --set-exit-if-changed .`, `dart analyze --fatal-infos`"
     runs-on: ubuntu-latest
     steps:
@@ -160,15 +179,15 @@ jobs:
         if: "always() && steps.protobuf_pub_upgrade.conclusion == 'success'"
         working-directory: protobuf
         run: dart analyze --fatal-infos
-  job_005:
+  job_006:
     name: "format_analyze; linux; Dart dev; PKG: protoc_plugin; `dart format --output=none --set-exit-if-changed .`, `./../tool/setup.sh`, `make protos`, `dart analyze --fatal-infos`"
     runs-on: ubuntu-latest
     steps:
       - name: Cache Pub hosted dependencies
         uses: actions/cache@v3
         with:
           path: "~/.pub-cache/hosted"
-          key: "os:ubuntu-latest;pub-cache-hosted;sdk:dev;packages:protoc_plugin;commands:format-command_0-command_2-analyze_0"
+          key: "os:ubuntu-latest;pub-cache-hosted;sdk:dev;packages:protoc_plugin;commands:format-command_0-command_3-analyze_0"
           restore-keys: |
             os:ubuntu-latest;pub-cache-hosted;sdk:dev;packages:protoc_plugin
             os:ubuntu-latest;pub-cache-hosted;sdk:dev
@@ -200,7 +219,7 @@ jobs:
         if: "always() && steps.protoc_plugin_pub_upgrade.conclusion == 'success'"
         working-directory: protoc_plugin
         run: dart analyze --fatal-infos
-  job_006:
+  job_007:
     name: "run_tests; linux; Dart 2.12.0; PKG: protobuf; `dart test`"
     runs-on: ubuntu-latest
     steps:
@@ -234,15 +253,16 @@ jobs:
       - job_003
       - job_004
       - job_005
-  job_007:
+      - job_006
+  job_008:
     name: "run_tests; linux; Dart 2.12.0; PKG: protoc_plugin; `./../tool/setup.sh`, `make protos`, `dart test`"
     runs-on: ubuntu-latest
     steps:
       - name: Cache Pub hosted dependencies
         uses: actions/cache@v3
         with:
           path: "~/.pub-cache/hosted"
-          key: "os:ubuntu-latest;pub-cache-hosted;sdk:2.12.0;packages:protoc_plugin;commands:command_0-command_2-test"
+          key: "os:ubuntu-latest;pub-cache-hosted;sdk:2.12.0;packages:protoc_plugin;commands:command_0-command_3-test"
           restore-keys: |
             os:ubuntu-latest;pub-cache-hosted;sdk:2.12.0;packages:protoc_plugin
             os:ubuntu-latest;pub-cache-hosted;sdk:2.12.0
@@ -276,7 +296,8 @@ jobs:
       - job_003
       - job_004
       - job_005
-  job_008:
+      - job_006
+  job_009:
     name: "run_tests; linux; Dart dev; PKG: protobuf; `dart test`"
     runs-on: ubuntu-latest
     steps:
@@ -310,15 +331,16 @@ jobs:
       - job_003
       - job_004
       - job_005
-  job_009:
+      - job_006
+  job_010:
     name: "run_tests; linux; Dart dev; PKG: protoc_plugin; `./../tool/setup.sh`, `make protos`, `dart test`"
     runs-on: ubuntu-latest
     steps:
       - name: Cache Pub hosted dependencies
         uses: actions/cache@v3
         with:
           path: "~/.pub-cache/hosted"
-          key: "os:ubuntu-latest;pub-cache-hosted;sdk:dev;packages:protoc_plugin;commands:command_0-command_2-test"
+          key: "os:ubuntu-latest;pub-cache-hosted;sdk:dev;packages:protoc_plugin;commands:command_0-command_3-test"
           restore-keys: |
             os:ubuntu-latest;pub-cache-hosted;sdk:dev;packages:protoc_plugin
             os:ubuntu-latest;pub-cache-hosted;sdk:dev
@@ -352,7 +374,8 @@ jobs:
       - job_003
       - job_004
       - job_005
-  job_010:
+      - job_006
+  job_011:
     name: "run_tests; osx; Dart 2.12.0; PKG: protobuf; `dart test`"
     runs-on: macos-latest
     steps:
@@ -386,7 +409,8 @@ jobs:
       - job_003
       - job_004
       - job_005
-  job_011:
+      - job_006
+  job_012:
     name: "run_tests; osx; Dart dev; PKG: protobuf; `dart test`"
     runs-on: macos-latest
     steps:
@@ -420,7 +444,8 @@ jobs:
       - job_003
       - job_004
       - job_005
-  job_012:
+      - job_006
+  job_013:
     name: "run_tests; windows; Dart 2.12.0; PKG: protobuf; `dart test`"
     runs-on: windows-latest
     steps:
@@ -444,7 +469,8 @@ jobs:
       - job_003
       - job_004
       - job_005
-  job_013:
+      - job_006
+  job_014:
     name: "run_tests; windows; Dart dev; PKG: protobuf; `dart test`"
     runs-on: windows-latest
     steps:
@@ -468,3 +494,4 @@ jobs:
       - job_003
       - job_004
       - job_005
+      - job_006

.gitignore

@@ -7,3 +7,4 @@ build/
 **/lib/generated/**
 packages
 pubspec.lock
+benchmarks/out

benchmarks/README.md

@@ -0,0 +1,20 @@
+## Running benchmarks
+
+- Compile protos with `./tool/compile_protos.sh`
+
+- **JIT:** Run benchmark programs in `bin/`, e.g. `dart bin/from_binary.dart`
+
+- **AOT and JS:**
+
+  - Compile benchmark programs to native, JS, and JIT snapshots with
+    `./tool/compile_benchmarks.dart`
+
+  - Run benchmark programs in `out/`:
+    - AOT: `./out/from_binary.exe`
+    - JS: `d8 $DART_SDK/lib/_internal/js_runtime/lib/preambles/d8.js out/from_binary.js`
+
+## Development
+
+`protoc_version` file specifies the version of protoc Golem will use when
+building the benchmarks, and allows updating the protoc version without
+changing Golem.

benchmarks/analysis_options.yaml

@@ -0,0 +1,5 @@
+include: package:lints/recommended.yaml
+
+linter:
+  rules:
+    - directives_ordering

benchmarks/bin/from_binary.dart

@@ -0,0 +1,44 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:typed_data';
+
+import 'package:protobuf_benchmarks/benchmark_base.dart';
+import 'package:protobuf_benchmarks/generated/google_message1_proto2.pb.dart'
+    as p2;
+import 'package:protobuf_benchmarks/generated/google_message1_proto3.pb.dart'
+    as p3;
+import 'package:protobuf_benchmarks/generated/google_message2.pb.dart';
+import 'package:protobuf_benchmarks/readfile.dart';
+
+class Benchmark extends BenchmarkBase {
+  final Uint8List _message1Proto2Input;
+  final Uint8List _message1Proto3Input;
+  final Uint8List _message2Input;
+
+  Benchmark(String name, List<int> message1Proto2Input,
+      List<int> message1Proto3Input, List<int> message2Input)
+      : _message1Proto2Input = Uint8List.fromList(message1Proto2Input),
+        _message1Proto3Input = Uint8List.fromList(message1Proto3Input),
+        _message2Input = Uint8List.fromList(message2Input),
+        super(name);
+
+  @override
+  void run() {
+    p2.GoogleMessage1.fromBuffer(_message1Proto2Input);
+    p3.GoogleMessage1.fromBuffer(_message1Proto3Input);
+    GoogleMessage2.fromBuffer(_message2Input);
+  }
+}
+
+void main() {
+  List<int> message1Proto2Input =
+      readfile('datasets/google_message1_proto2.pb');
+  List<int> message1Proto3Input =
+      readfile('datasets/google_message1_proto3.pb');
+  List<int> message2Input = readfile('datasets/google_message2.pb');
+  Benchmark('from_binary', message1Proto2Input, message1Proto3Input,
+          message2Input)
+      .report();
+}

benchmarks/bin/from_json_string.dart

@@ -0,0 +1,45 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:protobuf_benchmarks/benchmark_base.dart';
+import 'package:protobuf_benchmarks/generated/google_message1_proto2.pb.dart'
+    as p2;
+import 'package:protobuf_benchmarks/generated/google_message1_proto3.pb.dart'
+    as p3;
+import 'package:protobuf_benchmarks/generated/google_message2.pb.dart';
+import 'package:protobuf_benchmarks/readfile.dart';
+
+class Benchmark extends BenchmarkBase {
+  final String _message1Proto2JsonString;
+  final String _message1Proto3JsonString;
+  final String _message2JsonString;
+
+  Benchmark(String name, List<int> message1Proto2Input,
+      List<int> message1Proto3Input, List<int> message2Input)
+      : _message1Proto2JsonString =
+            p2.GoogleMessage1.fromBuffer(message1Proto2Input).writeToJson(),
+        _message1Proto3JsonString =
+            p3.GoogleMessage1.fromBuffer(message1Proto3Input).writeToJson(),
+        _message2JsonString =
+            GoogleMessage2.fromBuffer(message2Input).writeToJson(),
+        super(name);
+
+  @override
+  void run() {
+    p2.GoogleMessage1.fromJson(_message1Proto2JsonString);
+    p3.GoogleMessage1.fromJson(_message1Proto3JsonString);
+    GoogleMessage2.fromJson(_message2JsonString);
+  }
+}
+
+void main() {
+  List<int> message1Proto2Input =
+      readfile('datasets/google_message1_proto2.pb');
+  List<int> message1Proto3Input =
+      readfile('datasets/google_message1_proto3.pb');
+  List<int> message2Input = readfile('datasets/google_message2.pb');
+  Benchmark('from_json_string', message1Proto2Input, message1Proto3Input,
+          message2Input)
+      .report();
+}

benchmarks/bin/from_proto3_json_object.dart

@@ -0,0 +1,47 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:protobuf_benchmarks/benchmark_base.dart';
+import 'package:protobuf_benchmarks/generated/google_message1_proto2.pb.dart'
+    as p2;
+import 'package:protobuf_benchmarks/generated/google_message1_proto3.pb.dart'
+    as p3;
+import 'package:protobuf_benchmarks/generated/google_message2.pb.dart';
+import 'package:protobuf_benchmarks/readfile.dart';
+
+class Benchmark extends BenchmarkBase {
+  final Object? _message1Proto2Proto3JsonObject;
+  final Object? _message1Proto3Proto3JsonObject;
+  final Object? _message2Proto3JsonObject;
+
+  Benchmark(String name, List<int> message1Proto2Input,
+      List<int> message1Proto3Input, List<int> message2Input)
+      : _message1Proto2Proto3JsonObject =
+            p2.GoogleMessage1.fromBuffer(message1Proto2Input).toProto3Json(),
+        _message1Proto3Proto3JsonObject =
+            p3.GoogleMessage1.fromBuffer(message1Proto3Input).toProto3Json(),
+        _message2Proto3JsonObject =
+            GoogleMessage2.fromBuffer(message2Input).toProto3Json(),
+        super(name);
+
+  @override
+  void run() {
+    p2.GoogleMessage1.create()
+        .mergeFromProto3Json(_message1Proto2Proto3JsonObject);
+    p3.GoogleMessage1.create()
+        .mergeFromProto3Json(_message1Proto3Proto3JsonObject);
+    GoogleMessage2.create().mergeFromProto3Json(_message2Proto3JsonObject);
+  }
+}
+
+void main() {
+  List<int> message1Proto2Input =
+      readfile('datasets/google_message1_proto2.pb');
+  List<int> message1Proto3Input =
+      readfile('datasets/google_message1_proto3.pb');
+  List<int> message2Input = readfile('datasets/google_message2.pb');
+  Benchmark('from_proto3_json_object', message1Proto2Input, message1Proto3Input,
+          message2Input)
+      .report();
+}