From f21252860b268b0fae31022687991f944505ada2 Mon Sep 17 00:00:00 2001 From: William Hesse Date: Thu, 7 Dec 2023 18:14:47 +0100 Subject: [PATCH 01/13] Add benchmark_base_perf.dart override of benchmark_harness that starts and stops perf tool. --- lib/benchmark_harness.dart | 3 ++- lib/src/benchmark_base.dart | 16 ++++++++++++-- lib/src/benchmark_base_perf.dart | 37 ++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 lib/src/benchmark_base_perf.dart diff --git a/lib/benchmark_harness.dart b/lib/benchmark_harness.dart index ee1563c..1f38dd2 100644 --- a/lib/benchmark_harness.dart +++ b/lib/benchmark_harness.dart @@ -3,5 +3,6 @@ // BSD-style license that can be found in the LICENSE file. export 'src/async_benchmark_base.dart'; -export 'src/benchmark_base.dart'; +export 'src/benchmark_base.dart' + if (dart.library.io) 'src/benchmark_base_perf.dart'; export 'src/score_emitter.dart'; diff --git a/lib/src/benchmark_base.dart b/lib/src/benchmark_base.dart index ad1bb91..568b3a9 100644 --- a/lib/src/benchmark_base.dart +++ b/lib/src/benchmark_base.dart @@ -38,6 +38,13 @@ class BenchmarkBase { /// Not measured teardown code executed after the benchmark runs. void teardown() {} + /// Not measured code run just before starting the timed runs + void beforeTimedRuns() {} + + /// Not measured code run just after the timed runs finish. + /// Receives the total number of iterations run. + void afterTimedRuns(int totalIterations) {} + /// Measures the score for this benchmark by executing it enough times /// to reach [minimumMillis]. static _Measurement _measureForImpl(void Function() f, int minimumMillis) { @@ -47,6 +54,7 @@ class BenchmarkBase { final allowedJitter = minimumMillis < 1000 ? 0 : (minimumMicros * 0.1).floor(); var iter = 2; + var totalIterations = iter; final watch = Stopwatch()..start(); while (true) { watch.reset(); @@ -54,13 +62,14 @@ class BenchmarkBase { f(); } final elapsed = watch.elapsedMicroseconds; - final measurement = _Measurement(elapsed, iter); + final measurement = _Measurement(elapsed, iter, totalIterations); if (measurement.elapsedMicros >= (minimumMicros - allowedJitter)) { return measurement; } iter = measurement.estimateIterationsNeededToReach( minimumMicros: minimumMicros); + totalIterations += iter; } } @@ -74,8 +83,10 @@ class BenchmarkBase { setup(); // Warmup for at least 100ms. Discard result. _measureForImpl(warmup, 100); + beforeTimedRuns(); // Run the benchmark for at least 2000ms. var result = _measureForImpl(exercise, _minimumMeasureDurationMillis); + afterTimedRuns(result.totalIterations); teardown(); return result.score; } @@ -88,8 +99,9 @@ class BenchmarkBase { class _Measurement { final int elapsedMicros; final int iterations; + final int totalIterations; - _Measurement(this.elapsedMicros, this.iterations); + _Measurement(this.elapsedMicros, this.iterations, this.totalIterations); double get score => elapsedMicros / iterations; diff --git a/lib/src/benchmark_base_perf.dart b/lib/src/benchmark_base_perf.dart new file mode 100644 index 0000000..f9459bb --- /dev/null +++ b/lib/src/benchmark_base_perf.dart @@ -0,0 +1,37 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'benchmark_base.dart' as base; +import 'score_emitter.dart'; + +const perfControlFifoVariable = 'PERF_CONTROL_FIFO'; + +class BenchmarkBase extends base.BenchmarkBase { + BenchmarkBase(super.name, {super.emitter = const PrintEmitter()}); + + String? perfControlFifo; + late RandomAccessFile openedFifo; + + @override + void beforeTimedRuns() { + perfControlFifo = Platform.environment[perfControlFifoVariable]; + if (perfControlFifo != null) { + openedFifo = File(perfControlFifo!).openSync(); + openedFifo.writeStringSync('enable\n'); + // TODO: read 'ack\n' from second ack fifo, before proceeding. + } + } + + @override + void afterTimedRuns(int totalIterations) { + if (perfControlFifo != null) { + openedFifo.writeStringSync('disable\n'); + } + // TODO: await ack. + openedFifo.closeSync(); + emitter.emit('$name.totalIterations', totalIterations.toDouble()); + } +} From 8a2c57f1ff67e223662e39cfe82d237bca614e26 Mon Sep 17 00:00:00 2001 From: William Hesse Date: Fri, 8 Dec 2023 12:09:22 +0100 Subject: [PATCH 02/13] Add handling of "ack" message from perf communication channel --- lib/src/benchmark_base_perf.dart | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/lib/src/benchmark_base_perf.dart b/lib/src/benchmark_base_perf.dart index f9459bb..c774689 100644 --- a/lib/src/benchmark_base_perf.dart +++ b/lib/src/benchmark_base_perf.dart @@ -8,20 +8,42 @@ import 'benchmark_base.dart' as base; import 'score_emitter.dart'; const perfControlFifoVariable = 'PERF_CONTROL_FIFO'; +const perfControlAckVariable = 'PERF_CONTROL_ACK'; class BenchmarkBase extends base.BenchmarkBase { BenchmarkBase(super.name, {super.emitter = const PrintEmitter()}); String? perfControlFifo; late RandomAccessFile openedFifo; + String? perfControlAck; + late RandomAccessFile openedAck; @override void beforeTimedRuns() { perfControlFifo = Platform.environment[perfControlFifoVariable]; + perfControlAck = Platform.environment[perfControlAckVariable]; if (perfControlFifo != null) { - openedFifo = File(perfControlFifo!).openSync(); - openedFifo.writeStringSync('enable\n'); - // TODO: read 'ack\n' from second ack fifo, before proceeding. + openedFifo = File(perfControlFifo!).openSync(mode: FileMode.writeOnly); + if (perfControlAck != null) { + openedAck = File(perfControlAck!).openSync(); + openedFifo.writeStringSync('enable\n'); + + var ack = [...openedAck.readSync(4)]; + while (ack.length < 4) { + ack.addAll(openedAck.readSync(4 - ack.length)); + print('reading $ack'); + } + if (String.fromCharCodes(ack) != 'ack\n') { + print('Ack was $ack'); + } + /* var ackLength = 0; + while (ackLength < 4) { + ackLength += openedAck.readSync(4 - ackLength).length; + print('reading $ackLength'); + }*/ + } else { + openedFifo.writeStringSync('enable\n'); + } } } @@ -29,9 +51,9 @@ class BenchmarkBase extends base.BenchmarkBase { void afterTimedRuns(int totalIterations) { if (perfControlFifo != null) { openedFifo.writeStringSync('disable\n'); + openedFifo.closeSync(); + emitter.emit('$name.totalIterations', totalIterations.toDouble()); } // TODO: await ack. - openedFifo.closeSync(); - emitter.emit('$name.totalIterations', totalIterations.toDouble()); } } From 3010760072e6b716ffdc15e5b368f0ca098e73d4 Mon Sep 17 00:00:00 2001 From: William Hesse Date: Mon, 22 Jan 2024 10:57:40 +0100 Subject: [PATCH 03/13] Clean up ack handling, fix test --- lib/src/benchmark_base_perf.dart | 31 ++++++++++++++++--------------- test/result_emitter_test.dart | 2 +- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/lib/src/benchmark_base_perf.dart b/lib/src/benchmark_base_perf.dart index c774689..f329a4f 100644 --- a/lib/src/benchmark_base_perf.dart +++ b/lib/src/benchmark_base_perf.dart @@ -27,20 +27,7 @@ class BenchmarkBase extends base.BenchmarkBase { if (perfControlAck != null) { openedAck = File(perfControlAck!).openSync(); openedFifo.writeStringSync('enable\n'); - - var ack = [...openedAck.readSync(4)]; - while (ack.length < 4) { - ack.addAll(openedAck.readSync(4 - ack.length)); - print('reading $ack'); - } - if (String.fromCharCodes(ack) != 'ack\n') { - print('Ack was $ack'); - } - /* var ackLength = 0; - while (ackLength < 4) { - ackLength += openedAck.readSync(4 - ackLength).length; - print('reading $ackLength'); - }*/ + waitForAck(); } else { openedFifo.writeStringSync('enable\n'); } @@ -52,8 +39,22 @@ class BenchmarkBase extends base.BenchmarkBase { if (perfControlFifo != null) { openedFifo.writeStringSync('disable\n'); openedFifo.closeSync(); + if (perfControlAck != null) { + waitForAck(); + openedAck.closeSync(); + } emitter.emit('$name.totalIterations', totalIterations.toDouble()); } - // TODO: await ack. + } + + void waitForAck() { + var ack = [...openedAck.readSync(4)]; + while (ack.length < 4) { + ack.addAll(openedAck.readSync(4 - ack.length)); + print('reading $ack'); + } + if (String.fromCharCodes(ack) != 'ack\n') { + print('Ack was $ack'); + } } } diff --git a/test/result_emitter_test.dart b/test/result_emitter_test.dart index e2cd1ea..81d39b9 100644 --- a/test/result_emitter_test.dart +++ b/test/result_emitter_test.dart @@ -20,7 +20,7 @@ class MockResultEmitter extends ScoreEmitter { // Create a new benchmark which has an emitter. class BenchmarkWithResultEmitter extends BenchmarkBase { - const BenchmarkWithResultEmitter(ScoreEmitter emitter) + BenchmarkWithResultEmitter(ScoreEmitter emitter) : super('Template', emitter: emitter); @override From bb86037f34aea36392c9f393ebcd07297ff8f4db Mon Sep 17 00:00:00 2001 From: William Hesse Date: Thu, 8 Feb 2024 16:16:20 +0100 Subject: [PATCH 04/13] Put PerfBenchmarkBase into a new class, don't modify BenchmarkBase When PerfBenchmarkBase also starts, attaches, and stops the "perf stat" subprocess, measure() and report() will need to be async functions, so add measurePerf() and reportPerf() functions. This version still requires "perf stat" to be wrapped around the benchmark command. --- lib/benchmark_harness.dart | 5 ++- lib/src/benchmark_base.dart | 67 ++++++++++++++-------------- lib/src/benchmark_base_perf.dart | 1 + lib/src/perf_benchmark_base.dart | 76 ++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 35 deletions(-) create mode 100644 lib/src/perf_benchmark_base.dart diff --git a/lib/benchmark_harness.dart b/lib/benchmark_harness.dart index 1f38dd2..92c88dd 100644 --- a/lib/benchmark_harness.dart +++ b/lib/benchmark_harness.dart @@ -3,6 +3,7 @@ // BSD-style license that can be found in the LICENSE file. export 'src/async_benchmark_base.dart'; -export 'src/benchmark_base.dart' - if (dart.library.io) 'src/benchmark_base_perf.dart'; +export 'src/benchmark_base.dart' show BenchmarkBase; +export 'src/perf_benchmark_base_stub.dart' + if (dart.library.io) 'src/perf_benchmark_base.dart'; export 'src/score_emitter.dart'; diff --git a/lib/src/benchmark_base.dart b/lib/src/benchmark_base.dart index 568b3a9..300d70b 100644 --- a/lib/src/benchmark_base.dart +++ b/lib/src/benchmark_base.dart @@ -6,7 +6,7 @@ import 'dart:math' as math; import 'score_emitter.dart'; -const int _minimumMeasureDurationMillis = 2000; +const int minimumMeasureDurationMillis = 2000; class BenchmarkBase { final String name; @@ -47,46 +47,19 @@ class BenchmarkBase { /// Measures the score for this benchmark by executing it enough times /// to reach [minimumMillis]. - static _Measurement _measureForImpl(void Function() f, int minimumMillis) { - final minimumMicros = minimumMillis * 1000; - // If running a long measurement permit some amount of measurement jitter - // to avoid discarding results that are almost good, but not quite there. - final allowedJitter = - minimumMillis < 1000 ? 0 : (minimumMicros * 0.1).floor(); - var iter = 2; - var totalIterations = iter; - final watch = Stopwatch()..start(); - while (true) { - watch.reset(); - for (var i = 0; i < iter; i++) { - f(); - } - final elapsed = watch.elapsedMicroseconds; - final measurement = _Measurement(elapsed, iter, totalIterations); - if (measurement.elapsedMicros >= (minimumMicros - allowedJitter)) { - return measurement; - } - - iter = measurement.estimateIterationsNeededToReach( - minimumMicros: minimumMicros); - totalIterations += iter; - } - } /// Measures the score for this benchmark by executing it repeatedly until /// time minimum has been reached. static double measureFor(void Function() f, int minimumMillis) => - _measureForImpl(f, minimumMillis).score; + measureForImpl(f, minimumMillis).score; /// Measures the score for the benchmark and returns it. double measure() { setup(); // Warmup for at least 100ms. Discard result. - _measureForImpl(warmup, 100); - beforeTimedRuns(); + measureForImpl(warmup, 100); // Run the benchmark for at least 2000ms. - var result = _measureForImpl(exercise, _minimumMeasureDurationMillis); - afterTimedRuns(result.totalIterations); + var result = measureForImpl(exercise, minimumMeasureDurationMillis); teardown(); return result.score; } @@ -96,12 +69,40 @@ class BenchmarkBase { } } -class _Measurement { +/// Measures the score for this benchmark by executing it enough times +/// to reach [minimumMillis]. +Measurement measureForImpl(void Function() f, int minimumMillis) { + final minimumMicros = minimumMillis * 1000; + // If running a long measurement permit some amount of measurement jitter + // to avoid discarding results that are almost good, but not quite there. + final allowedJitter = + minimumMillis < 1000 ? 0 : (minimumMicros * 0.1).floor(); + var iter = 2; + var totalIterations = iter; + final watch = Stopwatch()..start(); + while (true) { + watch.reset(); + for (var i = 0; i < iter; i++) { + f(); + } + final elapsed = watch.elapsedMicroseconds; + final measurement = Measurement(elapsed, iter, totalIterations); + if (measurement.elapsedMicros >= (minimumMicros - allowedJitter)) { + return measurement; + } + + iter = measurement.estimateIterationsNeededToReach( + minimumMicros: minimumMicros); + totalIterations += iter; + } +} + +class Measurement { final int elapsedMicros; final int iterations; final int totalIterations; - _Measurement(this.elapsedMicros, this.iterations, this.totalIterations); + Measurement(this.elapsedMicros, this.iterations, this.totalIterations); double get score => elapsedMicros / iterations; diff --git a/lib/src/benchmark_base_perf.dart b/lib/src/benchmark_base_perf.dart index f329a4f..e4e1f4a 100644 --- a/lib/src/benchmark_base_perf.dart +++ b/lib/src/benchmark_base_perf.dart @@ -17,6 +17,7 @@ class BenchmarkBase extends base.BenchmarkBase { late RandomAccessFile openedFifo; String? perfControlAck; late RandomAccessFile openedAck; + late Process perfProcess; @override void beforeTimedRuns() { diff --git a/lib/src/perf_benchmark_base.dart b/lib/src/perf_benchmark_base.dart new file mode 100644 index 0000000..897e72f --- /dev/null +++ b/lib/src/perf_benchmark_base.dart @@ -0,0 +1,76 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'benchmark_base.dart'; +import 'score_emitter.dart'; + +const perfControlFifoVariable = 'PERF_CONTROL_FIFO'; +const perfControlAckVariable = 'PERF_CONTROL_ACK'; + +class PerfBenchmarkBase extends BenchmarkBase { + PerfBenchmarkBase(super.name, {super.emitter = const PrintEmitter()}); + + String? perfControlFifo; + late RandomAccessFile openedFifo; + String? perfControlAck; + late RandomAccessFile openedAck; + late Process perfProcess; + + Future _startPerfStat() async { + perfControlFifo = Platform.environment[perfControlFifoVariable]; + perfControlAck = Platform.environment[perfControlAckVariable]; + print(perfControlFifo); + if (perfControlFifo != null) { + openedFifo = File(perfControlFifo!).openSync(mode: FileMode.writeOnly); + if (perfControlAck != null) { + openedAck = File(perfControlAck!).openSync(); + openedFifo.writeStringSync('enable\n'); + _waitForAck(); + } else { + openedFifo.writeStringSync('enable\n'); + } + } + } + + Future _stopPerfStat(int totalIterations) async { + if (perfControlFifo != null) { + openedFifo.writeStringSync('disable\n'); + openedFifo.closeSync(); + if (perfControlAck != null) { + _waitForAck(); + openedAck.closeSync(); + } + emitter.emit('$name.totalIterations', totalIterations.toDouble()); + } + } + + /// Measures the score for the benchmark and returns it. + Future measurePerf() async { + setup(); + // Warmup for at least 100ms. Discard result. + measureForImpl(warmup, 100); + await _startPerfStat(); + // Run the benchmark for at least 2000ms. + var result = measureForImpl(exercise, minimumMeasureDurationMillis); + await _stopPerfStat(result.totalIterations); + teardown(); + return result.score; + } + + Future reportPerf() async { + emitter.emit(name, await measurePerf()); + } + + void _waitForAck() { + var ack = [...openedAck.readSync(5)]; + while (ack.length < 5) { + ack.addAll(openedAck.readSync(5 - ack.length)); + } + if (String.fromCharCodes(ack) != 'ack\n\x00') { + print('Ack was $ack'); + } + } +} From 0176c8a1a4c3aeb198095e67bbfcca1ad328f714 Mon Sep 17 00:00:00 2001 From: William Hesse Date: Thu, 8 Feb 2024 16:20:43 +0100 Subject: [PATCH 05/13] Add lib/src/perf_benchmark_base_stub.dart --- lib/src/perf_benchmark_base_stub.dart | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 lib/src/perf_benchmark_base_stub.dart diff --git a/lib/src/perf_benchmark_base_stub.dart b/lib/src/perf_benchmark_base_stub.dart new file mode 100644 index 0000000..81aa0ea --- /dev/null +++ b/lib/src/perf_benchmark_base_stub.dart @@ -0,0 +1,18 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'benchmark_base.dart'; +import 'score_emitter.dart'; + +class PerfBenchmarkBase extends BenchmarkBase { + PerfBenchmarkBase(super.name, {super.emitter = const PrintEmitter()}); + + Future measurePerf() async { + return super.measure(); + } + + Future reportPerf() async { + super.report(); + } +} From 73f42b00faa7442604e841c7c7e9f17188846c4e Mon Sep 17 00:00:00 2001 From: William Hesse Date: Thu, 8 Feb 2024 16:26:24 +0100 Subject: [PATCH 06/13] Remove file from previous version --- lib/src/benchmark_base_perf.dart | 61 -------------------------------- test/result_emitter_test.dart | 2 +- 2 files changed, 1 insertion(+), 62 deletions(-) delete mode 100644 lib/src/benchmark_base_perf.dart diff --git a/lib/src/benchmark_base_perf.dart b/lib/src/benchmark_base_perf.dart deleted file mode 100644 index e4e1f4a..0000000 --- a/lib/src/benchmark_base_perf.dart +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:io'; - -import 'benchmark_base.dart' as base; -import 'score_emitter.dart'; - -const perfControlFifoVariable = 'PERF_CONTROL_FIFO'; -const perfControlAckVariable = 'PERF_CONTROL_ACK'; - -class BenchmarkBase extends base.BenchmarkBase { - BenchmarkBase(super.name, {super.emitter = const PrintEmitter()}); - - String? perfControlFifo; - late RandomAccessFile openedFifo; - String? perfControlAck; - late RandomAccessFile openedAck; - late Process perfProcess; - - @override - void beforeTimedRuns() { - perfControlFifo = Platform.environment[perfControlFifoVariable]; - perfControlAck = Platform.environment[perfControlAckVariable]; - if (perfControlFifo != null) { - openedFifo = File(perfControlFifo!).openSync(mode: FileMode.writeOnly); - if (perfControlAck != null) { - openedAck = File(perfControlAck!).openSync(); - openedFifo.writeStringSync('enable\n'); - waitForAck(); - } else { - openedFifo.writeStringSync('enable\n'); - } - } - } - - @override - void afterTimedRuns(int totalIterations) { - if (perfControlFifo != null) { - openedFifo.writeStringSync('disable\n'); - openedFifo.closeSync(); - if (perfControlAck != null) { - waitForAck(); - openedAck.closeSync(); - } - emitter.emit('$name.totalIterations', totalIterations.toDouble()); - } - } - - void waitForAck() { - var ack = [...openedAck.readSync(4)]; - while (ack.length < 4) { - ack.addAll(openedAck.readSync(4 - ack.length)); - print('reading $ack'); - } - if (String.fromCharCodes(ack) != 'ack\n') { - print('Ack was $ack'); - } - } -} diff --git a/test/result_emitter_test.dart b/test/result_emitter_test.dart index 81d39b9..e2cd1ea 100644 --- a/test/result_emitter_test.dart +++ b/test/result_emitter_test.dart @@ -20,7 +20,7 @@ class MockResultEmitter extends ScoreEmitter { // Create a new benchmark which has an emitter. class BenchmarkWithResultEmitter extends BenchmarkBase { - BenchmarkWithResultEmitter(ScoreEmitter emitter) + const BenchmarkWithResultEmitter(ScoreEmitter emitter) : super('Template', emitter: emitter); @override From a9858d9e6889f5044a8dc440427da175281178ff Mon Sep 17 00:00:00 2001 From: William Hesse Date: Thu, 8 Feb 2024 19:26:07 +0100 Subject: [PATCH 07/13] Create perf stat subprocess and attach it to the current benchmark --- lib/src/benchmark_base.dart | 7 ------ lib/src/perf_benchmark_base.dart | 38 ++++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/lib/src/benchmark_base.dart b/lib/src/benchmark_base.dart index 300d70b..bc874f5 100644 --- a/lib/src/benchmark_base.dart +++ b/lib/src/benchmark_base.dart @@ -38,13 +38,6 @@ class BenchmarkBase { /// Not measured teardown code executed after the benchmark runs. void teardown() {} - /// Not measured code run just before starting the timed runs - void beforeTimedRuns() {} - - /// Not measured code run just after the timed runs finish. - /// Receives the total number of iterations run. - void afterTimedRuns(int totalIterations) {} - /// Measures the score for this benchmark by executing it enough times /// to reach [minimumMillis]. diff --git a/lib/src/perf_benchmark_base.dart b/lib/src/perf_benchmark_base.dart index 897e72f..37a903b 100644 --- a/lib/src/perf_benchmark_base.dart +++ b/lib/src/perf_benchmark_base.dart @@ -2,6 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:convert'; import 'dart:io'; import 'benchmark_base.dart'; @@ -20,10 +21,22 @@ class PerfBenchmarkBase extends BenchmarkBase { late Process perfProcess; Future _startPerfStat() async { + // TODO: Create these fifos here instead of getting them through env variables. perfControlFifo = Platform.environment[perfControlFifoVariable]; perfControlAck = Platform.environment[perfControlAckVariable]; - print(perfControlFifo); if (perfControlFifo != null) { + perfProcess = await Process.start('perf', [ + 'stat', + '--delay', + '-1', + '--control', + 'fifo:$perfControlFifo,$perfControlAck', + '-j', + '-p', + '$pid' + ]); + await Future.delayed(const Duration(seconds: 2)); + openedFifo = File(perfControlFifo!).openSync(mode: FileMode.writeOnly); if (perfControlAck != null) { openedAck = File(perfControlAck!).openSync(); @@ -43,7 +56,15 @@ class PerfBenchmarkBase extends BenchmarkBase { _waitForAck(); openedAck.closeSync(); } - emitter.emit('$name.totalIterations', totalIterations.toDouble()); + perfProcess.kill(ProcessSignal.sigint); + final lines = + utf8.decoder.bind(perfProcess.stderr).transform(const LineSplitter()); + final events = [ + await for (final line in lines) + if (line.startsWith('{"counter-value" : ')) + jsonDecode(line) as Map + ]; + _reportPerfStats(events, totalIterations); } } @@ -73,4 +94,17 @@ class PerfBenchmarkBase extends BenchmarkBase { print('Ack was $ack'); } } + + void _reportPerfStats(List> events, int iterations) { + for (final {'event': String event, 'counter-value': String counterString} + in events) { + final metric = + {'cycles:u': 'CpuCycles', 'page-faults:u': 'MajorPageFaults'}[event]; + if (metric != null) { + emitter.emit( + '$name($metric)', double.parse(counterString) / iterations); + } + } + emitter.emit('$name.totalIterations', iterations.toDouble()); + } } From 11d8eab7ed4e6bf4bc36d00f9a34e84225bc2283 Mon Sep 17 00:00:00 2001 From: William Hesse Date: Tue, 20 Feb 2024 10:29:20 +0100 Subject: [PATCH 08/13] Add integration test, put PerfBenchmarkBase in a separate public library --- integration_test/perf_benchmark_test.dart | 28 +++++++++++++++++++++++ lib/benchmark_harness.dart | 2 -- lib/perf_benchmark_harness.dart | 7 ++++++ lib/src/perf_benchmark_base.dart | 3 ++- 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 integration_test/perf_benchmark_test.dart create mode 100644 lib/perf_benchmark_harness.dart diff --git a/integration_test/perf_benchmark_test.dart b/integration_test/perf_benchmark_test.dart new file mode 100644 index 0000000..7930f4f --- /dev/null +++ b/integration_test/perf_benchmark_test.dart @@ -0,0 +1,28 @@ +// Copyright 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; + +import 'package:benchmark_harness/perf_benchmark_harness.dart'; +import 'package:test/test.dart'; + +class PerfBenchmark extends PerfBenchmarkBase { + PerfBenchmark(super.name); + int runCount = 0; + + @override + void run() { + runCount++; + for (final i in List.filled(1000, 7)) { + runCount += i - i; + } + } +} + +void main() { + test('run is called', () async { + final benchmark = PerfBenchmark('ForLoop'); + await benchmark.reportPerf(); + }); +} diff --git a/lib/benchmark_harness.dart b/lib/benchmark_harness.dart index 92c88dd..b46a36f 100644 --- a/lib/benchmark_harness.dart +++ b/lib/benchmark_harness.dart @@ -4,6 +4,4 @@ export 'src/async_benchmark_base.dart'; export 'src/benchmark_base.dart' show BenchmarkBase; -export 'src/perf_benchmark_base_stub.dart' - if (dart.library.io) 'src/perf_benchmark_base.dart'; export 'src/score_emitter.dart'; diff --git a/lib/perf_benchmark_harness.dart b/lib/perf_benchmark_harness.dart new file mode 100644 index 0000000..3de8329 --- /dev/null +++ b/lib/perf_benchmark_harness.dart @@ -0,0 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +export 'src/perf_benchmark_base_stub.dart' + if (dart.library.io) 'src/perf_benchmark_base.dart'; +export 'src/score_emitter.dart'; diff --git a/lib/src/perf_benchmark_base.dart b/lib/src/perf_benchmark_base.dart index 37a903b..a2a8a19 100644 --- a/lib/src/perf_benchmark_base.dart +++ b/lib/src/perf_benchmark_base.dart @@ -21,7 +21,8 @@ class PerfBenchmarkBase extends BenchmarkBase { late Process perfProcess; Future _startPerfStat() async { - // TODO: Create these fifos here instead of getting them through env variables. + // TODO: Create these fifo files here, in a temp directory, instead of + // getting their paths passed in through environment variables. perfControlFifo = Platform.environment[perfControlFifoVariable]; perfControlAck = Platform.environment[perfControlAckVariable]; if (perfControlFifo != null) { From 221124a683f595ce70e6490bd6b50d8d8397b1cc Mon Sep 17 00:00:00 2001 From: William Hesse Date: Tue, 20 Feb 2024 14:22:19 +0100 Subject: [PATCH 09/13] Create fifo named pipes in a temporary directory. Create the temporary pipes used to communicate with perf in a temporary directory in the system temp directory, and clean them up afterwards. Remove the conditional code based on environment variables, and use the perf stat wrapper around the benchmark unconditionally. --- lib/src/perf_benchmark_base.dart | 89 ++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/lib/src/perf_benchmark_base.dart b/lib/src/perf_benchmark_base.dart index a2a8a19..615af38 100644 --- a/lib/src/perf_benchmark_base.dart +++ b/lib/src/perf_benchmark_base.dart @@ -8,24 +8,41 @@ import 'dart:io'; import 'benchmark_base.dart'; import 'score_emitter.dart'; -const perfControlFifoVariable = 'PERF_CONTROL_FIFO'; -const perfControlAckVariable = 'PERF_CONTROL_ACK'; - class PerfBenchmarkBase extends BenchmarkBase { PerfBenchmarkBase(super.name, {super.emitter = const PrintEmitter()}); - String? perfControlFifo; - late RandomAccessFile openedFifo; - String? perfControlAck; - late RandomAccessFile openedAck; - late Process perfProcess; + late final Directory fifoDir; + late final String perfControlFifo; + late final RandomAccessFile openedFifo; + late final String perfControlAck; + late final RandomAccessFile openedAck; + late final Process perfProcess; + + Future _createFifos() async { + fifoDir = await Directory.systemTemp.createTemp('fifo'); + perfControlFifo = '${fifoDir.path}/perf_control_fifo'; + perfControlAck = '${fifoDir.path}/perf_control_ack'; + + try { + final fifoResult = await Process.run('mkfifo', [perfControlFifo]); + if (fifoResult.exitCode != 0) { + throw ProcessException('mkfifo', [perfControlFifo], + 'Cannot create fifo: ${fifoResult.stderr}', fifoResult.exitCode); + } + final ackResult = await Process.run('mkfifo', [perfControlAck]); + if (ackResult.exitCode != 0) { + throw ProcessException('mkfifo', [perfControlAck], + 'Cannot create fifo: ${ackResult.stderr}', ackResult.exitCode); + } + } catch (e) { + await fifoDir.delete(recursive: true); + rethrow; + } + } Future _startPerfStat() async { - // TODO: Create these fifo files here, in a temp directory, instead of - // getting their paths passed in through environment variables. - perfControlFifo = Platform.environment[perfControlFifoVariable]; - perfControlAck = Platform.environment[perfControlAckVariable]; - if (perfControlFifo != null) { + await _createFifos(); + try { perfProcess = await Process.start('perf', [ 'stat', '--delay', @@ -36,36 +53,54 @@ class PerfBenchmarkBase extends BenchmarkBase { '-p', '$pid' ]); - await Future.delayed(const Duration(seconds: 2)); + //await Future.delayed(const Duration(seconds: 2)); - openedFifo = File(perfControlFifo!).openSync(mode: FileMode.writeOnly); - if (perfControlAck != null) { - openedAck = File(perfControlAck!).openSync(); - openedFifo.writeStringSync('enable\n'); - _waitForAck(); - } else { - openedFifo.writeStringSync('enable\n'); - } + openedFifo = File(perfControlFifo).openSync(mode: FileMode.writeOnly); + + openedAck = File(perfControlAck).openSync(); + openedFifo.writeStringSync('enable\n'); + _waitForAck(); + } catch (e) { + await fifoDir.delete(recursive: true); + rethrow; } } Future _stopPerfStat(int totalIterations) async { - if (perfControlFifo != null) { + try { openedFifo.writeStringSync('disable\n'); openedFifo.closeSync(); - if (perfControlAck != null) { - _waitForAck(); - openedAck.closeSync(); - } + _waitForAck(); + openedAck.closeSync(); perfProcess.kill(ProcessSignal.sigint); final lines = utf8.decoder.bind(perfProcess.stderr).transform(const LineSplitter()); + // Exit code from perf is -2 when terminated with SIGINT. + final exitCode = await perfProcess.exitCode; + if (exitCode != 0 && exitCode != -2) { + throw ProcessException( + 'perf', + [ + 'stat', + '--delay', + '-1', + '--control', + 'fifo:$perfControlFifo,$perfControlAck', + '-j', + '-p', + '$pid' + ], + (await lines.toList()).join('\n'), + exitCode); + } final events = [ await for (final line in lines) if (line.startsWith('{"counter-value" : ')) jsonDecode(line) as Map ]; _reportPerfStats(events, totalIterations); + } finally { + await fifoDir.delete(recursive: true); } } From 166a5b83d51789e9e2c263b8282c6832ff476598 Mon Sep 17 00:00:00 2001 From: William Hesse Date: Mon, 11 Mar 2024 15:39:26 +0100 Subject: [PATCH 10/13] Refactor to address review comments --- lib/src/perf_benchmark_base.dart | 146 ++++++++++++++----------------- 1 file changed, 64 insertions(+), 82 deletions(-) diff --git a/lib/src/perf_benchmark_base.dart b/lib/src/perf_benchmark_base.dart index 615af38..2fa6059 100644 --- a/lib/src/perf_benchmark_base.dart +++ b/lib/src/perf_benchmark_base.dart @@ -17,103 +17,86 @@ class PerfBenchmarkBase extends BenchmarkBase { late final String perfControlAck; late final RandomAccessFile openedAck; late final Process perfProcess; + late final List perfProcessArgs; Future _createFifos() async { - fifoDir = await Directory.systemTemp.createTemp('fifo'); perfControlFifo = '${fifoDir.path}/perf_control_fifo'; perfControlAck = '${fifoDir.path}/perf_control_ack'; - try { - final fifoResult = await Process.run('mkfifo', [perfControlFifo]); - if (fifoResult.exitCode != 0) { - throw ProcessException('mkfifo', [perfControlFifo], - 'Cannot create fifo: ${fifoResult.stderr}', fifoResult.exitCode); - } - final ackResult = await Process.run('mkfifo', [perfControlAck]); - if (ackResult.exitCode != 0) { - throw ProcessException('mkfifo', [perfControlAck], - 'Cannot create fifo: ${ackResult.stderr}', ackResult.exitCode); - } - } catch (e) { - await fifoDir.delete(recursive: true); - rethrow; + final fifoResult = await Process.run('mkfifo', [perfControlFifo]); + if (fifoResult.exitCode != 0) { + throw ProcessException('mkfifo', [perfControlFifo], + 'Cannot create fifo: ${fifoResult.stderr}', fifoResult.exitCode); + } + final ackResult = await Process.run('mkfifo', [perfControlAck]); + if (ackResult.exitCode != 0) { + throw ProcessException('mkfifo', [perfControlAck], + 'Cannot create fifo: ${ackResult.stderr}', ackResult.exitCode); } } Future _startPerfStat() async { await _createFifos(); - try { - perfProcess = await Process.start('perf', [ - 'stat', - '--delay', - '-1', - '--control', - 'fifo:$perfControlFifo,$perfControlAck', - '-j', - '-p', - '$pid' - ]); - //await Future.delayed(const Duration(seconds: 2)); + perfProcessArgs = [ + 'stat', + '--delay', + '-1', + '--control', + 'fifo:$perfControlFifo,$perfControlAck', + '-j', + '-p', + '$pid', + ]; + perfProcess = await Process.start('perf', perfProcessArgs); - openedFifo = File(perfControlFifo).openSync(mode: FileMode.writeOnly); + openedFifo = File(perfControlFifo).openSync(mode: FileMode.writeOnly); - openedAck = File(perfControlAck).openSync(); - openedFifo.writeStringSync('enable\n'); - _waitForAck(); - } catch (e) { - await fifoDir.delete(recursive: true); - rethrow; - } + openedAck = File(perfControlAck).openSync(); + openedFifo.writeStringSync('enable\n'); + _waitForAck(); } Future _stopPerfStat(int totalIterations) async { - try { - openedFifo.writeStringSync('disable\n'); - openedFifo.closeSync(); - _waitForAck(); - openedAck.closeSync(); - perfProcess.kill(ProcessSignal.sigint); - final lines = - utf8.decoder.bind(perfProcess.stderr).transform(const LineSplitter()); - // Exit code from perf is -2 when terminated with SIGINT. - final exitCode = await perfProcess.exitCode; - if (exitCode != 0 && exitCode != -2) { - throw ProcessException( - 'perf', - [ - 'stat', - '--delay', - '-1', - '--control', - 'fifo:$perfControlFifo,$perfControlAck', - '-j', - '-p', - '$pid' - ], - (await lines.toList()).join('\n'), - exitCode); - } - final events = [ - await for (final line in lines) - if (line.startsWith('{"counter-value" : ')) - jsonDecode(line) as Map - ]; - _reportPerfStats(events, totalIterations); - } finally { - await fifoDir.delete(recursive: true); + openedFifo.writeStringSync('disable\n'); + openedFifo.closeSync(); + _waitForAck(); + openedAck.closeSync(); + perfProcess.kill(ProcessSignal.sigint); + final lines = + utf8.decoder.bind(perfProcess.stderr).transform(const LineSplitter()); + // Exit code from perf is -2 when terminated with SIGINT. + final exitCode = await perfProcess.exitCode; + if (exitCode != 0 && exitCode != -2) { + throw ProcessException( + 'perf', perfProcessArgs, (await lines.toList()).join('\n'), exitCode); } + final events = [ + await for (final line in lines) + if (line.contains('"counter-value"')) + jsonDecode(line) as Map + ]; + _reportPerfStats(events, totalIterations); } /// Measures the score for the benchmark and returns it. Future measurePerf() async { + Measurement result; setup(); - // Warmup for at least 100ms. Discard result. - measureForImpl(warmup, 100); - await _startPerfStat(); - // Run the benchmark for at least 2000ms. - var result = measureForImpl(exercise, minimumMeasureDurationMillis); - await _stopPerfStat(result.totalIterations); - teardown(); + try { + fifoDir = await Directory.systemTemp.createTemp('fifo'); + try { + // Warmup for at least 100ms. Discard result. + measureForImpl(warmup, 100); + await _startPerfStat(); + // Run the benchmark for at least 2000ms. + result = measureForImpl(exercise, minimumMeasureDurationMillis); + await _stopPerfStat(result.totalIterations); + } finally { + await fifoDir.delete(recursive: true); + } + } finally { + teardown(); + } return result.score; } @@ -122,12 +105,11 @@ class PerfBenchmarkBase extends BenchmarkBase { } void _waitForAck() { - var ack = [...openedAck.readSync(5)]; - while (ack.length < 5) { - ack.addAll(openedAck.readSync(5 - ack.length)); - } - if (String.fromCharCodes(ack) != 'ack\n\x00') { - print('Ack was $ack'); + // Perf writes 'ack\n\x00' to the acknowledgement fifo. + const ackLength = 'ack\n\x00'.length; + var ack = [...openedAck.readSync(ackLength)]; + while (ack.length < ackLength) { + ack.addAll(openedAck.readSync(ackLength - ack.length)); } } From a687636dde38f91efd6012642c293168ae7d5060 Mon Sep 17 00:00:00 2001 From: William Hesse Date: Mon, 11 Mar 2024 15:42:01 +0100 Subject: [PATCH 11/13] Remove unused import --- integration_test/perf_benchmark_test.dart | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration_test/perf_benchmark_test.dart b/integration_test/perf_benchmark_test.dart index 7930f4f..339777f 100644 --- a/integration_test/perf_benchmark_test.dart +++ b/integration_test/perf_benchmark_test.dart @@ -2,8 +2,6 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -import 'dart:async'; - import 'package:benchmark_harness/perf_benchmark_harness.dart'; import 'package:test/test.dart'; From be1a2c41adde471d7f96a8335eab0aedddac7d19 Mon Sep 17 00:00:00 2001 From: William Hesse Date: Tue, 16 Apr 2024 15:34:12 +0200 Subject: [PATCH 12/13] Add metric and unit parameters to ScoreEmitter.emit method --- lib/src/async_benchmark_base.dart | 2 +- lib/src/benchmark_base.dart | 2 +- lib/src/perf_benchmark_base.dart | 10 +++++----- lib/src/score_emitter.dart | 8 +++++--- test/result_emitter_test.dart | 3 ++- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/src/async_benchmark_base.dart b/lib/src/async_benchmark_base.dart index 1472ee7..b342a3d 100644 --- a/lib/src/async_benchmark_base.dart +++ b/lib/src/async_benchmark_base.dart @@ -64,6 +64,6 @@ class AsyncBenchmarkBase { /// Run the benchmark and report results on the [emitter]. Future report() async { - emitter.emit(name, await measure()); + emitter.emit(name, await measure(), unit: 'us.'); } } diff --git a/lib/src/benchmark_base.dart b/lib/src/benchmark_base.dart index bc874f5..51a89bb 100644 --- a/lib/src/benchmark_base.dart +++ b/lib/src/benchmark_base.dart @@ -58,7 +58,7 @@ class BenchmarkBase { } void report() { - emitter.emit(name, measure()); + emitter.emit(name, measure(), unit: 'us.'); } } diff --git a/lib/src/perf_benchmark_base.dart b/lib/src/perf_benchmark_base.dart index 2fa6059..8534d0f 100644 --- a/lib/src/perf_benchmark_base.dart +++ b/lib/src/perf_benchmark_base.dart @@ -48,7 +48,6 @@ class PerfBenchmarkBase extends BenchmarkBase { '$pid', ]; perfProcess = await Process.start('perf', perfProcessArgs); - openedFifo = File(perfControlFifo).openSync(mode: FileMode.writeOnly); openedAck = File(perfControlAck).openSync(); @@ -101,7 +100,7 @@ class PerfBenchmarkBase extends BenchmarkBase { } Future reportPerf() async { - emitter.emit(name, await measurePerf()); + emitter.emit(name, await measurePerf(), unit: 'us.'); } void _waitForAck() { @@ -119,10 +118,11 @@ class PerfBenchmarkBase extends BenchmarkBase { final metric = {'cycles:u': 'CpuCycles', 'page-faults:u': 'MajorPageFaults'}[event]; if (metric != null) { - emitter.emit( - '$name($metric)', double.parse(counterString) / iterations); + emitter.emit(name, double.parse(counterString) / iterations, + metric: metric); } } - emitter.emit('$name.totalIterations', iterations.toDouble()); + emitter.emit('$name.totalIterations', iterations.toDouble(), + metric: 'Count'); } } diff --git a/lib/src/score_emitter.dart b/lib/src/score_emitter.dart index f7138d3..43ea7aa 100644 --- a/lib/src/score_emitter.dart +++ b/lib/src/score_emitter.dart @@ -3,14 +3,16 @@ // BSD-style license that can be found in the LICENSE file. abstract class ScoreEmitter { - void emit(String testName, double value); + void emit(String testName, double value, + {String metric = 'RunTime', String unit}); } class PrintEmitter implements ScoreEmitter { const PrintEmitter(); @override - void emit(String testName, double value) { - print('$testName(RunTime): $value us.'); + void emit(String testName, double value, + {String metric = 'RunTime', String unit = ''}) { + print(['$testName($metric):', value, if (unit.isNotEmpty) unit].join(' ')); } } diff --git a/test/result_emitter_test.dart b/test/result_emitter_test.dart index e2cd1ea..bfbce4e 100644 --- a/test/result_emitter_test.dart +++ b/test/result_emitter_test.dart @@ -13,7 +13,8 @@ class MockResultEmitter extends ScoreEmitter { int emitCount = 0; @override - void emit(String name, double value) { + void emit(String name, double value, + {String metric = 'RunTime', String unit = ''}) { emitCount++; } } From 931e4e777cff7f5d7a2abf77249b4d3907e95be5 Mon Sep 17 00:00:00 2001 From: William Hesse Date: Thu, 18 Apr 2024 17:58:14 +0200 Subject: [PATCH 13/13] Switch to tab-separated output, update CHANGELOG and version number The json output format is only in the most recent versions of perf stat. Use the separated value output format instead. Address review comments, ensure perf process is terminated if failures occur. --- CHANGELOG.md | 6 +- lib/src/perf_benchmark_base.dart | 100 ++++++++++++++++--------------- pubspec.yaml | 2 +- 3 files changed, 58 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce0c0eb..ec69baf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ -## 2.2.3-wip +## 2.2.3 - Require Dart 3.2. +- Add `PerfBenchmarkBase` class which runs the 'perf stat' command from +linux-tools on a benchmark and reports metrics from the hardware +performance counters and the iteration count, as well as the run time +measurement reported by `BenchmarkBase`. ## 2.2.2 diff --git a/lib/src/perf_benchmark_base.dart b/lib/src/perf_benchmark_base.dart index 8534d0f..3c4a5a1 100644 --- a/lib/src/perf_benchmark_base.dart +++ b/lib/src/perf_benchmark_base.dart @@ -2,6 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:async'; import 'dart:convert'; import 'dart:io'; @@ -9,8 +10,6 @@ import 'benchmark_base.dart'; import 'score_emitter.dart'; class PerfBenchmarkBase extends BenchmarkBase { - PerfBenchmarkBase(super.name, {super.emitter = const PrintEmitter()}); - late final Directory fifoDir; late final String perfControlFifo; late final RandomAccessFile openedFifo; @@ -19,19 +18,17 @@ class PerfBenchmarkBase extends BenchmarkBase { late final Process perfProcess; late final List perfProcessArgs; + PerfBenchmarkBase(super.name, {super.emitter = const PrintEmitter()}); + Future _createFifos() async { perfControlFifo = '${fifoDir.path}/perf_control_fifo'; perfControlAck = '${fifoDir.path}/perf_control_ack'; - - final fifoResult = await Process.run('mkfifo', [perfControlFifo]); - if (fifoResult.exitCode != 0) { - throw ProcessException('mkfifo', [perfControlFifo], - 'Cannot create fifo: ${fifoResult.stderr}', fifoResult.exitCode); - } - final ackResult = await Process.run('mkfifo', [perfControlAck]); - if (ackResult.exitCode != 0) { - throw ProcessException('mkfifo', [perfControlAck], - 'Cannot create fifo: ${ackResult.stderr}', ackResult.exitCode); + for (final path in [perfControlFifo, perfControlAck]) { + final fifoResult = await Process.run('mkfifo', [path]); + if (fifoResult.exitCode != 0) { + throw ProcessException('mkfifo', [path], + 'Cannot create fifo: ${fifoResult.stderr}', fifoResult.exitCode); + } } } @@ -39,17 +36,16 @@ class PerfBenchmarkBase extends BenchmarkBase { await _createFifos(); perfProcessArgs = [ 'stat', - '--delay', - '-1', - '--control', - 'fifo:$perfControlFifo,$perfControlAck', - '-j', - '-p', - '$pid', + '--delay=-1', + '--control=fifo:$perfControlFifo,$perfControlAck', + '-x\\t', + '--pid=$pid', ]; perfProcess = await Process.start('perf', perfProcessArgs); - openedFifo = File(perfControlFifo).openSync(mode: FileMode.writeOnly); + } + void _enablePerf() { + openedFifo = File(perfControlFifo).openSync(mode: FileMode.writeOnly); openedAck = File(perfControlAck).openSync(); openedFifo.writeStringSync('enable\n'); _waitForAck(); @@ -61,20 +57,36 @@ class PerfBenchmarkBase extends BenchmarkBase { _waitForAck(); openedAck.closeSync(); perfProcess.kill(ProcessSignal.sigint); - final lines = - utf8.decoder.bind(perfProcess.stderr).transform(const LineSplitter()); - // Exit code from perf is -2 when terminated with SIGINT. + unawaited(perfProcess.stdout.drain()); + final lines = await perfProcess.stderr + .transform(utf8.decoder) + .transform(const LineSplitter()) + .toList(); final exitCode = await perfProcess.exitCode; - if (exitCode != 0 && exitCode != -2) { + // Exit code from perf is -SIGINT when terminated with SIGINT. + if (exitCode != 0 && exitCode != -ProcessSignal.sigint.signalNumber) { throw ProcessException( - 'perf', perfProcessArgs, (await lines.toList()).join('\n'), exitCode); + 'perf', perfProcessArgs, lines.join('\n'), exitCode); } - final events = [ - await for (final line in lines) - if (line.contains('"counter-value"')) - jsonDecode(line) as Map - ]; - _reportPerfStats(events, totalIterations); + + const metrics = { + 'cycles': 'CpuCycles', + 'page-faults': 'MajorPageFaults', + }; + for (final line in lines) { + if (line.split('\t') + case [ + String counter, + _, + String event && ('cycles' || 'page-faults'), + ... + ]) { + emitter.emit(name, double.parse(counter) / totalIterations, + metric: metrics[event]!); + } + } + emitter.emit('$name.totalIterations', totalIterations.toDouble(), + metric: 'Count'); } /// Measures the score for the benchmark and returns it. @@ -87,9 +99,15 @@ class PerfBenchmarkBase extends BenchmarkBase { // Warmup for at least 100ms. Discard result. measureForImpl(warmup, 100); await _startPerfStat(); - // Run the benchmark for at least 2000ms. - result = measureForImpl(exercise, minimumMeasureDurationMillis); - await _stopPerfStat(result.totalIterations); + try { + _enablePerf(); + // Run the benchmark for at least 2000ms. + result = measureForImpl(exercise, minimumMeasureDurationMillis); + await _stopPerfStat(result.totalIterations); + } catch (_) { + perfProcess.kill(ProcessSignal.sigkill); + rethrow; + } } finally { await fifoDir.delete(recursive: true); } @@ -111,18 +129,4 @@ class PerfBenchmarkBase extends BenchmarkBase { ack.addAll(openedAck.readSync(ackLength - ack.length)); } } - - void _reportPerfStats(List> events, int iterations) { - for (final {'event': String event, 'counter-value': String counterString} - in events) { - final metric = - {'cycles:u': 'CpuCycles', 'page-faults:u': 'MajorPageFaults'}[event]; - if (metric != null) { - emitter.emit(name, double.parse(counterString) / iterations, - metric: metric); - } - } - emitter.emit('$name.totalIterations', iterations.toDouble(), - metric: 'Count'); - } } diff --git a/pubspec.yaml b/pubspec.yaml index adc5f6d..465d274 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,5 +1,5 @@ name: benchmark_harness -version: 2.2.3-wip +version: 2.2.3 description: The official Dart project benchmark harness. repository: https://github.com/dart-lang/benchmark_harness