metrics: fix CPU usage collection

Author: vaind

packages/replay/metrics/configs/ci/collect.ts

@@ -26,12 +26,22 @@ const result = await collector.execute({
   tries: 10,
   async shouldAccept(results: Metrics[]): Promise<boolean> {
     const stats = new MetricsStats(results);
-    return true
-      && checkStdDev(stats, 'lcp', MetricsStats.lcp, 30)
-      && checkStdDev(stats, 'cls', MetricsStats.cls, 0.1)
-      && checkStdDev(stats, 'cpu', MetricsStats.cpu, 10)
-      && checkStdDev(stats, 'memory-mean', MetricsStats.memoryMean, 30 * 1024)
-      && checkStdDev(stats, 'memory-max', MetricsStats.memoryMax, 100 * 1024);
+    if (!checkStdDev(stats, 'lcp', MetricsStats.lcp, 30)
+      || !checkStdDev(stats, 'cls', MetricsStats.cls, 0.1)
+      || !checkStdDev(stats, 'cpu', MetricsStats.cpu, 10)
+      || !checkStdDev(stats, 'memory-mean', MetricsStats.memoryMean, 30 * 1024)
+      || !checkStdDev(stats, 'memory-max', MetricsStats.memoryMax, 100 * 1024)) {
+      return false;
+    }
+
+    const cpuUsage = stats.mean(MetricsStats.cpu)!;
+    if (cpuUsage > 0.9) {
+      console.error(`CPU usage too high to be accurate: ${(cpuUsage * 100).toFixed(2)} %.`,
+        'Consider simplifying the scenario or changing the CPU throttling factor.');
+      return false;
+    }
+
+    return true;
   },
 });
 

packages/replay/metrics/configs/dev/collect.ts

@@ -1,4 +1,5 @@
 import { Metrics, MetricsCollector } from '../../src/collector.js';
+import { MetricsStats } from '../../src/results/metrics-stats.js';
 import { JankTestScenario } from '../../src/scenarios.js';
 import { latestResultFile } from './env.js';
 
@@ -9,7 +10,14 @@ const result = await collector.execute({
   b: new JankTestScenario(true),
   runs: 1,
   tries: 1,
-  async shouldAccept(_results: Metrics[]): Promise<boolean> {
+  async shouldAccept(results: Metrics[]): Promise<boolean> {
+    const stats = new MetricsStats(results);
+    const cpuUsage = stats.mean(MetricsStats.cpu)!;
+    if (cpuUsage > 0.9) {
+      console.error(`CPU usage too high to be accurate: ${(cpuUsage * 100).toFixed(2)} %.`,
+        'Consider simplifying the scenario or changing the CPU throttling factor.');
+      return false;
+    }
     return true;
   },
 });

packages/replay/metrics/src/perf/sampler.ts

@@ -33,8 +33,7 @@ export class PerfMetrics {
   }
 
   public get Duration(): number {
-    // TODO check if any of `Duration` fields is maybe a sum of the others. E.g. verify the measured CPU usage manually.
-    return this._metrics.reduce((sum, metric) => metric.name.endsWith('Duration') ? sum + metric.value : sum, 0);
+    return this._find('TaskDuration');
   }
 
   public get JSHeapUsedSize(): number {
@@ -46,16 +45,17 @@ export class PerfMetricsSampler {
   private _consumers: PerfMetricsConsumer[] = [];
   private _timer!: NodeJS.Timer;
 
+  private constructor(private _cdp: playwright.CDPSession) { }
+
   public static async create(cdp: playwright.CDPSession, interval: number): Promise<PerfMetricsSampler> {
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    const self = new PerfMetricsSampler(cdp);
     await cdp.send('Performance.enable', { timeDomain: 'timeTicks' })
 
-    const self = new PerfMetricsSampler();
+    // collect first sample immediately
+    void self._collectSample();
 
-    self._timer = setInterval(async () => {
-      const metrics = await cdp.send('Performance.getMetrics').then((v) => v.metrics);
-      self._consumers.forEach((cb) => cb(new PerfMetrics(metrics)).catch(console.log));
-    }, interval);
+    // and set up automatic collection in the given interval
+    self._timer = setInterval(self._collectSample.bind(self), interval);
 
     return self;
   }
@@ -67,4 +67,10 @@ export class PerfMetricsSampler {
   public stop(): void {
     clearInterval(this._timer);
   }
+
+  private async _collectSample(): Promise<void> {
+    const response = await this._cdp.send('Performance.getMetrics');
+    const metrics = new PerfMetrics(response.metrics);
+    this._consumers.forEach(cb => cb(metrics).catch(console.error));
+  }
 }