ci(replay): Collect replay network metrics (#6881)

Adds network info to the collected metrics - it shows the amount of data sent (Net Tx) and received (Net Rx) by the browser. It works by intercepting the network requests with a custom handler that logs the transmission stats. Because Sentry SDK is the only application code sending requests, it equals the amount of data sent & received by the SDK. You can see in the example below that because there are no errors triggered, there's no data sent/received with just the plain Sentry SDK; there's network traffic only when Replay is added.

Author: vaind

packages/replay/metrics/src/collector.ts

@@ -3,6 +3,7 @@ import * as playwright from 'playwright';
 
 import { CpuUsage, CpuUsageSampler, CpuUsageSerialized } from './perf/cpu.js';
 import { JsHeapUsage, JsHeapUsageSampler, JsHeapUsageSerialized } from './perf/memory.js';
+import { NetworkUsage, NetworkUsageCollector, NetworkUsageSerialized } from './perf/network.js';
 import { PerfMetricsSampler } from './perf/sampler.js';
 import { Result } from './results/result.js';
 import { Scenario, TestCase } from './scenarios.js';
@@ -28,13 +29,24 @@ const PredefinedNetworkConditions = Object.freeze({
 });
 
 export class Metrics {
-  constructor(public readonly vitals: WebVitals, public readonly cpu: CpuUsage, public readonly memory: JsHeapUsage) { }
-
-  public static fromJSON(data: Partial<{ vitals: Partial<WebVitals>, cpu: CpuUsageSerialized, memory: JsHeapUsageSerialized }>): Metrics {
+  constructor(
+    public readonly vitals: WebVitals,
+    public readonly cpu: CpuUsage,
+    public readonly memory: JsHeapUsage,
+    public readonly network: NetworkUsage) { }
+
+  public static fromJSON(
+    data: Partial<{
+      vitals: Partial<WebVitals>,
+      cpu: CpuUsageSerialized,
+      memory: JsHeapUsageSerialized,
+      network: NetworkUsageSerialized
+    }>): Metrics {
     return new Metrics(
       WebVitals.fromJSON(data.vitals || {}),
       CpuUsage.fromJSON(data.cpu || {}),
       JsHeapUsage.fromJSON(data.memory || {}),
+      NetworkUsage.fromJSON(data.network || {}),
     );
   }
 }
@@ -131,7 +143,9 @@ export class MetricsCollector {
         const cpuSampler = new CpuUsageSampler(perfSampler);
         const memSampler = new JsHeapUsageSampler(perfSampler);
 
+        const networkCollector = await NetworkUsageCollector.create(page);
         const vitalsCollector = await WebVitalsCollector.create(page);
+
         await scenario.run(browser, page);
 
         // NOTE: FID needs some interaction to actually show a value
@@ -141,7 +155,11 @@ export class MetricsCollector {
           throw `Error logs in browser console:\n\t\t${errorLogs.join('\n\t\t')}`;
         }
 
-        return new Metrics(vitals, cpuSampler.getData(), memSampler.getData());
+        return new Metrics(
+          vitals,
+          cpuSampler.getData(),
+          memSampler.getData(),
+          networkCollector.getData());
       })(), {
         milliseconds: 60 * 1000,
       });

packages/replay/metrics/src/perf/network.ts

@@ -0,0 +1,71 @@
+import * as playwright from 'playwright';
+
+export class NetworkEvent {
+  constructor(
+    public url: string | undefined,
+    public requestSize: number | undefined,
+    public responseSize: number | undefined,
+    public requestTimeNs: bigint | undefined,
+    public responseTimeNs: bigint | undefined) { }
+
+  public static fromJSON(data: Partial<NetworkEvent>): NetworkEvent {
+    return new NetworkEvent(
+      data.url as string,
+      data.requestSize as number,
+      data.responseSize as number,
+      data.requestTimeNs == undefined ? undefined : BigInt(data.requestTimeNs),
+      data.responseTimeNs == undefined ? undefined : BigInt(data.responseTimeNs),
+    );
+  }
+}
+
+export type NetworkUsageSerialized = Partial<{ events: Array<NetworkEvent> }>;
+
+export class NetworkUsage {
+  public constructor(public events: Array<NetworkEvent>) { }
+
+  public static fromJSON(data: NetworkUsageSerialized): NetworkUsage {
+    return new NetworkUsage(data.events?.map(NetworkEvent.fromJSON) || []);
+  }
+}
+
+export class NetworkUsageCollector {
+  private _events = new Array<NetworkEvent>();
+
+  public static async create(page: playwright.Page): Promise<NetworkUsageCollector> {
+    const self = new NetworkUsageCollector();
+    await page.route(_ => true, self._captureRequest.bind(self));
+    return self;
+  }
+
+  public getData(): NetworkUsage {
+    return new NetworkUsage(this._events);
+  }
+
+  private async _captureRequest(
+    route: playwright.Route, request: playwright.Request): Promise<void> {
+    const url = request.url();
+    try {
+      const event = new NetworkEvent(
+        url,
+        request.postDataBuffer()?.length,
+        undefined,
+        process.hrtime.bigint(),
+        undefined
+      );
+      this._events.push(event);
+      // Note: playwright would error out on file:/// requests. They are used to access local test app resources.
+      if (url.startsWith('file:///')) {
+        route.continue();
+      } else {
+        const response = await route.fetch();
+        const body = await response.body();
+        route.fulfill({ response, body });
+        event.responseTimeNs = process.hrtime.bigint();
+        event.responseSize = body.length;
+      }
+    } catch (e) {
+      console.log(`Error when capturing request: ${request.method()} ${url} - ${e}`)
+    }
+  }
+}

packages/replay/metrics/src/results/analyzer.ts

@@ -60,6 +60,10 @@ export class ResultsAnalyzer {
     pushIfDefined(AnalyzerItemMetric.cpu, AnalyzerItemUnit.ratio, MetricsStats.cpu, MetricsStats.mean);
     pushIfDefined(AnalyzerItemMetric.memoryAvg, AnalyzerItemUnit.bytes, MetricsStats.memoryMean, MetricsStats.mean);
     pushIfDefined(AnalyzerItemMetric.memoryMax, AnalyzerItemUnit.bytes, MetricsStats.memoryMax, MetricsStats.max);
+    pushIfDefined(AnalyzerItemMetric.netTx, AnalyzerItemUnit.bytes, MetricsStats.netTx, MetricsStats.mean);
+    pushIfDefined(AnalyzerItemMetric.netRx, AnalyzerItemUnit.bytes, MetricsStats.netRx, MetricsStats.mean);
+    pushIfDefined(AnalyzerItemMetric.netCount, AnalyzerItemUnit.integer, MetricsStats.netCount, MetricsStats.mean);
+    pushIfDefined(AnalyzerItemMetric.netTime, AnalyzerItemUnit.ms, MetricsStats.netTime, MetricsStats.mean);
 
     return items;
   }
@@ -69,6 +73,7 @@ export enum AnalyzerItemUnit {
   ms,
   ratio, // 1.0 == 100 %
   bytes,
+  integer,
 }
 
 export interface AnalyzerItemValues {
@@ -115,6 +120,8 @@ class AnalyzerItemNumberValues implements AnalyzerItemValues {
         return filesize(value) as string;
       case AnalyzerItemUnit.ratio:
         return `${(value * 100).toFixed(2)} ${isDiff ? 'pp' : '%'}`;
+      case AnalyzerItemUnit.integer:
+        return `${value}`;
       default:
         return `${value.toFixed(2)} ${AnalyzerItemUnit[this._unit]}`;
     }
@@ -127,6 +134,10 @@ export enum AnalyzerItemMetric {
   cpu,
   memoryAvg,
   memoryMax,
+  netTx,
+  netRx,
+  netCount,
+  netTime,
 }
 
 export interface AnalyzerItem {

packages/replay/metrics/src/results/metrics-stats.ts

@@ -11,6 +11,10 @@ export class MetricsStats {
   static cpu: NumberProvider = metrics => metrics.cpu.average;
   static memoryMean: NumberProvider = metrics => ss.mean(Array.from(metrics.memory.snapshots.values()));
   static memoryMax: NumberProvider = metrics => ss.max(Array.from(metrics.memory.snapshots.values()));
+  static netTx: NumberProvider = metrics => ss.sum(metrics.network.events.map(e => e.requestSize || 0));
+  static netRx: NumberProvider = metrics => ss.sum(metrics.network.events.map(e => e.responseSize || 0));
+  static netCount: NumberProvider = metrics => ss.sum(metrics.network.events.map(e => e.requestTimeNs && e.responseTimeNs ? 1 : 0));
+  static netTime: NumberProvider = metrics => ss.sum(metrics.network.events.map(e => e.requestTimeNs && e.responseTimeNs ? Number(e.responseTimeNs - e.requestTimeNs) / 1e6 : 0));
 
   static mean: AnalyticsFunction = (items: Metrics[], dataProvider: NumberProvider) => {
     const numbers = MetricsStats._filteredValues(MetricsStats._collect(items, dataProvider));

packages/replay/metrics/src/scenarios.ts

@@ -42,6 +42,6 @@ export class JankTestScenario implements Scenario {
     url = `file:///${url.replace(/\\/g, '/')}`;
     console.log('Navigating to ', url);
     await page.goto(url, { waitUntil: 'load', timeout: 60000 });
-    await new Promise(resolve => setTimeout(resolve, 5000));
+    await new Promise(resolve => setTimeout(resolve, 12000));
   }
 }

packages/replay/metrics/src/util/console.ts

@@ -19,6 +19,10 @@ export function printStats(items: Metrics[]): void {
     cpu: `${((MetricsStats.mean(items, MetricsStats.cpu) || 0) * 100).toFixed(2)} %`,
     memoryMean: filesize(MetricsStats.mean(items, MetricsStats.memoryMean)),
     memoryMax: filesize(MetricsStats.max(items, MetricsStats.memoryMax)),
+    netTx: filesize(MetricsStats.mean(items, MetricsStats.netTx)),
+    netRx: filesize(MetricsStats.mean(items, MetricsStats.netRx)),
+    netCount: MetricsStats.mean(items, MetricsStats.netCount),
+    netTime: `${MetricsStats.mean(items, MetricsStats.netTime)?.toFixed(2)} ms`,
   });
 }
 

packages/replay/metrics/src/util/github.ts

@@ -59,7 +59,7 @@ async function tryAddOrUpdateComment(commentBuilder: PrCommentBuilder): Promise<
       ...defaultArgs,
       base: await Git.baseBranch,
       head: await Git.branch
-    })).data[0].number;
+    })).data[0]?.number;
     if (prNumber != undefined) {
       console.log(`Found PR number ${prNumber} based on base and head branches`);
     }

packages/replay/metrics/src/util/json.ts

@@ -7,6 +7,8 @@ export function JsonStringify<T>(object: T): string {
   return JSON.stringify(object, (_: unknown, value: any): unknown => {
     if (typeof value != 'undefined' && typeof value.toJSON == 'function') {
       return value.toJSON();
+    } else if (typeof value == 'bigint') {
+      return value.toString();
     } else {
       return value;
     }