build: Move replay metrics into dedicated package

Author: mydea

.github/workflows/build.yml

@@ -802,8 +802,8 @@ jobs:
         run: |
           echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1
 
-  replay_metrics:
-    name: Replay Metrics
+  overhead_metrics:
+    name: Overhead metrics
     needs: [job_get_metadata, job_build]
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -828,18 +828,14 @@ jobs:
           path: ${{ env.CACHED_BUILD_PATHS }}
           key: ${{ env.BUILD_CACHE_KEY }}
 
-      - name: Setup
-        run: yarn install
-        working-directory: packages/replay/metrics
-
       - name: Collect
         run: yarn ci:collect
-        working-directory: packages/replay/metrics
+        working-directory: packages/overhead-metrics
 
       - name: Process
         id: process
         run: yarn ci:process
-        working-directory: packages/replay/metrics
+        working-directory: packages/overhead-metrics
         # Don't run on forks - the PR comment cannot be added.
         if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
         env:

.vscode/launch.json

@@ -37,22 +37,6 @@
       "internalConsoleOptions": "openOnSessionStart",
       "outputCapture": "std"
     },
-    {
-        "type": "node",
-        "name": "Debug replay metrics collection script",
-        "request": "launch",
-        "cwd": "${workspaceFolder}/packages/replay/metrics/",
-        "program": "${workspaceFolder}/packages/replay/metrics/configs/dev/collect.ts",
-        "preLaunchTask": "Build Replay metrics script",
-    },
-    {
-        "type": "node",
-        "name": "Debug replay metrics processing script",
-        "request": "launch",
-        "cwd": "${workspaceFolder}/packages/replay/metrics/",
-        "program": "${workspaceFolder}/packages/replay/metrics/configs/dev/process.ts",
-        "preLaunchTask": "Build Replay metrics script",
-    },
     // Run rollup using the config file which is in the currently active tab.
     {
       "name": "Debug rollup (config from open file)",

.vscode/tasks.json

@@ -1,19 +1,13 @@
 {
   // See https://go.microsoft.com/fwlink/?LinkId=733558 for documentation about `tasks.json` syntax
-	"version": "2.0.0",
-	"tasks": [
-		{
-			"label": "Prepare nextjs integration test app for VSCode debugger",
-			"type": "npm",
-			"script": "predebug",
-			"path": "packages/nextjs/test/integration/",
-			"detail": "Link the SDK (if not already linked) and build test app",
-		},
-		{
-			"label": "Build Replay metrics script",
-			"type": "npm",
-			"script": "build",
-			"path": "packages/replay/metrics",
-		}
-	]
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "Prepare nextjs integration test app for VSCode debugger",
+      "type": "npm",
+      "script": "predebug",
+      "path": "packages/nextjs/test/integration/",
+      "detail": "Link the SDK (if not already linked) and build test app"
+    }
+  ]
 }

package.json

@@ -47,6 +47,7 @@
     "packages/hub",
     "packages/integration-tests",
     "packages/integrations",
+    "packages/overhead-metrics",
     "packages/nextjs",
     "packages/node",
     "packages/node-integration-tests",

packages/replay/metrics/.eslintrc.cjs renamed to packages/overhead-metrics/.eslintrc.cjs

@@ -1,5 +1,5 @@
 module.exports = {
-  extends: ['../.eslintrc.js'],
+  extends: ['../../.eslintrc.js'],
   ignorePatterns: ['test-apps'],
   overrides: [
     {
@@ -8,6 +8,9 @@ module.exports = {
         'no-console': 'off',
         '@typescript-eslint/no-non-null-assertion': 'off',
         'import/no-unresolved': 'off',
+        '@sentry-internal/sdk/no-optional-chaining': 'off',
+        '@sentry-internal/sdk/no-nullish-coalescing': 'off',
+        'jsdoc/require-jsdoc': 'off',
       },
     },
   ],

packages/replay/metrics/.gitignore renamed to packages/overhead-metrics/.gitignore

@@ -1,8 +1,8 @@
-# Replay performance metrics
+# Overhead performance metrics
 
-Evaluates Replay impact on website performance by running a web app in Chromium via Playwright and collecting various metrics.
+Evaluates Sentry & Replay impact on website performance by running a web app in Chromium via Playwright and collecting various metrics.
 
-The general idea is to run a web app without Sentry Replay and then run the same app again with Sentry and another one with Sentry+Replay included.
+The general idea is to run a web app without Sentry, and then run the same app again with Sentry and another one with Sentry+Replay included.
 For the three scenarios, we collect some metrics (CPU, memory, vitals) and later compare them and post as a comment in a PR.
 Changes in the metrics, compared to previous runs from the main branch, should be evaluated on case-by-case basis when preparing and reviewing the PR.
 

packages/replay/metrics/README.md renamed to packages/overhead-metrics/README.md

@@ -1,5 +1,7 @@
-import { Metrics, MetricsCollector } from '../../src/collector.js';
-import { MetricsStats, NumberProvider } from '../../src/results/metrics-stats.js';
+import type { Metrics } from '../../src/collector.js';
+import { MetricsCollector } from '../../src/collector.js';
+import type { NumberProvider } from '../../src/results/metrics-stats.js';
+import { MetricsStats } from '../../src/results/metrics-stats.js';
 import { JankTestScenario } from '../../src/scenarios.js';
 import { printStats } from '../../src/util/console.js';
 import { latestResultFile } from './env.js';
@@ -10,10 +12,12 @@ function checkStdDev(results: Metrics[], name: string, provider: NumberProvider,
     console.warn(`✗ | Discarding results because StandardDeviation(${name}) is undefined`);
     return false;
   } else if (value > max) {
-    console.warn(`✗ | Discarding results because StandardDeviation(${name}) is larger than ${max}. Actual value: ${value}`);
+    console.warn(
+      `✗ | Discarding results because StandardDeviation(${name}) is larger than ${max}. Actual value: ${value}`,
+    );
     return false;
   } else {
-    console.log(`✓ | StandardDeviation(${name}) is ${value} (<= ${max})`)
+    console.log(`✓ | StandardDeviation(${name}) is ${value} (<= ${max})`);
   }
   return true;
 }
@@ -31,19 +35,23 @@ const result = await collector.execute({
   async shouldAccept(results: Metrics[]): Promise<boolean> {
     await printStats(results);
 
-    if (!checkStdDev(results, 'lcp', MetricsStats.lcp, 50)
-      || !checkStdDev(results, 'cls', MetricsStats.cls, 0.1)
-      || !checkStdDev(results, 'cpu', MetricsStats.cpu, 1)
-      || !checkStdDev(results, 'memory-mean', MetricsStats.memoryMean, 1000 * 1024)
-      || !checkStdDev(results, 'memory-max', MetricsStats.memoryMax, 1000 * 1024)) {
+    if (
+      !checkStdDev(results, 'lcp', MetricsStats.lcp, 50) ||
+      !checkStdDev(results, 'cls', MetricsStats.cls, 0.1) ||
+      !checkStdDev(results, 'cpu', MetricsStats.cpu, 1) ||
+      !checkStdDev(results, 'memory-mean', MetricsStats.memoryMean, 1000 * 1024) ||
+      !checkStdDev(results, 'memory-max', MetricsStats.memoryMax, 1000 * 1024)
+    ) {
       return false;
     }
 
     const cpuUsage = MetricsStats.mean(results, MetricsStats.cpu)!;
     if (cpuUsage > 0.85) {
       // Note: complexity on the "JankTest" is defined by the `minimum = ...,` setting in app.js - specifying the number of animated elements.
-      console.warn(`✗ | Discarding results because CPU usage is too high and may be inaccurate: ${(cpuUsage * 100).toFixed(2)} %.`,
-        'Consider simplifying the scenario or changing the CPU throttling factor.');
+      console.warn(
+        `✗ | Discarding results because CPU usage is too high and may be inaccurate: ${(cpuUsage * 100).toFixed(2)} %.`,
+        'Consider simplifying the scenario or changing the CPU throttling factor.',
+      );
       return false;
     }
 

packages/replay/metrics/configs/README.md renamed to packages/overhead-metrics/configs/README.md

@@ -1,4 +1,4 @@
 export const previousResultsDir = 'out/previous-results';
 export const baselineResultsDir = 'out/baseline-results';
 export const latestResultFile = 'out/latest-result.json';
-export const artifactName = 'replay-sdk-metrics'
+export const artifactName = 'replay-sdk-metrics';