feat: analyze collected metrics

Author: vaind

packages/replay/metrics/configs/dev/process.ts

@@ -1,10 +1,22 @@
+import { AnalyzerItemMetric, ResultsAnalyzer } from '../../src/results/analyzer.js';
 import { Result } from '../../src/results/result.js';
 import { ResultsSet } from '../../src/results/results-set.js';
 import { latestResultFile, outDir } from './env.js';
 
 const resultsSet = new ResultsSet(outDir);
-
 const latestResult = Result.readFromFile(latestResultFile);
-console.log(latestResult);
 
-await resultsSet.add(latestResultFile);
+const analysis = ResultsAnalyzer.analyze(latestResult, resultsSet);
+
+const table: { [k: string]: any } = {};
+for (const item of analysis) {
+  const printable: { [k: string]: any } = {};
+  printable.value = item.value.asString();
+  if (item.other != undefined) {
+    printable.baseline = item.other.asString();
+  }
+  table[AnalyzerItemMetric[item.metric]] = printable;
+}
+console.table(table);
+
+await resultsSet.add(latestResultFile, true);

packages/replay/metrics/package.json

@@ -13,8 +13,10 @@
   },
   "dependencies": {
     "@types/node": "^18.11.17",
+    "filesize": "^10.0.6",
     "puppeteer": "^19.4.1",
     "simple-git": "^3.15.1",
+    "simple-statistics": "^7.8.0",
     "typescript": "^4.9.4"
   },
   "devDependencies": {

packages/replay/metrics/src/results/analyzer.ts

@@ -0,0 +1,104 @@
+import { GitHash } from '../util/git.js';
+import { Result } from './result.js';
+import { ResultsSet } from './results-set.js';
+import { MetricsStats } from './metrics-stats.js';
+import { filesize } from "filesize";
+
+// Compares latest result to previous/baseline results and produces the needed
+// info.
+export class ResultsAnalyzer {
+  public static analyze(currentResult: Result, baselineResults: ResultsSet): AnalyzerItem[] {
+    const items = new ResultsAnalyzer(currentResult).collect();
+
+    const baseline = baselineResults.find(
+      (other) => other.cpuThrottling == currentResult.cpuThrottling &&
+        other.name == currentResult.name &&
+        other.networkConditions == currentResult.networkConditions);
+
+    if (baseline != undefined) {
+      const baseItems = new ResultsAnalyzer(baseline[1]).collect();
+      // update items with baseline results
+      for (const base of baseItems) {
+        for (const item of items) {
+          if (item.metric == base.metric) {
+            item.other = base.value;
+            item.otherHash = baseline[0];
+          }
+        }
+      }
+    }
+
+    return items;
+  }
+
+  private constructor(private result: Result) { }
+
+  private collect(): AnalyzerItem[] {
+    const items = new Array<AnalyzerItem>();
+
+    const aStats = new MetricsStats(this.result.aResults);
+    const bStats = new MetricsStats(this.result.bResults);
+
+    const pushIfDefined = function (metric: AnalyzerItemMetric, unit: AnalyzerItemUnit, valueA?: number, valueB?: number) {
+      if (valueA == undefined || valueB == undefined) return;
+
+      items.push({
+        metric: metric,
+        value: {
+          unit: unit,
+          asDiff: () => valueB - valueA,
+          asRatio: () => valueB / valueA,
+          asString: () => {
+            const diff = valueB - valueA;
+            const prefix = diff >= 0 ? '+' : '';
+
+            switch (unit) {
+              case AnalyzerItemUnit.bytes:
+                return prefix + filesize(diff);
+              case AnalyzerItemUnit.ratio:
+                return prefix + (diff * 100).toFixed(2) + ' %';
+              default:
+                return prefix + diff.toFixed(2) + ' ' + AnalyzerItemUnit[unit];
+            }
+          }
+        }
+      })
+    }
+
+    pushIfDefined(AnalyzerItemMetric.lcp, AnalyzerItemUnit.ms, aStats.lcp, bStats.lcp);
+    pushIfDefined(AnalyzerItemMetric.cls, AnalyzerItemUnit.ms, aStats.cls, bStats.cls);
+    pushIfDefined(AnalyzerItemMetric.cpu, AnalyzerItemUnit.ratio, aStats.cpu, bStats.cpu);
+    pushIfDefined(AnalyzerItemMetric.memoryAvg, AnalyzerItemUnit.bytes, aStats.memoryAvg, bStats.memoryAvg);
+    pushIfDefined(AnalyzerItemMetric.memoryMax, AnalyzerItemUnit.bytes, aStats.memoryMax, bStats.memoryMax);
+
+    return items.filter((item) => item.value != undefined);
+  }
+}
+
+export enum AnalyzerItemUnit {
+  ms,
+  ratio, // 1.0 == 100 %
+  bytes,
+}
+
+export interface AnalyzerItemValue {
+  unit: AnalyzerItemUnit;
+  asString(): string;
+  asDiff(): number;
+  asRatio(): number; // 1.0 == 100 %
+}
+
+export enum AnalyzerItemMetric {
+  lcp,
+  cls,
+  cpu,
+  memoryAvg,
+  memoryMax,
+}
+
+export interface AnalyzerItem {
+  metric: AnalyzerItemMetric;
+  value: AnalyzerItemValue;
+  other?: AnalyzerItemValue;
+  otherHash?: GitHash;
+}

packages/replay/metrics/src/results/metrics-stats.ts

@@ -0,0 +1,41 @@
+import { Metrics } from '../collector';
+import * as ss from 'simple-statistics'
+
+export type NumberProvider = (metrics: Metrics) => number;
+
+export class MetricsStats {
+  constructor(private items: Metrics[]) { }
+
+  // See https://en.wikipedia.org/wiki/Interquartile_range#Outliers for details
+  public filterOutliers(dataProvider: NumberProvider): number[] {
+    let numbers = this.items.map(dataProvider);
+    // TODO implement, see https://github.com/getsentry/action-app-sdk-overhead-metrics/blob/9ce7d562ff79b317688d22bd5c0bb725cbdfdb81/src/test/kotlin/StartupTimeTest.kt#L27-L37
+    return numbers;
+  }
+
+  public filteredMean(dataProvider: NumberProvider): number | undefined {
+    const numbers = this.filterOutliers(dataProvider);
+    return numbers.length > 0 ? ss.mean(numbers) : undefined;
+  }
+
+  public get lcp(): number | undefined {
+    return this.filteredMean((metrics) => metrics.vitals.lcp);
+  }
+
+  public get cls(): number | undefined {
+    return this.filteredMean((metrics) => metrics.vitals.cls);
+  }
+
+  public get cpu(): number | undefined {
+    return this.filteredMean((metrics) => metrics.cpu.average);
+  }
+
+  public get memoryAvg(): number | undefined {
+    return this.filteredMean((metrics) => ss.mean(Array.from(metrics.memory.snapshots.values())));
+  }
+
+  public get memoryMax(): number | undefined {
+    const numbers = this.filterOutliers((metrics) => ss.max(Array.from(metrics.memory.snapshots.values())));
+    return numbers.length > 0 ? ss.max(numbers) : undefined;
+  }
+}

packages/replay/metrics/src/results/results-set.ts

@@ -1,7 +1,8 @@
 import assert from 'assert';
 import * as fs from 'fs';
 import path from 'path';
-import { Git } from '../git.js';
+import { Git, GitHash } from '../util/git.js';
+import { Result } from './result.js';
 
 const delimiter = '-';
 
@@ -16,7 +17,7 @@ export class ResultSetItem {
     return parseInt(this.parts[0]);
   }
 
-  public get hash(): string {
+  public get hash(): GitHash {
     return this.parts[1];
   }
 
@@ -38,23 +39,42 @@ export class ResultsSet {
     return this.items().length;
   }
 
+  public find(predicate: (value: Result) => boolean): [GitHash, Result] | undefined {
+    const items = this.items();
+    for (let i = 0; i < items.length; i++) {
+      const result = Result.readFromFile(items[i].path);
+      if (predicate(result)) {
+        return [items[i].hash, result];
+      }
+    }
+    return undefined;
+  }
+
   public items(): ResultSetItem[] {
     return this.files().map((file) => {
       return new ResultSetItem(path.join(this.directory, file.name));
     }).filter((item) => !isNaN(item.number));
   }
 
-  files(): fs.Dirent[] {
+  private files(): fs.Dirent[] {
     return fs.readdirSync(this.directory, { withFileTypes: true }).filter((v) => v.isFile())
   }
 
-  public async add(newFile: string): Promise<void> {
+  public async add(newFile: string, onlyIfDifferent: boolean = false): Promise<void> {
     console.log(`Preparing to add ${newFile} to ${this.directory}`);
     assert(fs.existsSync(newFile));
 
-    // Get the list of file sorted by the prefix number in the descending order.
+    // Get the list of file sorted by the prefix number in the descending order (starting with the oldest files).
     const files = this.items().sort((a, b) => b.number - a.number);
 
+    if (onlyIfDifferent && files.length > 0) {
+      const latestFile = files[files.length - 1];
+      if (fs.readFileSync(latestFile.path, { encoding: 'utf-8' }) == fs.readFileSync(newFile, { encoding: 'utf-8' })) {
+        console.log(`Skipping - it's already stored as ${latestFile.name}`);
+        return;
+      }
+    }
+
     // Rename all existing files, increasing the prefix
     for (const file of files) {
       const parts = file.name.split(delimiter);

packages/replay/metrics/src/git.ts renamed to packages/replay/metrics/src/util/git.ts

@@ -1,8 +1,10 @@
 import { simpleGit } from 'simple-git';
 
+export type GitHash = string;
+
 // A testing scenario we want to collect metrics for.
 export const Git = {
-  get hash(): Promise<string> {
+  get hash(): Promise<GitHash> {
     return (async () => {
       const git = simpleGit();
       let gitHash = await git.revparse('HEAD');

packages/replay/metrics/yarn.lock

@@ -278,6 +278,11 @@ fd-slicer@~1.1.0:
   dependencies:
     pend "~1.2.0"
 
+filesize@^10.0.6:
+  version "10.0.6"
+  resolved "https://registry.yarnpkg.com/filesize/-/filesize-10.0.6.tgz#5f4cd2721664cd925db3a7a5a87bbfd6ab5ebb1a"
+  integrity sha512-rzpOZ4C9vMFDqOa6dNpog92CoLYjD79dnjLk2TYDDtImRIyLTOzqojCb05Opd1WuiWjs+fshhCgTd8cl7y5t+g==
+
 fs-constants@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
@@ -521,6 +526,11 @@ simple-git@^3.15.1:
     "@kwsites/promise-deferred" "^1.1.1"
     debug "^4.3.4"
 
+simple-statistics@^7.8.0:
+  version "7.8.0"
+  resolved "https://registry.yarnpkg.com/simple-statistics/-/simple-statistics-7.8.0.tgz#1033d2d613656c7bd34f0e134fd7e69c803e6836"
+  integrity sha512-lTWbfJc0u6GZhBojLOrlHJMTHu6PdUjSsYLrpiH902dVBiYJyWlN/LdSoG8b5VvfG1D30gIBgarqMNeNmU5nAA==
+
 string_decoder@^1.1.1:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"