metrics: filter outliers

Author: vaind

packages/replay/metrics/configs/ci/collect.ts

@@ -27,12 +27,11 @@ const result = await collector.execute({
   async shouldAccept(results: Metrics[]): Promise<boolean> {
     const stats = new MetricsStats(results);
     return true
-      && checkStdDev(stats, 'lcp', MetricsStats.lcp, 10)
-      && checkStdDev(stats, 'cls', MetricsStats.cls, 10)
+      && checkStdDev(stats, 'lcp', MetricsStats.lcp, 30)
+      && checkStdDev(stats, 'cls', MetricsStats.cls, 0.1)
       && checkStdDev(stats, 'cpu', MetricsStats.cpu, 10)
-      && checkStdDev(stats, 'memory-mean', MetricsStats.memoryMean, 10000)
-      && checkStdDev(stats, 'memory-max', MetricsStats.memoryMax, 10000);
-    ;
+      && checkStdDev(stats, 'memory-mean', MetricsStats.memoryMean, 30 * 1024)
+      && checkStdDev(stats, 'memory-max', MetricsStats.memoryMax, 100 * 1024);
   },
 });
 

packages/replay/metrics/src/results/metrics-stats.ts

@@ -14,17 +14,29 @@ export class MetricsStats {
   static memoryMax: NumberProvider = metrics => ss.max(Array.from(metrics.memory.snapshots.values()));
 
   public mean(dataProvider: NumberProvider): number | undefined {
-    const numbers = this._items.map(dataProvider);
+    const numbers = this._filteredValues(dataProvider);
     return numbers.length > 0 ? ss.mean(numbers) : undefined;
   }
 
   public max(dataProvider: NumberProvider): number | undefined {
-    const numbers = this._items.map(dataProvider);
+    const numbers = this._filteredValues(dataProvider);
     return numbers.length > 0 ? ss.max(numbers) : undefined;
   }
 
   public stddev(dataProvider: NumberProvider): number | undefined {
-    const numbers = this._items.map(dataProvider);
+    const numbers = this._filteredValues(dataProvider);
     return numbers.length > 0 ? ss.standardDeviation(numbers) : undefined;
   }
+
+  // See https://en.wikipedia.org/wiki/Interquartile_range#Outliers for details on filtering.
+  private _filteredValues(dataProvider: NumberProvider): number[] {
+    const numbers = this._items.map(dataProvider);
+    numbers.sort((a, b) => a - b)
+
+    const q1 = ss.quantileSorted(numbers, 0.25);
+    const q3 = ss.quantileSorted(numbers, 0.75);
+    const iqr = q3 - q1
+
+    return numbers.filter(num => num >= (q1 - 1.5 * iqr) && num <= (q3 + 1.5 * iqr))
+  }
 }