fix flaky detector

Author: mydea

.github/workflows/flaky-test-detector.yml

@@ -4,6 +4,9 @@ on:
   pull_request:
     paths:
       - 'packages/browser-integration-tests/suites/**'
+    branches-ignore:
+      - master
+      - develop
 
 env:
   HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }}
@@ -77,5 +80,4 @@ jobs:
         working-directory: packages/browser-integration-tests
         env:
           CHANGED_TEST_PATHS: ${{ steps.changed.outputs.browser_integration_files }}
-          # Run 50 times when detecting changed test(s), else run all tests 5x
-          TEST_RUN_COUNT: ${{ steps.changed.outputs.browser_integration == 'true' && 50 || 5 }}
+          TEST_RUN_COUNT: 'AUTO'

packages/browser-integration-tests/scripts/detectFlakyTests.ts

@@ -23,8 +23,25 @@ ${changedPaths.join('\n')}
     }
   }
 
+  let runCount: number;
+  if (process.env.TEST_RUN_COUNT === 'AUTO') {
+    // No test paths detected: run everything 5x
+    runCount = 5;
+
+    if (testPaths.length > 0) {
+      // Run everything up to 100x, assuming that total runtime is less than 60min.
+      // We assume an average runtime of 3s per test, times 4 (for different browsers) = 12s per detected testPaths
+      // We want to keep overall runtime under 30min
+      const testCount = testPaths.length * 4;
+      const expectedRuntimePerTestPath = testCount * 3;
+      const expectedRuntime = Math.floor((30 * 60) / expectedRuntimePerTestPath);
+      runCount = Math.min(50, Math.max(expectedRuntime, 5));
+    }
+  } else {
+    runCount = parseInt(process.env.TEST_RUN_COUNT || '10');
+  }
+
   const cwd = path.join(__dirname, '../');
-  const runCount = parseInt(process.env.TEST_RUN_COUNT || '10');
 
   try {
     await new Promise<void>((resolve, reject) => {