ci: Make E2E tests required & check for PRs against master (#7670)

Author: mydea

.github/workflows/build.yml

@@ -140,7 +140,7 @@ jobs:
       is_develop: ${{ github.ref == 'refs/heads/develop' }}
       is_release: ${{ startsWith(github.ref, 'refs/heads/release/') }}
       # When merging into master, or from master
-      is_gitflow_sync: ${{ github.head_ref == 'refs/heads/master' || github.ref == 'refs/heads/master' }}
+      is_gitflow_sync: ${{ github.head_ref == 'master' || github.ref == 'refs/heads/master' }}
       has_gitflow_label:
         ${{ github.event_name == 'pull_request' && contains(steps.pr-labels.outputs.labels, ' Gitflow ') }}
       force_skip_cache:
@@ -181,6 +181,21 @@ jobs:
     outputs:
       dependency_cache_key: ${{ steps.compute_lockfile_hash.outputs.hash }}
 
+  job_check_branches:
+    name: Check PR branches
+    needs: job_get_metadata
+    runs-on: ubuntu-20.04
+    if: github.event_name == 'pull_request'
+    permissions:
+      pull-requests: write
+    steps:
+      - name: PR is opened against master
+        uses: mshick/add-pr-comment@a65df5f64fc741e91c59b8359a4bc56e57aaf5b1
+        if: ${{ github.base_ref == 'master' && !startsWith(github.ref, 'refs/heads/prepare-release/') }}
+        with:
+          message: |
+            ⚠️ This PR is opened against **master**. You probably want to open it against **develop**.
+
   job_build:
     name: Build
     needs: [job_get_metadata, job_install_deps]
@@ -709,6 +724,7 @@ jobs:
         job_browser_playwright_tests,
         job_browser_integration_tests,
         job_remix_integration_tests,
+        job_e2e_tests,
       ]
     # Always run this, even if a dependent job failed
     if: always()

.github/workflows/flaky-test-detector.yml

@@ -4,6 +4,9 @@ on:
   pull_request:
     paths:
       - 'packages/browser-integration-tests/suites/**'
+    branches-ignore:
+      - master
+      - develop
 
 env:
   HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }}
@@ -77,5 +80,4 @@ jobs:
         working-directory: packages/browser-integration-tests
         env:
           CHANGED_TEST_PATHS: ${{ steps.changed.outputs.browser_integration_files }}
-          # Run 50 times when detecting changed test(s), else run all tests 5x
-          TEST_RUN_COUNT: ${{ steps.changed.outputs.browser_integration == 'true' && 50 || 5 }}
+          TEST_RUN_COUNT: 'AUTO'

packages/browser-integration-tests/scripts/detectFlakyTests.ts

@@ -23,8 +23,25 @@ ${changedPaths.join('\n')}
     }
   }
 
+  let runCount: number;
+  if (process.env.TEST_RUN_COUNT === 'AUTO') {
+    // No test paths detected: run everything 5x
+    runCount = 5;
+
+    if (testPaths.length > 0) {
+      // Run everything up to 100x, assuming that total runtime is less than 60min.
+      // We assume an average runtime of 3s per test, times 4 (for different browsers) = 12s per detected testPaths
+      // We want to keep overall runtime under 30min
+      const testCount = testPaths.length * 4;
+      const expectedRuntimePerTestPath = testCount * 3;
+      const expectedRuntime = Math.floor((30 * 60) / expectedRuntimePerTestPath);
+      runCount = Math.min(50, Math.max(expectedRuntime, 5));
+    }
+  } else {
+    runCount = parseInt(process.env.TEST_RUN_COUNT || '10');
+  }
+
   const cwd = path.join(__dirname, '../');
-  const runCount = parseInt(process.env.TEST_RUN_COUNT || '10');
 
   try {
     await new Promise<void>((resolve, reject) => {