diff --git a/.github/workflows/flaky-test-detector.yml b/.github/workflows/flaky-test-detector.yml index bb414b553abd..9412da630775 100644 --- a/.github/workflows/flaky-test-detector.yml +++ b/.github/workflows/flaky-test-detector.yml @@ -77,5 +77,5 @@ jobs: working-directory: packages/browser-integration-tests env: CHANGED_TEST_PATHS: ${{ steps.changed.outputs.browser_integration_files }} - # Run 100 times when detecting changed test(s), else run all tests 5x - TEST_RUN_COUNT: ${{ steps.changed.outputs.browser_integration == 'true' && 100 || 5 }} + # Run 50 times when detecting changed test(s), else run all tests 5x + TEST_RUN_COUNT: ${{ steps.changed.outputs.browser_integration == 'true' && 50 || 5 }} diff --git a/packages/browser-integration-tests/scripts/detectFlakyTests.ts b/packages/browser-integration-tests/scripts/detectFlakyTests.ts index 22977fa3ed83..af0a5c86a18e 100644 --- a/packages/browser-integration-tests/scripts/detectFlakyTests.ts +++ b/packages/browser-integration-tests/scripts/detectFlakyTests.ts @@ -6,52 +6,72 @@ import { promisify } from 'util'; const exec = promisify(childProcess.exec); async function run(): Promise { - let testPaths = getTestPaths(); - let failed = []; + let testPaths: string[] = []; - try { - const changedPaths: string[] = process.env.CHANGED_TEST_PATHS ? JSON.parse(process.env.CHANGED_TEST_PATHS) : []; + const changedPaths: string[] = process.env.CHANGED_TEST_PATHS ? JSON.parse(process.env.CHANGED_TEST_PATHS) : []; - if (changedPaths.length > 0) { - console.log(`Detected changed test paths: + if (changedPaths.length > 0) { + console.log(`Detected changed test paths: ${changedPaths.join('\n')} `); - testPaths = testPaths.filter(p => changedPaths.some(changedPath => changedPath.includes(p))); + testPaths = getTestPaths().filter(p => changedPaths.some(changedPath => changedPath.includes(p))); + if (testPaths.length === 0) { + console.log('Could not find matching tests, aborting...'); + process.exit(1); } - } catch { - console.log('Could not detect changed test paths, running all tests.'); } const cwd = path.join(__dirname, '../'); const runCount = parseInt(process.env.TEST_RUN_COUNT || '10'); - for (const testPath of testPaths) { - console.log(`Running test: ${testPath}`); - const start = Date.now(); + try { + await new Promise((resolve, reject) => { + const cp = childProcess.spawn( + `yarn playwright test ${ + testPaths.length ? testPaths.join(' ') : './suites' + } --browser='all' --reporter='line' --repeat-each ${runCount}`, + { shell: true, cwd }, + ); + + let error: Error | undefined; + + cp.stdout.on('data', data => { + console.log(data ? (data as object).toString() : ''); + }); - try { - await exec(`yarn playwright test ${testPath} --browser='all' --repeat-each ${runCount}`, { - cwd, + cp.stderr.on('data', data => { + console.log(data ? (data as object).toString() : ''); }); - const end = Date.now(); - console.log(` ☑️ Passed ${runCount} times, avg. duration ${Math.ceil((end - start) / runCount)}ms`); - } catch (error) { - logError(error); - failed.push(testPath); - } - } - console.log(''); - console.log(''); + cp.on('error', e => { + console.error(e); + error = e; + }); - if (failed.length > 0) { - console.error(`⚠️ ${failed.length} test(s) failed.`); + cp.on('close', status => { + const err = error || (status !== 0 ? new Error(`Process exited with status ${status}`) : undefined); + + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + } catch (error) { + console.log(''); + console.log(''); + + console.error(`⚠️ Some tests failed.`); + console.error(error); process.exit(1); - } else { - console.log(`☑️ ${testPaths.length} test(s) passed.`); } + + console.log(''); + console.log(''); + console.log(`☑️ All tests passed.`); } function getTestPaths(): string[] {