From c333c88ef35ca9e939782da4c9429d13bf1f3213 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Mon, 6 Mar 2023 12:33:03 +0100 Subject: [PATCH 1/5] run test 100 times --- .../suites/replay/errors/droppedError/test.ts | 77 ++++++++++--------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/packages/integration-tests/suites/replay/errors/droppedError/test.ts b/packages/integration-tests/suites/replay/errors/droppedError/test.ts index c509dda8206e..a3ac85d8f5aa 100644 --- a/packages/integration-tests/suites/replay/errors/droppedError/test.ts +++ b/packages/integration-tests/suites/replay/errors/droppedError/test.ts @@ -17,49 +17,54 @@ import { getReplayEvent, shouldSkipReplayTest, waitForReplayRequest } from '../. * TODO: Once we have lifecycle hooks, we should revisit this test and make sure it behaves as expected. * This means that the recording should not be started or stopped if the error that triggered it is not sent. */ -sentryTest( - '[error-mode] should start recording if an error occurred although the error was dropped', - async ({ getLocalTestPath, page }) => { - if (shouldSkipReplayTest()) { - sentryTest.skip(); - } +for (let i = 0; i < 100; i++) { + sentryTest( + `[error-mode] should start recording if an error occurred although the error was dropped RUN ${i}`, + async ({ getLocalTestPath, page }) => { + if (shouldSkipReplayTest()) { + sentryTest.skip(); + } - let callsToSentry = 0; - const reqPromise0 = waitForReplayRequest(page, 0); + let callsToSentry = 0; + const reqPromise0 = waitForReplayRequest(page, 0); + const reqPromise1 = waitForReplayRequest(page, 1); - await page.route('https://dsn.ingest.sentry.io/**/*', route => { - callsToSentry++; - return route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ id: 'test-id' }), + await page.route('https://dsn.ingest.sentry.io/**/*', route => { + callsToSentry++; + return route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ id: 'test-id' }), + }); }); - }); - const url = await getLocalTestPath({ testDir: __dirname }); + const url = await getLocalTestPath({ testDir: __dirname }); - await page.goto(url); - await page.click('#go-background'); - expect(callsToSentry).toEqual(0); + await page.goto(url); + await page.click('#go-background'); + expect(callsToSentry).toEqual(0); - await page.click('#error'); - const req0 = await reqPromise0; + await page.click('#error'); + const req0 = await reqPromise0; - await page.click('#go-background'); - expect(callsToSentry).toEqual(2); // 2 replay events + await page.click('#go-background'); + await reqPromise1; - await page.click('#log'); - await page.click('#go-background'); + expect(callsToSentry).toEqual(2); // 2 replay events - const event0 = getReplayEvent(req0); + await page.click('#log'); + await page.click('#go-background'); - expect(event0).toEqual( - getExpectedReplayEvent({ - contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } }, - // This is by design. A dropped error shouldn't be in this list. - error_ids: [], - replay_type: 'error', - }), - ); - }, -); + const event0 = getReplayEvent(req0); + + expect(event0).toEqual( + getExpectedReplayEvent({ + contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } }, + // This is by design. A dropped error shouldn't be in this list. + error_ids: [], + replay_type: 'error', + }), + ); + }, + ); +} From 26a802d9dd0eff3a86fe3f5e4fcd5c8f40a50c53 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Mon, 6 Mar 2023 12:56:02 +0100 Subject: [PATCH 2/5] better test fix? --- .../suites/replay/errors/droppedError/test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/integration-tests/suites/replay/errors/droppedError/test.ts b/packages/integration-tests/suites/replay/errors/droppedError/test.ts index a3ac85d8f5aa..42cd184fd170 100644 --- a/packages/integration-tests/suites/replay/errors/droppedError/test.ts +++ b/packages/integration-tests/suites/replay/errors/droppedError/test.ts @@ -25,12 +25,11 @@ for (let i = 0; i < 100; i++) { sentryTest.skip(); } - let callsToSentry = 0; const reqPromise0 = waitForReplayRequest(page, 0); const reqPromise1 = waitForReplayRequest(page, 1); + const reqPromise2 = waitForReplayRequest(page, 2); await page.route('https://dsn.ingest.sentry.io/**/*', route => { - callsToSentry++; return route.fulfill({ status: 200, contentType: 'application/json', @@ -50,10 +49,11 @@ for (let i = 0; i < 100; i++) { await page.click('#go-background'); await reqPromise1; - expect(callsToSentry).toEqual(2); // 2 replay events - await page.click('#log'); await page.click('#go-background'); + await reqPromise2; + + // Note: The fact that reqPromise1/reqPromise2 are fulfilled prooves that the recording continues const event0 = getReplayEvent(req0); From 01982b5489a815d4daffc3fa7479f7e91c9cc987 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Mon, 6 Mar 2023 13:16:52 +0100 Subject: [PATCH 3/5] fix test --- .../suites/replay/errors/droppedError/test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/integration-tests/suites/replay/errors/droppedError/test.ts b/packages/integration-tests/suites/replay/errors/droppedError/test.ts index 42cd184fd170..9db69dd7eae9 100644 --- a/packages/integration-tests/suites/replay/errors/droppedError/test.ts +++ b/packages/integration-tests/suites/replay/errors/droppedError/test.ts @@ -29,7 +29,11 @@ for (let i = 0; i < 100; i++) { const reqPromise1 = waitForReplayRequest(page, 1); const reqPromise2 = waitForReplayRequest(page, 2); + let callsToSentry = 0; + await page.route('https://dsn.ingest.sentry.io/**/*', route => { + callsToSentry++; + return route.fulfill({ status: 200, contentType: 'application/json', From a3c7814b5ea1b3ec003487f94e6952855b373c37 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Mon, 6 Mar 2023 16:15:14 +0100 Subject: [PATCH 4/5] try use 3 workers? --- packages/integration-tests/playwright.config.ts | 6 +++--- packages/nextjs/playwright.config.ts | 6 +++--- packages/remix/playwright.config.ts | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/integration-tests/playwright.config.ts b/packages/integration-tests/playwright.config.ts index 7445d985c1ba..3baf3937c1f7 100644 --- a/packages/integration-tests/playwright.config.ts +++ b/packages/integration-tests/playwright.config.ts @@ -4,8 +4,8 @@ const config: PlaywrightTestConfig = { retries: 0, // Run tests inside of a single file in parallel fullyParallel: true, - // Use 5 workers on CI, else use defaults (based on available CPU cores) - // Note that 5 is a random number selected to work well with our CI setup - workers: process.env.CI ? 5 : undefined, + // Use 3 workers on CI, else use defaults (based on available CPU cores) + // Note that 3 is a random number selected to work well with our CI setup + workers: process.env.CI ? 3 : undefined, }; export default config; diff --git a/packages/nextjs/playwright.config.ts b/packages/nextjs/playwright.config.ts index d0cf5c2fa767..d8f73993c815 100644 --- a/packages/nextjs/playwright.config.ts +++ b/packages/nextjs/playwright.config.ts @@ -8,9 +8,9 @@ const config: PlaywrightTestConfig = { }, // Run tests inside of a single file in parallel fullyParallel: true, - // Use 5 workers on CI, else use defaults (based on available CPU cores) - // Note that 5 is a random number selected to work well with our CI setup - workers: process.env.CI ? 5 : undefined, + // Use 3 workers on CI, else use defaults (based on available CPU cores) + // Note that 3 is a random number selected to work well with our CI setup + workers: process.env.CI ? 3 : undefined, webServer: { cwd: path.join(__dirname, 'test', 'integration'), command: 'yarn start', diff --git a/packages/remix/playwright.config.ts b/packages/remix/playwright.config.ts index 88ba7b451389..63de33ff74ed 100644 --- a/packages/remix/playwright.config.ts +++ b/packages/remix/playwright.config.ts @@ -7,9 +7,9 @@ const config: PlaywrightTestConfig = { }, // Run tests inside of a single file in parallel fullyParallel: true, - // Use 5 workers on CI, else use defaults (based on available CPU cores) - // Note that 5 is a random number selected to work well with our CI setup - workers: process.env.CI ? 5 : undefined, + // Use 3 workers on CI, else use defaults (based on available CPU cores) + // Note that 3 is a random number selected to work well with our CI setup + workers: process.env.CI ? 3 : undefined, webServer: { command: '(cd test/integration/ && yarn build && yarn start)', port: 3000, From 15e783e7067036297255f68cdf86640bb9296efa Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Mon, 6 Mar 2023 17:20:04 +0100 Subject: [PATCH 5/5] test --- .../suites/replay/errors/droppedError/test.ts | 82 +++++++++---------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/packages/integration-tests/suites/replay/errors/droppedError/test.ts b/packages/integration-tests/suites/replay/errors/droppedError/test.ts index 9db69dd7eae9..5bcb75bfe619 100644 --- a/packages/integration-tests/suites/replay/errors/droppedError/test.ts +++ b/packages/integration-tests/suites/replay/errors/droppedError/test.ts @@ -17,58 +17,56 @@ import { getReplayEvent, shouldSkipReplayTest, waitForReplayRequest } from '../. * TODO: Once we have lifecycle hooks, we should revisit this test and make sure it behaves as expected. * This means that the recording should not be started or stopped if the error that triggered it is not sent. */ -for (let i = 0; i < 100; i++) { - sentryTest( - `[error-mode] should start recording if an error occurred although the error was dropped RUN ${i}`, - async ({ getLocalTestPath, page }) => { - if (shouldSkipReplayTest()) { - sentryTest.skip(); - } +sentryTest( + '[error-mode] should start recording if an error occurred although the error was dropped', + async ({ getLocalTestPath, page }) => { + if (shouldSkipReplayTest()) { + sentryTest.skip(); + } - const reqPromise0 = waitForReplayRequest(page, 0); - const reqPromise1 = waitForReplayRequest(page, 1); - const reqPromise2 = waitForReplayRequest(page, 2); + const reqPromise0 = waitForReplayRequest(page, 0); + const reqPromise1 = waitForReplayRequest(page, 1); + const reqPromise2 = waitForReplayRequest(page, 2); - let callsToSentry = 0; + let callsToSentry = 0; - await page.route('https://dsn.ingest.sentry.io/**/*', route => { - callsToSentry++; + await page.route('https://dsn.ingest.sentry.io/**/*', route => { + callsToSentry++; - return route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ id: 'test-id' }), - }); + return route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ id: 'test-id' }), }); + }); - const url = await getLocalTestPath({ testDir: __dirname }); + const url = await getLocalTestPath({ testDir: __dirname }); - await page.goto(url); - await page.click('#go-background'); - expect(callsToSentry).toEqual(0); + await page.goto(url); + await page.click('#go-background'); + expect(callsToSentry).toEqual(0); - await page.click('#error'); - const req0 = await reqPromise0; + await page.click('#error'); + const req0 = await reqPromise0; - await page.click('#go-background'); - await reqPromise1; + await page.click('#go-background'); + await reqPromise1; - await page.click('#log'); - await page.click('#go-background'); - await reqPromise2; + await page.click('#log'); + await page.click('#go-background'); + await reqPromise2; - // Note: The fact that reqPromise1/reqPromise2 are fulfilled prooves that the recording continues + // Note: The fact that reqPromise1/reqPromise2 are fulfilled prooves that the recording continues - const event0 = getReplayEvent(req0); + const event0 = getReplayEvent(req0); - expect(event0).toEqual( - getExpectedReplayEvent({ - contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } }, - // This is by design. A dropped error shouldn't be in this list. - error_ids: [], - replay_type: 'error', - }), - ); - }, - ); -} + expect(event0).toEqual( + getExpectedReplayEvent({ + contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } }, + // This is by design. A dropped error shouldn't be in this list. + error_ids: [], + replay_type: 'error', + }), + ); + }, +);