From 03de1c2030d9ea9debc98eda35936890d3b8801f Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 15 Mar 2023 09:56:43 +0100 Subject: [PATCH 01/14] test: Make E2E tests async & parallelizable --- packages/e2e-tests/.eslintrc.js | 2 +- packages/e2e-tests/.gitignore | 1 + packages/e2e-tests/lib/buildApp.ts | 80 ++++ .../e2e-tests/lib/buildRecipeInstances.ts | 47 ++ packages/e2e-tests/lib/constants.ts | 6 + packages/e2e-tests/lib/runAllTestApps.ts | 84 ++++ packages/e2e-tests/lib/runTestApp.ts | 54 +++ packages/e2e-tests/lib/testApp.ts | 50 ++ packages/e2e-tests/lib/types.ts | 49 ++ packages/e2e-tests/lib/utils.ts | 69 +++ packages/e2e-tests/lib/validate.ts | 33 ++ packages/e2e-tests/package.json | 2 + packages/e2e-tests/registrySetup.ts | 96 ++++ packages/e2e-tests/run.ts | 444 +----------------- .../create-next-app/playwright.config.ts | 4 +- .../create-next-app/tsconfig.json | 3 +- .../nextjs-app-dir/playwright.config.ts | 6 +- .../nextjs-app-dir/tsconfig.json | 1 - .../node-express-app/playwright.config.ts | 4 +- .../node-express-app/src/app.ts | 2 +- .../node-express-app/test-recipe.json | 2 +- .../playwright.config.ts | 4 +- packages/e2e-tests/test-recipe-schema.json | 6 +- yarn.lock | 18 +- 24 files changed, 615 insertions(+), 452 deletions(-) create mode 100644 packages/e2e-tests/lib/buildApp.ts create mode 100644 packages/e2e-tests/lib/buildRecipeInstances.ts create mode 100644 packages/e2e-tests/lib/constants.ts create mode 100644 packages/e2e-tests/lib/runAllTestApps.ts create mode 100644 packages/e2e-tests/lib/runTestApp.ts create mode 100644 packages/e2e-tests/lib/testApp.ts create mode 100644 packages/e2e-tests/lib/types.ts create mode 100644 packages/e2e-tests/lib/utils.ts create mode 100644 packages/e2e-tests/lib/validate.ts create mode 100644 packages/e2e-tests/registrySetup.ts diff --git a/packages/e2e-tests/.eslintrc.js b/packages/e2e-tests/.eslintrc.js index 8c585e48f252..5ba76262baaf 100644 --- a/packages/e2e-tests/.eslintrc.js +++ b/packages/e2e-tests/.eslintrc.js @@ -3,7 +3,7 @@ module.exports = { node: true, }, extends: ['../../.eslintrc.js'], - ignorePatterns: ['test-applications/**'], + ignorePatterns: ['test-applications/**', 'tmp/**'], parserOptions: { sourceType: 'module', }, diff --git a/packages/e2e-tests/.gitignore b/packages/e2e-tests/.gitignore index 4c49bd78f1d0..a16add542a3f 100644 --- a/packages/e2e-tests/.gitignore +++ b/packages/e2e-tests/.gitignore @@ -1 +1,2 @@ .env +tmp diff --git a/packages/e2e-tests/lib/buildApp.ts b/packages/e2e-tests/lib/buildApp.ts new file mode 100644 index 000000000000..e802d691e96a --- /dev/null +++ b/packages/e2e-tests/lib/buildApp.ts @@ -0,0 +1,80 @@ +/* eslint-disable no-console */ + +import * as fs from 'fs-extra'; +import * as path from 'path'; + +import { DEFAULT_BUILD_TIMEOUT_SECONDS } from './constants'; +import type { Env, RecipeInstance } from './types'; +import { spawnAsync } from './utils'; + +export async function buildApp(appDir: string, recipeInstance: RecipeInstance, env: Env): Promise { + const { recipe, label, dependencyOverrides } = recipeInstance; + + const packageJsonPath = path.resolve(appDir, 'package.json'); + + if (dependencyOverrides) { + // Override dependencies + const packageJson: { dependencies?: Record } = JSON.parse( + fs.readFileSync(packageJsonPath, { encoding: 'utf-8' }), + ); + packageJson.dependencies = packageJson.dependencies + ? { ...packageJson.dependencies, ...dependencyOverrides } + : dependencyOverrides; + fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2), { + encoding: 'utf-8', + }); + } + + if (recipe.buildCommand) { + console.log(`Running build command for test application "${label}"`); + + const buildResult = await spawnAsync(recipe.buildCommand, { + cwd: appDir, + timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000, + env: { + ...process.env, + ...env, + } as unknown as NodeJS.ProcessEnv, + }); + + if (buildResult.error) { + console.log(`Build failed for test application "${label}"`); + + // Prepends some text to the output build command's output so we can distinguish it from logging in this script + console.log(buildResult.stdout.replace(/^/gm, ' [BUILD OUTPUT] ')); + console.log(buildResult.stderr.replace(/^/gm, ' [BUILD OUTPUT] ')); + + console.log('[BUILD ERROR] ', buildResult.error); + throw buildResult.error; + } + + if (recipe.buildAssertionCommand) { + console.log(`Running build assertion for test application "${label}"`); + + const buildAssertionResult = await spawnAsync( + recipe.buildAssertionCommand, + { + cwd: appDir, + timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000, + env: { + ...process.env, + ...env, + } as unknown as NodeJS.ProcessEnv, + }, + buildResult.stdout, + ); + + if (buildAssertionResult.error) { + console.log(`Build assertion failed for test application "${label}"`); + + // Prepends some text to the output build command's output so we can distinguish it from logging in this script + console.log(buildAssertionResult.stdout.replace(/^/gm, ' [BUILD ASSERTION OUTPUT] ')); + console.log(buildAssertionResult.stderr.replace(/^/gm, ' [BUILD ASSERTION OUTPUT] ')); + + console.log('[BUILD ASSERTION ERROR] ', buildAssertionResult.error); + + throw buildAssertionResult.error; + } + } + } +} diff --git a/packages/e2e-tests/lib/buildRecipeInstances.ts b/packages/e2e-tests/lib/buildRecipeInstances.ts new file mode 100644 index 000000000000..2820c2e055ce --- /dev/null +++ b/packages/e2e-tests/lib/buildRecipeInstances.ts @@ -0,0 +1,47 @@ +import * as fs from 'fs'; + +import type { Recipe, RecipeInput, RecipeInstance } from './types'; + +export function buildRecipeInstances(recipePaths: string[]): RecipeInstance[] { + const recipes = buildRecipes(recipePaths); + const recipeInstances: RecipeInstance[] = []; + + const basePort = 3001; + + recipes.forEach((recipe, i) => { + recipe.versions.forEach(version => { + const dependencyOverrides = + Object.keys(version.dependencyOverrides).length > 0 ? version.dependencyOverrides : undefined; + const dependencyOverridesInformationString = dependencyOverrides + ? ` (Dependency overrides: ${JSON.stringify(dependencyOverrides)})` + : ''; + + recipeInstances.push({ + label: `${recipe.testApplicationName}${dependencyOverridesInformationString}`, + recipe, + dependencyOverrides, + port: basePort + i, + }); + }); + }); + + return recipeInstances; +} + +function buildRecipes(recipePaths: string[]): Recipe[] { + return recipePaths.map(recipePath => buildRecipe(recipePath)); +} + +function buildRecipe(recipePath: string): Recipe { + const recipe: RecipeInput = JSON.parse(fs.readFileSync(recipePath, 'utf-8')); + + const versions = process.env.CANARY_E2E_TEST + ? recipe.canaryVersions ?? [] + : recipe.versions ?? [{ dependencyOverrides: {} }]; + + return { + ...recipe, + path: recipePath, + versions, + }; +} diff --git a/packages/e2e-tests/lib/constants.ts b/packages/e2e-tests/lib/constants.ts new file mode 100644 index 000000000000..3198b652e0b1 --- /dev/null +++ b/packages/e2e-tests/lib/constants.ts @@ -0,0 +1,6 @@ +export const TEST_REGISTRY_CONTAINER_NAME = 'verdaccio-e2e-test-registry'; +export const DEFAULT_BUILD_TIMEOUT_SECONDS = 60 * 5; +export const DEFAULT_TEST_TIMEOUT_SECONDS = 60 * 2; +export const VERDACCIO_VERSION = '5.15.3'; +export const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages'; +export const TMP_DIR = 'tmp'; diff --git a/packages/e2e-tests/lib/runAllTestApps.ts b/packages/e2e-tests/lib/runAllTestApps.ts new file mode 100644 index 000000000000..b12a0ccb2bdb --- /dev/null +++ b/packages/e2e-tests/lib/runAllTestApps.ts @@ -0,0 +1,84 @@ +/* eslint-disable no-console */ +import { buildRecipeInstances } from './buildRecipeInstances'; +import { buildAndTestApp } from './runTestApp'; +import type { RecipeInstance, RecipeTestResult } from './types'; + +export async function runAllTestApps( + recipePaths: string[], + envVarsToInject: Record, +): Promise { + const maxParallel = process.env.CI ? 2 : 5; + + const recipeInstances = buildRecipeInstances(recipePaths); + + const results = await shardPromises( + recipeInstances, + recipeInstance => buildAndTestApp(recipeInstance, envVarsToInject), + maxParallel, + ); + + console.log('--------------------------------------'); + console.log('Test Result Summary:'); + + results.forEach(result => { + if (result.buildFailed) { + console.log(`● BUILD FAILED - ${result.label} (${result.recipe.path}`); + } else { + console.log(`● BUILD SUCCEEDED - ${result.label}`); + result.tests.forEach(testResult => { + console.log(` ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`); + }); + } + }); + + const failed = results.filter(result => result.buildFailed || result.testFailed); + + if (failed.length) { + console.log(`${failed.length} test(s) failed.`); + process.exit(1); + } + + console.log('All tests succeeded. 🎉'); +} + +// Always run X promises at a time +function shardPromises( + recipes: RecipeInstance[], + callback: (recipe: RecipeInstance) => Promise, + maxParallel: number, +): Promise { + return new Promise(resolve => { + console.log(`Running a total of ${recipes.length} jobs, with up to ${maxParallel} jobs in parallel...`); + const results: RecipeTestResult[] = []; + const remaining = recipes.slice(); + const running: Promise[] = []; + + function runNext(): void { + if (running.length < maxParallel && remaining.length > 0) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const next = remaining.shift()!; + const promise = callback(next); + + console.log(`Running job ${next.label}, ${remaining.length} remaining...`); + + running.push(promise); + + promise + .then(result => results.push(result)) + .finally(() => { + const pos = running.indexOf(promise); + running.splice(pos, 1); + + runNext(); + }); + } else if (remaining.length === 0 && running.length === 0) { + resolve(results); + } + } + + // Initial runs + for (let i = 0; i < maxParallel; i++) { + runNext(); + } + }); +} diff --git a/packages/e2e-tests/lib/runTestApp.ts b/packages/e2e-tests/lib/runTestApp.ts new file mode 100644 index 000000000000..dac0228567e7 --- /dev/null +++ b/packages/e2e-tests/lib/runTestApp.ts @@ -0,0 +1,54 @@ +/* eslint-disable no-console */ + +import { uuid4 } from '@sentry/utils'; +import * as fs from 'fs-extra'; +import * as path from 'path'; + +import { buildApp } from './buildApp'; +import { TMP_DIR } from './constants'; +import { testApp } from './testApp'; +import type { Env, RecipeInstance, RecipeTestResult } from './types'; + +// This should never throw, we always return a result here +export async function buildAndTestApp( + recipeInstance: RecipeInstance, + envVarsToInject: Record, +): Promise { + const { recipe, port } = recipeInstance; + const recipeDirname = path.dirname(recipe.path); + + const targetDir = path.join(TMP_DIR, `${recipe.testApplicationName}-${uuid4()}`); + + await fs.copy(recipeDirname, targetDir); + + const env: Env = { + ...envVarsToInject, + PORT: port.toString(), + }; + + try { + await buildApp(targetDir, recipeInstance, env); + } catch (error) { + await fs.remove(targetDir); + + return { + ...recipeInstance, + buildFailed: true, + testFailed: false, + tests: [], + }; + } + + // This cannot throw, we always return a result here + const results = await testApp(targetDir, recipeInstance, env); + + // Cleanup + await fs.remove(targetDir); + + return { + ...recipeInstance, + buildFailed: false, + testFailed: results.some(result => result.result !== 'PASS'), + tests: results, + }; +} diff --git a/packages/e2e-tests/lib/testApp.ts b/packages/e2e-tests/lib/testApp.ts new file mode 100644 index 000000000000..e25418662c38 --- /dev/null +++ b/packages/e2e-tests/lib/testApp.ts @@ -0,0 +1,50 @@ +/* eslint-disable no-console */ + +import { DEFAULT_TEST_TIMEOUT_SECONDS } from './constants'; +import type { Env, RecipeInstance, TestDef, TestResult } from './types'; +import { spawnAsync } from './utils'; + +export async function testApp(appDir: string, recipeInstance: RecipeInstance, env: Env): Promise { + const { recipe } = recipeInstance; + + const results: TestResult[] = []; + for (const test of recipe.tests) { + results.push(await runTest(appDir, recipeInstance, test, env)); + } + + return results; +} + +async function runTest(appDir: string, recipeInstance: RecipeInstance, test: TestDef, env: Env): Promise { + const { recipe, label } = recipeInstance; + console.log(`Running test command for test application "${label}", test "${test.testName}"`); + + const testResult = await spawnAsync(test.testCommand, { + cwd: appDir, + timeout: (recipe.testTimeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000, + env: { + ...process.env, + ...env, + } as unknown as NodeJS.ProcessEnv, + }); + + if (testResult.error) { + console.log(`Test failed for test application "${label}", test "${test.testName}"`); + + // Prepends some text to the output test command's output so we can distinguish it from logging in this script + console.log(testResult.stdout.replace(/^/gm, ' [TEST OUTPUT] ')); + console.log(testResult.stderr.replace(/^/gm, ' [TEST OUTPUT] ')); + + console.log('[TEST ERROR] ', testResult.error); + + return { + testName: test.testName, + result: testResult.error?.message.includes('ETDIMEDOUT') ? 'TIMEOUT' : 'FAIL', + }; + } + + return { + testName: test.testName, + result: 'PASS', + }; +} diff --git a/packages/e2e-tests/lib/types.ts b/packages/e2e-tests/lib/types.ts new file mode 100644 index 000000000000..c78bc45b6bf5 --- /dev/null +++ b/packages/e2e-tests/lib/types.ts @@ -0,0 +1,49 @@ +export type TestResult = { + testName: string; + result: 'PASS' | 'FAIL' | 'TIMEOUT'; +}; + +type DependencyOverrides = Record; + +export interface TestDef { + testName: string; + testCommand: string; + timeoutSeconds?: number; +} + +export interface RecipeInput { + testApplicationName: string; + buildCommand?: string; + buildAssertionCommand?: string; + buildTimeoutSeconds?: number; + testTimeoutSeconds?: number; + tests: TestDef[]; + versions?: { dependencyOverrides: DependencyOverrides }[]; + canaryVersions?: { dependencyOverrides: DependencyOverrides }[]; +} + +export interface Recipe { + path: string; + testApplicationName: string; + buildCommand?: string; + buildAssertionCommand?: string; + buildTimeoutSeconds?: number; + testTimeoutSeconds?: number; + tests: TestDef[]; + versions: { dependencyOverrides: DependencyOverrides }[]; +} + +export interface RecipeInstance { + label: string; + recipe: Recipe; + dependencyOverrides?: DependencyOverrides; + port: number; +} + +export interface RecipeTestResult extends RecipeInstance { + buildFailed: boolean; + testFailed: boolean; + tests: TestResult[]; +} + +export type Env = Record; diff --git a/packages/e2e-tests/lib/utils.ts b/packages/e2e-tests/lib/utils.ts new file mode 100644 index 000000000000..caf2cf424171 --- /dev/null +++ b/packages/e2e-tests/lib/utils.ts @@ -0,0 +1,69 @@ +/* eslint-disable no-console */ +import * as childProcess from 'child_process'; + +// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message +export function printCIErrorMessage(message: string): void { + if (process.env.CI) { + console.log(`::error::${message}`); + } else { + console.log(message); + } +} + +interface SpawnAsync { + stdout: string; + stderr: string; + error?: Error; + status: number | null; +} + +export function spawnAsync( + cmd: string, + options?: + | childProcess.SpawnOptionsWithoutStdio + | childProcess.SpawnOptionsWithStdioTuple, + input?: string, +): Promise { + const start = Date.now(); + + return new Promise(resolve => { + const cp = childProcess.spawn(cmd, { shell: true, ...options }); + + const stderr: unknown[] = []; + const stdout: string[] = []; + let error: Error | undefined; + + cp.stdout.on('data', data => { + stdout.push(data ? (data as object).toString() : ''); + }); + + cp.stderr.on('data', data => { + stderr.push(data ? (data as object).toString() : ''); + }); + + cp.on('error', e => { + error = e; + }); + + cp.on('close', status => { + const end = Date.now(); + + // We manually mark this as timed out if the process takes too long + if (!error && status === 1 && options?.timeout && end >= start + options.timeout) { + error = new Error(`ETDIMEDOUT: Process timed out after ${options.timeout} ms.`); + } + + resolve({ + stdout: stdout.join(''), + stderr: stderr.join(''), + error: error || (status !== 0 ? new Error(`Process exited with status ${status}`) : undefined), + status, + }); + }); + + if (input) { + cp.stdin.write(input); + cp.stdin.end(); + } + }); +} diff --git a/packages/e2e-tests/lib/validate.ts b/packages/e2e-tests/lib/validate.ts new file mode 100644 index 000000000000..2abb4974ce70 --- /dev/null +++ b/packages/e2e-tests/lib/validate.ts @@ -0,0 +1,33 @@ +/* eslint-disable no-console */ + +export function validate(): boolean { + let missingEnvVar = false; + + if (!process.env.E2E_TEST_AUTH_TOKEN) { + console.log( + "No auth token configured! Please configure the E2E_TEST_AUTH_TOKEN environment variable with an auth token that has the scope 'project:read'!", + ); + missingEnvVar = true; + } + + if (!process.env.E2E_TEST_DSN) { + console.log('No DSN configured! Please configure the E2E_TEST_DSN environment variable with a DSN!'); + missingEnvVar = true; + } + + if (!process.env.E2E_TEST_SENTRY_ORG_SLUG) { + console.log( + 'No Sentry organization slug configured! Please configure the E2E_TEST_SENTRY_ORG_SLUG environment variable with a Sentry organization slug!', + ); + missingEnvVar = true; + } + + if (!process.env.E2E_TEST_SENTRY_TEST_PROJECT) { + console.log( + 'No Sentry project configured! Please configure the E2E_TEST_SENTRY_TEST_PROJECT environment variable with a Sentry project slug!', + ); + missingEnvVar = true; + } + + return !missingEnvVar; +} diff --git a/packages/e2e-tests/package.json b/packages/e2e-tests/package.json index 1028f20d7374..82c00f5fd7b9 100644 --- a/packages/e2e-tests/package.json +++ b/packages/e2e-tests/package.json @@ -19,9 +19,11 @@ "test:validate-test-app-setups": "ts-node validate-test-app-setups.ts" }, "devDependencies": { + "@sentry/utils": "7.43.0", "@types/glob": "8.0.0", "@types/node": "^14.6.4", "dotenv": "16.0.3", + "fs-extra": "11.1.0", "glob": "8.0.3", "ts-node": "10.9.1", "typescript": "3.8.3", diff --git a/packages/e2e-tests/registrySetup.ts b/packages/e2e-tests/registrySetup.ts new file mode 100644 index 000000000000..ea7dc3decca5 --- /dev/null +++ b/packages/e2e-tests/registrySetup.ts @@ -0,0 +1,96 @@ +/* eslint-disable no-console */ +import * as childProcess from 'child_process'; +import * as path from 'path'; + +import { PUBLISH_PACKAGES_DOCKER_IMAGE_NAME, TEST_REGISTRY_CONTAINER_NAME, VERDACCIO_VERSION } from './lib/constants'; + +const publishScriptNodeVersion = process.env.E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION; +const repositoryRoot = path.resolve(__dirname, '../..'); + +// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines +function groupCIOutput(groupTitle: string, fn: () => void): void { + if (process.env.CI) { + console.log(`::group::${groupTitle}`); + fn(); + console.log('::endgroup::'); + } else { + fn(); + } +} + +export function registrySetup(): void { + groupCIOutput('Test Registry Setup', () => { + // Stop test registry container (Verdaccio) if it was already running + childProcess.spawnSync('docker', ['stop', TEST_REGISTRY_CONTAINER_NAME], { stdio: 'ignore' }); + console.log('Stopped previously running test registry'); + + // Start test registry (Verdaccio) + const startRegistryProcessResult = childProcess.spawnSync( + 'docker', + [ + 'run', + '--detach', + '--rm', + '--name', + TEST_REGISTRY_CONTAINER_NAME, + '-p', + '4873:4873', + '-v', + `${__dirname}/verdaccio-config:/verdaccio/conf`, + `verdaccio/verdaccio:${VERDACCIO_VERSION}`, + ], + { encoding: 'utf8', stdio: 'inherit' }, + ); + + if (startRegistryProcessResult.status !== 0) { + throw new Error('Start Registry Process failed.'); + } + + // Build container image that is uploading our packages to fake registry with specific Node.js/npm version + const buildPublishImageProcessResult = childProcess.spawnSync( + 'docker', + [ + 'build', + '--tag', + PUBLISH_PACKAGES_DOCKER_IMAGE_NAME, + '--file', + './Dockerfile.publish-packages', + ...(publishScriptNodeVersion ? ['--build-arg', `NODE_VERSION=${publishScriptNodeVersion}`] : []), + '.', + ], + { + encoding: 'utf8', + stdio: 'inherit', + }, + ); + + if (buildPublishImageProcessResult.status !== 0) { + throw new Error('Build Publish Image failed.'); + } + + // Run container that uploads our packages to fake registry + const publishImageContainerRunProcess = childProcess.spawnSync( + 'docker', + [ + 'run', + '--rm', + '-v', + `${repositoryRoot}:/sentry-javascript`, + '--network', + 'host', + PUBLISH_PACKAGES_DOCKER_IMAGE_NAME, + ], + { + encoding: 'utf8', + stdio: 'inherit', + }, + ); + + if (publishImageContainerRunProcess.status !== 0) { + throw new Error('Publish Image Container failed.'); + } + }); + + console.log(''); + console.log(''); +} diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts index 842911863f02..a4fec2c480dc 100644 --- a/packages/e2e-tests/run.ts +++ b/packages/e2e-tests/run.ts @@ -1,441 +1,35 @@ /* eslint-disable max-lines */ /* eslint-disable no-console */ -import * as childProcess from 'child_process'; import * as dotenv from 'dotenv'; -import * as fs from 'fs'; import * as glob from 'glob'; -import * as path from 'path'; -// Load environment variables from .env file locally -dotenv.config(); +import { runAllTestApps } from './lib/runAllTestApps'; +import { validate } from './lib/validate'; +import { registrySetup } from './registrySetup'; -const repositoryRoot = path.resolve(__dirname, '../..'); +async function run(): Promise { + // Load environment variables from .env file locally + dotenv.config(); -const TEST_REGISTRY_CONTAINER_NAME = 'verdaccio-e2e-test-registry'; -const VERDACCIO_VERSION = '5.15.3'; - -const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages'; - -const publishScriptNodeVersion = process.env.E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION; - -const DEFAULT_BUILD_TIMEOUT_SECONDS = 60 * 5; -const DEFAULT_TEST_TIMEOUT_SECONDS = 60 * 2; - -let missingEnvVar = false; - -if (!process.env.E2E_TEST_AUTH_TOKEN) { - console.log( - "No auth token configured! Please configure the E2E_TEST_AUTH_TOKEN environment variable with an auth token that has the scope 'project:read'!", - ); - missingEnvVar = true; -} - -if (!process.env.E2E_TEST_DSN) { - console.log('No DSN configured! Please configure the E2E_TEST_DSN environment variable with a DSN!'); - missingEnvVar = true; -} - -if (!process.env.E2E_TEST_SENTRY_ORG_SLUG) { - console.log( - 'No Sentry organization slug configured! Please configure the E2E_TEST_SENTRY_ORG_SLUG environment variable with a Sentry organization slug!', - ); - missingEnvVar = true; -} - -if (!process.env.E2E_TEST_SENTRY_TEST_PROJECT) { - console.log( - 'No Sentry project configured! Please configure the E2E_TEST_SENTRY_TEST_PROJECT environment variable with a Sentry project slug!', - ); - missingEnvVar = true; -} - -if (missingEnvVar) { - process.exit(1); -} - -const envVarsToInject = { - REACT_APP_E2E_TEST_DSN: process.env.E2E_TEST_DSN, - NEXT_PUBLIC_E2E_TEST_DSN: process.env.E2E_TEST_DSN, -}; - -// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines -function groupCIOutput(groupTitle: string, fn: () => void): void { - if (process.env.CI) { - console.log(`::group::${groupTitle}`); - fn(); - console.log('::endgroup::'); - } else { - fn(); - } -} - -// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message -function printCIErrorMessage(message: string): void { - if (process.env.CI) { - console.log(`::error::${message}`); - } else { - console.log(message); - } -} - -groupCIOutput('Test Registry Setup', () => { - // Stop test registry container (Verdaccio) if it was already running - childProcess.spawnSync('docker', ['stop', TEST_REGISTRY_CONTAINER_NAME], { stdio: 'ignore' }); - console.log('Stopped previously running test registry'); - - // Start test registry (Verdaccio) - const startRegistryProcessResult = childProcess.spawnSync( - 'docker', - [ - 'run', - '--detach', - '--rm', - '--name', - TEST_REGISTRY_CONTAINER_NAME, - '-p', - '4873:4873', - '-v', - `${__dirname}/verdaccio-config:/verdaccio/conf`, - `verdaccio/verdaccio:${VERDACCIO_VERSION}`, - ], - { encoding: 'utf8', stdio: 'inherit' }, - ); - - if (startRegistryProcessResult.status !== 0) { + if (!validate()) { process.exit(1); } - // Build container image that is uploading our packages to fake registry with specific Node.js/npm version - const buildPublishImageProcessResult = childProcess.spawnSync( - 'docker', - [ - 'build', - '--tag', - PUBLISH_PACKAGES_DOCKER_IMAGE_NAME, - '--file', - './Dockerfile.publish-packages', - ...(publishScriptNodeVersion ? ['--build-arg', `NODE_VERSION=${publishScriptNodeVersion}`] : []), - '.', - ], - { - encoding: 'utf8', - stdio: 'inherit', - }, - ); + const envVarsToInject = { + REACT_APP_E2E_TEST_DSN: process.env.E2E_TEST_DSN, + NEXT_PUBLIC_E2E_TEST_DSN: process.env.E2E_TEST_DSN, + }; - if (buildPublishImageProcessResult.status !== 0) { - process.exit(1); - } + try { + registrySetup(); - // Run container that uploads our packages to fake registry - const publishImageContainerRunProcess = childProcess.spawnSync( - 'docker', - [ - 'run', - '--rm', - '-v', - `${repositoryRoot}:/sentry-javascript`, - '--network', - 'host', - PUBLISH_PACKAGES_DOCKER_IMAGE_NAME, - ], - { - encoding: 'utf8', - stdio: 'inherit', - }, - ); + const recipePaths = glob.sync(`${__dirname}/test-applications/*/test-recipe.json`, { absolute: true }); - if (publishImageContainerRunProcess.status !== 0) { + await runAllTestApps(recipePaths, envVarsToInject); + } catch (error) { + console.error(error); process.exit(1); } -}); - -const recipePaths = glob.sync(`${__dirname}/test-applications/*/test-recipe.json`, { absolute: true }); - -let processShouldExitWithError = false; - -type TestResult = { - testName: string; - result: 'PASS' | 'FAIL' | 'TIMEOUT'; -}; - -type VersionResult = { - dependencyOverrides?: Record; - buildFailed: boolean; - testResults: TestResult[]; -}; - -type RecipeResult = { - testApplicationName: string; - testApplicationPath: string; - versionResults: VersionResult[]; -}; - -type Recipe = { - testApplicationName: string; - buildCommand?: string; - buildAssertionCommand?: string; - buildTimeoutSeconds?: number; - tests: { - testName: string; - testCommand: string; - timeoutSeconds?: number; - }[]; - versions?: { dependencyOverrides: Record }[]; - canaryVersions?: { dependencyOverrides: Record }[]; -}; - -const recipeResults: RecipeResult[] = recipePaths.map(recipePath => { - const recipe: Recipe = JSON.parse(fs.readFileSync(recipePath, 'utf-8')); - const recipeDirname = path.dirname(recipePath); - - function runRecipe(dependencyOverrides: Record | undefined): VersionResult { - const dependencyOverridesInformationString = dependencyOverrides - ? ` (Dependency overrides: ${JSON.stringify(dependencyOverrides)})` - : ''; - - if (recipe.buildCommand) { - console.log( - `Running E2E test build command for test application "${recipe.testApplicationName}"${dependencyOverridesInformationString}`, - ); - const buildCommandProcess = childProcess.spawnSync(recipe.buildCommand, { - cwd: path.dirname(recipePath), - encoding: 'utf8', - shell: true, // needed so we can pass the build command in as whole without splitting it up into args - timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000, - env: { - ...process.env, - ...envVarsToInject, - }, - }); - - // Prepends some text to the output build command's output so we can distinguish it from logging in this script - console.log(buildCommandProcess.stdout.replace(/^/gm, '[BUILD OUTPUT] ')); - console.log(buildCommandProcess.stderr.replace(/^/gm, '[BUILD OUTPUT] ')); - - const buildCommandProcessError: undefined | (Error & { code?: string }) = buildCommandProcess.error; - - if (buildCommandProcessError?.code === 'ETIMEDOUT') { - processShouldExitWithError = true; - - printCIErrorMessage( - `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) timed out!`, - ); - - return { - dependencyOverrides, - buildFailed: true, - testResults: [], - }; - } else if (buildCommandProcess.status !== 0) { - processShouldExitWithError = true; - - printCIErrorMessage( - `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) failed!`, - ); - - return { - dependencyOverrides, - buildFailed: true, - testResults: [], - }; - } - - if (recipe.buildAssertionCommand) { - console.log( - `Running E2E test build assertion for test application "${recipe.testApplicationName}"${dependencyOverridesInformationString}`, - ); - const buildAssertionCommandProcess = childProcess.spawnSync(recipe.buildAssertionCommand, { - cwd: path.dirname(recipePath), - input: buildCommandProcess.stdout, - encoding: 'utf8', - shell: true, // needed so we can pass the build command in as whole without splitting it up into args - timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000, - env: { - ...process.env, - ...envVarsToInject, - }, - }); - - // Prepends some text to the output build command's output so we can distinguish it from logging in this script - console.log(buildAssertionCommandProcess.stdout.replace(/^/gm, '[BUILD ASSERTION OUTPUT] ')); - console.log(buildAssertionCommandProcess.stderr.replace(/^/gm, '[BUILD ASSERTION OUTPUT] ')); - - const buildAssertionCommandProcessError: undefined | (Error & { code?: string }) = - buildAssertionCommandProcess.error; - - if (buildAssertionCommandProcessError?.code === 'ETIMEDOUT') { - processShouldExitWithError = true; - - printCIErrorMessage( - `Build assertion in test application "${recipe.testApplicationName}" (${path.dirname( - recipePath, - )}) timed out!`, - ); - - return { - dependencyOverrides, - buildFailed: true, - testResults: [], - }; - } else if (buildAssertionCommandProcess.status !== 0) { - processShouldExitWithError = true; - - printCIErrorMessage( - `Build assertion in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) failed!`, - ); - - return { - dependencyOverrides, - buildFailed: true, - testResults: [], - }; - } - } - } - - const testResults: TestResult[] = recipe.tests.map(test => { - console.log( - `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"${dependencyOverridesInformationString}`, - ); - - const testProcessResult = childProcess.spawnSync(test.testCommand, { - cwd: path.dirname(recipePath), - timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000, - encoding: 'utf8', - shell: true, // needed so we can pass the test command in as whole without splitting it up into args - env: { - ...process.env, - ...envVarsToInject, - }, - }); - - // Prepends some text to the output test command's output so we can distinguish it from logging in this script - console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] ')); - console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] ')); - - const error: undefined | (Error & { code?: string }) = testProcessResult.error; - - if (error?.code === 'ETIMEDOUT') { - processShouldExitWithError = true; - printCIErrorMessage( - `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( - recipePath, - )}) timed out.`, - ); - return { - testName: test.testName, - result: 'TIMEOUT', - }; - } else if (testProcessResult.status !== 0) { - processShouldExitWithError = true; - printCIErrorMessage( - `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( - recipePath, - )}) failed.`, - ); - return { - testName: test.testName, - result: 'FAIL', - }; - } else { - console.log( - `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( - recipePath, - )}) succeeded.`, - ); - return { - testName: test.testName, - result: 'PASS', - }; - } - }); - - return { - dependencyOverrides, - buildFailed: false, - testResults, - }; - } - - const versionsToRun: { - dependencyOverrides?: Record; - }[] = process.env.CANARY_E2E_TEST ? recipe.canaryVersions ?? [] : recipe.versions ?? [{}]; - - const versionResults = versionsToRun.map(({ dependencyOverrides }) => { - const packageJsonPath = path.resolve(recipeDirname, 'package.json'); - const packageJsonBackupPath = path.resolve(recipeDirname, 'package.json.bak'); - - if (dependencyOverrides) { - // Back up original package.json - fs.copyFileSync(packageJsonPath, packageJsonBackupPath); - - // Override dependencies - const packageJson: { dependencies?: Record } = JSON.parse( - fs.readFileSync(packageJsonPath, { encoding: 'utf-8' }), - ); - packageJson.dependencies = packageJson.dependencies - ? { ...packageJson.dependencies, ...dependencyOverrides } - : dependencyOverrides; - fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2), { - encoding: 'utf-8', - }); - } - - try { - return runRecipe(dependencyOverrides); - } finally { - if (dependencyOverrides) { - // Restore original package.json - fs.rmSync(packageJsonPath, { force: true }); - fs.copyFileSync(packageJsonBackupPath, packageJsonPath); - fs.rmSync(packageJsonBackupPath, { force: true }); - } - } - }); - - return { - testApplicationName: recipe.testApplicationName, - testApplicationPath: recipePath, - versionResults, - }; -}); - -console.log('--------------------------------------'); -console.log('Test Result Summary:'); - -recipeResults.forEach(recipeResult => { - recipeResult.versionResults.forEach(versionResult => { - const dependencyOverridesInformationString = versionResult.dependencyOverrides - ? ` (Dependency overrides: ${JSON.stringify(versionResult.dependencyOverrides)})` - : ''; - - if (versionResult.buildFailed) { - console.log( - `● BUILD FAILED - ${recipeResult.testApplicationName} (${path.dirname( - recipeResult.testApplicationPath, - )})${dependencyOverridesInformationString}`, - ); - } else { - console.log( - `● BUILD SUCCEEDED - ${recipeResult.testApplicationName} (${path.dirname( - recipeResult.testApplicationPath, - )})${dependencyOverridesInformationString}`, - ); - versionResult.testResults.forEach(testResult => { - console.log(` ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`); - }); - } - }); -}); - -// Stop test registry -childProcess.spawnSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' }); -console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own - -if (processShouldExitWithError) { - console.log('Not all tests succeeded.'); - process.exit(1); -} else { - console.log('All tests succeeded.'); } + +void run(); diff --git a/packages/e2e-tests/test-applications/create-next-app/playwright.config.ts b/packages/e2e-tests/test-applications/create-next-app/playwright.config.ts index 289976304812..b2c8ace9d92d 100644 --- a/packages/e2e-tests/test-applications/create-next-app/playwright.config.ts +++ b/packages/e2e-tests/test-applications/create-next-app/playwright.config.ts @@ -29,8 +29,6 @@ const config: PlaywrightTestConfig = { use: { /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ actionTimeout: 0, - /* Base URL to use in actions like `await page.goto('/')`. */ - // baseURL: 'http://localhost:3000', /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: 'on-first-retry', @@ -62,7 +60,7 @@ const config: PlaywrightTestConfig = { /* Run your local dev server before starting the tests */ webServer: { command: process.env.TEST_MODE === 'prod' ? 'yarn start' : 'yarn dev', - port: 3000, + port: process.env.PORT ? parseInt(process.env.PORT) : 3000, }, }; diff --git a/packages/e2e-tests/test-applications/create-next-app/tsconfig.json b/packages/e2e-tests/test-applications/create-next-app/tsconfig.json index 3ff0501fdb85..7686eb53f6ac 100644 --- a/packages/e2e-tests/test-applications/create-next-app/tsconfig.json +++ b/packages/e2e-tests/test-applications/create-next-app/tsconfig.json @@ -12,8 +12,7 @@ "moduleResolution": "node", "resolveJsonModule": true, "isolatedModules": true, - "jsx": "preserve", - "incremental": true + "jsx": "preserve" }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "next.config.js"], "exclude": ["node_modules"] diff --git a/packages/e2e-tests/test-applications/nextjs-app-dir/playwright.config.ts b/packages/e2e-tests/test-applications/nextjs-app-dir/playwright.config.ts index ae466dab4350..2bacdad88e2e 100644 --- a/packages/e2e-tests/test-applications/nextjs-app-dir/playwright.config.ts +++ b/packages/e2e-tests/test-applications/nextjs-app-dir/playwright.config.ts @@ -7,6 +7,8 @@ if (!testEnv) { throw new Error('No test env defined'); } +const port = process.env.PORT ? parseInt(process.env.PORT) : 3000; + /** * See https://playwright.dev/docs/test-configuration. */ @@ -34,7 +36,7 @@ const config: PlaywrightTestConfig = { /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ actionTimeout: 0, /* Base URL to use in actions like `await page.goto('/')`. */ - baseURL: 'http://localhost:3000', + baseURL: `http://localhost:${port}`, /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: 'on-first-retry', @@ -54,7 +56,7 @@ const config: PlaywrightTestConfig = { webServer: [ { command: testEnv === 'development' ? 'yarn dev' : 'yarn start', - port: 3000, + port, }, { command: 'yarn ts-node-script start-event-proxy.ts', diff --git a/packages/e2e-tests/test-applications/nextjs-app-dir/tsconfig.json b/packages/e2e-tests/test-applications/nextjs-app-dir/tsconfig.json index bacd391b697e..e78ff0d9c0e7 100644 --- a/packages/e2e-tests/test-applications/nextjs-app-dir/tsconfig.json +++ b/packages/e2e-tests/test-applications/nextjs-app-dir/tsconfig.json @@ -13,7 +13,6 @@ "resolveJsonModule": true, "isolatedModules": true, "jsx": "preserve", - "incremental": true, "plugins": [ { "name": "next" diff --git a/packages/e2e-tests/test-applications/node-express-app/playwright.config.ts b/packages/e2e-tests/test-applications/node-express-app/playwright.config.ts index d665bba2d978..ef7bf3704715 100644 --- a/packages/e2e-tests/test-applications/node-express-app/playwright.config.ts +++ b/packages/e2e-tests/test-applications/node-express-app/playwright.config.ts @@ -27,8 +27,6 @@ const config: PlaywrightTestConfig = { use: { /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ actionTimeout: 0, - /* Base URL to use in actions like `await page.goto('/')`. */ - // baseURL: 'http://localhost:3000', /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: 'on-first-retry', @@ -60,7 +58,7 @@ const config: PlaywrightTestConfig = { /* Run your local dev server before starting the tests */ webServer: { command: 'yarn start', - port: 3000, + port: process.env.PORT ? parseInt(process.env.PORT) : 3000, }, }; diff --git a/packages/e2e-tests/test-applications/node-express-app/src/app.ts b/packages/e2e-tests/test-applications/node-express-app/src/app.ts index e128e6c33882..ec4a5c73fcb6 100644 --- a/packages/e2e-tests/test-applications/node-express-app/src/app.ts +++ b/packages/e2e-tests/test-applications/node-express-app/src/app.ts @@ -17,7 +17,7 @@ Sentry.init({ }); const app = express(); -const port = 3000; +const port = process.env.PORT ? parseInt(process.env.PORT) : 3000; app.use(Sentry.Handlers.requestHandler()); app.use(Sentry.Handlers.tracingHandler()); diff --git a/packages/e2e-tests/test-applications/node-express-app/test-recipe.json b/packages/e2e-tests/test-applications/node-express-app/test-recipe.json index e3038c8d0459..039049258171 100644 --- a/packages/e2e-tests/test-applications/node-express-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/node-express-app/test-recipe.json @@ -1,6 +1,6 @@ { "$schema": "../../test-recipe-schema.json", - "testApplicationName": "Node Express App", + "testApplicationName": "node-express-app", "buildCommand": "yarn install && yarn build", "tests": [ { diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/playwright.config.ts b/packages/e2e-tests/test-applications/standard-frontend-react/playwright.config.ts index 9249280b4c34..1a6a49424672 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react/playwright.config.ts +++ b/packages/e2e-tests/test-applications/standard-frontend-react/playwright.config.ts @@ -29,8 +29,6 @@ const config: PlaywrightTestConfig = { use: { /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ actionTimeout: 0, - /* Base URL to use in actions like `await page.goto('/')`. */ - // baseURL: 'http://localhost:3000', /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: 'on-first-retry', @@ -62,7 +60,7 @@ const config: PlaywrightTestConfig = { /* Run your local dev server before starting the tests */ webServer: { command: 'yarn start', - port: 3000, + port: process.env.PORT ? parseInt(process.env.PORT) : 3000, }, }; diff --git a/packages/e2e-tests/test-recipe-schema.json b/packages/e2e-tests/test-recipe-schema.json index 178ca96ab3d0..ee14c2eb4f82 100644 --- a/packages/e2e-tests/test-recipe-schema.json +++ b/packages/e2e-tests/test-recipe-schema.json @@ -17,7 +17,11 @@ }, "buildTimeoutSeconds": { "type": "number", - "description": "Timeout for the build command in seconds. Default: 60" + "description": "Timeout for the build command in seconds. Default: 5 minutes" + }, + "testTimeoutSeconds": { + "type": "number", + "description": "Timeout for the test command in seconds. Default: 2 minutes" }, "tests": { "type": "array", diff --git a/yarn.lock b/yarn.lock index d39f972d5311..b07e4ba1aa5f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13445,6 +13445,15 @@ fs-constants@^1.0.0: resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== +fs-extra@11.1.0, fs-extra@^11.1.0: + version "11.1.0" + resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.0.tgz#5784b102104433bb0e090f48bfc4a30742c357ed" + integrity sha512-0rcTq621PD5jM/e0a3EJoGC/1TC5ZBCERW82LQuwfGnCa1V8w7dpYH1yNu+SLb6E5dkeCBzKEyLGlFrnr+dUyw== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + fs-extra@4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-4.0.2.tgz#f91704c53d1b461f893452b0c307d9997647ab6b" @@ -13483,15 +13492,6 @@ fs-extra@^10.0.0, fs-extra@^10.1.0: jsonfile "^6.0.1" universalify "^2.0.0" -fs-extra@^11.1.0: - version "11.1.0" - resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.0.tgz#5784b102104433bb0e090f48bfc4a30742c357ed" - integrity sha512-0rcTq621PD5jM/e0a3EJoGC/1TC5ZBCERW82LQuwfGnCa1V8w7dpYH1yNu+SLb6E5dkeCBzKEyLGlFrnr+dUyw== - dependencies: - graceful-fs "^4.2.0" - jsonfile "^6.0.1" - universalify "^2.0.0" - fs-extra@^4.0.2, fs-extra@^4.0.3: version "4.0.3" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-4.0.3.tgz#0d852122e5bc5beb453fb028e9c0c9bf36340c94" From c63e83e573e6cf41f35c6ca4c045fcaa11255c8d Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 15 Mar 2023 13:34:47 +0100 Subject: [PATCH 02/14] update verdaccio --- packages/e2e-tests/lib/constants.ts | 2 +- packages/e2e-tests/package.json | 3 ++- packages/e2e-tests/verdaccio-config/config.yaml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/e2e-tests/lib/constants.ts b/packages/e2e-tests/lib/constants.ts index 3198b652e0b1..fe24b98841fd 100644 --- a/packages/e2e-tests/lib/constants.ts +++ b/packages/e2e-tests/lib/constants.ts @@ -1,6 +1,6 @@ export const TEST_REGISTRY_CONTAINER_NAME = 'verdaccio-e2e-test-registry'; export const DEFAULT_BUILD_TIMEOUT_SECONDS = 60 * 5; export const DEFAULT_TEST_TIMEOUT_SECONDS = 60 * 2; -export const VERDACCIO_VERSION = '5.15.3'; +export const VERDACCIO_VERSION = '5.22.1'; export const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages'; export const TMP_DIR = 'tmp'; diff --git a/packages/e2e-tests/package.json b/packages/e2e-tests/package.json index 82c00f5fd7b9..c011af47b136 100644 --- a/packages/e2e-tests/package.json +++ b/packages/e2e-tests/package.json @@ -16,7 +16,8 @@ "test:e2e": "run-s test:validate-configuration test:validate-test-app-setups test:run", "test:run": "ts-node run.ts", "test:validate-configuration": "ts-node validate-verdaccio-configuration.ts", - "test:validate-test-app-setups": "ts-node validate-test-app-setups.ts" + "test:validate-test-app-setups": "ts-node validate-test-app-setups.ts", + "clean": "rimraf tmp test-applications/**/node_modules test-applications/**/dist" }, "devDependencies": { "@sentry/utils": "7.43.0", diff --git a/packages/e2e-tests/verdaccio-config/config.yaml b/packages/e2e-tests/verdaccio-config/config.yaml index a5b4f7919cc8..7c5cd16df28d 100644 --- a/packages/e2e-tests/verdaccio-config/config.yaml +++ b/packages/e2e-tests/verdaccio-config/config.yaml @@ -204,7 +204,7 @@ middlewares: # https://verdaccio.org/docs/logger # log settings -logs: { type: stdout, format: pretty, level: http } +log: { type: stdout, format: pretty, level: http } #experiments: # # support for npm token command # token: false From 800ad7e24c0a66cb67a6f220774a164e8568c29a Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 15 Mar 2023 15:12:35 +0100 Subject: [PATCH 03/14] ensure volta is setup --- .../e2e-tests/test-applications/create-next-app/package.json | 4 ++++ .../e2e-tests/test-applications/create-react-app/package.json | 4 ++++ .../e2e-tests/test-applications/nextjs-app-dir/package.json | 4 ++++ .../e2e-tests/test-applications/node-express-app/package.json | 4 ++++ .../test-applications/standard-frontend-react/package.json | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/packages/e2e-tests/test-applications/create-next-app/package.json b/packages/e2e-tests/test-applications/create-next-app/package.json index 5d5e829bace6..dee9275c2bdc 100644 --- a/packages/e2e-tests/test-applications/create-next-app/package.json +++ b/packages/e2e-tests/test-applications/create-next-app/package.json @@ -24,5 +24,9 @@ }, "devDependencies": { "@playwright/test": "^1.27.1" + }, + "volta": { + "node": "16.19.0", + "yarn": "1.22.19" } } diff --git a/packages/e2e-tests/test-applications/create-react-app/package.json b/packages/e2e-tests/test-applications/create-react-app/package.json index 201f636468ad..e0fc502238d0 100644 --- a/packages/e2e-tests/test-applications/create-react-app/package.json +++ b/packages/e2e-tests/test-applications/create-react-app/package.json @@ -41,5 +41,9 @@ "last 1 firefox version", "last 1 safari version" ] + }, + "volta": { + "node": "16.19.0", + "yarn": "1.22.19" } } diff --git a/packages/e2e-tests/test-applications/nextjs-app-dir/package.json b/packages/e2e-tests/test-applications/nextjs-app-dir/package.json index 5256c1aa8cdb..064e5d396d1f 100644 --- a/packages/e2e-tests/test-applications/nextjs-app-dir/package.json +++ b/packages/e2e-tests/test-applications/nextjs-app-dir/package.json @@ -24,5 +24,9 @@ "devDependencies": { "ts-node": "10.9.1", "@playwright/test": "^1.27.1" + }, + "volta": { + "node": "16.19.0", + "yarn": "1.22.19" } } diff --git a/packages/e2e-tests/test-applications/node-express-app/package.json b/packages/e2e-tests/test-applications/node-express-app/package.json index 4c07eb8a1028..3bcdb79df6ba 100644 --- a/packages/e2e-tests/test-applications/node-express-app/package.json +++ b/packages/e2e-tests/test-applications/node-express-app/package.json @@ -19,5 +19,9 @@ }, "devDependencies": { "@playwright/test": "^1.27.1" + }, + "volta": { + "node": "16.19.0", + "yarn": "1.22.19" } } diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/package.json b/packages/e2e-tests/test-applications/standard-frontend-react/package.json index 2752d1ddb4de..1b932fa4cf30 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react/package.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react/package.json @@ -46,5 +46,9 @@ "@playwright/test": "1.26.1", "axios": "1.1.2", "serve": "14.0.1" + }, + "volta": { + "node": "16.19.0", + "yarn": "1.22.19" } } From f149db94f6b49d7ea291f68f43f5af76240fb6a1 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 15 Mar 2023 15:51:15 +0100 Subject: [PATCH 04/14] no yarn pure-lockfile --- .../test-applications/create-next-app/test-recipe.json | 2 +- .../test-applications/create-react-app/test-recipe.json | 2 +- .../e2e-tests/test-applications/nextjs-app-dir/test-recipe.json | 2 +- .../test-applications/standard-frontend-react/test-recipe.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/e2e-tests/test-applications/create-next-app/test-recipe.json b/packages/e2e-tests/test-applications/create-next-app/test-recipe.json index cf4bf4431929..38b4bbc06af0 100644 --- a/packages/e2e-tests/test-applications/create-next-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/create-next-app/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "create-next-app", - "buildCommand": "yarn install --pure-lockfile && npx playwright install && yarn build", + "buildCommand": "yarn install && npx playwright install && yarn build", "tests": [ { "testName": "Playwright tests - Prod Mode", diff --git a/packages/e2e-tests/test-applications/create-react-app/test-recipe.json b/packages/e2e-tests/test-applications/create-react-app/test-recipe.json index d21ed4a6ace1..ee9c8e1dc40c 100644 --- a/packages/e2e-tests/test-applications/create-react-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/create-react-app/test-recipe.json @@ -1,6 +1,6 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "create-react-app", - "buildCommand": "yarn install --pure-lockfile && yarn build", + "buildCommand": "yarn install && yarn build", "tests": [] } diff --git a/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json b/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json index b3ec72add40f..b711dd6e922c 100644 --- a/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json +++ b/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "nextjs-13-app-dir", - "buildCommand": "yarn install --pure-lockfile && npx playwright install && yarn build", + "buildCommand": "yarn install && npx playwright install && yarn build", "buildAssertionCommand": "yarn ts-node --script-mode assert-build.ts", "tests": [ { diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json index 95ff8d535877..76916d74d280 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "standard-frontend-react", - "buildCommand": "yarn install --pure-lockfile && npx playwright install && yarn build", + "buildCommand": "yarn install && npx playwright install && yarn build", "tests": [ { "testName": "Playwright tests", From b7a356456ac0b01090cbd3d71dc8b4c383558a5b Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 29 Mar 2023 13:02:03 +0000 Subject: [PATCH 05/14] try fix --- packages/e2e-tests/lib/constants.ts | 1 - packages/e2e-tests/lib/runTestApp.ts | 7 ++++--- packages/e2e-tests/package.json | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/e2e-tests/lib/constants.ts b/packages/e2e-tests/lib/constants.ts index fe24b98841fd..bdc550009148 100644 --- a/packages/e2e-tests/lib/constants.ts +++ b/packages/e2e-tests/lib/constants.ts @@ -3,4 +3,3 @@ export const DEFAULT_BUILD_TIMEOUT_SECONDS = 60 * 5; export const DEFAULT_TEST_TIMEOUT_SECONDS = 60 * 2; export const VERDACCIO_VERSION = '5.22.1'; export const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages'; -export const TMP_DIR = 'tmp'; diff --git a/packages/e2e-tests/lib/runTestApp.ts b/packages/e2e-tests/lib/runTestApp.ts index dac0228567e7..9c91a0f14482 100644 --- a/packages/e2e-tests/lib/runTestApp.ts +++ b/packages/e2e-tests/lib/runTestApp.ts @@ -1,14 +1,15 @@ /* eslint-disable no-console */ -import { uuid4 } from '@sentry/utils'; import * as fs from 'fs-extra'; +import * as os from 'os'; import * as path from 'path'; import { buildApp } from './buildApp'; -import { TMP_DIR } from './constants'; import { testApp } from './testApp'; import type { Env, RecipeInstance, RecipeTestResult } from './types'; +let tmpDirCount = 0; + // This should never throw, we always return a result here export async function buildAndTestApp( recipeInstance: RecipeInstance, @@ -17,7 +18,7 @@ export async function buildAndTestApp( const { recipe, port } = recipeInstance; const recipeDirname = path.dirname(recipe.path); - const targetDir = path.join(TMP_DIR, `${recipe.testApplicationName}-${uuid4()}`); + const targetDir = path.join(os.tmpdir(), `${recipe.testApplicationName}-${tmpDirCount++}`); await fs.copy(recipeDirname, targetDir); diff --git a/packages/e2e-tests/package.json b/packages/e2e-tests/package.json index fcf0fa486b22..73f13d1c5174 100644 --- a/packages/e2e-tests/package.json +++ b/packages/e2e-tests/package.json @@ -20,7 +20,6 @@ "clean": "rimraf tmp test-applications/**/node_modules test-applications/**/dist" }, "devDependencies": { - "@sentry/utils": "7.43.0", "@types/glob": "8.0.0", "@types/node": "^14.6.4", "dotenv": "16.0.3", From b8a9efe8c3150b5066dc5849a1ba8a30fdc6a8c3 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 29 Mar 2023 13:29:37 +0000 Subject: [PATCH 06/14] Bump build timeout --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4ac4a75e150b..c23196e5ae80 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -185,7 +185,7 @@ jobs: name: Build needs: [job_get_metadata, job_install_deps] runs-on: ubuntu-20.04 - timeout-minutes: 20 + timeout-minutes: 30 steps: - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) uses: actions/checkout@v3 From 12a8c09aba4e12c81442c1ede04efcba368db522 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 29 Mar 2023 14:07:33 +0000 Subject: [PATCH 07/14] undo tmpdir change --- packages/e2e-tests/lib/constants.ts | 1 + packages/e2e-tests/lib/runTestApp.ts | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/e2e-tests/lib/constants.ts b/packages/e2e-tests/lib/constants.ts index bdc550009148..fe24b98841fd 100644 --- a/packages/e2e-tests/lib/constants.ts +++ b/packages/e2e-tests/lib/constants.ts @@ -3,3 +3,4 @@ export const DEFAULT_BUILD_TIMEOUT_SECONDS = 60 * 5; export const DEFAULT_TEST_TIMEOUT_SECONDS = 60 * 2; export const VERDACCIO_VERSION = '5.22.1'; export const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages'; +export const TMP_DIR = 'tmp'; diff --git a/packages/e2e-tests/lib/runTestApp.ts b/packages/e2e-tests/lib/runTestApp.ts index 9c91a0f14482..f38740fe5dd0 100644 --- a/packages/e2e-tests/lib/runTestApp.ts +++ b/packages/e2e-tests/lib/runTestApp.ts @@ -1,10 +1,10 @@ /* eslint-disable no-console */ import * as fs from 'fs-extra'; -import * as os from 'os'; import * as path from 'path'; import { buildApp } from './buildApp'; +import { TMP_DIR } from './constants'; import { testApp } from './testApp'; import type { Env, RecipeInstance, RecipeTestResult } from './types'; @@ -18,7 +18,7 @@ export async function buildAndTestApp( const { recipe, port } = recipeInstance; const recipeDirname = path.dirname(recipe.path); - const targetDir = path.join(os.tmpdir(), `${recipe.testApplicationName}-${tmpDirCount++}`); + const targetDir = path.join(TMP_DIR, `${recipe.testApplicationName}-${tmpDirCount++}`); await fs.copy(recipeDirname, targetDir); From 45c67c808d6a5d34589758990567eab2cf377bdd Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 29 Mar 2023 14:11:12 +0000 Subject: [PATCH 08/14] fix test --- .../tests/fixtures/ReplayRecordingData.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/tests/fixtures/ReplayRecordingData.ts b/packages/e2e-tests/test-applications/standard-frontend-react/tests/fixtures/ReplayRecordingData.ts index 6648bfd059a4..da5ba529edd7 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react/tests/fixtures/ReplayRecordingData.ts +++ b/packages/e2e-tests/test-applications/standard-frontend-react/tests/fixtures/ReplayRecordingData.ts @@ -2,7 +2,11 @@ import { expect } from '@playwright/test'; export const ReplayRecordingData = [ [ - { type: 4, data: { href: 'http://localhost:3000/', width: 1280, height: 720 }, timestamp: expect.any(Number) }, + { + type: 4, + data: { href: expect.stringMatching(/http:\/\/localhost:\d+\//), width: 1280, height: 720 }, + timestamp: expect.any(Number), + }, { type: 2, data: { @@ -105,7 +109,7 @@ export const ReplayRecordingData = [ node: { type: 2, tagName: 'a', - attributes: { id: 'navigation', href: 'http://localhost:3000/user/5' }, + attributes: { id: 'navigation', href: expect.stringMatching(/http:\/\/localhost:\d+\/user\/5/) }, childNodes: [], id: 14, }, @@ -140,7 +144,7 @@ export const ReplayRecordingData = [ tag: 'performanceSpan', payload: { op: 'navigation.navigate', - description: 'http://localhost:3000/', + description: expect.stringMatching(/http:\/\/localhost:\d+\//), startTimestamp: expect.any(Number), endTimestamp: expect.any(Number), data: { @@ -166,7 +170,7 @@ export const ReplayRecordingData = [ tag: 'performanceSpan', payload: { op: 'resource.script', - description: expect.stringMatching(/http:\/\/localhost:3000\/static\/js\/main.(\w+).js/), + description: expect.stringMatching(/http:\/\/localhost:\d+\/static\/js\/main.(\w+).js/), startTimestamp: expect.any(Number), endTimestamp: expect.any(Number), data: { From ecd3eb81e756ce074e12355a78896fd4cd26d945 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 30 Mar 2023 05:14:51 +0000 Subject: [PATCH 09/14] Dont cache in verdaccio --- packages/e2e-tests/verdaccio-config/config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/e2e-tests/verdaccio-config/config.yaml b/packages/e2e-tests/verdaccio-config/config.yaml index 7c5cd16df28d..8619f3932da1 100644 --- a/packages/e2e-tests/verdaccio-config/config.yaml +++ b/packages/e2e-tests/verdaccio-config/config.yaml @@ -21,6 +21,7 @@ auth: uplinks: npmjs: url: https://registry.npmjs.org/ + cache: false # Learn how to protect your packages # https://verdaccio.org/docs/protect-your-dependencies/ @@ -196,7 +197,7 @@ packages: # A value of 0 makes the http server behave similarly to Node.js versions prior to 8.0.0, which did not have a keep-alive timeout. # WORKAROUND: Through given configuration you can workaround following issue https://github.com/verdaccio/verdaccio/issues/301. Set to 0 in case 60 is not enough. server: - keepAliveTimeout: 60 + keepAliveTimeout: 0 middlewares: audit: From 42b3e961cc4b6744b71fbd3040d235496e0ef276 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 30 Mar 2023 07:11:32 +0000 Subject: [PATCH 10/14] fix new suite --- .../standard-frontend-react-tracing-import/package.json | 4 ++++ .../playwright.config.ts | 4 +--- .../standard-frontend-react-tracing-import/test-recipe.json | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/package.json b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/package.json index cead7143f1ed..f1e97a01925a 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/package.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/package.json @@ -46,5 +46,9 @@ "@playwright/test": "1.26.1", "axios": "1.1.2", "serve": "14.0.1" + }, + "volta": { + "node": "16.19.0", + "yarn": "1.22.19" } } diff --git a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/playwright.config.ts b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/playwright.config.ts index 9249280b4c34..1a6a49424672 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/playwright.config.ts +++ b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/playwright.config.ts @@ -29,8 +29,6 @@ const config: PlaywrightTestConfig = { use: { /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ actionTimeout: 0, - /* Base URL to use in actions like `await page.goto('/')`. */ - // baseURL: 'http://localhost:3000', /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: 'on-first-retry', @@ -62,7 +60,7 @@ const config: PlaywrightTestConfig = { /* Run your local dev server before starting the tests */ webServer: { command: 'yarn start', - port: 3000, + port: process.env.PORT ? parseInt(process.env.PORT) : 3000, }, }; diff --git a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json index 1a8f03c6bbae..864736daaad8 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "standard-frontend-react-tracing-import", - "buildCommand": "yarn install --pure-lockfile && npx playwright install && yarn build", + "buildCommand": "yarn install && npx playwright install && yarn build", "tests": [ { "testName": "Playwright tests", From f99ee18973146e1046c91cfc55c58cfc4e5e574f Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 30 Mar 2023 07:20:45 +0000 Subject: [PATCH 11/14] hmmm --- .github/workflows/build.yml | 1 + packages/e2e-tests/verdaccio-config/config.yaml | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c23196e5ae80..1d04bb34f0bf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -692,6 +692,7 @@ jobs: E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests' + YARN_CACHE_FOLDER: '/var/tmp/e2e-test-yarn-cache' run: | cd packages/e2e-tests yarn test:e2e diff --git a/packages/e2e-tests/verdaccio-config/config.yaml b/packages/e2e-tests/verdaccio-config/config.yaml index 8619f3932da1..7c5cd16df28d 100644 --- a/packages/e2e-tests/verdaccio-config/config.yaml +++ b/packages/e2e-tests/verdaccio-config/config.yaml @@ -21,7 +21,6 @@ auth: uplinks: npmjs: url: https://registry.npmjs.org/ - cache: false # Learn how to protect your packages # https://verdaccio.org/docs/protect-your-dependencies/ @@ -197,7 +196,7 @@ packages: # A value of 0 makes the http server behave similarly to Node.js versions prior to 8.0.0, which did not have a keep-alive timeout. # WORKAROUND: Through given configuration you can workaround following issue https://github.com/verdaccio/verdaccio/issues/301. Set to 0 in case 60 is not enough. server: - keepAliveTimeout: 0 + keepAliveTimeout: 60 middlewares: audit: From e0151ec910245fe59e0f94c4ed3daad7398badf1 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 30 Mar 2023 08:06:53 +0000 Subject: [PATCH 12/14] hmmm v2 --- .github/workflows/build.yml | 1 - packages/e2e-tests/README.md | 2 +- .../test-applications/create-next-app/test-recipe.json | 2 +- .../test-applications/create-react-app/test-recipe.json | 2 +- .../e2e-tests/test-applications/nextjs-app-dir/test-recipe.json | 2 +- .../test-applications/node-express-app/test-recipe.json | 2 +- .../standard-frontend-react-tracing-import/test-recipe.json | 2 +- .../test-applications/standard-frontend-react/test-recipe.json | 2 +- 8 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1d04bb34f0bf..c23196e5ae80 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -692,7 +692,6 @@ jobs: E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests' - YARN_CACHE_FOLDER: '/var/tmp/e2e-test-yarn-cache' run: | cd packages/e2e-tests yarn test:e2e diff --git a/packages/e2e-tests/README.md b/packages/e2e-tests/README.md index 2a30943f52f6..d8227481a25a 100644 --- a/packages/e2e-tests/README.md +++ b/packages/e2e-tests/README.md @@ -54,7 +54,7 @@ To get you started with the recipe, you can copy the following into `test-recipe { "$schema": "../../test-recipe-schema.json", "testApplicationName": "My New Test Application", - "buildCommand": "yarn install --pure-lockfile", + "buildCommand": "yarn install --network-concurrency 1", "tests": [ { "testName": "My new test", diff --git a/packages/e2e-tests/test-applications/create-next-app/test-recipe.json b/packages/e2e-tests/test-applications/create-next-app/test-recipe.json index 38b4bbc06af0..141f9e1489c1 100644 --- a/packages/e2e-tests/test-applications/create-next-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/create-next-app/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "create-next-app", - "buildCommand": "yarn install && npx playwright install && yarn build", + "buildCommand": "yarn install --network-concurrency 1 && npx playwright install && yarn build", "tests": [ { "testName": "Playwright tests - Prod Mode", diff --git a/packages/e2e-tests/test-applications/create-react-app/test-recipe.json b/packages/e2e-tests/test-applications/create-react-app/test-recipe.json index ee9c8e1dc40c..3f3c496c4857 100644 --- a/packages/e2e-tests/test-applications/create-react-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/create-react-app/test-recipe.json @@ -1,6 +1,6 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "create-react-app", - "buildCommand": "yarn install && yarn build", + "buildCommand": "yarn install --network-concurrency 1 && yarn build", "tests": [] } diff --git a/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json b/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json index b711dd6e922c..4f6290d444d0 100644 --- a/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json +++ b/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "nextjs-13-app-dir", - "buildCommand": "yarn install && npx playwright install && yarn build", + "buildCommand": "yarn install --network-concurrency 1 && npx playwright install && yarn build", "buildAssertionCommand": "yarn ts-node --script-mode assert-build.ts", "tests": [ { diff --git a/packages/e2e-tests/test-applications/node-express-app/test-recipe.json b/packages/e2e-tests/test-applications/node-express-app/test-recipe.json index 039049258171..4859973cd360 100644 --- a/packages/e2e-tests/test-applications/node-express-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/node-express-app/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "node-express-app", - "buildCommand": "yarn install && yarn build", + "buildCommand": "yarn install --network-concurrency 1 && yarn build", "tests": [ { "testName": "Test express server", diff --git a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json index 864736daaad8..fce2379a280f 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "standard-frontend-react-tracing-import", - "buildCommand": "yarn install && npx playwright install && yarn build", + "buildCommand": "yarn install --network-concurrency 1 && npx playwright install && yarn build", "tests": [ { "testName": "Playwright tests", diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json index 76916d74d280..207dd5409b50 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "standard-frontend-react", - "buildCommand": "yarn install && npx playwright install && yarn build", + "buildCommand": "yarn install --network-concurrency 1 && npx playwright install && yarn build", "tests": [ { "testName": "Playwright tests", From 45c023a1e99df4090e0c98162ae6529bbffdc26c Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 30 Mar 2023 09:37:59 +0000 Subject: [PATCH 13/14] network mutex --- .github/workflows/build.yml | 2 ++ packages/e2e-tests/README.md | 2 +- .../e2e-tests/test-applications/create-next-app/package.json | 1 + .../test-applications/create-next-app/test-recipe.json | 2 +- .../test-applications/create-react-app/test-recipe.json | 2 +- .../e2e-tests/test-applications/nextjs-app-dir/package.json | 1 + .../e2e-tests/test-applications/nextjs-app-dir/test-recipe.json | 2 +- .../e2e-tests/test-applications/node-express-app/package.json | 1 + .../test-applications/node-express-app/test-recipe.json | 2 +- .../standard-frontend-react-tracing-import/test-recipe.json | 2 +- .../test-applications/standard-frontend-react/test-recipe.json | 2 +- 11 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c23196e5ae80..5c23a0da850c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -692,8 +692,10 @@ jobs: E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests' + # yarn network mutex is necessary because we run multiple yarn install commands at the same time and they might conflict run: | cd packages/e2e-tests + yarn config set -- --mutex network yarn test:e2e job_required_tests: diff --git a/packages/e2e-tests/README.md b/packages/e2e-tests/README.md index d8227481a25a..2d8db0f5b41f 100644 --- a/packages/e2e-tests/README.md +++ b/packages/e2e-tests/README.md @@ -54,7 +54,7 @@ To get you started with the recipe, you can copy the following into `test-recipe { "$schema": "../../test-recipe-schema.json", "testApplicationName": "My New Test Application", - "buildCommand": "yarn install --network-concurrency 1", + "buildCommand": "yarn install", "tests": [ { "testName": "My new test", diff --git a/packages/e2e-tests/test-applications/create-next-app/package.json b/packages/e2e-tests/test-applications/create-next-app/package.json index dee9275c2bdc..d7fd81efd220 100644 --- a/packages/e2e-tests/test-applications/create-next-app/package.json +++ b/packages/e2e-tests/test-applications/create-next-app/package.json @@ -23,6 +23,7 @@ "typescript": "4.9.4" }, "devDependencies": { + "axios": "1.1.2", "@playwright/test": "^1.27.1" }, "volta": { diff --git a/packages/e2e-tests/test-applications/create-next-app/test-recipe.json b/packages/e2e-tests/test-applications/create-next-app/test-recipe.json index 141f9e1489c1..38b4bbc06af0 100644 --- a/packages/e2e-tests/test-applications/create-next-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/create-next-app/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "create-next-app", - "buildCommand": "yarn install --network-concurrency 1 && npx playwright install && yarn build", + "buildCommand": "yarn install && npx playwright install && yarn build", "tests": [ { "testName": "Playwright tests - Prod Mode", diff --git a/packages/e2e-tests/test-applications/create-react-app/test-recipe.json b/packages/e2e-tests/test-applications/create-react-app/test-recipe.json index 3f3c496c4857..ee9c8e1dc40c 100644 --- a/packages/e2e-tests/test-applications/create-react-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/create-react-app/test-recipe.json @@ -1,6 +1,6 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "create-react-app", - "buildCommand": "yarn install --network-concurrency 1 && yarn build", + "buildCommand": "yarn install && yarn build", "tests": [] } diff --git a/packages/e2e-tests/test-applications/nextjs-app-dir/package.json b/packages/e2e-tests/test-applications/nextjs-app-dir/package.json index 064e5d396d1f..eff411391d04 100644 --- a/packages/e2e-tests/test-applications/nextjs-app-dir/package.json +++ b/packages/e2e-tests/test-applications/nextjs-app-dir/package.json @@ -22,6 +22,7 @@ "typescript": "4.9.4" }, "devDependencies": { + "axios": "1.1.2", "ts-node": "10.9.1", "@playwright/test": "^1.27.1" }, diff --git a/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json b/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json index 4f6290d444d0..b711dd6e922c 100644 --- a/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json +++ b/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "nextjs-13-app-dir", - "buildCommand": "yarn install --network-concurrency 1 && npx playwright install && yarn build", + "buildCommand": "yarn install && npx playwright install && yarn build", "buildAssertionCommand": "yarn ts-node --script-mode assert-build.ts", "tests": [ { diff --git a/packages/e2e-tests/test-applications/node-express-app/package.json b/packages/e2e-tests/test-applications/node-express-app/package.json index 3bcdb79df6ba..605a3b8d65b0 100644 --- a/packages/e2e-tests/test-applications/node-express-app/package.json +++ b/packages/e2e-tests/test-applications/node-express-app/package.json @@ -18,6 +18,7 @@ "typescript": "4.9.5" }, "devDependencies": { + "axios": "1.1.2", "@playwright/test": "^1.27.1" }, "volta": { diff --git a/packages/e2e-tests/test-applications/node-express-app/test-recipe.json b/packages/e2e-tests/test-applications/node-express-app/test-recipe.json index 4859973cd360..039049258171 100644 --- a/packages/e2e-tests/test-applications/node-express-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/node-express-app/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "node-express-app", - "buildCommand": "yarn install --network-concurrency 1 && yarn build", + "buildCommand": "yarn install && yarn build", "tests": [ { "testName": "Test express server", diff --git a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json index fce2379a280f..864736daaad8 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "standard-frontend-react-tracing-import", - "buildCommand": "yarn install --network-concurrency 1 && npx playwright install && yarn build", + "buildCommand": "yarn install && npx playwright install && yarn build", "tests": [ { "testName": "Playwright tests", diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json index 207dd5409b50..76916d74d280 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "standard-frontend-react", - "buildCommand": "yarn install --network-concurrency 1 && npx playwright install && yarn build", + "buildCommand": "yarn install && npx playwright install && yarn build", "tests": [ { "testName": "Playwright tests", From fba6c4542a23fa3bc71ffaa47578d0d5b3d0af51 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 30 Mar 2023 10:43:49 +0000 Subject: [PATCH 14/14] Revert "network mutex" This reverts commit 45c023a1e99df4090e0c98162ae6529bbffdc26c. --- .github/workflows/build.yml | 2 -- packages/e2e-tests/README.md | 2 +- .../e2e-tests/test-applications/create-next-app/package.json | 1 - .../test-applications/create-next-app/test-recipe.json | 2 +- .../test-applications/create-react-app/test-recipe.json | 2 +- .../e2e-tests/test-applications/nextjs-app-dir/package.json | 1 - .../e2e-tests/test-applications/nextjs-app-dir/test-recipe.json | 2 +- .../e2e-tests/test-applications/node-express-app/package.json | 1 - .../test-applications/node-express-app/test-recipe.json | 2 +- .../standard-frontend-react-tracing-import/test-recipe.json | 2 +- .../test-applications/standard-frontend-react/test-recipe.json | 2 +- 11 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c23a0da850c..c23196e5ae80 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -692,10 +692,8 @@ jobs: E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests' - # yarn network mutex is necessary because we run multiple yarn install commands at the same time and they might conflict run: | cd packages/e2e-tests - yarn config set -- --mutex network yarn test:e2e job_required_tests: diff --git a/packages/e2e-tests/README.md b/packages/e2e-tests/README.md index 2d8db0f5b41f..d8227481a25a 100644 --- a/packages/e2e-tests/README.md +++ b/packages/e2e-tests/README.md @@ -54,7 +54,7 @@ To get you started with the recipe, you can copy the following into `test-recipe { "$schema": "../../test-recipe-schema.json", "testApplicationName": "My New Test Application", - "buildCommand": "yarn install", + "buildCommand": "yarn install --network-concurrency 1", "tests": [ { "testName": "My new test", diff --git a/packages/e2e-tests/test-applications/create-next-app/package.json b/packages/e2e-tests/test-applications/create-next-app/package.json index d7fd81efd220..dee9275c2bdc 100644 --- a/packages/e2e-tests/test-applications/create-next-app/package.json +++ b/packages/e2e-tests/test-applications/create-next-app/package.json @@ -23,7 +23,6 @@ "typescript": "4.9.4" }, "devDependencies": { - "axios": "1.1.2", "@playwright/test": "^1.27.1" }, "volta": { diff --git a/packages/e2e-tests/test-applications/create-next-app/test-recipe.json b/packages/e2e-tests/test-applications/create-next-app/test-recipe.json index 38b4bbc06af0..141f9e1489c1 100644 --- a/packages/e2e-tests/test-applications/create-next-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/create-next-app/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "create-next-app", - "buildCommand": "yarn install && npx playwright install && yarn build", + "buildCommand": "yarn install --network-concurrency 1 && npx playwright install && yarn build", "tests": [ { "testName": "Playwright tests - Prod Mode", diff --git a/packages/e2e-tests/test-applications/create-react-app/test-recipe.json b/packages/e2e-tests/test-applications/create-react-app/test-recipe.json index ee9c8e1dc40c..3f3c496c4857 100644 --- a/packages/e2e-tests/test-applications/create-react-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/create-react-app/test-recipe.json @@ -1,6 +1,6 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "create-react-app", - "buildCommand": "yarn install && yarn build", + "buildCommand": "yarn install --network-concurrency 1 && yarn build", "tests": [] } diff --git a/packages/e2e-tests/test-applications/nextjs-app-dir/package.json b/packages/e2e-tests/test-applications/nextjs-app-dir/package.json index eff411391d04..064e5d396d1f 100644 --- a/packages/e2e-tests/test-applications/nextjs-app-dir/package.json +++ b/packages/e2e-tests/test-applications/nextjs-app-dir/package.json @@ -22,7 +22,6 @@ "typescript": "4.9.4" }, "devDependencies": { - "axios": "1.1.2", "ts-node": "10.9.1", "@playwright/test": "^1.27.1" }, diff --git a/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json b/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json index b711dd6e922c..4f6290d444d0 100644 --- a/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json +++ b/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "nextjs-13-app-dir", - "buildCommand": "yarn install && npx playwright install && yarn build", + "buildCommand": "yarn install --network-concurrency 1 && npx playwright install && yarn build", "buildAssertionCommand": "yarn ts-node --script-mode assert-build.ts", "tests": [ { diff --git a/packages/e2e-tests/test-applications/node-express-app/package.json b/packages/e2e-tests/test-applications/node-express-app/package.json index 605a3b8d65b0..3bcdb79df6ba 100644 --- a/packages/e2e-tests/test-applications/node-express-app/package.json +++ b/packages/e2e-tests/test-applications/node-express-app/package.json @@ -18,7 +18,6 @@ "typescript": "4.9.5" }, "devDependencies": { - "axios": "1.1.2", "@playwright/test": "^1.27.1" }, "volta": { diff --git a/packages/e2e-tests/test-applications/node-express-app/test-recipe.json b/packages/e2e-tests/test-applications/node-express-app/test-recipe.json index 039049258171..4859973cd360 100644 --- a/packages/e2e-tests/test-applications/node-express-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/node-express-app/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "node-express-app", - "buildCommand": "yarn install && yarn build", + "buildCommand": "yarn install --network-concurrency 1 && yarn build", "tests": [ { "testName": "Test express server", diff --git a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json index 864736daaad8..fce2379a280f 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "standard-frontend-react-tracing-import", - "buildCommand": "yarn install && npx playwright install && yarn build", + "buildCommand": "yarn install --network-concurrency 1 && npx playwright install && yarn build", "tests": [ { "testName": "Playwright tests", diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json index 76916d74d280..207dd5409b50 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "standard-frontend-react", - "buildCommand": "yarn install && npx playwright install && yarn build", + "buildCommand": "yarn install --network-concurrency 1 && npx playwright install && yarn build", "tests": [ { "testName": "Playwright tests",