From 14af937e2b52f9a0dd257b955afd03b8bb05eec5 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 21 Apr 2023 12:05:00 +0000 Subject: [PATCH 1/8] test(e2e): Use pnpm for e2e tests --- .github/workflows/build.yml | 3 ++ .github/workflows/canary.yml | 3 ++ packages/e2e-tests/lib/runAllTestApps.ts | 2 +- .../create-next-app/test-recipe.json | 2 +- .../create-react-app/test-recipe.json | 2 +- .../nextjs-app-dir/test-recipe.json | 2 +- .../nextjs-edge/test-recipe.json | 36 +++++++++++++++++++ .../node-express-app/test-recipe.json | 2 +- .../test-recipe.json | 2 +- .../standard-frontend-react/test-recipe.json | 2 +- 10 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 packages/e2e-tests/test-applications/nextjs-edge/test-recipe.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index acf25d7b5cc0..bac522026228 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -768,6 +768,9 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ env.HEAD_COMMIT }} + - uses: pnpm/action-setup@v2 + with: + version: 7 - name: Set up Node uses: actions/setup-node@v3 with: diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index d3968a0dfb75..cdb19c3dc857 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -26,6 +26,9 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ env.HEAD_COMMIT }} + - uses: pnpm/action-setup@v2 + with: + version: 7 - name: Set up Node uses: actions/setup-node@v3 with: diff --git a/packages/e2e-tests/lib/runAllTestApps.ts b/packages/e2e-tests/lib/runAllTestApps.ts index 4f2e405d58aa..d0cef6f51def 100644 --- a/packages/e2e-tests/lib/runAllTestApps.ts +++ b/packages/e2e-tests/lib/runAllTestApps.ts @@ -11,7 +11,7 @@ export async function runAllTestApps( recipePaths: string[], envVarsToInject: Record, ): Promise { - const maxParallel = process.env.CI ? 1 : 1; // For now we are disabling parallel execution because it was causing problems (runners were too slow and timeouts happened) + const maxParallel = process.env.CI ? 2 : 4; // For now we are disabling parallel execution because it was causing problems (runners were too slow and timeouts happened) const recipeInstances = constructRecipeInstances(recipePaths); diff --git a/packages/e2e-tests/test-applications/create-next-app/test-recipe.json b/packages/e2e-tests/test-applications/create-next-app/test-recipe.json index 38b4bbc06af0..546f98446933 100644 --- a/packages/e2e-tests/test-applications/create-next-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/create-next-app/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "create-next-app", - "buildCommand": "yarn install && npx playwright install && yarn build", + "buildCommand": "pnpm install && npx playwright install && pnpm build", "tests": [ { "testName": "Playwright tests - Prod Mode", diff --git a/packages/e2e-tests/test-applications/create-react-app/test-recipe.json b/packages/e2e-tests/test-applications/create-react-app/test-recipe.json index ee9c8e1dc40c..1cd9ea8a4660 100644 --- a/packages/e2e-tests/test-applications/create-react-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/create-react-app/test-recipe.json @@ -1,6 +1,6 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "create-react-app", - "buildCommand": "yarn install && yarn build", + "buildCommand": "pnpm install && npx playwright install && pnpm build", "tests": [] } diff --git a/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json b/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json index b711dd6e922c..3fe2f95b324c 100644 --- a/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json +++ b/packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "nextjs-13-app-dir", - "buildCommand": "yarn install && npx playwright install && yarn build", + "buildCommand": "pnpm install && npx playwright install && pnpm build", "buildAssertionCommand": "yarn ts-node --script-mode assert-build.ts", "tests": [ { diff --git a/packages/e2e-tests/test-applications/nextjs-edge/test-recipe.json b/packages/e2e-tests/test-applications/nextjs-edge/test-recipe.json new file mode 100644 index 000000000000..3f0bb12c14dc --- /dev/null +++ b/packages/e2e-tests/test-applications/nextjs-edge/test-recipe.json @@ -0,0 +1,36 @@ +{ + "$schema": "../../test-recipe-schema.json", + "testApplicationName": "nextjs-edge", + "buildCommand": "pnpm install && npx playwright install && pnpm build", + "buildAssertionCommand": "yarn ts-node --script-mode assert-build.ts", + "tests": [ + { + "testName": "Test", + "testCommand": "yarn test" + } + ], + "versions": [ + { + "dependencyOverrides": { + "next": "^12" + } + }, + { + "dependencyOverrides": { + "next": "^13" + } + } + ], + "canaryVersions": [ + { + "dependencyOverrides": { + "next": "latest" + } + }, + { + "dependencyOverrides": { + "next": "canary" + } + } + ] +} diff --git a/packages/e2e-tests/test-applications/node-express-app/test-recipe.json b/packages/e2e-tests/test-applications/node-express-app/test-recipe.json index 039049258171..c899ef8cfe52 100644 --- a/packages/e2e-tests/test-applications/node-express-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/node-express-app/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "node-express-app", - "buildCommand": "yarn install && yarn build", + "buildCommand": "pnpm install && pnpm build", "tests": [ { "testName": "Test express server", diff --git a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json index 864736daaad8..98d86d555161 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "standard-frontend-react-tracing-import", - "buildCommand": "yarn install && npx playwright install && yarn build", + "buildCommand": "pnpm install && npx playwright install && pnpm build", "tests": [ { "testName": "Playwright tests", diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json index 76916d74d280..f3a16c838d13 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json @@ -1,7 +1,7 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "standard-frontend-react", - "buildCommand": "yarn install && npx playwright install && yarn build", + "buildCommand": "pnpm install && npx playwright install && pnpm build", "tests": [ { "testName": "Playwright tests", From 7eecf5e52394db6ab0ce93196e94e5564a8b2992 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 21 Apr 2023 12:05:32 +0000 Subject: [PATCH 2/8] . --- .../e2e-tests/test-applications/create-react-app/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/e2e-tests/test-applications/create-react-app/tsconfig.json b/packages/e2e-tests/test-applications/create-react-app/tsconfig.json index 1d693b2b06ac..30153e418edf 100644 --- a/packages/e2e-tests/test-applications/create-react-app/tsconfig.json +++ b/packages/e2e-tests/test-applications/create-react-app/tsconfig.json @@ -16,5 +16,6 @@ "noEmit": true, "jsx": "react" }, - "include": ["src"] + "include": ["src"], + "exclude": ["**.test.tsx"] } From 9de484b3b9eda0eb886f12247106dba731b29cf0 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 21 Apr 2023 12:06:45 +0000 Subject: [PATCH 3/8] . --- packages/e2e-tests/lib/buildApp.ts | 4 ---- packages/e2e-tests/lib/runAllTestApps.ts | 2 -- 2 files changed, 6 deletions(-) diff --git a/packages/e2e-tests/lib/buildApp.ts b/packages/e2e-tests/lib/buildApp.ts index 15b9e0497cdc..2d53488b1785 100644 --- a/packages/e2e-tests/lib/buildApp.ts +++ b/packages/e2e-tests/lib/buildApp.ts @@ -29,13 +29,9 @@ export async function buildApp(appDir: string, recipeInstance: RecipeInstance, e if (recipe.buildCommand) { console.log(`Running build command for test application "${label}"`); - fs.mkdirSync(path.join(os.tmpdir(), 'e2e-test-yarn-caches'), { recursive: true }); - const tempYarnCache = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-test-yarn-caches', 'cache-')); - const env = { ...process.env, ...envVars, - YARN_CACHE_FOLDER: tempYarnCache, // Use a separate yarn cache for each build commmand because multiple yarn commands running at the same time may corrupt the cache }; const buildResult = await spawnAsync(recipe.buildCommand, { diff --git a/packages/e2e-tests/lib/runAllTestApps.ts b/packages/e2e-tests/lib/runAllTestApps.ts index d0cef6f51def..844420e75298 100644 --- a/packages/e2e-tests/lib/runAllTestApps.ts +++ b/packages/e2e-tests/lib/runAllTestApps.ts @@ -37,8 +37,6 @@ export async function runAllTestApps( const failed = results.filter(result => result.buildFailed || result.testFailed); - fs.rmSync(path.join(os.tmpdir(), 'e2e-test-yarn-caches'), { force: true, recursive: true }); - if (failed.length) { console.log(`${failed.length} test(s) failed.`); process.exit(1); From f05c6a027ea6b6a71becc13ff56b7d8ac06b3f0b Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 21 Apr 2023 12:16:21 +0000 Subject: [PATCH 4/8] . --- packages/e2e-tests/lib/constants.ts | 1 - packages/e2e-tests/lib/runTestApp.ts | 17 +++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/packages/e2e-tests/lib/constants.ts b/packages/e2e-tests/lib/constants.ts index fe24b98841fd..bdc550009148 100644 --- a/packages/e2e-tests/lib/constants.ts +++ b/packages/e2e-tests/lib/constants.ts @@ -3,4 +3,3 @@ export const DEFAULT_BUILD_TIMEOUT_SECONDS = 60 * 5; export const DEFAULT_TEST_TIMEOUT_SECONDS = 60 * 2; export const VERDACCIO_VERSION = '5.22.1'; export const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages'; -export const TMP_DIR = 'tmp'; diff --git a/packages/e2e-tests/lib/runTestApp.ts b/packages/e2e-tests/lib/runTestApp.ts index 22dbdb9c5693..b07296a6d426 100644 --- a/packages/e2e-tests/lib/runTestApp.ts +++ b/packages/e2e-tests/lib/runTestApp.ts @@ -1,15 +1,12 @@ -/* eslint-disable no-console */ - -import * as fs from 'fs-extra'; +import * as fs from 'fs'; +import * as fsExtra from 'fs-extra'; +import * as os from 'os'; import * as path from 'path'; import { buildApp } from './buildApp'; -import { TMP_DIR } from './constants'; import { testApp } from './testApp'; import type { Env, RecipeInstance, RecipeTestResult } from './types'; -let tmpDirCount = 0; - // This should never throw, we always return a result here export async function buildAndTestApp( recipeInstance: RecipeInstance, @@ -18,9 +15,9 @@ export async function buildAndTestApp( const { recipe, portModulo, portGap } = recipeInstance; const recipeDirname = path.dirname(recipe.path); - const targetDir = path.join(TMP_DIR, `${recipe.testApplicationName}-${tmpDirCount++}`); + const targetDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'e2e-test-temp-app-dir-')); - await fs.copy(recipeDirname, targetDir); + await fsExtra.copy(recipeDirname, targetDir); const env: Env = { ...envVarsToInject, @@ -31,7 +28,7 @@ export async function buildAndTestApp( try { await buildApp(targetDir, recipeInstance, env); } catch (error) { - await fs.remove(targetDir); + await fsExtra.remove(targetDir); return { ...recipeInstance, @@ -45,7 +42,7 @@ export async function buildAndTestApp( const results = await testApp(targetDir, recipeInstance, env); // Cleanup - await fs.remove(targetDir); + await fsExtra.remove(targetDir); return { ...recipeInstance, From 35dee0cde260c890285186dd8a5b2d37b6cf573e Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 21 Apr 2023 12:43:11 +0000 Subject: [PATCH 5/8] . --- packages/e2e-tests/README.md | 4 +-- packages/e2e-tests/lib/buildApp.ts | 1 - packages/e2e-tests/lib/runAllTestApps.ts | 3 -- packages/e2e-tests/lib/runTestApp.ts | 29 ++++++++++--------- .../create-react-app/src/App.test.tsx | 9 ------ .../create-react-app/tsconfig.json | 2 +- 6 files changed, 19 insertions(+), 29 deletions(-) delete mode 100644 packages/e2e-tests/test-applications/create-react-app/src/App.test.tsx diff --git a/packages/e2e-tests/README.md b/packages/e2e-tests/README.md index 2d8db0f5b41f..316d03997483 100644 --- a/packages/e2e-tests/README.md +++ b/packages/e2e-tests/README.md @@ -54,11 +54,11 @@ To get you started with the recipe, you can copy the following into `test-recipe { "$schema": "../../test-recipe-schema.json", "testApplicationName": "My New Test Application", - "buildCommand": "yarn install", + "buildCommand": "pnpm install", "tests": [ { "testName": "My new test", - "testCommand": "yarn test", + "testCommand": "pnpm test", "timeoutSeconds": 60 } ] diff --git a/packages/e2e-tests/lib/buildApp.ts b/packages/e2e-tests/lib/buildApp.ts index 2d53488b1785..ab8f95668d6a 100644 --- a/packages/e2e-tests/lib/buildApp.ts +++ b/packages/e2e-tests/lib/buildApp.ts @@ -1,7 +1,6 @@ /* eslint-disable no-console */ import * as fs from 'fs-extra'; -import * as os from 'os'; import * as path from 'path'; import { DEFAULT_BUILD_TIMEOUT_SECONDS } from './constants'; diff --git a/packages/e2e-tests/lib/runAllTestApps.ts b/packages/e2e-tests/lib/runAllTestApps.ts index 844420e75298..c48ff79aaa7f 100644 --- a/packages/e2e-tests/lib/runAllTestApps.ts +++ b/packages/e2e-tests/lib/runAllTestApps.ts @@ -1,7 +1,4 @@ /* eslint-disable no-console */ -import * as fs from 'fs'; -import * as os from 'os'; -import * as path from 'path'; import { constructRecipeInstances } from './constructRecipeInstances'; import { buildAndTestApp } from './runTestApp'; diff --git a/packages/e2e-tests/lib/runTestApp.ts b/packages/e2e-tests/lib/runTestApp.ts index b07296a6d426..6a5366dc5b05 100644 --- a/packages/e2e-tests/lib/runTestApp.ts +++ b/packages/e2e-tests/lib/runTestApp.ts @@ -1,6 +1,5 @@ import * as fs from 'fs'; import * as fsExtra from 'fs-extra'; -import * as os from 'os'; import * as path from 'path'; import { buildApp } from './buildApp'; @@ -15,7 +14,9 @@ export async function buildAndTestApp( const { recipe, portModulo, portGap } = recipeInstance; const recipeDirname = path.dirname(recipe.path); - const targetDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'e2e-test-temp-app-dir-')); + const tmpFolder = path.join(__dirname, '..', 'tmp'); + await fs.promises.mkdir(tmpFolder, { recursive: true }); + const targetDir = await fs.promises.mkdtemp(path.join(tmpFolder, 'tmp-app-')); await fsExtra.copy(recipeDirname, targetDir); @@ -39,15 +40,17 @@ export async function buildAndTestApp( } // This cannot throw, we always return a result here - const results = await testApp(targetDir, recipeInstance, env); - - // Cleanup - await fsExtra.remove(targetDir); - - return { - ...recipeInstance, - buildFailed: false, - testFailed: results.some(result => result.result !== 'PASS'), - tests: results, - }; + return testApp(targetDir, recipeInstance, env) + .finally(() => { + // Cleanup + void fsExtra.remove(targetDir); + }) + .then(results => { + return { + ...recipeInstance, + buildFailed: false, + testFailed: results.some(result => result.result !== 'PASS'), + tests: results, + }; + }); } diff --git a/packages/e2e-tests/test-applications/create-react-app/src/App.test.tsx b/packages/e2e-tests/test-applications/create-react-app/src/App.test.tsx deleted file mode 100644 index 2a68616d9846..000000000000 --- a/packages/e2e-tests/test-applications/create-react-app/src/App.test.tsx +++ /dev/null @@ -1,9 +0,0 @@ -import React from 'react'; -import { render, screen } from '@testing-library/react'; -import App from './App'; - -test('renders learn react link', () => { - render(); - const linkElement = screen.getByText(/learn react/i); - expect(linkElement).toBeInTheDocument(); -}); diff --git a/packages/e2e-tests/test-applications/create-react-app/tsconfig.json b/packages/e2e-tests/test-applications/create-react-app/tsconfig.json index 30153e418edf..4bd4dd6a0417 100644 --- a/packages/e2e-tests/test-applications/create-react-app/tsconfig.json +++ b/packages/e2e-tests/test-applications/create-react-app/tsconfig.json @@ -17,5 +17,5 @@ "jsx": "react" }, "include": ["src"], - "exclude": ["**.test.tsx"] + "exclude": ["src/**/*.test.tsx"] } From de345ed8b5794065401b2484b767ec0c5eeeba02 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 21 Apr 2023 12:55:00 +0000 Subject: [PATCH 6/8] . --- .../nextjs-edge/test-recipe.json | 36 ------------------- 1 file changed, 36 deletions(-) delete mode 100644 packages/e2e-tests/test-applications/nextjs-edge/test-recipe.json diff --git a/packages/e2e-tests/test-applications/nextjs-edge/test-recipe.json b/packages/e2e-tests/test-applications/nextjs-edge/test-recipe.json deleted file mode 100644 index 3f0bb12c14dc..000000000000 --- a/packages/e2e-tests/test-applications/nextjs-edge/test-recipe.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "$schema": "../../test-recipe-schema.json", - "testApplicationName": "nextjs-edge", - "buildCommand": "pnpm install && npx playwright install && pnpm build", - "buildAssertionCommand": "yarn ts-node --script-mode assert-build.ts", - "tests": [ - { - "testName": "Test", - "testCommand": "yarn test" - } - ], - "versions": [ - { - "dependencyOverrides": { - "next": "^12" - } - }, - { - "dependencyOverrides": { - "next": "^13" - } - } - ], - "canaryVersions": [ - { - "dependencyOverrides": { - "next": "latest" - } - }, - { - "dependencyOverrides": { - "next": "canary" - } - } - ] -} From 49ff9dc84a8a061b74a3b767bb027528a3e842c6 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 21 Apr 2023 15:34:01 +0200 Subject: [PATCH 7/8] Update packages/e2e-tests/lib/runAllTestApps.ts --- packages/e2e-tests/lib/runAllTestApps.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/e2e-tests/lib/runAllTestApps.ts b/packages/e2e-tests/lib/runAllTestApps.ts index c48ff79aaa7f..ad30c34e4738 100644 --- a/packages/e2e-tests/lib/runAllTestApps.ts +++ b/packages/e2e-tests/lib/runAllTestApps.ts @@ -8,7 +8,7 @@ export async function runAllTestApps( recipePaths: string[], envVarsToInject: Record, ): Promise { - const maxParallel = process.env.CI ? 2 : 4; // For now we are disabling parallel execution because it was causing problems (runners were too slow and timeouts happened) + const maxParallel = process.env.CI ? 3 : 6; const recipeInstances = constructRecipeInstances(recipePaths); From 2a609ab850e02505a87dd6f4cd7016d448e398d1 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Mon, 24 Apr 2023 15:04:55 +0200 Subject: [PATCH 8/8] undo unnecessary change --- .../test-applications/create-react-app/test-recipe.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/e2e-tests/test-applications/create-react-app/test-recipe.json b/packages/e2e-tests/test-applications/create-react-app/test-recipe.json index 1cd9ea8a4660..a2b1bd98ea61 100644 --- a/packages/e2e-tests/test-applications/create-react-app/test-recipe.json +++ b/packages/e2e-tests/test-applications/create-react-app/test-recipe.json @@ -1,6 +1,6 @@ { "$schema": "../../test-recipe-schema.json", "testApplicationName": "create-react-app", - "buildCommand": "pnpm install && npx playwright install && pnpm build", + "buildCommand": "pnpm install && pnpm build", "tests": [] }