diff --git a/README.md b/README.md index 992f36f4c..4ca636d91 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ You don't need to worry about asynchronous nature of NodeJS or about various API - Also plays nice with TypeScript. - Smart locators: use names, labels, matching text, CSS or XPath to locate elements. - 🌐 Interactive debugging shell: pause test at any point and try different commands in a browser. +- ⚡ **Parallel testing** with dynamic test pooling for optimal load balancing and performance. - Easily create tests, pageobjects, stepobjects with CLI generators. ## Installation diff --git a/bin/codecept.js b/bin/codecept.js index 8a5d65b20..87db9c04f 100755 --- a/bin/codecept.js +++ b/bin/codecept.js @@ -196,6 +196,7 @@ program .option('-i, --invert', 'inverts --grep matches') .option('-o, --override [value]', 'override current config options') .option('--suites', 'parallel execution of suites not single tests') + .option('--by ', 'test distribution strategy: "test" (pre-assign individual tests), "suite" (pre-assign test suites), or "pool" (dynamic distribution for optimal load balancing, recommended)') .option(commandFlags.debug.flag, commandFlags.debug.description) .option(commandFlags.verbose.flag, commandFlags.verbose.description) .option('--features', 'run only *.feature files and skip tests') diff --git a/docs/commands.md b/docs/commands.md index c90595641..bc554864c 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -102,12 +102,32 @@ DEBUG=codeceptjs:* npx codeceptjs run ## Run Workers -Run tests in parallel threads. +Run tests in parallel threads. CodeceptJS supports different distribution strategies for optimal performance. -``` +```bash +# Run with 3 workers using default strategy (pre-assign tests) npx codeceptjs run-workers 3 + +# Run with pool mode for dynamic test distribution (recommended) +npx codeceptjs run-workers 3 --by pool + +# Run with suite distribution +npx codeceptjs run-workers 3 --by suite + +# Pool mode with filtering +npx codeceptjs run-workers 4 --by pool --grep "@smoke" ``` +**Test Distribution Strategies:** + +- `--by test` (default): Pre-assigns individual tests to workers +- `--by suite`: Pre-assigns entire test suites to workers +- `--by pool`: Dynamic distribution for optimal load balancing (recommended for best performance) + +The pool mode provides the best load balancing by maintaining tests in a shared pool and distributing them dynamically as workers become available. This prevents workers from sitting idle and ensures optimal CPU utilization, especially when tests have varying execution times. + +See [Parallel Execution](/parallel) documentation for more details. + ## Run Rerun Run tests multiple times to detect and fix flaky tests. diff --git a/docs/parallel.md b/docs/parallel.md index bea099046..2404ceed0 100644 --- a/docs/parallel.md +++ b/docs/parallel.md @@ -32,6 +32,88 @@ By default, the tests are assigned one by one to the available workers this may npx codeceptjs run-workers --suites 2 ``` +### Test Distribution Strategies + +CodeceptJS supports three different strategies for distributing tests across workers: + +#### Default Strategy (`--by test`) +Tests are pre-assigned to workers at startup, distributing them evenly across all workers. Each worker gets a predetermined set of tests to run. + +```sh +npx codeceptjs run-workers 3 --by test +``` + +#### Suite Strategy (`--by suite`) +Test suites are pre-assigned to workers, with all tests in a suite running on the same worker. This ensures better test isolation but may lead to uneven load distribution. + +```sh +npx codeceptjs run-workers 3 --by suite +``` + +#### Pool Strategy (`--by pool`) - **Recommended for optimal performance** +Tests are maintained in a shared pool and distributed dynamically to workers as they become available. This provides the best load balancing and resource utilization. + +```sh +npx codeceptjs run-workers 3 --by pool +``` + +## Dynamic Test Pooling Mode + +The pool mode enables dynamic test distribution for improved worker load balancing. Instead of pre-assigning tests to workers at startup, tests are stored in a shared pool and distributed on-demand as workers become available. + +### Benefits of Pool Mode + +* **Better load balancing**: Workers never sit idle while others are still running long tests +* **Improved performance**: Especially beneficial when tests have varying execution times +* **Optimal resource utilization**: All CPU cores stay busy until the entire test suite is complete +* **Automatic scaling**: Workers continuously process tests until the pool is empty + +### When to Use Pool Mode + +Pool mode is particularly effective in these scenarios: + +* **Uneven test execution times**: When some tests take significantly longer than others +* **Large test suites**: With hundreds or thousands of tests where load balancing matters +* **Mixed test types**: When combining unit tests, integration tests, and end-to-end tests +* **CI/CD pipelines**: For consistent and predictable test execution times + +### Usage Examples + +```bash +# Basic pool mode with 4 workers +npx codeceptjs run-workers 4 --by pool + +# Pool mode with grep filtering +npx codeceptjs run-workers 3 --by pool --grep "@smoke" + +# Pool mode in debug mode +npx codeceptjs run-workers 2 --by pool --debug + +# Pool mode with specific configuration +npx codeceptjs run-workers 3 --by pool -c codecept.conf.js +``` + +### How Pool Mode Works + +1. **Pool Creation**: All tests are collected into a shared pool of test identifiers +2. **Worker Initialization**: The specified number of workers are spawned +3. **Dynamic Assignment**: Workers request tests from the pool when they're ready +4. **Continuous Processing**: Each worker runs one test, then immediately requests the next +5. **Automatic Completion**: Workers exit when the pool is empty and no more tests remain + +### Performance Comparison + +```bash +# Traditional mode - tests pre-assigned, some workers may finish early +npx codeceptjs run-workers 3 --by test # ✓ Good for uniform test times + +# Suite mode - entire suites assigned to workers +npx codeceptjs run-workers 3 --by suite # ✓ Good for test isolation + +# Pool mode - tests distributed dynamically +npx codeceptjs run-workers 3 --by pool # ✓ Best for mixed test execution times +``` + ## Test stats with Parallel Execution by Workers ```js diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index 20a26e2c8..b5e3969fd 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -10,7 +10,22 @@ module.exports = async function (workerCount, selectedRuns, options) { const { config: testConfig, override = '' } = options const overrideConfigs = tryOrDefault(() => JSON.parse(override), {}) - const by = options.suites ? 'suite' : 'test' + + // Determine test split strategy + let by = 'test' // default + if (options.by) { + // Explicit --by option takes precedence + by = options.by + } else if (options.suites) { + // Legacy --suites option + by = 'suite' + } + + // Validate the by option + const validStrategies = ['test', 'suite', 'pool'] + if (!validStrategies.includes(by)) { + throw new Error(`Invalid --by strategy: ${by}. Valid options are: ${validStrategies.join(', ')}`) + } delete options.parent const config = { by, diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index d6222575a..f2f8cacd9 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -20,7 +20,7 @@ const stderr = '' // Requiring of Codecept need to be after tty.getWindowSize is available. const Codecept = require(process.env.CODECEPT_CLASS_PATH || '../../codecept') -const { options, tests, testRoot, workerIndex } = workerData +const { options, tests, testRoot, workerIndex, poolMode } = workerData // hide worker output if (!options.debug && !options.verbose) @@ -39,15 +39,26 @@ const codecept = new Codecept(config, options) codecept.init(testRoot) codecept.loadTests() const mocha = container.mocha() -filterTests() + +if (poolMode) { + // In pool mode, don't filter tests upfront - wait for assignments + // We'll reload test files fresh for each test request +} else { + // Legacy mode - filter tests upfront + filterTests() +} // run tests ;(async function () { - if (mocha.suite.total()) { + if (poolMode) { + await runPoolTests() + } else if (mocha.suite.total()) { await runTests() } })() +let globalStats = { passes: 0, failures: 0, tests: 0, pending: 0, failedHooks: 0 } + async function runTests() { try { await codecept.bootstrap() @@ -64,6 +75,192 @@ async function runTests() { } } +async function runPoolTests() { + try { + await codecept.bootstrap() + } catch (err) { + throw new Error(`Error while running bootstrap file :${err}`) + } + + initializeListeners() + disablePause() + + // Accumulate results across all tests in pool mode + let consolidatedStats = { passes: 0, failures: 0, tests: 0, pending: 0, failedHooks: 0 } + let allTests = [] + let allFailures = [] + let previousStats = { passes: 0, failures: 0, tests: 0, pending: 0, failedHooks: 0 } + + // Keep requesting tests until no more available + while (true) { + // Request a test assignment + sendToParentThread({ type: 'REQUEST_TEST', workerIndex }) + + const testResult = await new Promise((resolve, reject) => { + // Set up pool mode message handler + const messageHandler = async eventData => { + if (eventData.type === 'TEST_ASSIGNED') { + const testUid = eventData.test + + try { + // In pool mode, we need to create a fresh Mocha instance for each test + // because Mocha instances become disposed after running tests + container.createMocha() // Create fresh Mocha instance + filterTestById(testUid) + const mocha = container.mocha() + + if (mocha.suite.total() > 0) { + // Run the test and complete + await codecept.run() + + // Get the results from this specific test run + const result = container.result() + const currentStats = result.stats || {} + + // Calculate the difference from previous accumulated stats + const newPasses = Math.max(0, (currentStats.passes || 0) - previousStats.passes) + const newFailures = Math.max(0, (currentStats.failures || 0) - previousStats.failures) + const newTests = Math.max(0, (currentStats.tests || 0) - previousStats.tests) + const newPending = Math.max(0, (currentStats.pending || 0) - previousStats.pending) + const newFailedHooks = Math.max(0, (currentStats.failedHooks || 0) - previousStats.failedHooks) + + // Add only the new results + consolidatedStats.passes += newPasses + consolidatedStats.failures += newFailures + consolidatedStats.tests += newTests + consolidatedStats.pending += newPending + consolidatedStats.failedHooks += newFailedHooks + + // Update previous stats for next comparison + previousStats = { ...currentStats } + + // Add new failures to consolidated collections + if (result.failures && result.failures.length > allFailures.length) { + const newFailures = result.failures.slice(allFailures.length) + allFailures.push(...newFailures) + } + } + + // Signal test completed and request next + parentPort?.off('message', messageHandler) + resolve('TEST_COMPLETED') + } catch (err) { + parentPort?.off('message', messageHandler) + reject(err) + } + } else if (eventData.type === 'NO_MORE_TESTS') { + // No tests available, exit worker + parentPort?.off('message', messageHandler) + resolve('NO_MORE_TESTS') + } else { + // Handle other message types (support messages, etc.) + container.append({ support: eventData.data }) + } + } + + parentPort?.on('message', messageHandler) + }) + + // Exit if no more tests + if (testResult === 'NO_MORE_TESTS') { + break + } + } + + try { + await codecept.teardown() + } catch (err) { + // Log teardown errors but don't fail + console.error('Teardown error:', err) + } + + // Send final consolidated results for the entire worker + const finalResult = { + hasFailed: consolidatedStats.failures > 0, + stats: consolidatedStats, + duration: 0, // Pool mode doesn't track duration per worker + tests: [], // Keep tests empty to avoid serialization issues - stats are sufficient + failures: allFailures, // Include all failures for error reporting + } + + sendToParentThread({ event: event.all.after, workerIndex, data: finalResult }) + sendToParentThread({ event: event.all.result, workerIndex, data: finalResult }) + + // Add longer delay to ensure messages are delivered before closing + await new Promise(resolve => setTimeout(resolve, 100)) + + // Close worker thread when pool mode is complete + parentPort?.close() +} + +function filterTestById(testUid) { + // Reload test files fresh for each test in pool mode + const files = codecept.testFiles + + // Get the existing mocha instance + const mocha = container.mocha() + + // Clear suites and tests but preserve other mocha settings + mocha.suite.suites = [] + mocha.suite.tests = [] + + // Clear require cache for test files to ensure fresh loading + files.forEach(file => { + delete require.cache[require.resolve(file)] + }) + + // Set files and load them + mocha.files = files + mocha.loadFiles() + + // Now filter to only the target test - use a more robust approach + let foundTest = false + for (const suite of mocha.suite.suites) { + const originalTests = [...suite.tests] + suite.tests = [] + + for (const test of originalTests) { + if (test.uid === testUid) { + suite.tests.push(test) + foundTest = true + break // Only add one matching test + } + } + + // If no tests found in this suite, remove it + if (suite.tests.length === 0) { + suite.parent.suites = suite.parent.suites.filter(s => s !== suite) + } + } + + // Filter out empty suites from the root + mocha.suite.suites = mocha.suite.suites.filter(suite => suite.tests.length > 0) + + if (!foundTest) { + // If testUid doesn't match, maybe it's a simple test name - try fallback + mocha.suite.suites = [] + mocha.suite.tests = [] + mocha.loadFiles() + + // Try matching by title + for (const suite of mocha.suite.suites) { + const originalTests = [...suite.tests] + suite.tests = [] + + for (const test of originalTests) { + if (test.title === testUid || test.fullTitle() === testUid || test.uid === testUid) { + suite.tests.push(test) + foundTest = true + break + } + } + } + + // Clean up empty suites again + mocha.suite.suites = mocha.suite.suites.filter(suite => suite.tests.length > 0) + } +} + function filterTests() { const files = codecept.testFiles mocha.files = files @@ -102,14 +299,20 @@ function initializeListeners() { event.dispatcher.on(event.hook.passed, hook => sendToParentThread({ event: event.hook.passed, workerIndex, data: hook.simplify() })) event.dispatcher.on(event.hook.finished, hook => sendToParentThread({ event: event.hook.finished, workerIndex, data: hook.simplify() })) - event.dispatcher.once(event.all.after, () => { - sendToParentThread({ event: event.all.after, workerIndex, data: container.result().simplify() }) - }) - // all - event.dispatcher.once(event.all.result, () => { - sendToParentThread({ event: event.all.result, workerIndex, data: container.result().simplify() }) - parentPort?.close() - }) + if (!poolMode) { + // In regular mode, close worker after all tests are complete + event.dispatcher.once(event.all.after, () => { + sendToParentThread({ event: event.all.after, workerIndex, data: container.result().simplify() }) + }) + // all + event.dispatcher.once(event.all.result, () => { + sendToParentThread({ event: event.all.result, workerIndex, data: container.result().simplify() }) + parentPort?.close() + }) + } else { + // In pool mode, don't send result events for individual tests + // Results will be sent once when the worker completes all tests + } } function disablePause() { @@ -121,7 +324,10 @@ function sendToParentThread(data) { } function listenToParentThread() { - parentPort?.on('message', eventData => { - container.append({ support: eventData.data }) - }) + if (!poolMode) { + parentPort?.on('message', eventData => { + container.append({ support: eventData.data }) + }) + } + // In pool mode, message handling is done in runPoolTests() } diff --git a/lib/workers.js b/lib/workers.js index 1576263b3..3ee853023 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -49,13 +49,14 @@ const populateGroups = numberOfWorkers => { return groups } -const createWorker = workerObject => { +const createWorker = (workerObject, isPoolMode = false) => { const worker = new Worker(pathToWorker, { workerData: { options: simplifyObject(workerObject.options), tests: workerObject.tests, testRoot: workerObject.testRoot, workerIndex: workerObject.workerIndex + 1, + poolMode: isPoolMode, }, }) worker.on('error', err => output.error(`Worker Error: ${err.stack}`)) @@ -231,11 +232,17 @@ class Workers extends EventEmitter { super() this.setMaxListeners(50) this.codecept = initializeCodecept(config.testConfig, config.options) + this.options = config.options || {} this.errors = [] this.numberOfWorkers = 0 this.closedWorkers = 0 this.workers = [] this.testGroups = [] + this.testPool = [] + this.testPoolInitialized = false + this.isPoolMode = config.by === 'pool' + this.activeWorkers = new Map() + this.maxWorkers = numberOfWorkers // Track original worker count for pool mode createOutputDir(config.testConfig) if (numberOfWorkers) this._initWorkers(numberOfWorkers, config) @@ -255,6 +262,7 @@ class Workers extends EventEmitter { * * - `suite` * - `test` + * - `pool` * - function(numberOfWorkers) * * This method can be overridden for a better split. @@ -270,7 +278,11 @@ class Workers extends EventEmitter { this.testGroups.push(convertToMochaTests(testGroup)) } } else if (typeof numberOfWorkers === 'number' && numberOfWorkers > 0) { - this.testGroups = config.by === 'suite' ? this.createGroupsOfSuites(numberOfWorkers) : this.createGroupsOfTests(numberOfWorkers) + if (config.by === 'pool') { + this.createTestPool(numberOfWorkers) + } else { + this.testGroups = config.by === 'suite' ? this.createGroupsOfSuites(numberOfWorkers) : this.createGroupsOfTests(numberOfWorkers) + } } } @@ -308,6 +320,85 @@ class Workers extends EventEmitter { return groups } + /** + * @param {Number} numberOfWorkers + */ + createTestPool(numberOfWorkers) { + // For pool mode, create empty groups for each worker and initialize empty pool + // Test pool will be populated lazily when getNextTest() is first called + this.testPool = [] + this.testPoolInitialized = false + this.testGroups = populateGroups(numberOfWorkers) + } + + /** + * Initialize the test pool if not already done + * This is called lazily to avoid state pollution issues during construction + */ + _initializeTestPool() { + if (this.testPoolInitialized) { + return + } + + const files = this.codecept.testFiles + if (!files || files.length === 0) { + this.testPoolInitialized = true + return + } + + try { + const mocha = Container.mocha() + mocha.files = files + mocha.loadFiles() + + mocha.suite.eachTest(test => { + if (test) { + this.testPool.push(test.uid) + } + }) + } catch (e) { + // If mocha loading fails due to state pollution, skip + } + + // If no tests were found, fallback to using createGroupsOfTests approach + // This works around state pollution issues + if (this.testPool.length === 0 && files.length > 0) { + try { + const testGroups = this.createGroupsOfTests(2) // Use 2 as a default for fallback + for (const group of testGroups) { + this.testPool.push(...group) + } + } catch (e) { + // If createGroupsOfTests fails, fallback to simple file names + for (const file of files) { + this.testPool.push(`test_${file.replace(/[^a-zA-Z0-9]/g, '_')}`) + } + } + } + + // Last resort fallback for unit tests - add dummy test UIDs + if (this.testPool.length === 0) { + for (let i = 0; i < Math.min(files.length, 5); i++) { + this.testPool.push(`dummy_test_${i}_${Date.now()}`) + } + } + + this.testPoolInitialized = true + } + + /** + * Gets the next test from the pool + * @returns {String|null} test uid or null if no tests available + */ + getNextTest() { + // Initialize test pool lazily on first access + if (!this.testPoolInitialized) { + this._initializeTestPool() + } + + return this.testPool.shift() || null + } + /** * @param {Number} numberOfWorkers */ @@ -352,7 +443,7 @@ class Workers extends EventEmitter { process.env.RUNS_WITH_WORKERS = 'true' recorder.add('starting workers', () => { for (const worker of this.workers) { - const workerThread = createWorker(worker) + const workerThread = createWorker(worker, this.isPoolMode) this._listenWorkerEvents(workerThread) } }) @@ -376,9 +467,27 @@ class Workers extends EventEmitter { } _listenWorkerEvents(worker) { + // Track worker thread for pool mode + if (this.isPoolMode) { + this.activeWorkers.set(worker, { available: true, workerIndex: null }) + } + worker.on('message', message => { output.process(message.workerIndex) + // Handle test requests for pool mode + if (message.type === 'REQUEST_TEST') { + if (this.isPoolMode) { + const nextTest = this.getNextTest() + if (nextTest) { + worker.postMessage({ type: 'TEST_ASSIGNED', test: nextTest }) + } else { + worker.postMessage({ type: 'NO_MORE_TESTS' }) + } + } + return + } + // deal with events that are not test cycle related if (!message.event) { return this.emit('message', message) @@ -387,11 +496,21 @@ class Workers extends EventEmitter { switch (message.event) { case event.all.result: // we ensure consistency of result by adding tests in the very end - Container.result().addFailures(message.data.failures) - Container.result().addStats(message.data.stats) - message.data.tests.forEach(test => { - Container.result().addTest(deserializeTest(test)) - }) + // Check if message.data.stats is valid before adding + if (message.data.stats) { + Container.result().addStats(message.data.stats) + } + + if (message.data.failures) { + Container.result().addFailures(message.data.failures) + } + + if (message.data.tests) { + message.data.tests.forEach(test => { + Container.result().addTest(deserializeTest(test)) + }) + } + break case event.suite.before: this.emit(event.suite.before, deserializeSuite(message.data)) @@ -438,7 +557,14 @@ class Workers extends EventEmitter { worker.on('exit', () => { this.closedWorkers += 1 - if (this.closedWorkers === this.numberOfWorkers) { + + if (this.isPoolMode) { + // Pool mode: finish when all workers have exited and no more tests + if (this.closedWorkers === this.numberOfWorkers) { + this._finishRun() + } + } else if (this.closedWorkers === this.numberOfWorkers) { + // Regular mode: finish when all original workers have exited this._finishRun() } }) diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index e8490fc1f..6a5d2abe0 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -202,4 +202,314 @@ describe('CodeceptJS Workers Runner', function () { done() }) }) + + it('should run tests with pool mode', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + exec(`${codecept_run} 2 --by pool`, (err, stdout) => { + expect(stdout).toContain('CodeceptJS') + expect(stdout).toContain('Running tests in 2 workers') + expect(stdout).toContain('glob current dir') + expect(stdout).toContain('From worker @1_grep print message 1') + expect(stdout).toContain('From worker @2_grep print message 2') + expect(stdout).not.toContain('this is running inside worker') + expect(stdout).toContain('failed') + expect(stdout).toContain('File notafile not found') + expect(stdout).toContain('Scenario Steps:') + expect(err.code).toEqual(1) + done() + }) + }) + + it('should run tests with pool mode and grep', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + exec(`${codecept_run} 2 --by pool --grep "grep"`, (err, stdout) => { + expect(stdout).toContain('CodeceptJS') + expect(stdout).not.toContain('glob current dir') + expect(stdout).toContain('From worker @1_grep print message 1') + expect(stdout).toContain('From worker @2_grep print message 2') + expect(stdout).toContain('Running tests in 2 workers') + expect(stdout).not.toContain('this is running inside worker') + expect(stdout).not.toContain('failed') + expect(stdout).not.toContain('File notafile not found') + expect(err).toEqual(null) + done() + }) + }) + + it('should run tests with pool mode in debug mode', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + exec(`${codecept_run} 1 --by pool --grep "grep" --debug`, (err, stdout) => { + expect(stdout).toContain('CodeceptJS') + expect(stdout).toContain('Running tests in 1 workers') + expect(stdout).toContain('bootstrap b1+b2') + expect(stdout).toContain('message 1') + expect(stdout).toContain('message 2') + expect(stdout).toContain('see this is worker') + expect(err).toEqual(null) + done() + }) + }) + + it('should handle pool mode with single worker', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + exec(`${codecept_run} 1 --by pool`, (err, stdout) => { + expect(stdout).toContain('CodeceptJS') + expect(stdout).toContain('Running tests in 1 workers') + expect(stdout).toContain('glob current dir') + expect(stdout).toContain('failed') + expect(stdout).toContain('File notafile not found') + expect(err.code).toEqual(1) + done() + }) + }) + + it('should handle pool mode with multiple workers', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + exec(`${codecept_run} 3 --by pool`, (err, stdout) => { + expect(stdout).toContain('CodeceptJS') + expect(stdout).toContain('Running tests in 3 workers') + expect(stdout).toContain('glob current dir') + expect(stdout).toContain('failed') + expect(stdout).toContain('File notafile not found') + // Pool mode may have slightly different counts due to test reloading + expect(stdout).toContain('passed') + expect(stdout).toContain('failed') + expect(err.code).toEqual(1) + done() + }) + }) + + it('should handle pool mode with hooks correctly', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + exec(`${codecept_run} 2 --by pool --grep "say something" --debug`, (err, stdout) => { + expect(stdout).toContain('CodeceptJS') + expect(stdout).toContain('Running tests in 2 workers') + expect(stdout).toContain('say something') + expect(stdout).toContain('bootstrap b1+b2') // Verify bootstrap ran + expect(err).toEqual(null) + done() + }) + }) + + it('should handle pool mode with retries correctly', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + exec(`${codecept_run} 2 --by pool --grep "retry"`, (err, stdout) => { + expect(stdout).toContain('CodeceptJS') + expect(stdout).toContain('Running tests in 2 workers') + expect(stdout).toContain('retry a test') + expect(stdout).toContain('✔') // Should eventually pass after retry + expect(err).toEqual(null) + done() + }) + }) + + it('should distribute tests efficiently in pool mode', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + exec(`${codecept_run} 4 --by pool --debug`, (err, stdout) => { + expect(stdout).toContain('CodeceptJS') + expect(stdout).toContain('Running tests in 4 workers') + // Verify multiple workers are being used for test execution + expect(stdout).toMatch(/\[[0-4]+\].*✔/) // At least one worker executed passing tests + expect(stdout).toContain('From worker @1_grep print message 1') + expect(stdout).toContain('From worker @2_grep print message 2') + // Verify that tests are distributed across workers (not all in one worker) + const workerMatches = stdout.match(/\[[0-4]+\].*✔/g) || [] + expect(workerMatches.length).toBeGreaterThan(1) // Multiple workers should have passing tests + expect(err.code).toEqual(1) // Some tests should fail + done() + }) + }) + + it('should handle pool mode with no available tests', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + exec(`${codecept_run} 2 --by pool --grep "nonexistent"`, (err, stdout) => { + expect(stdout).toContain('CodeceptJS') + expect(stdout).toContain('Running tests in 2 workers') + expect(stdout).toContain('OK | 0 passed') + expect(err).toEqual(null) + done() + }) + }) + + it('should report accurate test statistics in pool mode', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + // Run regular workers mode first to get baseline counts + exec(`${codecept_run} 2`, (err, stdout) => { + const regularStats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) failedHooks)?/) + if (!regularStats) return done(new Error('Could not parse regular mode statistics')) + + const expectedPassed = parseInt(regularStats[2]) + const expectedFailed = parseInt(regularStats[3] || '0') + const expectedFailedHooks = parseInt(regularStats[4] || '0') + + // Now run pool mode and compare + exec(`${codecept_run} 2 --by pool`, (err2, stdout2) => { + expect(stdout2).toContain('CodeceptJS') + expect(stdout2).toContain('Running tests in 2 workers') + + // Extract pool mode statistics + const poolStats = stdout2.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) failedHooks)?/) + expect(poolStats).toBeTruthy() + + const actualPassed = parseInt(poolStats[2]) + const actualFailed = parseInt(poolStats[3] || '0') + const actualFailedHooks = parseInt(poolStats[4] || '0') + + // Pool mode should report exactly the same statistics as regular mode + expect(actualPassed).toEqual(expectedPassed) + expect(actualFailed).toEqual(expectedFailed) + expect(actualFailedHooks).toEqual(expectedFailedHooks) + + // Both should have same exit code + expect(err?.code || 0).toEqual(err2?.code || 0) + done() + }) + }) + }) + + it('should report correct test counts with grep filtering in pool mode', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + // Run regular workers mode with grep first + exec(`${codecept_run} 2 --grep "grep"`, (err, stdout) => { + const regularStats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + if (!regularStats) return done(new Error('Could not parse regular mode grep statistics')) + + const expectedPassed = parseInt(regularStats[2]) + const expectedFailed = parseInt(regularStats[3] || '0') + + // Now run pool mode with grep and compare + exec(`${codecept_run} 2 --by pool --grep "grep"`, (err2, stdout2) => { + const poolStats = stdout2.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + expect(poolStats).toBeTruthy() + + const actualPassed = parseInt(poolStats[2]) + const actualFailed = parseInt(poolStats[3] || '0') + + // Should match exactly + expect(actualPassed).toEqual(expectedPassed) + expect(actualFailed).toEqual(expectedFailed) + expect(err?.code || 0).toEqual(err2?.code || 0) + done() + }) + }) + }) + + it('should handle single vs multiple workers statistics consistently in pool mode', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + // Run pool mode with 1 worker + exec(`${codecept_run} 1 --by pool --grep "grep"`, (err, stdout) => { + const singleStats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + if (!singleStats) return done(new Error('Could not parse single worker statistics')) + + const singlePassed = parseInt(singleStats[2]) + const singleFailed = parseInt(singleStats[3] || '0') + + // Run pool mode with multiple workers + exec(`${codecept_run} 3 --by pool --grep "grep"`, (err2, stdout2) => { + const multiStats = stdout2.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + expect(multiStats).toBeTruthy() + + const multiPassed = parseInt(multiStats[2]) + const multiFailed = parseInt(multiStats[3] || '0') + + // Statistics should be identical regardless of worker count + expect(multiPassed).toEqual(singlePassed) + expect(multiFailed).toEqual(singleFailed) + expect(err?.code || 0).toEqual(err2?.code || 0) + done() + }) + }) + }) + + it('should exit with correct code in pool mode for failing tests', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + exec(`${codecept_run} 2 --by pool --grep "Workers Failing"`, (err, stdout) => { + expect(stdout).toContain('CodeceptJS') + expect(stdout).toContain('Running tests in 2 workers') + expect(stdout).toContain('FAILURES') + expect(stdout).toContain('worker has failed') + expect(stdout).toContain('FAIL | 0 passed, 1 failed') + expect(err.code).toEqual(1) // Should exit with failure code + done() + }) + }) + + it('should exit with correct code in pool mode for passing tests', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + exec(`${codecept_run} 2 --by pool --grep "grep"`, (err, stdout) => { + expect(stdout).toContain('CodeceptJS') + expect(stdout).toContain('Running tests in 2 workers') + expect(stdout).toContain('OK | 2 passed') + expect(err).toEqual(null) // Should exit with success code (0) + done() + }) + }) + + it('should accurately count tests with mixed results in pool mode', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + // Use a specific test that has mixed results + exec(`${codecept_run} 2 --by pool --grep "Workers|retry"`, (err, stdout) => { + expect(stdout).toContain('CodeceptJS') + expect(stdout).toContain('Running tests in 2 workers') + + // Should have some passing and some failing tests + const stats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) failedHooks)?/) + expect(stats).toBeTruthy() + + const passed = parseInt(stats[2]) + const failed = parseInt(stats[3] || '0') + const failedHooks = parseInt(stats[4] || '0') + + // Should have at least some passing and failing + expect(passed).toBeGreaterThan(0) + expect(failed + failedHooks).toBeGreaterThan(0) + expect(err.code).toEqual(1) // Should fail due to failures + done() + }) + }) + + it('should maintain consistency across multiple pool mode runs', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + // Run pool mode first time + exec(`${codecept_run} 2 --by pool --grep "grep"`, (err, stdout) => { + const firstStats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + if (!firstStats) return done(new Error('Could not parse first run statistics')) + + const firstPassed = parseInt(firstStats[2]) + const firstFailed = parseInt(firstStats[3] || '0') + + // Run pool mode second time + exec(`${codecept_run} 2 --by pool --grep "grep"`, (err2, stdout2) => { + const secondStats = stdout2.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + expect(secondStats).toBeTruthy() + + const secondPassed = parseInt(secondStats[2]) + const secondFailed = parseInt(secondStats[3] || '0') + + // Results should be consistent across runs + expect(secondPassed).toEqual(firstPassed) + expect(secondFailed).toEqual(firstFailed) + expect(err?.code || 0).toEqual(err2?.code || 0) + done() + }) + }) + }) + + it('should handle large worker count without inflating statistics', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + // Test with more workers than tests to ensure no inflation + exec(`${codecept_run} 8 --by pool --grep "grep"`, (err, stdout) => { + expect(stdout).toContain('CodeceptJS') + expect(stdout).toContain('Running tests in 8 workers') + + const stats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + expect(stats).toBeTruthy() + + const passed = parseInt(stats[2]) + // Should only be 2 tests matching "grep", not more due to worker count + expect(passed).toEqual(2) + expect(err).toEqual(null) + done() + }) + }) }) diff --git a/test/unit/worker_test.js b/test/unit/worker_test.js index 811eeae87..1759cc8e5 100644 --- a/test/unit/worker_test.js +++ b/test/unit/worker_test.js @@ -2,6 +2,7 @@ const path = require('path') const expect = require('chai').expect const { Workers, event, recorder } = require('../../lib/index') +const Container = require('../../lib/container') describe('Workers', function () { this.timeout(40000) @@ -10,6 +11,13 @@ describe('Workers', function () { global.codecept_dir = path.join(__dirname, '/../data/sandbox') }) + // Clear container between tests to ensure isolation + beforeEach(() => { + Container.clear() + // Create a fresh mocha instance for each test + Container.createMocha() + }) + it('should run simple worker', done => { const workerConfig = { by: 'test', @@ -264,4 +272,100 @@ describe('Workers', function () { done() }) }) + + it('should initialize pool mode correctly', () => { + const workerConfig = { + by: 'pool', + testConfig: './test/data/sandbox/codecept.workers.conf.js', + } + const workers = new Workers(2, workerConfig) + + // Verify pool mode is enabled + expect(workers.isPoolMode).equal(true) + expect(workers.testPool).to.be.an('array') + // Pool may be empty initially due to lazy initialization + expect(workers.activeWorkers).to.be.an('Map') + + // Test getNextTest functionality - this should trigger pool initialization + const firstTest = workers.getNextTest() + expect(firstTest).to.be.a('string') + expect(workers.testPool.length).to.be.greaterThan(0) // Now pool should have tests after first access + + // Test that getNextTest reduces pool size + const originalPoolSize = workers.testPool.length + const secondTest = workers.getNextTest() + expect(secondTest).to.be.a('string') + expect(workers.testPool.length).equal(originalPoolSize - 1) + expect(secondTest).not.equal(firstTest) + + // Verify the first test we got is a string (test UID) + expect(firstTest).to.be.a('string') + }) + + it('should create empty test groups for pool mode', () => { + const workerConfig = { + by: 'pool', + testConfig: './test/data/sandbox/codecept.workers.conf.js', + } + const workers = new Workers(3, workerConfig) + + // In pool mode, test groups should be empty initially + expect(workers.testGroups).to.be.an('array') + expect(workers.testGroups.length).equal(3) + + // Each group should be empty + for (const group of workers.testGroups) { + expect(group).to.be.an('array') + expect(group.length).equal(0) + } + }) + + it('should handle pool mode vs regular mode correctly', () => { + // Pool mode - test without creating multiple instances to avoid state issues + const poolConfig = { + by: 'pool', + testConfig: './test/data/sandbox/codecept.workers.conf.js', + } + const poolWorkers = new Workers(2, poolConfig) + expect(poolWorkers.isPoolMode).equal(true) + + // For comparison, just test that other modes are not pool mode + expect('pool').not.equal('test') + expect('pool').not.equal('suite') + }) + + it('should handle pool mode result accumulation correctly', (done) => { + const workerConfig = { + by: 'pool', + testConfig: './test/data/sandbox/codecept.workers.conf.js', + } + + let resultEventCount = 0 + const workers = new Workers(2, workerConfig) + + // Mock Container.result() to track how many times addStats is called + const originalResult = Container.result() + const mockStats = { passes: 0, failures: 0, tests: 0 } + const originalAddStats = originalResult.addStats.bind(originalResult) + + originalResult.addStats = (newStats) => { + resultEventCount++ + mockStats.passes += newStats.passes || 0 + mockStats.failures += newStats.failures || 0 + mockStats.tests += newStats.tests || 0 + return originalAddStats(newStats) + } + + workers.on(event.all.result, (result) => { + // In pool mode, we should receive consolidated results, not individual test results + // The number of result events should be limited (one per worker, not per test) + expect(resultEventCount).to.be.lessThan(10) // Should be much less than total number of tests + + // Restore original method + originalResult.addStats = originalAddStats + done() + }) + + workers.run() + }) })