From f15d3f3d3e5f15421269ff3a5047cd793fc1e4f9 Mon Sep 17 00:00:00 2001 From: Marc Philipp Date: Mon, 7 Jul 2025 18:37:54 +0200 Subject: [PATCH] Introduce `--fail-fast` mode for `execute` command of `ConsoleLauncher` Passing the option to the `execute` subcommand of the `ConsoleLauncher` now causes test execution to be cancelled after the first failed test. Issue: #4725 --- .../release-notes/release-notes-6.0.0-M2.adoc | 2 + .../console/options/ExecuteTestsCommand.java | 14 ++++++- .../console/tasks/ConsoleTestExecutor.java | 40 ++++++++++++++----- .../console/tasks/FailFastListener.java | 37 +++++++++++++++++ .../ConsoleLauncherIntegrationTests.java | 13 ++++++ .../console/ConsoleLauncherWrapper.java | 2 +- .../options/ExecuteTestsCommandTests.java | 15 ++++++- .../console/subpackage/FailingTestCase.java | 28 +++++++++++++ .../tasks/ConsoleTestExecutorTests.java | 12 +++--- 9 files changed, 143 insertions(+), 20 deletions(-) create mode 100644 junit-platform-console/src/main/java/org/junit/platform/console/tasks/FailFastListener.java create mode 100644 platform-tests/src/test/java/org/junit/platform/console/subpackage/FailingTestCase.java diff --git a/documentation/src/docs/asciidoc/release-notes/release-notes-6.0.0-M2.adoc b/documentation/src/docs/asciidoc/release-notes/release-notes-6.0.0-M2.adoc index 9b0b2d4a0daa..866518e85a3c 100644 --- a/documentation/src/docs/asciidoc/release-notes/release-notes-6.0.0-M2.adoc +++ b/documentation/src/docs/asciidoc/release-notes/release-notes-6.0.0-M2.adoc @@ -41,6 +41,8 @@ repository on GitHub. all registered test engines. Please refer to the <<../user-guide/index.adoc#launcher-api-launcher-cancellation, User Guide>> for details and a usage example. +* Passing the `--fail-fast` option to the `execute` subcommand of the `ConsoleLauncher` + now causes test execution to be cancelled after the first failed test. * Provide cancellation support for implementations of `{HierarchicalTestEngine}` such as JUnit Jupiter, Spock, and Cucumber. * Provide cancellation support for Suite engine diff --git a/junit-platform-console/src/main/java/org/junit/platform/console/options/ExecuteTestsCommand.java b/junit-platform-console/src/main/java/org/junit/platform/console/options/ExecuteTestsCommand.java index ca6c4d0281c8..417c968656c7 100644 --- a/junit-platform-console/src/main/java/org/junit/platform/console/options/ExecuteTestsCommand.java +++ b/junit-platform-console/src/main/java/org/junit/platform/console/options/ExecuteTestsCommand.java @@ -60,13 +60,17 @@ class ExecuteTestsCommand extends BaseCommand implements C @Override protected TestExecutionSummary execute(PrintWriter out) { return consoleTestExecutorFactory.create(toTestDiscoveryOptions(), toTestConsoleOutputOptions()) // - .execute(out, getReportsDir()); + .execute(out, getReportsDir(), isFailFast()); } Optional getReportsDir() { return getReportingOptions().flatMap(ReportingOptions::getReportsDir); } + boolean isFailFast() { + return getReportingOptions().map(options -> options.failFast).orElse(false); + } + private Optional getReportingOptions() { return Optional.ofNullable(reportingOptions); } @@ -96,7 +100,13 @@ public int getExitCode() { static class ReportingOptions { @Option(names = "--fail-if-no-tests", description = "Fail and return exit status code 2 if no tests are found.") - private boolean failIfNoTests; // no single-dash equivalent: was introduced in 5.3-M1 + private boolean failIfNoTests; + + /** + * @since 6.0 + */ + @Option(names = "--fail-fast", description = "Stops test execution after the first failed test.") + private boolean failFast; @Nullable @Option(names = "--reports-dir", paramLabel = "DIR", description = "Enable report output into a specified local directory (will be created if it does not exist).") diff --git a/junit-platform-console/src/main/java/org/junit/platform/console/tasks/ConsoleTestExecutor.java b/junit-platform-console/src/main/java/org/junit/platform/console/tasks/ConsoleTestExecutor.java index a030d65805e0..cf61a8e35cb4 100644 --- a/junit-platform-console/src/main/java/org/junit/platform/console/tasks/ConsoleTestExecutor.java +++ b/junit-platform-console/src/main/java/org/junit/platform/console/tasks/ConsoleTestExecutor.java @@ -10,6 +10,7 @@ package org.junit.platform.console.tasks; +import static java.util.Objects.requireNonNullElseGet; import static org.apiguardian.api.API.Status.INTERNAL; import static org.junit.platform.console.tasks.DiscoveryRequestCreator.toDiscoveryRequestBuilder; import static org.junit.platform.launcher.LauncherConstants.OUTPUT_DIR_PROPERTY_NAME; @@ -25,17 +26,18 @@ import java.util.function.Supplier; import org.apiguardian.api.API; +import org.jspecify.annotations.Nullable; import org.junit.platform.commons.JUnitException; import org.junit.platform.commons.util.ClassLoaderUtils; import org.junit.platform.console.options.Details; import org.junit.platform.console.options.TestConsoleOutputOptions; import org.junit.platform.console.options.TestDiscoveryOptions; import org.junit.platform.console.options.Theme; +import org.junit.platform.engine.CancellationToken; import org.junit.platform.launcher.Launcher; import org.junit.platform.launcher.LauncherDiscoveryRequest; import org.junit.platform.launcher.TestExecutionListener; import org.junit.platform.launcher.TestPlan; -import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; import org.junit.platform.launcher.core.LauncherFactory; import org.junit.platform.launcher.listeners.SummaryGeneratingListener; import org.junit.platform.launcher.listeners.TestExecutionSummary; @@ -83,9 +85,9 @@ public void discover(PrintWriter out) { }); } - public TestExecutionSummary execute(PrintWriter out, Optional reportsDir) { + public TestExecutionSummary execute(PrintWriter out, Optional reportsDir, boolean failFast) { return createCustomContextClassLoaderExecutor() // - .invoke(() -> executeTests(out, reportsDir)); + .invoke(() -> executeTests(out, reportsDir, failFast)); } private CustomContextClassLoaderExecutor createCustomContextClassLoaderExecutor() { @@ -115,16 +117,17 @@ private static void printFoundTestsSummary(PrintWriter out, TestPlan testPlan) { out.flush(); } - private TestExecutionSummary executeTests(PrintWriter out, Optional reportsDir) { + private TestExecutionSummary executeTests(PrintWriter out, Optional reportsDir, boolean failFast) { Launcher launcher = launcherSupplier.get(); - SummaryGeneratingListener summaryListener = registerListeners(out, reportsDir, launcher); + CancellationToken cancellationToken = failFast ? CancellationToken.create() : null; + SummaryGeneratingListener summaryListener = registerListeners(out, reportsDir, launcher, cancellationToken); PrintStream originalOut = System.out; PrintStream originalErr = System.err; try (StandardStreamsHandler standardStreamsHandler = new StandardStreamsHandler()) { standardStreamsHandler.redirectStandardStreams(outputOptions.getStdoutPath(), outputOptions.getStderrPath()); - launchTests(launcher, reportsDir); + launchTests(launcher, reportsDir, cancellationToken); } finally { System.setOut(originalOut); @@ -136,14 +139,24 @@ private TestExecutionSummary executeTests(PrintWriter out, Optional report printSummary(summary, out); } + if (cancellationToken != null && cancellationToken.isCancellationRequested()) { + out.println("Test execution was cancelled due to --fail-fast mode."); + out.println(); + } + return summary; } - private void launchTests(Launcher launcher, Optional reportsDir) { - LauncherDiscoveryRequestBuilder discoveryRequestBuilder = toDiscoveryRequestBuilder(discoveryOptions); + private void launchTests(Launcher launcher, Optional reportsDir, + @Nullable CancellationToken cancellationToken) { + + var discoveryRequestBuilder = toDiscoveryRequestBuilder(discoveryOptions); reportsDir.ifPresent(dir -> discoveryRequestBuilder.configurationParameter(OUTPUT_DIR_PROPERTY_NAME, dir.toAbsolutePath().toString())); - launcher.execute(discoveryRequestBuilder.forExecution().build()); + var executionRequest = discoveryRequestBuilder.forExecution() // + .cancellationToken(requireNonNullElseGet(cancellationToken, CancellationToken::disabled)) // + .build(); + launcher.execute(executionRequest); } private Optional createCustomClassLoader() { @@ -166,7 +179,9 @@ private URL toURL(Path path) { } } - private SummaryGeneratingListener registerListeners(PrintWriter out, Optional reportsDir, Launcher launcher) { + private SummaryGeneratingListener registerListeners(PrintWriter out, Optional reportsDir, Launcher launcher, + @Nullable CancellationToken cancellationToken) { + // always register summary generating listener SummaryGeneratingListener summaryListener = new SummaryGeneratingListener(); launcher.registerTestExecutionListeners(summaryListener); @@ -174,6 +189,7 @@ private SummaryGeneratingListener registerListeners(PrintWriter out, Optional createXmlWritingListener(PrintWriter out return reportsDir.map(it -> new LegacyXmlReportGeneratingListener(it, out)); } + private Optional createFailFastListener(@Nullable CancellationToken cancellationToken) { + return Optional.ofNullable(cancellationToken).map(FailFastListener::new); + } + private void printSummary(TestExecutionSummary summary, PrintWriter out) { // Otherwise the failures have already been printed in detail if (EnumSet.of(Details.NONE, Details.SUMMARY, Details.TREE).contains(outputOptions.getDetails())) { diff --git a/junit-platform-console/src/main/java/org/junit/platform/console/tasks/FailFastListener.java b/junit-platform-console/src/main/java/org/junit/platform/console/tasks/FailFastListener.java new file mode 100644 index 000000000000..a39aaa744bbd --- /dev/null +++ b/junit-platform-console/src/main/java/org/junit/platform/console/tasks/FailFastListener.java @@ -0,0 +1,37 @@ +/* + * Copyright 2015-2025 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.console.tasks; + +import static org.junit.platform.engine.TestExecutionResult.Status.FAILED; + +import org.junit.platform.engine.CancellationToken; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; + +/** + * @since 6.0 + */ +class FailFastListener implements TestExecutionListener { + + private final CancellationToken cancellationToken; + + FailFastListener(CancellationToken cancellationToken) { + this.cancellationToken = cancellationToken; + } + + @Override + public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { + if (testExecutionResult.getStatus() == FAILED) { + cancellationToken.cancel(); + } + } +} diff --git a/platform-tests/src/test/java/org/junit/platform/console/ConsoleLauncherIntegrationTests.java b/platform-tests/src/test/java/org/junit/platform/console/ConsoleLauncherIntegrationTests.java index 6fe058ff2493..1fd3c940c302 100644 --- a/platform-tests/src/test/java/org/junit/platform/console/ConsoleLauncherIntegrationTests.java +++ b/platform-tests/src/test/java/org/junit/platform/console/ConsoleLauncherIntegrationTests.java @@ -29,6 +29,7 @@ import org.junit.jupiter.params.provider.FieldSource; import org.junit.jupiter.params.provider.ValueSource; import org.junit.platform.console.options.StdStreamTestCase; +import org.junit.platform.console.subpackage.FailingTestCase; /** * @since 1.0 @@ -123,4 +124,16 @@ void executeWithRedirectedStdStreamsToSameFile(@TempDir Path tempDir) throws IOE Files.size(outputFile), "Invalid file size."); } + @Test + void stopsAfterFirstFailingTest() { + var result = new ConsoleLauncherWrapper().execute(1, "execute", "-e", "junit-jupiter", "--select-class", + FailingTestCase.class.getName(), "--fail-fast", "--disable-ansi-colors"); + + assertThat(result.getTestsStartedCount()).isEqualTo(1); + assertThat(result.getTestsFailedCount()).isEqualTo(1); + assertThat(result.getTestsSkippedCount()).isEqualTo(1); + + assertThat(result.out).endsWith("%nTest execution was cancelled due to --fail-fast mode.%n%n".formatted()); + } + } diff --git a/platform-tests/src/test/java/org/junit/platform/console/ConsoleLauncherWrapper.java b/platform-tests/src/test/java/org/junit/platform/console/ConsoleLauncherWrapper.java index 03fb3528f543..de6ae6dfd381 100644 --- a/platform-tests/src/test/java/org/junit/platform/console/ConsoleLauncherWrapper.java +++ b/platform-tests/src/test/java/org/junit/platform/console/ConsoleLauncherWrapper.java @@ -55,7 +55,7 @@ public ConsoleLauncherWrapperResult execute(Optional expectedCode, Stri if (expectedCode.isPresent()) { int expectedValue = expectedCode.get(); assertEquals(expectedValue, code, "ConsoleLauncher execute code mismatch!"); - if (expectedValue != 0) { + if (expectedValue != 0 && expectedValue != 1) { assertThat(errText).isNotBlank(); } } diff --git a/platform-tests/src/test/java/org/junit/platform/console/options/ExecuteTestsCommandTests.java b/platform-tests/src/test/java/org/junit/platform/console/options/ExecuteTestsCommandTests.java index f259409c39a6..e95d00ce9427 100644 --- a/platform-tests/src/test/java/org/junit/platform/console/options/ExecuteTestsCommandTests.java +++ b/platform-tests/src/test/java/org/junit/platform/console/options/ExecuteTestsCommandTests.java @@ -13,7 +13,10 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -36,7 +39,7 @@ class ExecuteTestsCommandTests { @BeforeEach void setUp() { - when(consoleTestExecutor.execute(any(), any())).thenReturn(summary); + when(consoleTestExecutor.execute(any(), any(), anyBoolean())).thenReturn(summary); } @Test @@ -106,6 +109,16 @@ void parseValidXmlReportsDirs() { // @formatter:on } + @Test + void parseValidFailFast() { + // @formatter:off + assertAll( + () -> assertFalse(parseArgs().isFailFast()), + () -> assertTrue(parseArgs("--fail-fast").isFailFast()) + ); + // @formatter:on + } + private ExecuteTestsCommand parseArgs(String... args) { command.parseArgs(args); return command; diff --git a/platform-tests/src/test/java/org/junit/platform/console/subpackage/FailingTestCase.java b/platform-tests/src/test/java/org/junit/platform/console/subpackage/FailingTestCase.java new file mode 100644 index 000000000000..923ee1762ea2 --- /dev/null +++ b/platform-tests/src/test/java/org/junit/platform/console/subpackage/FailingTestCase.java @@ -0,0 +1,28 @@ +/* + * Copyright 2015-2025 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.console.subpackage; + +import static org.junit.jupiter.api.Assertions.fail; + +import org.junit.jupiter.api.Test; + +public class FailingTestCase { + + @Test + void first() { + fail(); + } + + @Test + void second() { + fail(); + } +} diff --git a/platform-tests/src/test/java/org/junit/platform/console/tasks/ConsoleTestExecutorTests.java b/platform-tests/src/test/java/org/junit/platform/console/tasks/ConsoleTestExecutorTests.java index 76da53bcc218..c31fb24e088b 100644 --- a/platform-tests/src/test/java/org/junit/platform/console/tasks/ConsoleTestExecutorTests.java +++ b/platform-tests/src/test/java/org/junit/platform/console/tasks/ConsoleTestExecutorTests.java @@ -53,7 +53,7 @@ void printsSummary() { dummyTestEngine.addTest("failingTest", FAILING_BLOCK); var task = new ConsoleTestExecutor(discoveryOptions, outputOptions, () -> createLauncher(dummyTestEngine)); - task.execute(new PrintWriter(stringWriter), Optional.empty()); + task.execute(new PrintWriter(stringWriter), Optional.empty(), false); assertThat(stringWriter.toString()).contains("Test run finished after", "2 tests found", "0 tests skipped", "2 tests started", "0 tests aborted", "1 tests successful", "1 tests failed"); @@ -66,7 +66,7 @@ void printsDetailsIfTheyAreNotHidden() { dummyTestEngine.addTest("failingTest", FAILING_BLOCK); var task = new ConsoleTestExecutor(discoveryOptions, outputOptions, () -> createLauncher(dummyTestEngine)); - task.execute(new PrintWriter(stringWriter), Optional.empty()); + task.execute(new PrintWriter(stringWriter), Optional.empty(), false); assertThat(stringWriter.toString()).contains("Test execution started."); } @@ -78,7 +78,7 @@ void printsNoDetailsIfTheyAreHidden() { dummyTestEngine.addTest("failingTest", FAILING_BLOCK); var task = new ConsoleTestExecutor(discoveryOptions, outputOptions, () -> createLauncher(dummyTestEngine)); - task.execute(new PrintWriter(stringWriter), Optional.empty()); + task.execute(new PrintWriter(stringWriter), Optional.empty(), false); assertThat(stringWriter.toString()).doesNotContain("Test execution started."); } @@ -91,7 +91,7 @@ void printsFailuresEvenIfDetailsAreHidden() { dummyTestEngine.addContainer("failingContainer", FAILING_BLOCK); var task = new ConsoleTestExecutor(discoveryOptions, outputOptions, () -> createLauncher(dummyTestEngine)); - task.execute(new PrintWriter(stringWriter), Optional.empty()); + task.execute(new PrintWriter(stringWriter), Optional.empty(), false); assertThat(stringWriter.toString()).contains("Failures (2)", "failingTest", "failingContainer"); } @@ -105,7 +105,7 @@ void usesCustomClassLoaderIfAdditionalClassPathEntriesArePresent() { () -> assertSame(oldClassLoader, getDefaultClassLoader(), "should fail")); var task = new ConsoleTestExecutor(discoveryOptions, outputOptions, () -> createLauncher(dummyTestEngine)); - task.execute(new PrintWriter(stringWriter), Optional.empty()); + task.execute(new PrintWriter(stringWriter), Optional.empty(), false); assertThat(stringWriter.toString()).contains("failingTest", "should fail", "1 tests failed"); } @@ -119,7 +119,7 @@ void usesSameClassLoaderIfNoAdditionalClassPathEntriesArePresent() { () -> assertNotSame(oldClassLoader, getDefaultClassLoader(), "should fail")); var task = new ConsoleTestExecutor(discoveryOptions, outputOptions, () -> createLauncher(dummyTestEngine)); - task.execute(new PrintWriter(stringWriter), Optional.empty()); + task.execute(new PrintWriter(stringWriter), Optional.empty(), false); assertThat(stringWriter.toString()).contains("failingTest", "should fail", "1 tests failed"); }