From b6ef217c2b2c78fcb2b8b815d5f2e74532360475 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sat, 13 Jul 2024 11:36:29 -0400 Subject: [PATCH 1/6] Emit a specific exit code when no tests match inputs to `swift test`. This PR modifies the entry point function used by Swift Package Manager so that when no tests match the inputs (or, in the case of `swift test list`, when there are no Swift Testing tests in the test target), Swift Package Manager can detect it and respond appropriately. Because exit codes exist in a flat namespace, and because on POSIX-y systems only the low 8 bits of an exit code are reliable, our options here are limited. I have selected the `EX_UNAVAILABLE` code from sysexits.h to represent this state. On Windows, all 32 bits of the exit code are propagated, but there is no `EX_UNAVAILABLE` value, so I've used the Win32 `ERROR_NOT_FOUND` error code instead. My assumption is that neither of these codes is likely to _actually_ be passed to `exit()` by test code (because why would they be?) A separate PR is required in Swift Package Manager to correctly handle this exit code and emit the `noMatchingTests` diagnostic that's currently emitted only for XCTest. --- .../ABI/EntryPoints/ABIEntryPoint.swift | 5 ++++ .../Testing/ABI/EntryPoints/EntryPoint.swift | 19 ++++++++++++++- .../ABI/EntryPoints/SwiftPMEntryPoint.swift | 23 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift index b18e51e50..81988b126 100644 --- a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift @@ -118,6 +118,11 @@ private func entryPoint( let eventHandler = try eventHandlerForStreamingEvents(version: args?.eventStreamVersion, forwardingTo: recordHandler) let exitCode = await entryPoint(passing: args, eventHandler: eventHandler) + + // To maintain compatibility with Xcode 16 Beta 1, suppress custom exit codes. + if exitCode == EXIT_NO_TESTS_FOUND { + return EXIT_SUCCESS + } return exitCode } #endif diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index b10f31c8e..6ed1bbe87 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -74,8 +74,13 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha } } + // The set of matching tests (or, in the case of `swift test list`, the set + // of all tests.) + let tests: [Test] + if args.listTests ?? false { - let tests = await Test.all + tests = await Array(Test.all) + if args.verbosity > .min { for testID in listTestsForEntryPoint(tests) { // Print the test ID to stdout (classical CLI behavior.) @@ -95,8 +100,20 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha } else { // Run the tests. let runner = await Runner(configuration: configuration) + tests = runner.tests await runner.run() } + + // If there were no matching tests, exit with a dedicated exit code so that + // the caller (assumed to be Swift Package Manager) can implement special + // handling. + if tests.isEmpty { + exitCode.withLock { exitCode in + if exitCode == EXIT_SUCCESS { + exitCode = EXIT_NO_TESTS_FOUND + } + } + } } catch { #if !SWT_NO_FILE_IO try? FileHandle.stderr.write(String(describing: error)) diff --git a/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift index 118d3a335..90c00d0d5 100644 --- a/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift @@ -14,6 +14,29 @@ private import _TestingInternals #endif +/// The exit code returned to Swift Package Manager when no tests matched the +/// inputs specified by the developer (or, for the case of `swift test list`, +/// when no tests were found.) +/// +/// Because Swift Package Manager does not directly link to the testing library, +/// it duplicates the definition of this constant in its own source. Any changes +/// to this constant in either package must be mirrored in the other. +/// +/// Tools authors using the ABI entry point function can determine if no tests +/// matched the developer's inputs by counting the number of test records passed +/// to the event handler or written to the event stream output path. +/// +/// This constant is not part of the public interface of the testing library. +var EXIT_NO_TESTS_FOUND: CInt { +#if SWT_TARGET_OS_APPLE || os(Linux) + EX_UNAVAILABLE +#elseif os(Windows) + CInt(bitPattern: ERROR_NOT_FOUND) +#else +#warning("Platform-specific implementation missing: value for EXIT_NO_TESTS_FOUND unavailable") +#endif +} + /// The entry point to the testing library used by Swift Package Manager. /// /// - Parameters: From 0f5a9227cc55f20ababe8afd8a285ca2612394cf Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sat, 13 Jul 2024 11:50:26 -0400 Subject: [PATCH 2/6] Unit test --- Tests/TestingTests/SwiftPMTests.swift | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 4c4547179..a59f53408 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -88,6 +88,14 @@ struct SwiftPMTests { } } + @Test("--filter with no matches") + func filterWithNoMatches() async { + var args = __CommandLineArguments_v0() + args.filter = ["NOTHING_MATCHES_THIS_TEST_NAME_HOPEFULLY"] + let exitCode = await __swiftPMEntryPoint(passing: args) as CInt + #expect(exitCode == EXIT_NO_TESTS_FOUND) + } + @Test("--skip argument") @available(_regexAPI, *) func skip() async throws { From a97a2c50f72df8f5c48191b34771849046d520e2 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sat, 13 Jul 2024 16:56:52 -0400 Subject: [PATCH 3/6] Fix Windows and Linux builds --- Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift | 2 +- Sources/_TestingInternals/include/Includes.h | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift index 90c00d0d5..e6da35d9f 100644 --- a/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift @@ -31,7 +31,7 @@ var EXIT_NO_TESTS_FOUND: CInt { #if SWT_TARGET_OS_APPLE || os(Linux) EX_UNAVAILABLE #elseif os(Windows) - CInt(bitPattern: ERROR_NOT_FOUND) + ERROR_NOT_FOUND #else #warning("Platform-specific implementation missing: value for EXIT_NO_TESTS_FOUND unavailable") #endif diff --git a/Sources/_TestingInternals/include/Includes.h b/Sources/_TestingInternals/include/Includes.h index c856d4155..94015f47a 100644 --- a/Sources/_TestingInternals/include/Includes.h +++ b/Sources/_TestingInternals/include/Includes.h @@ -103,6 +103,10 @@ #include #endif +#if __has_include() +#include +#endif + #if defined(_WIN32) #define WIN32_LEAN_AND_MEAN #define NOMINMAX From e72d97d63850f197cc2f91418b2432e7491d9e8f Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sun, 14 Jul 2024 09:23:15 -0400 Subject: [PATCH 4/6] Add unit test for EXIT_NO_TESTS_FOUND value --- Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift | 1 + Tests/TestingTests/SwiftPMTests.swift | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift index e6da35d9f..a5dc872ae 100644 --- a/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift @@ -34,6 +34,7 @@ var EXIT_NO_TESTS_FOUND: CInt { ERROR_NOT_FOUND #else #warning("Platform-specific implementation missing: value for EXIT_NO_TESTS_FOUND unavailable") + 2 // We're assuming that EXIT_SUCCESS = 0 and EXIT_FAILURE = 1. #endif } diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index a59f53408..6a4ccbc37 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -29,6 +29,12 @@ struct SwiftPMTests { #expect(!CommandLine.arguments.isEmpty) } + @Test("EXIT_NO_TESTS_FOUND is unique") + func valueOfEXIT_NO_TESTS_FOUND() { + #expect(EXIT_NO_TESTS_FOUND != EXIT_SUCCESS) + #expect(EXIT_NO_TESTS_FOUND != EXIT_FAILURE) + } + @Test("--parallel/--no-parallel argument") func parallel() throws { var configuration = try configurationForEntryPoint(withArguments: ["PATH"]) From d0ba21219ae7ea117851338738c590c8ccb6c980 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sun, 14 Jul 2024 09:23:59 -0400 Subject: [PATCH 5/6] Typo --- Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift index a5dc872ae..a4ae25bb1 100644 --- a/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift @@ -34,7 +34,7 @@ var EXIT_NO_TESTS_FOUND: CInt { ERROR_NOT_FOUND #else #warning("Platform-specific implementation missing: value for EXIT_NO_TESTS_FOUND unavailable") - 2 // We're assuming that EXIT_SUCCESS = 0 and EXIT_FAILURE = 1. + return 2 // We're assuming that EXIT_SUCCESS = 0 and EXIT_FAILURE = 1. #endif } From fce4630a25f8cf0634006572ad9fbed65711c75b Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sun, 14 Jul 2024 10:59:42 -0400 Subject: [PATCH 6/6] Tweak comment --- Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift index a4ae25bb1..7237b95d4 100644 --- a/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift @@ -14,9 +14,9 @@ private import _TestingInternals #endif -/// The exit code returned to Swift Package Manager when no tests matched the -/// inputs specified by the developer (or, for the case of `swift test list`, -/// when no tests were found.) +/// The exit code returned to Swift Package Manager by Swift Testing when no +/// tests matched the inputs specified by the developer (or, for the case of +/// `swift test list`, when no tests were found.) /// /// Because Swift Package Manager does not directly link to the testing library, /// it duplicates the definition of this constant in its own source. Any changes