Emit ABIv0 JSON objects for tests when listing them.

This PR changes the behavior of `swift test list` and its various synonyms to allow reporting the list of tests via the ABI-stable JSON mechanism described in #479.

As it is not currently possible to directly call `swift test list --experimental-event-stream-output ... --experimental-event-stream-version 0`, it's a bit hard to test this code. However, it is possible to opt into this mode using `--experimental-configuration-path` and passing a path to a JSON file that includes `"listTests": true` (as noted by @allevato.)

Resolves #506.
Resolves rdar://130627856.

Author: grynspan

Sources/Testing/ABI/EntryPoints/EntryPoint.swift

@@ -31,58 +31,66 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha
   let exitCode = Locked(rawValue: EXIT_SUCCESS)
 
   do {
-    let args = try args ?? parseCommandLineArguments(from: CommandLine.arguments)
-    if args.listTests ?? false {
-      for testID in await listTestsForEntryPoint(Test.all) {
-#if SWT_TARGET_OS_APPLE && !SWT_NO_FILE_IO
-        try? FileHandle.stdout.write("\(testID)\n")
-#else
-        print(testID)
-#endif
-      }
-    } else {
 #if !SWT_NO_EXIT_TESTS
       // If an exit test was specified, run it. `exitTest` returns `Never`.
       if let exitTest = ExitTest.findInEnvironmentForEntryPoint() {
         await exitTest()
       }
 #endif
 
-      // Configure the test runner.
-      var configuration = try configurationForEntryPoint(from: args)
+    let args = try args ?? parseCommandLineArguments(from: CommandLine.arguments)
+    // Configure the test runner.
+    var configuration = try configurationForEntryPoint(from: args)
 
-      // Set up the event handler.
-      configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
-        if case let .issueRecorded(issue) = event.kind, !issue.isKnown {
-          exitCode.withLock { exitCode in
-            exitCode = EXIT_FAILURE
-          }
+    // Set up the event handler.
+    configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
+      if case let .issueRecorded(issue) = event.kind, !issue.isKnown {
+        exitCode.withLock { exitCode in
+          exitCode = EXIT_FAILURE
         }
-        oldEventHandler(event, context)
       }
+      oldEventHandler(event, context)
+    }
 
 #if !SWT_NO_FILE_IO
-      // Configure the event recorder to write events to stderr.
-      var options = Event.ConsoleOutputRecorder.Options()
-      options = .for(.stderr)
-      options.verbosity = args.verbosity
-      let eventRecorder = Event.ConsoleOutputRecorder(options: options) { string in
-        try? FileHandle.stderr.write(string)
-      }
+    // Configure the event recorder to write events to stderr.
+    var options = Event.ConsoleOutputRecorder.Options()
+    options = .for(.stderr)
+    options.verbosity = args.verbosity
+    let eventRecorder = Event.ConsoleOutputRecorder(options: options) { string in
+      try? FileHandle.stderr.write(string)
+    }
+    configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
+      eventRecorder.record(event, in: context)
+      oldEventHandler(event, context)
+    }
+#endif
+
+    // If the caller specified an alternate event handler, hook it up too.
+    if let eventHandler {
       configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
-        eventRecorder.record(event, in: context)
+        eventHandler(event, context)
         oldEventHandler(event, context)
       }
-#endif
+    }
 
-      // If the caller specified an alternate event handler, hook it up too.
-      if let eventHandler {
-        configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
-          eventHandler(event, context)
-          oldEventHandler(event, context)
-        }
+    if args.listTests ?? false {
+      let tests = await Test.all
+      for testID in listTestsForEntryPoint(tests) {
+        // Print the test ID to stdout (classical CLI behavior.)
+#if SWT_TARGET_OS_APPLE && !SWT_NO_FILE_IO
+        try? FileHandle.stdout.write("\(testID)\n")
+#else
+        print(testID)
+#endif
       }
 
+      // Post an event for every discovered test. These events are turned into
+      // JSON objects if JSON output is enabled.
+      for test in tests {
+        Event.post(.testDiscovered, for: test, testCase: nil, configuration: configuration)
+      }
+    } else {
       // Run the tests.
       let runner = await Runner(configuration: configuration)
       await runner.run()

Sources/Testing/ABI/v0/ABIv0.Record+Streaming.swift

@@ -31,18 +31,15 @@ extension ABIv0.Record {
   ) -> Event.Handler {
     let humanReadableOutputRecorder = Event.HumanReadableOutputRecorder()
     return { event, context in
-      if case let .runStarted(plan) = event.kind {
-        // Gather all tests in the run and forward records for them.
-        for test in plan.steps.lazy.map(\.test) {
-          try? JSON.withEncoding(of: Self(encoding: test)) { testJSON in
-            eventHandler(testJSON)
-          }
+      if case .testDiscovered = event.kind, let test = context.test {
+        try? JSON.withEncoding(of: Self(encoding: test)) { testJSON in
+          eventHandler(testJSON)
+        }
+      } else {
+        let messages = humanReadableOutputRecorder.record(event, in: context)
+        if let eventRecord = Self(encoding: event, in: context, messages: messages) {
+          try? JSON.withEncoding(of: eventRecord, eventHandler)
         }
-      }
-
-      let messages = humanReadableOutputRecorder.record(event, in: context)
-      if let eventRecord = Self(encoding: event, in: context, messages: messages) {
-        try? JSON.withEncoding(of: eventRecord, eventHandler)
       }
     }
   }
@@ -92,6 +89,12 @@ func eventHandlerForStreamingEventSnapshots(
   to eventHandler: @escaping @Sendable (_ eventAndContextJSON: UnsafeRawBufferPointer) -> Void
 ) -> Event.Handler {
   return { event, context in
+    if case .testDiscovered = event.kind {
+      // Discard events of this kind rather than forwarding them to avoid a
+      // crash in Xcode 16 Beta 1 (which does not expect any events to occur
+      // before .runStarted.)
+      return
+    }
     let snapshot = EventAndContextSnapshot(
       event: Event.Snapshot(snapshotting: event),
       eventContext: Event.Context.Snapshot(snapshotting: context)

Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedTest.swift

@@ -81,7 +81,7 @@ extension ABIv0 {
         let testIsParameterized = test.isParameterized
         isParameterized = testIsParameterized
         if testIsParameterized {
-          _testCases = test.testCases?.map(EncodedTestCase.init(encoding:))
+          _testCases = test.uncheckedTestCases?.map(EncodedTestCase.init(encoding:))
         }
       }
       name = test.name

Sources/Testing/Events/Event.swift

@@ -13,13 +13,18 @@
 public struct Event: Sendable {
   /// An enumeration describing the various kinds of event that can be observed.
   public enum Kind: Sendable {
-    /// A test run started.
+    /// A test was discovered during test run planning.
     ///
-    /// - Parameters:
-    ///   - plan: The test plan of the run that started.
+    /// This event is recorded once per discovered test when an instance of
+    /// ``Runner`` is initialized, _prior to_ ``Runner/run()`` being called. It
+    /// does not indicate whether or not a test will run—only that the test was
+    /// found by the testing library.
+    case testDiscovered
+
+    /// A test run started.
     ///
     /// This event is the first event posted after ``Runner/run()`` is called.
-    indirect case runStarted(_ plan: Runner.Plan)
+    case runStarted
 
     /// An iteration of the test run started.
     ///
@@ -318,6 +323,14 @@ extension Event {
 extension Event.Kind {
   /// A serializable enumeration describing the various kinds of event that can be observed.
   public enum Snapshot: Sendable, Codable {
+    /// A test was discovered during test run planning.
+    ///
+    /// This event is recorded once per discovered test when an instance of
+    /// ``Runner`` is initialized, _prior to_ ``Runner/run()`` being called. It
+    /// does not indicate whether or not a test will run—only that the test was
+    /// found by the testing library.
+    case testDiscovered
+
     /// A test run started.
     ///
     /// This is the first event posted after ``Runner/run()`` is called.
@@ -420,6 +433,8 @@ extension Event.Kind {
     /// - Parameter kind: The original ``Event.Kind`` to snapshot.
     public init(snapshotting kind: Event.Kind) {
       switch kind {
+      case .testDiscovered:
+        self = .testDiscovered
       case .runStarted:
         self = .runStarted
       case let .iterationStarted(index):

Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift

@@ -296,6 +296,11 @@ extension Event.HumanReadableOutputRecorder {
 
     // Finally, produce any messages for the event.
     switch event.kind {
+    case .testDiscovered:
+      // Suppress events of this kind from output as they are not generally
+      // interesting in human-readable output.
+      break
+
     case .runStarted:
       var comments = [Comment]()
       if verbosity > 0 {

Sources/Testing/Running/Runner.swift

@@ -27,8 +27,8 @@ public struct Runner: Sendable {
   ///   - tests: The tests to run.
   ///   - configuration: The configuration to use for running.
   public init(testing tests: [Test], configuration: Configuration = .init()) async {
-    self.plan = await Plan(tests: tests, configuration: configuration)
-    self.configuration = configuration
+    let plan = await Plan(tests: tests, configuration: configuration)
+    self.init(plan: plan, configuration: configuration)
   }
 
   /// Initialize an instance of this type that runs the tests in the specified
@@ -40,6 +40,12 @@ public struct Runner: Sendable {
   public init(plan: Plan, configuration: Configuration = .init()) {
     self.plan = plan
     self.configuration = configuration
+
+    // Post an event for every test in the resulting test plan. These events are
+    // turned into JSON objects if JSON output is enabled.
+    for test in plan.steps.lazy.map(\.test) {
+      Event.post(.testDiscovered, for: test, testCase: nil, configuration: configuration)
+    }
   }
 
   /// Initialize an instance of this type that runs all tests found in the
@@ -343,7 +349,7 @@ extension Runner {
     }
 
     await Configuration.withCurrent(runner.configuration) {
-      Event.post(.runStarted(runner.plan), for: nil, testCase: nil, configuration: runner.configuration)
+      Event.post(.runStarted, for: nil, testCase: nil, configuration: runner.configuration)
       defer {
         Event.post(.runEnded, for: nil, testCase: nil, configuration: runner.configuration)
       }

Sources/Testing/Test.swift

@@ -127,6 +127,23 @@ public struct Test: Sendable {
       return testCases
     }
   }
+  
+  /// Equivalent to ``testCases``, but without requiring that the test cases be
+  /// evaluated first.
+  ///
+  /// Most callers should not use this property and should prefer ``testCases``
+  /// since it will help catch logic errors in the testing library. Use this
+  /// property if you are interested in the test's test cases, but the test has
+  /// not been evaluated by an instance of ``Runner/Plan`` (e.g. if you are
+  /// implementing `swift test list`.)
+  var uncheckedTestCases: (some Sequence<Test.Case>)? {
+    testCasesState.flatMap { testCasesState in
+      if case let .evaluated(testCases) = testCasesState {
+        return testCases
+      }
+      return nil
+    }
+  }
 
   /// Evaluate this test's cases if they have not been evaluated yet.
   ///

Tests/TestingTests/ABIEntryPointTests.swift

@@ -109,6 +109,25 @@ struct ABIEntryPointTests {
     #expect(result)
   }
 
+  @Test("v0 entry point listing tests only")
+  func v0_listingTestsOnly() async throws {
+    var arguments = __CommandLineArguments_v0()
+    arguments.listTests = true
+    arguments.eventStreamVersion = 0
+    arguments.verbosity = .min
+
+    try await confirmation("Test matched", expectedCount: 1...) { testMatched in
+      _ = try await _invokeEntryPointV0(passing: arguments) { recordJSON in
+        let record = try! JSON.decode(ABIv0.Record.self, from: recordJSON)
+        if case .test = record.kind {
+          testMatched()
+        } else {
+          Issue.record("Unexpected record \(record)")
+        }
+      }
+    }
+  }
+
   private func _invokeEntryPointV0(
     passing arguments: __CommandLineArguments_v0,
     recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void = { _ in }

Tests/TestingTests/EventTests.swift

@@ -48,7 +48,7 @@ struct EventTests {
                 sourceLocation: nil)
             )
           ),
-          Event.Kind.runStarted(Runner.Plan(steps: [])),
+          Event.Kind.runStarted,
           Event.Kind.runEnded,
           Event.Kind.testCaseStarted,
           Event.Kind.testCaseEnded,

Tests/TestingTests/SwiftPMTests.swift

@@ -155,7 +155,7 @@ struct SwiftPMTests {
     do {
       let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--xunit-output", temporaryFilePath])
       let eventContext = Event.Context()
-      configuration.eventHandler(Event(.runStarted(Runner.Plan(steps: [])), testID: nil, testCaseID: nil), eventContext)
+      configuration.eventHandler(Event(.runStarted, testID: nil, testCaseID: nil), eventContext)
       configuration.eventHandler(Event(.runEnded, testID: nil, testCaseID: nil), eventContext)
     }
 
@@ -222,15 +222,11 @@ struct SwiftPMTests {
     }
     do {
       let configuration = try configurationForEntryPoint(withArguments: ["PATH", outputArgumentName, temporaryFilePath, versionArgumentName, version])
-      let eventContext = Event.Context()
-
       let test = Test {}
-      let plan = Runner.Plan(
-        steps: [
-          Runner.Plan.Step(test: test, action: .run(options: .init(isParallelizationEnabled: true)))
-        ]
-      )
-      configuration.handleEvent(Event(.runStarted(plan), testID: nil, testCaseID: nil), in: eventContext)
+      let eventContext = Event.Context(test: test)
+
+      configuration.handleEvent(Event(.testDiscovered, testID: test.id, testCaseID: nil), in: eventContext)
+      configuration.handleEvent(Event(.runStarted, testID: nil, testCaseID: nil), in: eventContext)
       do {
         let eventContext = Event.Context(test: test)
         configuration.handleEvent(Event(.testStarted, testID: test.id, testCaseID: nil), in: eventContext)