Testing: Run all test SPM executable actions in a operation queue (#9243)

Add an operation queue around execution of swiftPM process execution.

**swift-testing** based test will run all tests in parallel at the same
time, causing each invocation to spawn a process which will then spawn
many other subprocesses. This is compounded by tests that use arguments,
as each combination will create another test that will run in parallel.

These leads to a process bomb consuming all CPU and causes massive I/O
blocking.

Add a limit to how many swiftPM top level processes can run at the same
time during testing. This is similar behaviour to XCtest based testing
with the **--num-workers** option limiting the number of occurrent
tests.

Author: kcieplak

Sources/_InternalTestSupport/SwiftPMProduct.swift

@@ -18,6 +18,9 @@ import struct Basics.AsyncProcessResult
 
 import enum TSCBasic.ProcessEnv
 
+// Fan out from invocation of SPM 'swift-*' commands can be quite large.  Limit the number of concurrent tasks to a fraction of total CPUs.
+private let swiftPMExecutionQueue = AsyncOperationQueue(concurrentTasks: Int(Double(ProcessInfo.processInfo.activeProcessorCount) * 0.5))
+
 /// Defines the executables used by SwiftPM.
 /// Contains path to the currently built executable and
 /// helper method to execute them.
@@ -82,28 +85,36 @@ extension SwiftPM {
         env: Environment? = nil,
         throwIfCommandFails: Bool = true
     ) async throws -> (stdout: String, stderr: String) {
-        let result = try await executeProcess(
-            args,
-            packagePath: packagePath,
-            env: env
-        )
-        //Remove /r from stdout/stderr so that tests do not have to deal with them
-        let stdout = try String(decoding: result.output.get().filter( { $0 != 13 }), as: Unicode.UTF8.self)
-        let stderr = try String(decoding: result.stderrOutput.get().filter( { $0 != 13 }), as: Unicode.UTF8.self)
-        
-        let returnValue = (stdout: stdout, stderr: stderr)
-        if (!throwIfCommandFails) { return returnValue }
-
-        if result.exitStatus == .terminated(code: 0) {
-            return returnValue
+        // Swift Testing uses Swift concurrency for test execution and creates a task for each test to run in parallel.
+        // A single invocation of "swift build" can spawn a large number of subprocesses.
+        // When this pattern is repeated across many tests, thousands of processes compete for
+        // CPU/disk/network resources. Tests can take thousands of seconds to complete, with periods
+        // of no stdout/stderr output that can cause activity timeouts in CI pipelines.
+        // Run all SPM executions under a queue to limit the maximum number of concurrent SPM processes.
+        try await swiftPMExecutionQueue.withOperation {
+            let result = try await executeProcess(
+                args,
+                packagePath: packagePath,
+                env: env
+            )
+            // Remove /r from stdout/stderr so that tests do not have to deal with them
+            let stdout = try String(decoding: result.output.get().filter { $0 != 13 }, as: Unicode.UTF8.self)
+            let stderr = try String(decoding: result.stderrOutput.get().filter { $0 != 13 }, as: Unicode.UTF8.self)
+
+            let returnValue = (stdout: stdout, stderr: stderr)
+            if !throwIfCommandFails { return returnValue }
+
+            if result.exitStatus == .terminated(code: 0) {
+                return returnValue
+            }
+            throw SwiftPMError.executionFailure(
+                underlying: AsyncProcessResult.Error.nonZeroExit(result),
+                stdout: stdout,
+                stderr: stderr
+            )
         }
-        throw SwiftPMError.executionFailure(
-            underlying: AsyncProcessResult.Error.nonZeroExit(result),
-            stdout: stdout,
-            stderr: stderr
-        )
     }
-    
+
     private func executeProcess(
         _ args: [String],
         packagePath: AbsolutePath? = nil,