From 5da76a36d630d4689f1946323281bfba92883f25 Mon Sep 17 00:00:00 2001 From: Kris Cieplak Date: Fri, 10 Oct 2025 10:19:23 -0400 Subject: [PATCH 1/2] [WIP] Experiment with adding a operation queue Add an operation queue around execution of swiftPM process execution. swift-testing based test will run all tests in parallel at the same time, causing each invocation to spawn a process which will then spawn many other subprocesses. These leads to a process bomb and consumes all CPU and causes massive I/O blocking. Add a limit to how many swiftPM top level processes at the same time. --- .../_InternalTestSupport/SwiftPMProduct.swift | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/Sources/_InternalTestSupport/SwiftPMProduct.swift b/Sources/_InternalTestSupport/SwiftPMProduct.swift index 0608fe4ca47..00b5a5eb7ce 100644 --- a/Sources/_InternalTestSupport/SwiftPMProduct.swift +++ b/Sources/_InternalTestSupport/SwiftPMProduct.swift @@ -18,6 +18,8 @@ import struct Basics.AsyncProcessResult import enum TSCBasic.ProcessEnv +private let swiftPMExecutionQueue = AsyncOperationQueue(concurrentTasks: 6) + /// Defines the executables used by SwiftPM. /// Contains path to the currently built executable and /// helper method to execute them. @@ -82,28 +84,30 @@ extension SwiftPM { env: Environment? = nil, throwIfCommandFails: Bool = true ) async throws -> (stdout: String, stderr: String) { - let result = try await executeProcess( - args, - packagePath: packagePath, - env: env - ) - //Remove /r from stdout/stderr so that tests do not have to deal with them - let stdout = try String(decoding: result.output.get().filter( { $0 != 13 }), as: Unicode.UTF8.self) - let stderr = try String(decoding: result.stderrOutput.get().filter( { $0 != 13 }), as: Unicode.UTF8.self) - - let returnValue = (stdout: stdout, stderr: stderr) - if (!throwIfCommandFails) { return returnValue } - - if result.exitStatus == .terminated(code: 0) { - return returnValue + try await swiftPMExecutionQueue.withOperation { + let result = try await executeProcess( + args, + packagePath: packagePath, + env: env + ) + // Remove /r from stdout/stderr so that tests do not have to deal with them + let stdout = try String(decoding: result.output.get().filter { $0 != 13 }, as: Unicode.UTF8.self) + let stderr = try String(decoding: result.stderrOutput.get().filter { $0 != 13 }, as: Unicode.UTF8.self) + + let returnValue = (stdout: stdout, stderr: stderr) + if !throwIfCommandFails { return returnValue } + + if result.exitStatus == .terminated(code: 0) { + return returnValue + } + throw SwiftPMError.executionFailure( + underlying: AsyncProcessResult.Error.nonZeroExit(result), + stdout: stdout, + stderr: stderr + ) } - throw SwiftPMError.executionFailure( - underlying: AsyncProcessResult.Error.nonZeroExit(result), - stdout: stdout, - stderr: stderr - ) } - + private func executeProcess( _ args: [String], packagePath: AbsolutePath? = nil, From e99a75c14f85f32a55d07d550fc4f44b7bf81dfb Mon Sep 17 00:00:00 2001 From: Kris Cieplak Date: Tue, 14 Oct 2025 13:37:08 -0400 Subject: [PATCH 2/2] Updates based on PR review * Initialize the queue size to some fraction of number of CPUs. * Add a comment on why we are running executions on a queue. --- Sources/_InternalTestSupport/SwiftPMProduct.swift | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Sources/_InternalTestSupport/SwiftPMProduct.swift b/Sources/_InternalTestSupport/SwiftPMProduct.swift index 00b5a5eb7ce..86422039ebc 100644 --- a/Sources/_InternalTestSupport/SwiftPMProduct.swift +++ b/Sources/_InternalTestSupport/SwiftPMProduct.swift @@ -18,7 +18,8 @@ import struct Basics.AsyncProcessResult import enum TSCBasic.ProcessEnv -private let swiftPMExecutionQueue = AsyncOperationQueue(concurrentTasks: 6) +// Fan out from invocation of SPM 'swift-*' commands can be quite large. Limit the number of concurrent tasks to a fraction of total CPUs. +private let swiftPMExecutionQueue = AsyncOperationQueue(concurrentTasks: Int(Double(ProcessInfo.processInfo.activeProcessorCount) * 0.5)) /// Defines the executables used by SwiftPM. /// Contains path to the currently built executable and @@ -84,6 +85,12 @@ extension SwiftPM { env: Environment? = nil, throwIfCommandFails: Bool = true ) async throws -> (stdout: String, stderr: String) { + // Swift Testing uses Swift concurrency for test execution and creates a task for each test to run in parallel. + // A single invocation of "swift build" can spawn a large number of subprocesses. + // When this pattern is repeated across many tests, thousands of processes compete for + // CPU/disk/network resources. Tests can take thousands of seconds to complete, with periods + // of no stdout/stderr output that can cause activity timeouts in CI pipelines. + // Run all SPM executions under a queue to limit the maximum number of concurrent SPM processes. try await swiftPMExecutionQueue.withOperation { let result = try await executeProcess( args,