Provide library paths to plugin tools when run as custom tasks (#8767)

Linux build tool plugins won't work because they require a library
path to the Swift standard libraries. Add the toolchain's library path
to the environment for the custom task. Enable the tests that cover
this area for non-macOS platforms.

Author: cmcgee1024

Sources/Basics/Environment/EnvironmentKey.swift

@@ -24,6 +24,16 @@ public struct EnvironmentKey {
 extension EnvironmentKey {
     package static let path: Self = "PATH"
 
+    package static var libraryPath: Self {
+        #if os(Windows)
+        path
+        #elseif canImport(Darwin)
+        "DYLD_LIBRARY_PATH"
+        #else
+        "LD_LIBRARY_PATH"
+        #endif
+    }
+
     /// A set of known keys which should not be included in cache keys.
     package static let nonCachable: Set<Self> = [
         "TERM",

Sources/PackageModel/Toolchain.swift

@@ -35,6 +35,9 @@ public protocol Toolchain {
     /// An array of paths to search for libraries at link time.
     var librarySearchPaths: [AbsolutePath] { get }
 
+    /// An array of paths to use with binaries produced by this toolchain at run time.
+    var runtimeLibraryPaths: [AbsolutePath] { get }
+
     /// Configuration from the used toolchain.
     var installedSwiftPMConfiguration: InstalledSwiftPMConfiguration { get }
 

Sources/PackageModel/UserToolchain.swift

@@ -42,6 +42,9 @@ public final class UserToolchain: Toolchain {
     /// An array of paths to search for libraries at link time.
     public let librarySearchPaths: [AbsolutePath]
 
+    /// An array of paths to use with binaries produced by this toolchain at run time.
+    public let runtimeLibraryPaths: [AbsolutePath]
+
     /// Path containing Swift resources for dynamic linking.
     public var swiftResourcesPath: AbsolutePath? {
         swiftSDK.pathsConfiguration.swiftResourcesPath
@@ -206,6 +209,24 @@ public final class UserToolchain: Toolchain {
         }
     }
 
+    private static func computeRuntimeLibraryPaths(targetInfo: JSON) throws -> [AbsolutePath] {
+        var libraryPaths: [AbsolutePath] = []
+
+        for runtimeLibPath in (try? (try? targetInfo.get("paths"))?.getArray("runtimeLibraryPaths")) ?? [] {
+            guard case .string(let value) = runtimeLibPath else {
+                continue
+            }
+
+            guard let path = try? AbsolutePath(validating: value) else {
+                continue
+            }
+
+            libraryPaths.append(path)
+        }
+
+        return libraryPaths
+    }
+
     private static func computeSwiftCompilerVersion(targetInfo: JSON) -> String? {
         // Get the compiler version from the target info
         let compilerVersion: String
@@ -701,6 +722,9 @@ public final class UserToolchain: Toolchain {
         // Get compiler version information from target info
         self.swiftCompilerVersion = Self.computeSwiftCompilerVersion(targetInfo: targetInfo)
 
+        // Get the list of runtime libraries from the target info
+        self.runtimeLibraryPaths = try Self.computeRuntimeLibraryPaths(targetInfo: targetInfo)
+
         // Use the triple from Swift SDK or compute the host triple from the target info
         var triple = try swiftSDK.targetTriple ?? Self.getHostTriple(targetInfo: targetInfo)
 

Sources/SwiftBuildSupport/PIFBuilder.swift

@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-import Basics
+@_spi(SwiftPMInternal) import Basics
 import Foundation
 import PackageGraph
 import PackageLoading
@@ -357,9 +357,12 @@ public final class PIFBuilder {
                         let result2 = PackagePIFBuilder.BuildToolPluginInvocationResult(
                             prebuildCommandOutputPaths: result.prebuildCommands.map( { $0.outputFilesDirectory } ),
                             buildCommands: result.buildCommands.map( { buildCommand in
-                                var env: [String: String] = [:]
-                                for (key, value) in buildCommand.configuration.environment {
-                                    env[key.rawValue] = value
+                                var newEnv: Environment = buildCommand.configuration.environment
+
+                                if let runtimeLibPaths = try? buildParameters.toolchain.runtimeLibraryPaths {
+                                    for libPath in runtimeLibPaths {
+                                        newEnv.appendPath(key: .libraryPath, value: libPath.pathString)
+                                    }
                                 }
 
                                 let workingDir = buildCommand.configuration.workingDirectory
@@ -370,7 +373,7 @@ public final class PIFBuilder {
                                     displayName: buildCommand.configuration.displayName,
                                     executable: buildCommand.configuration.executable.pathString,
                                     arguments: buildCommand.configuration.arguments,
-                                    environment: env,
+                                    environment: .init(newEnv),
                                     workingDir: workingDir,
                                     inputPaths: buildCommand.inputFiles,
                                     outputPaths: buildCommand.outputFiles.map(\.pathString),

Sources/_InternalTestSupport/MockBuildTestHelper.swift

@@ -31,6 +31,7 @@ public struct MockToolchain: PackageModel.Toolchain {
     public let swiftCompilerPath = AbsolutePath("/fake/path/to/swiftc")
     public let includeSearchPaths = [AbsolutePath]()
     public let librarySearchPaths = [AbsolutePath]()
+    public let runtimeLibraryPaths: [AbsolutePath] = [AbsolutePath]()
     public let swiftResourcesPath: AbsolutePath?
     public let swiftStaticResourcesPath: AbsolutePath? = nil
     public let sdkRootPath: AbsolutePath? = nil

Tests/CommandsTests/SwiftSDKCommandTests.swift

@@ -66,7 +66,7 @@ final class SwiftSDKCommandTests: CommandsTestCase {
 
                     // We only expect tool's output on the stdout stream.
                     XCTAssertMatch(
-                        stdout,
+                        stdout + "\nstderr:\n" + stderr,
                         .contains("\(bundle)` successfully installed as test-sdk.artifactbundle.")
                     )
 

Tests/FunctionalTests/PluginTests.swift

@@ -70,7 +70,7 @@ final class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("Build of product 'MyTool' complete!"), "stdout:\n\(stdout)")
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if !os(Windows) // https://github.com/swiftlang/swift-package-manager/issues/8774
         // Try again with the Swift Build build system
         try await fixture(name: "Miscellaneous/Plugins") { fixturePath in
             let (stdout, _) = try await executeSwiftBuild(fixturePath.appending("MySourceGenClient"), configuration: .Debug, extraArgs: ["--build-system", "swiftbuild", "--product", "MyTool"])
@@ -92,7 +92,7 @@ final class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("Build of product 'MyOtherLocalTool' complete!"), "stdout:\n\(stdout)")
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if !os(Windows) // https://github.com/swiftlang/swift-package-manager/issues/8774
         try await fixture(name: "Miscellaneous/Plugins") { fixturePath in
             let (stdout, _) = try await executeSwiftBuild(fixturePath.appending("MySourceGenPlugin"), configuration: .Debug, extraArgs: ["--build-system", "swiftbuild", "--product", "MyOtherLocalTool"])
             XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")
@@ -115,7 +115,7 @@ final class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if !os(Windows) // https://github.com/swiftlang/swift-package-manager/issues/8774
         // Try again with the Swift Build build system
         try await fixture(name: "Miscellaneous/Plugins") { fixturePath in
             let (stdout, _) = try await executeSwiftBuild(fixturePath.appending("ClientOfPluginWithInternalExecutable"), extraArgs: ["--build-system", "swiftbuild"])
@@ -136,13 +136,11 @@ final class PluginTests: XCTestCase {
             }
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
         try await fixture(name: "Miscellaneous/Plugins") { fixturePath in
             await XCTAssertThrowsCommandExecutionError(try await executeSwiftBuild(fixturePath.appending("InvalidUseOfInternalPluginExecutable")), "Illegally used internal executable"
 ) { error in
             }
         }
-#endif
     }
 
     func testLocalBuildToolPluginUsingRemoteExecutable() async throws {
@@ -159,7 +157,7 @@ final class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if !os(Windows) // https://github.com/swiftlang/swift-package-manager/issues/8774
         // Try again with the Swift Build build system
         try await fixture(name: "Miscellaneous/Plugins") { fixturePath in
             let (stdout, _) = try await executeSwiftBuild(fixturePath.appending("LibraryWithLocalBuildToolPluginUsingRemoteTool"), extraArgs: ["--build-system", "swiftbuild"])
@@ -182,7 +180,7 @@ final class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if os(macOS)
         // Try again with the Swift Build build system
         try await fixture(name: "Miscellaneous/Plugins") { fixturePath in
             let (stdout, _) = try await executeSwiftBuild(fixturePath.appending("MyBuildToolPluginDependencies"), extraArgs: ["--build-system", "swiftbuild"])
@@ -204,7 +202,7 @@ final class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("Build of product 'MyLocalTool' complete!"), "stdout:\n\(stdout)")
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if !os(Windows) // https://github.com/swiftlang/swift-package-manager/issues/8774
         try await fixture(name: "Miscellaneous/Plugins") { fixturePath in
             let (stdout, _) = try await executeSwiftBuild(fixturePath.appending("ContrivedTestPlugin"), configuration: .Debug, extraArgs: ["--build-system", "swiftbuild", "--product", "MyLocalTool", "--disable-sandbox"])
             XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")
@@ -1158,7 +1156,7 @@ final class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("type of snippet target: snippet"), "output:\n\(stderr)\n\(stdout)")
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if !os(Windows) // https://github.com/swiftlang/swift-package-manager/issues/8774
         // Try again with the Swift Build build system
         try await fixture(name: "Miscellaneous/Plugins") { path in
             let (stdout, stderr) = try await executeSwiftPackage(path.appending("PluginsAndSnippets"), configuration: .Debug, extraArgs: ["--build-system", "swiftbuild", "do-something"])
@@ -1175,7 +1173,7 @@ final class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("Build complete!"), "output:\n\(stderr)\n\(stdout)")
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if !os(Windows) // https://github.com/swiftlang/swift-package-manager/issues/8774
         // Try again with the Swift Build build system
         try await fixture(name: "Miscellaneous/Plugins") { path in
             let (stdout, stderr) = try await executeSwiftBuild(path.appending("IncorrectDependencies"), extraArgs: ["--build-system", "swiftbuild", "--build-tests"])
@@ -1219,7 +1217,7 @@ final class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if !os(Windows) // https://github.com/swiftlang/swift-package-manager/issues/8774
         // Try again with Swift Build build system
         try await fixture(name: "Miscellaneous/Plugins") { fixturePath in
             let (stdout, _) = try await executeSwiftBuild(fixturePath.appending("TransitivePluginOnlyDependency"), extraArgs: ["--build-system", "swiftbuild"])
@@ -1240,7 +1238,7 @@ final class PluginTests: XCTestCase {
             }
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if !os(Windows) // https://github.com/swiftlang/swift-package-manager/issues/8774
         try await fixture(name: "Miscellaneous/Plugins") { fixturePath in
             do {
                 try await executeSwiftBuild(fixturePath.appending("MissingPlugin"), extraArgs: ["--build-system", "swiftbuild"])
@@ -1262,7 +1260,7 @@ final class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if !os(Windows) // https://github.com/swiftlang/swift-package-manager/issues/8774
         // Try again with the Swift Build build system
         try await fixture(name: "Miscellaneous/Plugins") { fixturePath in
             let (stdout, _) = try await executeSwiftBuild(fixturePath.appending("PluginCanBeReferencedByProductName"), extraArgs: ["--build-system", "swiftbuild"])
@@ -1292,7 +1290,7 @@ final class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("Build of product 'MyLocalTool' complete!"), "stdout:\n\(stdout)")
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if os(macOS)
         try await fixture(name: "Miscellaneous/Plugins") { fixturePath in
             let (stdout, stderr) = try await executeSwiftBuild(
                 fixturePath.appending(component: "MySourceGenPlugin"),
@@ -1315,8 +1313,7 @@ final class PluginTests: XCTestCase {
             let (stdout, _) = try await executeSwiftBuild(fixturePath, configuration: .Debug)
             XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")
         }
-
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if os(macOS)
         // Try again with the Swift Build build system
         try await fixture(name: "Miscellaneous/Plugins/MySourceGenPluginUsingURLBasedAPI") { fixturePath in
             let (stdout, _) = try await executeSwiftBuild(fixturePath, configuration: .Debug, extraArgs: ["--build-system", "swiftbuild"])
@@ -1333,7 +1330,7 @@ final class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")
         }
 
-#if os(macOS) // See https://github.com/swiftlang/swift-package-manager/issues/8416 for errors running build tools on Linux
+#if !os(Windows) // https://github.com/swiftlang/swift-package-manager/issues/8774
         try await fixture(name: "Miscellaneous/Plugins/DependentPlugins") { fixturePath in
             let (stdout, _) = try await executeSwiftBuild(fixturePath, extraArgs: ["--build-system", "swiftbuild"])
             XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")