refactor toolchain SwiftPM libraries path

motivation: improve logic in how Manifest and Plugin APIs locations are derived

changes:
* rename fields on ToolchainConfiguration to better articulate their meaning
* change logic in UserToolchain to derive the SwiftPM libraries path more clearly and reliably
* adjust call-sites and test

Author: tomerd

CONTRIBUTING.md

@@ -336,7 +336,7 @@ $> SWIFT_EXEC=/path/to/my/built/swiftc swift build
 
 SwiftPM computes the path of its runtime libraries relative to where it is
 installed. This path can be overridden by setting the environment variable
-`SWIFTPM_PD_LIBS` to a directory containing the libraries, or a colon-separated list of
+`SWIFTPM_CUSTOM_LIBS_DIR` to a directory containing the libraries, or a colon-separated list of
 absolute search paths. SwiftPM will choose the first
 path which exists on disk. If none of the paths are present on disk, it will fall
 back to built-in computation.

Sources/Commands/SwiftRunTool.swift

@@ -136,7 +136,7 @@ public struct SwiftRunTool: SwiftCommand {
             let arguments = buildOp.buildPlan!.createREPLArguments()
             print("Launching Swift REPL with arguments: \(arguments.joined(separator: " "))")
             try run(
-                swiftTool.getToolchain().swiftInterpreter,
+                swiftTool.getToolchain().swiftInterpreterPath,
                 originalWorkingDirectory: swiftTool.originalWorkingDirectory,
                 arguments: arguments)
 
@@ -171,7 +171,7 @@ public struct SwiftRunTool: SwiftCommand {
             if let executable = options.executable, isValidSwiftFilePath(executable) {
                 swiftTool.diagnostics.emit(.runFileDeprecation)
                 // Redirect execution to the toolchain's swift executable.
-                let swiftInterpreterPath = try swiftTool.getToolchain().swiftInterpreter
+                let swiftInterpreterPath = try swiftTool.getToolchain().swiftInterpreterPath
                 // Prepend the script to interpret to the arguments.
                 let arguments = [executable] + options.arguments
                 try run(

Sources/PackageLoading/ManifestLoader.swift

@@ -689,21 +689,20 @@ public final class ManifestLoader: ManifestLoaderProtocol {
         let moduleCachePath = (ProcessEnv.vars["SWIFTPM_MODULECACHE_OVERRIDE"] ?? ProcessEnv.vars["SWIFTPM_TESTS_MODULECACHE"]).flatMap{ AbsolutePath.init($0) }
 
         var cmd: [String] = []
-        cmd += [self.toolchain.swiftCompiler.pathString]
+        cmd += [self.toolchain.swiftCompilerPath.pathString]
         cmd += verbosity.ccArgs
 
         let macOSPackageDescriptionPath: AbsolutePath
-        // If we got the binDir that means we could be developing SwiftPM in Xcode
+        // if runtimePath is set to "PackageFrameworks" that means we could be developing SwiftPM in Xcode
         // which produces a framework for dynamic package products.
-        let packageFrameworkPath = runtimePath.appending(component: "PackageFrameworks")
-        if self.toolchain.binDir != nil, localFileSystem.exists(packageFrameworkPath)  {
+        if runtimePath.basename == "PackageFrameworks" {
             cmd += [
-                "-F", packageFrameworkPath.pathString,
+                "-F", runtimePath.pathString,
                 "-framework", "PackageDescription",
-                "-Xlinker", "-rpath", "-Xlinker", packageFrameworkPath.pathString,
+                "-Xlinker", "-rpath", "-Xlinker", runtimePath.pathString,
             ]
 
-            macOSPackageDescriptionPath = packageFrameworkPath.appending(RelativePath("PackageDescription.framework/PackageDescription"))
+            macOSPackageDescriptionPath = runtimePath.appending(RelativePath("PackageDescription.framework/PackageDescription"))
         } else {
             cmd += [
                 "-L", runtimePath.pathString,
@@ -722,7 +721,7 @@ public final class ManifestLoader: ManifestLoaderProtocol {
         // Use the same minimum deployment target as the PackageDescription library (with a fallback of 10.15).
 #if os(macOS)
         let triple = Self._hostTriple.memoize {
-            Triple.getHostTriple(usingSwiftCompiler: self.toolchain.swiftCompiler)
+            Triple.getHostTriple(usingSwiftCompiler: self.toolchain.swiftCompilerPath)
         }
 
         let version = try Self._packageDescriptionMinimumDeploymentTarget.memoize {
@@ -853,9 +852,15 @@ public final class ManifestLoader: ManifestLoaderProtocol {
         var cmd = [String]()
         let runtimePath = self.runtimePath(for: toolsVersion)
         cmd += ["-swift-version", toolsVersion.swiftLanguageVersion.rawValue]
-        cmd += ["-I", runtimePath.pathString]
+        // if runtimePath is set to "PackageFrameworks" that means we could be developing SwiftPM in Xcode
+        // which produces a framework for dynamic package products.
+        if runtimePath.basename == "PackageFrameworks" {
+            cmd += ["-I", runtimePath.parentDirectory.pathString]
+        } else {
+            cmd += ["-I", runtimePath.pathString]
+        }
       #if os(macOS)
-        if let sdkRoot = self.toolchain.sdkRoot ?? self.sdkRoot() {
+        if let sdkRoot = self.toolchain.sdkRootPath ?? self.sdkRoot() {
             cmd += ["-sdk", sdkRoot.pathString]
         }
       #endif
@@ -865,19 +870,14 @@ public final class ManifestLoader: ManifestLoaderProtocol {
 
     /// Returns the runtime path given the manifest version and path to libDir.
     private func runtimePath(for version: ToolsVersion) -> AbsolutePath {
-        // Bin dir will be set when developing swiftpm without building all of the runtimes.
-        if let binDir = self.toolchain.binDir {
-            return binDir
-        }
-        
-        // Otherwise we use the standard location of the manifest API in the toolchain, if it exists.
-        let manifestAPIDir = self.toolchain.libDir.appending(component: "ManifestAPI")
+        let manifestAPIDir = self.toolchain.swiftPMLibrariesLocation.manifestAPI
         if localFileSystem.exists(manifestAPIDir) {
             return manifestAPIDir
         }
-        
-        // Otherwise, fall back on the old location (this would indicate that we're using an old toolchain).
-        return self.toolchain.libDir.appending(version.runtimeSubpath)
+
+        // FIXME: how do we test this?
+        // Fall back on the old location (this would indicate that we're using an old toolchain).
+        return self.toolchain.swiftPMLibrariesLocation.manifestAPI.parentDirectory.appending(version.runtimeSubpath)
     }
 
     /// Returns path to the manifest database inside the given cache directory.

Sources/PackageModel/ToolchainConfiguration.swift

@@ -16,34 +16,50 @@ import TSCBasic
 /// using the package manager with alternate toolchains in the future.
 public struct ToolchainConfiguration {
     /// The path of the swift compiler.
-    public let swiftCompiler: AbsolutePath
+    public let swiftCompilerPath: AbsolutePath
 
     /// Extra flags to pass the Swift compiler.
     public let swiftCompilerFlags: [String]
 
-    /// The path of the library resources.
-    public let libDir: AbsolutePath
-
-    /// The bin directory.
-    public let binDir: AbsolutePath?
+    /// SwiftPM library paths.
+    public let swiftPMLibrariesLocation: SwiftPMLibrariesLocation
 
     /// The path to SDK root.
     ///
     /// If provided, it will be passed to the swift interpreter.
-    public let sdkRoot: AbsolutePath?
+    public let sdkRootPath: AbsolutePath?
 
     /// XCTest Location
-    public let xctestLocation: AbsolutePath?
+    public let xctestPath: AbsolutePath?
 
     /// Creates the set of manifest resources associated with a `swiftc` executable.
     ///
     /// - Parameters:
-    ///     - swiftCompiler: The absolute path of the associated `swiftc` executable.
+    ///     - swiftCompilerPath: The absolute path of the associated `swiftc` executable.
     ///     - swiftCompilerFlags: Extra flags to pass the Swift compiler.: Extra flags to pass the Swift compiler.
-    ///     - libDir: The path of the library resources.
-    ///     - binDir: The bin directory.
-    ///     - sdkRoot: The path to SDK root.
-    ///     - xctestLocation: XCTest Location
+    ///     - swiftPMLibrariesRootPath: Custom path for SwiftPM libraries.
+    ///     - sdkRootPath: The path to SDK root.
+    ///     - xctestPath: XCTest Location
+    public init(
+        swiftCompilerPath: AbsolutePath,
+        swiftCompilerFlags: [String] = [],
+        swiftPMLibrariesLocation: SwiftPMLibrariesLocation? = nil,
+        sdkRootPath: AbsolutePath? = nil,
+        xctestPath: AbsolutePath? = nil
+    ) {
+        let swiftPMLibrariesLocation = swiftPMLibrariesLocation ?? {
+            return .init(swiftCompilerPath: swiftCompilerPath)
+        }()
+
+        self.swiftCompilerPath = swiftCompilerPath
+        self.swiftCompilerFlags = swiftCompilerFlags
+        self.swiftPMLibrariesLocation = swiftPMLibrariesLocation
+        self.sdkRootPath = sdkRootPath
+        self.xctestPath = xctestPath
+    }
+
+    // deprecated 8/2021
+    @available(*, deprecated, message: "use non-deprecated initializer instead")
     public init(
         swiftCompiler: AbsolutePath,
         swiftCompilerFlags: [String] = [],
@@ -52,15 +68,36 @@ public struct ToolchainConfiguration {
         sdkRoot: AbsolutePath? = nil,
         xctestLocation: AbsolutePath? = nil
     ) {
-        self.swiftCompiler = swiftCompiler
-        self.swiftCompilerFlags = swiftCompilerFlags
-        self.libDir = libDir ?? Self.libDir(forBinDir: swiftCompiler.parentDirectory)
-        self.binDir = binDir
-        self.sdkRoot = sdkRoot
-        self.xctestLocation = xctestLocation
+        self.init(
+            swiftCompilerPath: swiftCompiler,
+            swiftCompilerFlags: swiftCompilerFlags,
+            swiftPMLibrariesLocation: libDir.map { .init(root: $0) },
+            sdkRootPath: sdkRoot,
+            xctestPath: xctestLocation
+        )
     }
+}
+
+extension ToolchainConfiguration {
+    public struct SwiftPMLibrariesLocation {
+        public var manifestAPI: AbsolutePath
+        public var pluginAPI: AbsolutePath
+
+        public init(manifestAPI: AbsolutePath, pluginAPI: AbsolutePath) {
+            self.manifestAPI = manifestAPI
+            self.pluginAPI = pluginAPI
+        }
+
+        public init(root: AbsolutePath) {
+            self.init(
+                manifestAPI: root.appending(component: "ManifestAPI"),
+                pluginAPI: root.appending(component: "PluginAPI")
+            )
+        }
 
-    public static func libDir(forBinDir binDir: AbsolutePath) -> AbsolutePath {
-        return binDir.parentDirectory.appending(components: "lib", "swift", "pm")
+        fileprivate init(swiftCompilerPath: AbsolutePath) {
+            let rootPath = swiftCompilerPath.parentDirectory.parentDirectory.appending(components: "lib", "swift", "pm")
+            self.init(root: rootPath)
+        }
     }
 }

Sources/SPMTestSupport/Toolchain.swift

@@ -35,10 +35,9 @@ extension ToolchainConfiguration {
         get {
             let toolchain = UserToolchain.default
             return .init(
-                swiftCompiler: toolchain.configuration.swiftCompiler,
+                swiftCompilerPath: toolchain.configuration.swiftCompilerPath,
                 swiftCompilerFlags: [],
-                libDir: toolchain.configuration.libDir,
-                binDir: toolchain.configuration.binDir
+                swiftPMLibrariesLocation: toolchain.configuration.swiftPMLibrariesLocation
             )
         }
     }

Sources/Workspace/DefaultPluginScriptRunner.swift

@@ -40,36 +40,33 @@ public struct DefaultPluginScriptRunner: PluginScriptRunner {
 
     public var hostTriple: Triple {
         return Self._hostTriple.memoize {
-            Triple.getHostTriple(usingSwiftCompiler: self.toolchain.swiftCompiler)
+            Triple.getHostTriple(usingSwiftCompiler: self.toolchain.swiftCompilerPath)
         }
     }
 
     /// Helper function that compiles a plugin script as an executable and returns the path to it.
     fileprivate func compile(sources: Sources, toolsVersion: ToolsVersion, cacheDir: AbsolutePath) throws -> AbsolutePath {
         // FIXME: Much of this is copied from the ManifestLoader and should be consolidated.
 
-        // Bin dir will be set when developing swiftpm without building all of the runtimes.
-        let runtimePath = self.toolchain.binDir ?? self.toolchain.libDir.appending(component: "PluginAPI")
+        let runtimePath = self.toolchain.swiftPMLibrariesLocation.pluginAPI
 
         // Compile the package plugin script.
-        var command = [self.toolchain.swiftCompiler.pathString]
+        var command = [self.toolchain.swiftCompilerPath.pathString]
 
         // FIXME: Workaround for the module cache bug that's been haunting Swift CI
         // <rdar://problem/48443680>
         let moduleCachePath = ProcessEnv.vars["SWIFTPM_MODULECACHE_OVERRIDE"] ?? ProcessEnv.vars["SWIFTPM_TESTS_MODULECACHE"]
 
-        // If we got the binDir that means we could be developing SwiftPM in Xcode
-        // which produces a framework for dynamic package products.
-        let packageFrameworkPath = runtimePath.appending(component: "PackageFrameworks")
-
         let macOSPackageDescriptionPath: AbsolutePath
-        if self.toolchain.binDir != nil, localFileSystem.exists(packageFrameworkPath) {
+        // if runtimePath is set to "PackageFrameworks" that means we could be developing SwiftPM in Xcode
+        // which produces a framework for dynamic package products.
+        if runtimePath.basename == "PackageFrameworks" {
             command += [
-                "-F", packageFrameworkPath.pathString,
+                "-F", runtimePath.pathString,
                 "-framework", "PackagePlugin",
-                "-Xlinker", "-rpath", "-Xlinker", packageFrameworkPath.pathString,
+                "-Xlinker", "-rpath", "-Xlinker", runtimePath.pathString,
             ]
-            macOSPackageDescriptionPath = packageFrameworkPath.appending(RelativePath("PackagePlugin.framework/PackagePlugin"))
+            macOSPackageDescriptionPath = runtimePath.appending(RelativePath("PackagePlugin.framework/PackagePlugin"))
         } else {
             command += [
                 "-L", runtimePath.pathString,
@@ -99,9 +96,15 @@ public struct DefaultPluginScriptRunner: PluginScriptRunner {
         command += self.toolchain.swiftCompilerFlags
 
         command += ["-swift-version", toolsVersion.swiftLanguageVersion.rawValue]
-        command += ["-I", runtimePath.pathString]
+        // if runtimePath is set to "PackageFrameworks" that means we could be developing SwiftPM in Xcode
+        // which produces a framework for dynamic package products.
+        if runtimePath.basename == "PackageFrameworks" {
+            command += ["-I", runtimePath.parentDirectory.pathString]
+        } else {
+            command += ["-I", runtimePath.pathString]
+        }
         #if os(macOS)
-        if let sdkRoot = self.toolchain.sdkRoot ?? self.sdkRoot() {
+        if let sdkRoot = self.toolchain.sdkRootPath ?? self.sdkRoot() {
             command += ["-sdk", sdkRoot.pathString]
         }
         #endif

Sources/Workspace/Destination.swift

@@ -90,8 +90,10 @@ public struct Destination: Encodable, Equatable {
         environment: [String:String] = ProcessEnv.vars
     ) throws -> Destination {
         // Select the correct binDir.
-        let customBinDir = ProcessEnv
-            .vars["SWIFTPM_CUSTOM_BINDIR"]
+        if ProcessEnv.vars["SWIFTPM_CUSTOM_BINDIR"] != nil {
+            print("SWIFTPM_CUSTOM_BINDIR was deprecated in favor of SWIFTPM_CUSTOM_BIN_DIR")
+        }
+        let customBinDir = (ProcessEnv.vars["SWIFTPM_CUSTOM_BIN_DIR"] ?? ProcessEnv.vars["SWIFTPM_CUSTOM_BINDIR"])
             .flatMap{ try? AbsolutePath(validating: $0) }
         let binDir = try customBinDir ?? binDir ?? Destination.hostBinDir(
             originalWorkingDirectory: originalWorkingDirectory)