Merge pull request #1253 from artemcm/RefactorLibSwiftScanLookup

[Dependency Scanning] Move computation of the path for libSwiftScan to the toolchain

Author: artemcm

Sources/SwiftDriver/Driver/Driver.swift

@@ -650,11 +650,11 @@ public struct Driver {
                                                                                outputFileMap: outputFileMap)
 
     self.supportedFrontendFlags =
-      try Self.computeSupportedCompilerArgs(of: self.toolchain, hostTriple: self.hostTriple,
-                                                parsedOptions: &self.parsedOptions,
-                                                diagnosticsEngine: diagnosticEngine,
-                                                fileSystem: fileSystem, executor: executor,
-                                                env: env)
+      try Self.computeSupportedCompilerArgs(of: self.toolchain,
+                                            parsedOptions: &self.parsedOptions,
+                                            diagnosticsEngine: diagnosticEngine,
+                                            fileSystem: fileSystem,
+                                            executor: executor)
     let supportedFrontendFlagsLocal = self.supportedFrontendFlags
     self.savedUnknownDriverFlagsForSwiftFrontend = try self.parsedOptions.saveUnknownFlags {
       Driver.isOptionFound($0, allOpts: supportedFrontendFlagsLocal)

Sources/SwiftDriver/Driver/WindowsExtensions.swift

@@ -20,3 +20,27 @@ internal func executableName(_ name: String) -> String {
   return name
 #endif
 }
+
+@_spi(Testing) public func sharedLibraryName(_ name: String) -> String {
+#if canImport(Darwin)
+  let ext = ".dylib"
+#elseif os(Windows)
+  let ext = ".dll"
+#else
+  let ext = ".so"
+#endif
+  return name + ext
+}
+
+// FIXME: This can be subtly wrong, we should rather
+// try to get the client to provide this info or move to a better
+// path convention for where we keep compiler support libraries
+internal var compilerHostSupportLibraryOSComponent : String {
+#if canImport(Darwin)
+  return "macosx"
+#elseif os(Windows)
+  return "windows"
+#else
+  return "linux"
+#endif
+}

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -221,7 +221,7 @@ public extension Driver {
     // attempt to fallback to using `swift-frontend -scan-dependencies` invocations for dependency
     // scanning.
     var fallbackToFrontend = parsedOptions.hasArgument(.driverScanDependenciesNonLib)
-    let scanLibPath = try Self.getScanLibPath(of: toolchain, hostTriple: hostTriple, env: env)
+    let scanLibPath = try toolchain.lookupSwiftScanLib()
     if try interModuleDependencyOracle
         .verifyOrCreateScannerInstance(fileSystem: fileSystem,
                                        swiftScanLibPath: scanLibPath) == false {
@@ -490,46 +490,9 @@ public extension Driver {
                                                      useResponseFiles: useResponseFiles)
     return args
   }
-}
-
-@_spi(Testing) public extension Driver {
-  static func getScanLibPath(of toolchain: Toolchain, hostTriple: Triple,
-                             env: [String: String]) throws -> AbsolutePath {
-    if hostTriple.isWindows {
-      // no matter if we are in a build tree or an installed tree, the layout is
-      // always: `bin/_InternalSwiftScan.dll`
-      return try getRootPath(of: toolchain, env: env)
-                    .appending(component: "bin")
-                    .appending(component: "_InternalSwiftScan.dll")
-    }
-
-    let sharedLibExt: String
-    if hostTriple.isMacOSX {
-      sharedLibExt = ".dylib"
-    } else {
-      sharedLibExt = ".so"
-    }
-    let libScanner = "lib_InternalSwiftScan\(sharedLibExt)"
-    // We first look into position in toolchain
-    let libPath
-     = try getRootPath(of: toolchain, env: env).appending(component: "lib")
-      .appending(component: "swift")
-      .appending(component: hostTriple.osNameUnversioned)
-      .appending(component: libScanner)
-    if localFileSystem.exists(libPath) {
-        return libPath
-    }
-    // In case we are using a compiler from the build dir, we should also try
-    // this path.
-    return try getRootPath(of: toolchain, env: env).appending(component: "lib")
-      .appending(component: libScanner)
-  }
 
   static func getRootPath(of toolchain: Toolchain, env: [String: String])
   throws -> AbsolutePath {
-    if let overrideString = env["SWIFT_DRIVER_SWIFT_SCAN_TOOLCHAIN_PATH"] {
-      return try AbsolutePath(validating: overrideString)
-    }
     return try toolchain.getToolPath(.swiftCompiler)
       .parentDirectory // bin
       .parentDirectory // toolchain root

Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift

@@ -57,14 +57,15 @@ extension Toolchain {
 }
 
 extension Driver {
-  static func computeSupportedCompilerArgs(of toolchain: Toolchain, hostTriple: Triple,
+
+  static func computeSupportedCompilerArgs(of toolchain: Toolchain,
                                            parsedOptions: inout ParsedOptions,
                                            diagnosticsEngine: DiagnosticsEngine,
                                            fileSystem: FileSystem,
-                                           executor: DriverExecutor, env: [String: String])
+                                           executor: DriverExecutor)
   throws -> Set<String> {
-    if let supportedArgs = try querySupportedCompilerArgsInProcess(of: toolchain, hostTriple: hostTriple,
-                                                                   fileSystem: fileSystem, env: env) {
+    if let supportedArgs =
+        try querySupportedCompilerArgsInProcess(of: toolchain, fileSystem: fileSystem) {
       return supportedArgs
     }
 
@@ -84,13 +85,9 @@ extension Driver {
   }
 
   static func querySupportedCompilerArgsInProcess(of toolchain: Toolchain,
-                                                     hostTriple: Triple,
-                                                     fileSystem: FileSystem,
-                                                     env: [String: String])
+                                                  fileSystem: FileSystem)
   throws -> Set<String>? {
-    let swiftScanLibPath = try Self.getScanLibPath(of: toolchain,
-                                                   hostTriple: hostTriple,
-                                                   env: env)
+    let swiftScanLibPath = try toolchain.lookupSwiftScanLib()
     if fileSystem.exists(swiftScanLibPath) {
       let libSwiftScanInstance = try SwiftScan(dylib: swiftScanLibPath)
       if libSwiftScanInstance.canQuerySupportedArguments() {

Sources/SwiftDriver/SwiftScan/SwiftScan.swift

@@ -317,14 +317,10 @@ internal final class SwiftScan {
 @_spi(Testing) public extension Driver {
   func querySupportedArgumentsForTest() throws -> Set<String>? {
     // If a capable libSwiftScan is found, manually ensure we can get the supported arguments
-    let scanLibPath = try Self.getScanLibPath(of: toolchain,
-                                              hostTriple: hostTriple,
-                                              env: env)
-    if fileSystem.exists(scanLibPath) {
-      let libSwiftScanInstance = try SwiftScan(dylib: scanLibPath)
-      if libSwiftScanInstance.canQuerySupportedArguments() {
-        return try libSwiftScanInstance.querySupportedArguments()
-      }
+    let scanLibPath = try toolchain.lookupSwiftScanLib()
+    let libSwiftScanInstance = try SwiftScan(dylib: scanLibPath)
+    if libSwiftScanInstance.canQuerySupportedArguments() {
+      return try libSwiftScanInstance.querySupportedArguments()
     }
     return nil
   }

Sources/SwiftDriver/Toolchains/Toolchain.swift

@@ -245,6 +245,46 @@ extension Toolchain {
     }
   }
 
+  /// Looks for the executable in the `SWIFT_DRIVER_SWIFTSCAN_LIB` environment variable, if found nothing,
+  /// looks in the `lib` relative to the compiler executable.
+  /// TODO: If the driver needs to lookup other shared libraries, this is simple to generalize
+  @_spi(Testing) public func lookupSwiftScanLib() throws -> AbsolutePath {
+#if os(Windows)
+    // no matter if we are in a build tree or an installed tree, the layout is
+    // always: `bin/_InternalSwiftScan.dll`
+    return try toolchain.getToolPath(.swiftCompiler)
+                        .parentDirectory // bin
+                        .appending(component: "_InternalSwiftScan.dll")
+#else
+    let libraryName = sharedLibraryName("lib_InternalSwiftScan")
+    if let overrideString = env["SWIFT_DRIVER_SWIFTSCAN_LIB"],
+       let path = try? AbsolutePath(validating: overrideString) {
+      return path
+    } else {
+      let compilerPath = try getToolPath(.swiftCompiler)
+      let toolchainRootPath = compilerPath.parentDirectory // bin
+                                          .parentDirectory // toolchain root
+
+      let searchPaths = [toolchainRootPath.appending(component: "lib")
+                                          .appending(component: "swift")
+                                          .appending(component: compilerHostSupportLibraryOSComponent),
+                         toolchainRootPath.appending(component: "lib")
+                                          .appending(component: "swift")
+                                          .appending(component: "host"),
+                         // In case we are using a compiler from the build dir, we should also try
+                         // this path.
+                         toolchainRootPath.appending(component: "lib")]
+      for libraryPath in searchPaths.map({ $0.appending(component: libraryName) }) {
+        if fileSystem.isFile(libraryPath) {
+          return libraryPath
+        }
+      }
+    }
+
+    throw ToolchainError.unableToFind(tool: libraryName)
+#endif
+  }
+
   private func xcrunFind(executable: String) throws -> AbsolutePath {
     let xcrun = "xcrun"
     guard lookupExecutablePath(filename: xcrun, searchPaths: searchPaths) != nil else {

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -648,7 +648,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
   }
 
   func testModuleAliasingWithImportPrescan() throws {
-    let (_, _, toolchain, hostTriple) = try getDriverArtifactsForScanning()
+    let (_, _, toolchain, _) = try getDriverArtifactsForScanning()
 
     let dummyDriver = try Driver(args: ["swiftc", "-module-name", "dummyDriverCheck", "test.swift"])
     guard dummyDriver.isFrontendArgSupported(.moduleAlias) else {
@@ -658,9 +658,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
     // The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
     // queries.
     let dependencyOracle = InterModuleDependencyOracle()
-    let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                hostTriple: hostTriple,
-                                                env: ProcessEnv.vars)
+    let scanLibPath = try toolchain.lookupSwiftScanLib()
     guard try dependencyOracle
             .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
                                            swiftScanLibPath: scanLibPath) else {
@@ -840,9 +838,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
 
       // 2. Run a dependency scan to find the just-built module
       let dependencyOracle = InterModuleDependencyOracle()
-      let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                  hostTriple: hostTriple,
-                                                  env: ProcessEnv.vars)
+      let scanLibPath = try toolchain.lookupSwiftScanLib()
       guard try dependencyOracle
               .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
                                              swiftScanLibPath: scanLibPath) else {
@@ -939,14 +935,12 @@ final class ExplicitModuleBuildTests: XCTestCase {
 
   /// Test the libSwiftScan dependency scanning (import-prescan).
   func testDependencyImportPrescan() throws {
-    let (stdLibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
+    let (stdLibPath, shimsPath, toolchain, _) = try getDriverArtifactsForScanning()
 
     // The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
     // queries.
     let dependencyOracle = InterModuleDependencyOracle()
-    let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                hostTriple: hostTriple,
-                                                env: ProcessEnv.vars)
+    let scanLibPath = try toolchain.lookupSwiftScanLib()
     guard try dependencyOracle
             .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
                                            swiftScanLibPath: scanLibPath) else {
@@ -1020,14 +1014,12 @@ final class ExplicitModuleBuildTests: XCTestCase {
   }
 
   func testDependencyScanningFailure() throws {
-    let (stdlibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
+    let (stdlibPath, shimsPath, toolchain, _) = try getDriverArtifactsForScanning()
 
     // The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
     // queries.
     let dependencyOracle = InterModuleDependencyOracle()
-    let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                hostTriple: hostTriple,
-                                                env: ProcessEnv.vars)
+    let scanLibPath = try toolchain.lookupSwiftScanLib()
     guard try dependencyOracle
       .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
                                      swiftScanLibPath: scanLibPath) else {
@@ -1098,9 +1090,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
     // The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
     // queries.
     let dependencyOracle = InterModuleDependencyOracle()
-    let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                hostTriple: hostTriple,
-                                                env: ProcessEnv.vars)
+    let scanLibPath = try toolchain.lookupSwiftScanLib()
     guard try dependencyOracle
             .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
                                            swiftScanLibPath: scanLibPath) else {
@@ -1293,11 +1283,9 @@ final class ExplicitModuleBuildTests: XCTestCase {
   }
 
   func testDependencyGraphDotSerialization() throws {
-      let (stdlibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
+      let (stdlibPath, shimsPath, toolchain, _) = try getDriverArtifactsForScanning()
       let dependencyOracle = InterModuleDependencyOracle()
-      let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                  hostTriple: hostTriple,
-                                                  env: ProcessEnv.vars)
+      let scanLibPath = try toolchain.lookupSwiftScanLib()
       guard try dependencyOracle
               .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
                                              swiftScanLibPath: scanLibPath) else {
@@ -1358,7 +1346,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
 
   /// Test the libSwiftScan dependency scanning.
   func testDependencyScanReuseCache() throws {
-    let (stdlibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
+    let (stdlibPath, shimsPath, toolchain, _) = try getDriverArtifactsForScanning()
     try withTemporaryDirectory { path in
       let cacheSavePath = path.appending(component: "saved.moddepcache")
       let main = path.appending(component: "testDependencyScanning.swift")
@@ -1394,9 +1382,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
         scannerCommand.removeFirst()
       }
 
-      let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                  hostTriple: hostTriple,
-                                                  env: ProcessEnv.vars)
+      let scanLibPath = try toolchain.lookupSwiftScanLib()
       // Run the first scan and serialize the cache contents.
       let firstDependencyOracle = InterModuleDependencyOracle()
       guard try firstDependencyOracle

Tests/SwiftDriverTests/SwiftDriverTests.swift

@@ -6685,6 +6685,8 @@ final class SwiftDriverTests: XCTestCase {
     let PATH = "PATH"
 #endif
     let SWIFT_FRONTEND_EXEC = "SWIFT_DRIVER_SWIFT_FRONTEND_EXEC"
+    let SWIFT_SCANNER_LIB = "SWIFT_DRIVER_SWIFTSCAN_LIB"
+
 
     // Reset the environment to ensure tool resolution is exactly run against PATH.
     var driver = try Driver(args: ["swiftc", "-print-target-info"], env: [PATH: ProcessEnv.path!])
@@ -6695,6 +6697,7 @@ final class SwiftDriverTests: XCTestCase {
 
     try withTemporaryDirectory { toolsDirectory in
       let customSwiftFrontend = toolsDirectory.appending(component: executableName("swift-frontend"))
+      let customSwiftScan = toolsDirectory.appending(component: sharedLibraryName("lib_InternalSwiftScan"))
       try localFileSystem.createSymbolicLink(customSwiftFrontend, pointingAt: defaultSwiftFrontend, relative: false)
 
       try withTemporaryDirectory { tempDirectory in 
@@ -6707,7 +6710,9 @@ final class SwiftDriverTests: XCTestCase {
         // test if SWIFT_DRIVER_TOOLNAME_EXEC is respected
         do {
           var driver = try Driver(args: ["swiftc", "-print-target-info"],
-                                  env: [PATH: ProcessEnv.path!, SWIFT_FRONTEND_EXEC: customSwiftFrontend.pathString])
+                                  env: [PATH: ProcessEnv.path!,
+                                        SWIFT_FRONTEND_EXEC: customSwiftFrontend.pathString,
+                                        SWIFT_SCANNER_LIB: customSwiftScan.pathString])
           let jobs = try driver.planBuild()
           XCTAssertEqual(jobs.count, 1)
           XCTAssertEqual(jobs.first!.tool.name, customSwiftFrontend.pathString)
@@ -6716,15 +6721,16 @@ final class SwiftDriverTests: XCTestCase {
         // test if tools directory is respected
         do {
           var driver = try Driver(args: ["swiftc", "-print-target-info", "-tools-directory", toolsDirectory.pathString],
-                                  env: [PATH: ProcessEnv.path!])
+                                  env: [PATH: ProcessEnv.path!, SWIFT_SCANNER_LIB: customSwiftScan.pathString])
           let jobs = try driver.planBuild()
           XCTAssertEqual(jobs.count, 1)
           XCTAssertEqual(jobs.first!.tool.name, customSwiftFrontend.pathString)
         }
 
         // test if current working directory is searched before PATH
         do {
-          var driver = try Driver(args: ["swiftc", "-print-target-info"], env: [PATH: toolsDirectory.pathString])
+          var driver = try Driver(args: ["swiftc", "-print-target-info"],
+                                  env: [PATH: toolsDirectory.pathString, SWIFT_SCANNER_LIB: customSwiftScan.pathString])
           let jobs = try driver.planBuild()
           XCTAssertEqual(jobs.count, 1)
           XCTAssertEqual(jobs.first!.tool.name, anotherSwiftFrontend.pathString)