[Explicit Module Builds] Adopt dependency scanner Link Libraries support

- Adopt new libSwiftScan API to query link libraries for each module dependency
- Introduce '-explicit-auto-linking' to turn explicit dependency link library information into flags passed directly to the linker invocation

Author: artemcm

Sources/CSwiftScan/include/swiftscan_header.h

@@ -18,7 +18,7 @@
 #include <stdint.h>
 
 #define SWIFTSCAN_VERSION_MAJOR 0
-#define SWIFTSCAN_VERSION_MINOR 6
+#define SWIFTSCAN_VERSION_MINOR 10
 
 //=== Public Scanner Data Types -------------------------------------------===//
 
@@ -41,6 +41,7 @@ typedef enum {
 
 typedef struct swiftscan_module_details_s *swiftscan_module_details_t;
 typedef struct swiftscan_dependency_info_s *swiftscan_dependency_info_t;
+typedef struct swiftscan_link_library_info_s *swiftscan_link_library_info_t;
 typedef struct swiftscan_dependency_graph_s *swiftscan_dependency_graph_t;
 typedef struct swiftscan_import_set_s *swiftscan_import_set_t;
 typedef struct swiftscan_diagnostic_info_s *swiftscan_diagnostic_info_t;
@@ -60,6 +61,10 @@ typedef struct {
   swiftscan_dependency_info_t *modules;
   size_t count;
 } swiftscan_dependency_set_t;
+typedef struct {
+  swiftscan_link_library_info_t *link_libraries;
+  size_t count;
+} swiftscan_link_library_set_t;
 
 //=== Batch Scan Input Specification --------------------------------------===//
 
@@ -110,9 +115,19 @@ typedef struct {
   (*swiftscan_module_info_get_source_files)(swiftscan_dependency_info_t);
   swiftscan_string_set_t *
   (*swiftscan_module_info_get_direct_dependencies)(swiftscan_dependency_info_t);
+  swiftscan_link_library_set_t *
+  (*swiftscan_module_info_get_link_libraries)(swiftscan_dependency_graph_t);
   swiftscan_module_details_t
   (*swiftscan_module_info_get_details)(swiftscan_dependency_info_t);
 
+  //=== Link Library Info Functions ------------------------------------===//
+  swiftscan_string_ref_t
+  (*swiftscan_link_library_info_get_link_name)(swiftscan_link_library_info_t);
+  bool
+  (*swiftscan_link_library_info_get_is_framework)(swiftscan_link_library_info_t);
+  bool
+  (*swiftscan_link_library_info_get_should_force_load)(swiftscan_link_library_info_t);
+
   //=== Dependency Module Info Details Functions ----------------------------===//
   swiftscan_dependency_info_kind_t
   (*swiftscan_module_detail_get_kind)(swiftscan_module_details_t);

Sources/SwiftDriver/Driver/Driver.swift

@@ -3076,6 +3076,14 @@ extension Driver {
         diagnosticsEngine.emit(.error_hermetic_seal_requires_lto)
       }
     }
+
+    if parsedOptions.hasArgument(.explicitAutoLinking) {
+      if !parsedOptions.hasArgument(.driverExplicitModuleBuild) {
+        diagnosticsEngine.emit(.error(Error.optionRequiresAnother(Option.explicitAutoLinking.spelling,
+                                                                  Option.driverExplicitModuleBuild.spelling)),
+                              location: nil)
+      }
+    }
   }
 
   private static func validateSanitizerAddressUseOdrIndicatorFlag(

Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

@@ -34,7 +34,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
 /// that specify said explicit module dependencies.
 @_spi(Testing) public struct ExplicitDependencyBuildPlanner {
   /// The module dependency graph.
-  private let dependencyGraph: InterModuleDependencyGraph
+  @_spi(Testing) public let dependencyGraph: InterModuleDependencyGraph
 
   /// A set of direct and transitive dependencies for every module in the dependency graph
   private let reachabilityMap: [ModuleDependencyId : Set<ModuleDependencyId>]
@@ -421,6 +421,29 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
     }
   }
 
+  public func getLinkLibraryLoadCommandFlags(_ commandLine: inout [Job.ArgTemplate]) throws {
+    var allLinkLibraries = Set<LinkLibraryInfo>()
+    for (_, moduleInfo) in dependencyGraph.modules {
+      guard let moduleLinkLibraries = moduleInfo.linkLibraries else {
+        continue
+      }
+      for linkLibrary in moduleLinkLibraries {
+        allLinkLibraries.insert(linkLibrary)
+      }
+    }
+
+    for linkLibrary in allLinkLibraries {
+      if !linkLibrary.isFramework {
+        commandLine.appendFlag("-l\(linkLibrary.linkName)")
+      } else {
+        commandLine.appendFlag(.Xlinker)
+        commandLine.appendFlag("-framework")
+        commandLine.appendFlag(.Xlinker)
+        commandLine.appendFlag(linkLibrary.linkName)
+      }
+    }
+  }
+
   /// Resolve all module dependencies of the main module and add them to the lists of
   /// inputs and command line flags.
   public mutating func resolveMainModuleDependencies(inputs: inout [TypedVirtualPath],

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift

@@ -27,24 +27,26 @@ import func TSCBasic.topologicalSort
       if let currentInfo = modules[swiftModuleId],
          externalModuleId.moduleName != mainModuleName {
         let newExternalModuleDetails =
-        try SwiftPrebuiltExternalModuleDetails(compiledModulePath:
-                                                TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()),
-                                               isFramework: externalModuleDetails.isFramework)
+        SwiftPrebuiltExternalModuleDetails(compiledModulePath:
+                                            TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()),
+                                           isFramework: externalModuleDetails.isFramework)
         let newInfo = ModuleInfo(modulePath: TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()),
                                  sourceFiles: [],
                                  directDependencies: currentInfo.directDependencies,
+                                 linkLibraries: currentInfo.linkLibraries,
                                  details: .swiftPrebuiltExternal(newExternalModuleDetails))
         Self.replaceModule(originalId: swiftModuleId, replacementId: prebuiltModuleId,
                            replacementInfo: newInfo, in: &modules)
       } else if let currentPrebuiltInfo = modules[prebuiltModuleId] {
         // Just update the isFramework bit on this prebuilt module dependency
         let newExternalModuleDetails =
-        try SwiftPrebuiltExternalModuleDetails(compiledModulePath:
-                                                TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()),
-                                               isFramework: externalModuleDetails.isFramework)
+        SwiftPrebuiltExternalModuleDetails(compiledModulePath:
+                                            TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()),
+                                           isFramework: externalModuleDetails.isFramework)
         let newInfo = ModuleInfo(modulePath: TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()),
                                  sourceFiles: [],
                                  directDependencies: currentPrebuiltInfo.directDependencies,
+                                 linkLibraries: currentPrebuiltInfo.linkLibraries,
                                  details: .swiftPrebuiltExternal(newExternalModuleDetails))
         Self.replaceModule(originalId: prebuiltModuleId, replacementId: prebuiltModuleId,
                            replacementInfo: newInfo, in: &modules)
@@ -231,6 +233,9 @@ extension InterModuleDependencyGraph {
     let firstModuleDependencies = firstInfo.directDependencies ?? []
     let secondModuleDependencies = secondInfo.directDependencies ?? []
     let combinedDependencies = Array(Set(firstModuleDependencies + secondModuleDependencies))
+    let firstLinkLibraries = firstInfo.linkLibraries ?? []
+    let secondLinkLibraries = secondInfo.linkLibraries ?? []
+    let combinedLinkLibraries = Array(Set(firstLinkLibraries + secondLinkLibraries))
 
     let firstModuleCapturedPCMArgs = firstDetails.capturedPCMArgs ?? Set<[String]>()
     let secondModuleCapturedPCMArgs = secondDetails.capturedPCMArgs ?? Set<[String]>()
@@ -245,6 +250,7 @@ extension InterModuleDependencyGraph {
     return ModuleInfo(modulePath: firstInfo.modulePath,
                       sourceFiles: combinedSourceFiles,
                       directDependencies: combinedDependencies,
+                      linkLibraries: combinedLinkLibraries,
                       details: .clang(combinedModuleDetails))
   }
 }

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift

@@ -94,6 +94,13 @@ public struct BridgingHeader: Codable, Hashable {
   var moduleDependencies: [String]
 }
 
+/// Linked Library
+public struct LinkLibraryInfo: Codable, Hashable {
+  public var linkName: String
+  public var isFramework: Bool
+  public var shouldForceLoad: Bool
+}
+
 /// Details specific to Swift modules.
 public struct SwiftModuleDetails: Codable, Hashable {
   /// The module interface from which this module was built, if any.
@@ -203,6 +210,9 @@ public struct ModuleInfo: Codable, Hashable {
   /// The set of direct module dependencies of this module.
   public var directDependencies: [ModuleDependencyId]?
 
+  /// The set of libraries that need to be linked
+  public var linkLibraries: [LinkLibraryInfo]?
+
   /// Specific details of a particular kind of module.
   public var details: Details
 
@@ -226,10 +236,12 @@ public struct ModuleInfo: Codable, Hashable {
   public init(modulePath: TextualVirtualPath,
               sourceFiles: [String]?,
               directDependencies: [ModuleDependencyId]?,
+              linkLibraries: [LinkLibraryInfo]?,
               details: Details) {
     self.modulePath = modulePath
     self.sourceFiles = sourceFiles
     self.directDependencies = directDependencies
+    self.linkLibraries = linkLibraries
     self.details = details
   }
 }
@@ -306,7 +318,7 @@ public struct InterModuleDependencyGraph: Codable {
   public var mainModule: ModuleInfo { modules[.swift(mainModuleName)]! }
 }
 
-internal extension InterModuleDependencyGraph {
+@_spi(Testing) public  extension InterModuleDependencyGraph {
   func toJSONData() throws -> Data {
     let encoder = JSONEncoder()
 #if os(Linux) || os(Android)

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

@@ -174,6 +174,13 @@ public class InterModuleDependencyOracle {
     return swiftScan.supportsDiagnosticSourceLocations
   }
 
+  @_spi(Testing) public func supportsLinkLibraries() throws -> Bool {
+    guard let swiftScan = swiftScanLibInstance else {
+      fatalError("Attempting to query supported scanner API with no scanner instance.")
+    }
+    return swiftScan.supportsLinkLibraries
+  }
+
   @_spi(Testing) public func getScannerDiagnostics() throws -> [ScannerDiagnosticPayload]? {
     guard let swiftScan = swiftScanLibInstance else {
       fatalError("Attempting to reset scanner cache with no scanner instance.")

Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift

@@ -469,6 +469,12 @@ extension Driver {
       commandLine.appendFlag("-frontend-parseable-output")
     }
 
+    // If explicit auto-linking is enabled, ensure that compiler tasks do not produce
+    // auto-link load commands in resulting object files.
+    if parsedOptions.hasArgument(.explicitAutoLinking) {
+      commandLine.appendFlag(.disableAllAutolinking)
+    }
+
     savedUnknownDriverFlagsForSwiftFrontend.forEach {
       commandLine.appendFlag($0)
     }

Sources/SwiftDriver/Jobs/LinkJob.swift

@@ -75,6 +75,10 @@ extension Driver {
       targetInfo: frontendTargetInfo
     )
 
+    if parsedOptions.hasArgument(.explicitAutoLinking) {
+      try explicitDependencyBuildPlanner?.getLinkLibraryLoadCommandFlags(&commandLine)
+    }
+
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .link,

Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift

@@ -120,6 +120,24 @@ private extension SwiftScan {
       directDependencies = nil
     }
 
+    var linkLibraries: [LinkLibraryInfo] = []
+    if supportsLinkLibraries {
+      let linkLibrarySetRefOrNull = api.swiftscan_module_info_get_link_libraries(moduleInfoRef)
+      guard let linkLibrarySetRef = linkLibrarySetRefOrNull else {
+        throw DependencyScanningError.missingField("dependency_graph.link_libraries")
+      }
+      // Turn the `swiftscan_dependency_set_t` into an array of `swiftscan_dependency_info_t`
+      // references we can iterate through in order to construct `ModuleInfo` objects.
+      let linkLibraryRefArray = Array(UnsafeBufferPointer(start: linkLibrarySetRef.pointee.link_libraries,
+                                                          count: Int(linkLibrarySetRef.pointee.count)))
+      for linkLibraryRefOrNull in linkLibraryRefArray {
+        guard let linkLibraryRef = linkLibraryRefOrNull else {
+          throw DependencyScanningError.missingField("dependency_set_t.link_libraries[_]")
+        }
+        linkLibraries.append(try constructLinkLibrayInfo(from: linkLibraryRef))
+      }
+    }
+
     guard let moduleDetailsRef = api.swiftscan_module_info_get_details(moduleInfoRef) else {
       throw DependencyScanningError.missingField("modules[\(moduleId)].details")
     }
@@ -128,9 +146,16 @@ private extension SwiftScan {
 
     return (moduleId, ModuleInfo(modulePath: modulePath, sourceFiles: sourceFiles,
                                  directDependencies: directDependencies,
+                                 linkLibraries: linkLibraries,
                                  details: details))
   }
 
+  func constructLinkLibrayInfo(from linkLibraryInfoRef: swiftscan_link_library_info_t) throws -> LinkLibraryInfo {
+    return LinkLibraryInfo(linkName: try toSwiftString(api.swiftscan_link_library_info_get_link_name(linkLibraryInfoRef)),
+                             isFramework: api.swiftscan_link_library_info_get_is_framework(linkLibraryInfoRef),
+                             shouldForceLoad: api.swiftscan_link_library_info_get_should_force_load(linkLibraryInfoRef))
+  }
+
   /// From a reference to a binary-format module info details object info returned by libSwiftScan,
   /// construct an instance of an `ModuleInfo`.Details as used by the driver.
   /// The object returned by libSwiftScan is a union so ensure to execute dependency-specific queries.

Sources/SwiftDriver/SwiftScan/SwiftScan.swift

@@ -364,6 +364,13 @@ private extension String {
            api.swiftscan_source_location_get_column_number != nil
   }
 
+  @_spi(Testing) public var supportsLinkLibraries : Bool {
+    return api.swiftscan_module_info_get_link_libraries != nil &&
+           api.swiftscan_link_library_info_get_link_name != nil &&
+           api.swiftscan_link_library_info_get_is_framework != nil &&
+           api.swiftscan_link_library_info_get_should_force_load != nil
+  }
+
   func serializeScannerCache(to path: AbsolutePath) {
     api.swiftscan_scanner_cache_serialize(scanner,
                                           path.description.cString(using: String.Encoding.utf8))
@@ -636,6 +643,11 @@ private extension swiftscan_functions_t {
     self.swiftscan_source_location_get_line_number = try loadOptional("swiftscan_source_location_get_line_number")
     self.swiftscan_source_location_get_column_number = try loadOptional("swiftscan_source_location_get_column_number")
 
+    self.swiftscan_module_info_get_link_libraries = try loadOptional("swiftscan_module_info_get_link_libraries")
+    self.swiftscan_link_library_info_get_link_name = try loadOptional("swiftscan_link_library_info_get_link_name")
+    self.swiftscan_link_library_info_get_is_framework = try loadOptional("swiftscan_link_library_info_get_is_framework")
+    self.swiftscan_link_library_info_get_should_force_load = try loadOptional("swiftscan_link_library_info_get_should_force_load")
+
     // Swift Overlay Dependencies
     self.swiftscan_swift_textual_detail_get_swift_overlay_dependencies =
       try loadOptional("swiftscan_swift_textual_detail_get_swift_overlay_dependencies")