Merge pull request #781 from artemcm/ExplicitModuleCherryFarming

[5.5][Explicit Module Builds] Cherry-pick explicit module build changes

Author: artemcm

Sources/CSwiftScan/include/swiftscan_header.h

@@ -200,17 +200,20 @@ typedef struct {
   //=== Scanner Functions ---------------------------------------------------===//  
   swiftscan_scanner_t (*swiftscan_scanner_create)(void);
   void (*swiftscan_scanner_dispose)(swiftscan_scanner_t);
-
   swiftscan_dependency_graph_t
   (*swiftscan_dependency_graph_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t);
-
   swiftscan_batch_scan_result_t *
   (*swiftscan_batch_scan_result_create)(swiftscan_scanner_t,
                                         swiftscan_batch_scan_input_t *,
                                         swiftscan_scan_invocation_t);
-
   swiftscan_import_set_t
   (*swiftscan_import_set_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t);
+
+  //=== Scanner Cache Functions ---------------------------------------------===//
+  void (*swiftscan_scanner_cache_serialize)(swiftscan_scanner_t scanner, const char * path);
+  bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path);
+  void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner);
+
 } swiftscan_functions_t;
 
 #endif // SWIFT_C_DEPENDENCY_SCAN_H

Sources/SwiftDriver/CMakeLists.txt

@@ -10,7 +10,6 @@ add_library(SwiftDriver
   "ExplicitModuleBuilds/ClangVersionedDependencyResolution.swift"
   "ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift"
   "ExplicitModuleBuilds/ModuleDependencyScanning.swift"
-  "ExplicitModuleBuilds/PlaceholderDependencyResolution.swift"
   "ExplicitModuleBuilds/SerializableModuleArtifacts.swift"
   "ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift"
   "ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift"

Sources/SwiftDriver/Driver/Driver.swift

@@ -23,6 +23,7 @@ public struct Driver {
     case invalidArgumentValue(String, String)
     case relativeFrontendPath(String)
     case subcommandPassedToDriver
+    case externalTargetDetailsAPIError
     case integratedReplRemoved
     case cannotSpecify_OForMultipleOutputs
     case conflictingOptions(Option, Option)
@@ -38,6 +39,7 @@ public struct Driver {
     case malformedModuleDependency(String, String)
     case missingPCMArguments(String)
     case missingModuleDependency(String)
+    case missingContextHashOnSwiftDependency(String)
     case dependencyScanningFailure(Int, String)
     case missingExternalDependency(String)
 
@@ -56,6 +58,8 @@ public struct Driver {
         return "relative frontend path: \(path)"
       case .subcommandPassedToDriver:
         return "subcommand passed to driver"
+      case .externalTargetDetailsAPIError:
+        return "Cannot specify both: externalTargetModulePathMap and externalTargetModuleDetailsMap"
       case .integratedReplRemoved:
         return "Compiler-internal integrated REPL has been removed; use the LLDB-enhanced REPL instead."
       case .cannotSpecify_OForMultipleOutputs:
@@ -91,6 +95,8 @@ public struct Driver {
         return "Missing extraPcmArgs to build Clang module: \(moduleName)"
       case .missingModuleDependency(let moduleName):
         return "Missing Module Dependency Info: \(moduleName)"
+      case .missingContextHashOnSwiftDependency(let moduleName):
+        return "Missing Context Hash for Swift dependency: \(moduleName)"
       case .dependencyScanningFailure(let code, let error):
         return "Module Dependency Scanner returned with non-zero exit status: \(code), \(error)"
       case .unableToLoadOutputFileMap(let path):
@@ -302,12 +308,9 @@ public struct Driver {
   /// is shared across many targets; otherwise, a new instance is created by the driver itself.
   @_spi(Testing) public let interModuleDependencyOracle: InterModuleDependencyOracle
 
-  // TODO: Once the clients have transitioned to using the InterModuleDependencyOracle API,
-  // this must convey information about the externally-prebuilt targets only
-  /// All external artifacts a build system (e.g. SwiftPM) may pass in as input to the explicit
-  /// build of the current module. Consists of a map of externally-built targets, and a map of all previously
-  /// discovered/scanned modules and their infos.
-  @_spi(Testing) public var externalBuildArtifacts: ExternalBuildArtifacts? = nil
+  /// A dictionary of external targets that are a part of the same build, mapping to filesystem paths
+  /// of their module files
+  @_spi(Testing) public var externalTargetModuleDetailsMap: ExternalTargetModuleDetailsMap? = nil
 
   /// A collection of all the flags the selected toolchain's `swift-frontend` supports
   public let supportedFrontendFlags: Set<String>
@@ -384,9 +387,17 @@ public struct Driver {
   ///   expand response files, etc. By default this is the local filesystem.
   /// - Parameter executor: Used by the driver to execute jobs. The default argument
   ///   is present to streamline testing, it shouldn't be used in production.
-  /// - Parameter externalBuildArtifacts: All external artifacts a build system may pass in as input to the explicit
-  ///   build of the current module. Consists of a map of externally-built targets, and a map of all previously
-  ///   discovered/scanned modules.
+  /// - Parameter integratedDriver: Used to distinguish whether the driver is being used as
+  ///   an executable or as a library.
+  /// - Parameter compilerExecutableDir: Directory that contains the compiler executable to be used.
+  ///   Used when in `integratedDriver` mode as a substitute for the driver knowing its executable path.
+  /// - Parameter externalTargetModulePathMap: DEPRECATED: A dictionary of external targets
+  ///   that are a part of the same build, mapping to filesystem paths of their module files.
+  /// - Parameter externalTargetModuleDetailsMap: A dictionary of external targets that are a part of
+  ///   the same build, mapping to a details value which includes a filesystem path of their
+  ///   `.swiftmodule` and a flag indicating whether the external target is a framework.
+  /// - Parameter interModuleDependencyOracle: An oracle for querying inter-module dependencies,
+  ///   shared across different module builds by a build system.
   public init(
     args: [String],
     env: [String: String] = ProcessEnv.vars,
@@ -395,10 +406,9 @@ public struct Driver {
     executor: DriverExecutor,
     integratedDriver: Bool = true,
     compilerExecutableDir: AbsolutePath? = nil,
-    // FIXME: Duplication with externalBuildArtifacts and externalTargetModulePathMap
-    // is a temporary backwards-compatibility shim to help transition SwiftPM to the new API
-    externalBuildArtifacts: ExternalBuildArtifacts? = nil,
+    // Deprecated in favour of the below `externalTargetModuleDetailsMap`
     externalTargetModulePathMap: ExternalTargetModulePathMap? = nil,
+    externalTargetModuleDetailsMap: ExternalTargetModuleDetailsMap? = nil,
     interModuleDependencyOracle: InterModuleDependencyOracle? = nil
   ) throws {
     self.env = env
@@ -408,10 +418,15 @@ public struct Driver {
     self.diagnosticEngine = diagnosticsEngine
     self.executor = executor
 
-    if let externalArtifacts = externalBuildArtifacts {
-      self.externalBuildArtifacts = externalArtifacts
-    } else if let externalTargetPaths = externalTargetModulePathMap {
-      self.externalBuildArtifacts = (externalTargetPaths, [:])
+    if externalTargetModulePathMap != nil && externalTargetModuleDetailsMap != nil {
+      throw Error.externalTargetDetailsAPIError
+    }
+    if let externalTargetPaths = externalTargetModulePathMap {
+      self.externalTargetModuleDetailsMap = externalTargetPaths.mapValues {
+        ExternalTargetModuleDetails(path: $0, isFramework: false)
+      }
+    } else if let externalTargetDetails = externalTargetModuleDetailsMap {
+      self.externalTargetModuleDetailsMap = externalTargetDetails
     }
 
     if case .subcommand = try Self.invocationRunMode(forArgs: args).mode {
@@ -504,14 +519,6 @@ public struct Driver {
       self.interModuleDependencyOracle = dependencyOracle
     } else {
       self.interModuleDependencyOracle = InterModuleDependencyOracle()
-
-      // This is a shim for backwards-compatibility with ModuleInfoMap-based API
-      // used by SwiftPM
-      if let externalArtifacts = externalBuildArtifacts {
-        if !externalArtifacts.1.isEmpty {
-          try self.interModuleDependencyOracle.mergeModules(from: externalArtifacts.1)
-        }
-      }
     }
 
     self.fileListThreshold = try Self.computeFileListThreshold(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine)

Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

@@ -14,13 +14,21 @@ import TSCUtility
 import Foundation
 
 /// A map from a module identifier to a path to its .swiftmodule file.
+/// Deprecated in favour of the below `ExternalTargetModuleDetails`
 public typealias ExternalTargetModulePathMap = [ModuleDependencyId: AbsolutePath]
 
-// FIXME: ExternalBuildArtifacts is a temporary backwards-compatibility shim
-// to help transition SwiftPM to the new API.
-/// A tuple all external artifacts a build system may pass in as input to the explicit build of the current module
-/// Consists of a map of externally-built targets, and a map of all previously discovered/scanned modules.
-public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleInfoMap)
+/// Details about an external target, including the path to its .swiftmodule file
+/// and whether it is a framework.
+public struct ExternalTargetModuleDetails {
+  public init(path: AbsolutePath, isFramework: Bool) {
+    self.path = path
+    self.isFramework = isFramework
+  }
+  let path: AbsolutePath
+  let isFramework: Bool
+}
+
+public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalTargetModuleDetails]
 
 /// In Explicit Module Build mode, this planner is responsible for generating and providing
 /// build jobs for all module dependencies and providing compile command options
@@ -211,18 +219,20 @@ public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleIn
                                               inputs: &inputs,
                                               commandLine: &commandLine)
 
+        let moduleMapPath = moduleDetails.moduleMapPath.path
         // Encode the target triple pcm args into the output `.pcm` filename
         let targetEncodedModulePath =
           try targetEncodedClangModuleFilePath(for: moduleInfo,
-                                               hashParts: getPCMHashParts(pcmArgs: pcmArgs))
+                                               hashParts: getPCMHashParts(pcmArgs: pcmArgs,
+                                                                          moduleMapPath: moduleMapPath.description))
         outputs.append(TypedVirtualPath(file: targetEncodedModulePath, type: .pcm))
         commandLine.appendFlags("-emit-pcm", "-module-name", moduleId.moduleName,
                                 "-o", targetEncodedModulePath.description)
 
         // The only required input is the .modulemap for this module.
         // Command line options in the dependency scanner output will include the
         // required modulemap, so here we must only add it to the list of inputs.
-        inputs.append(TypedVirtualPath(file: moduleDetails.moduleMapPath.path,
+        inputs.append(TypedVirtualPath(file: moduleMapPath,
                                        type: .clangModuleMap))
 
         jobs.append(Job(
@@ -246,7 +256,7 @@ public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleIn
                                                           commandLine: inout [Job.ArgTemplate]) throws {
     // Prohibit the frontend from implicitly building textual modules into binary modules.
     commandLine.appendFlags("-disable-implicit-swift-modules", "-Xcc", "-Xclang", "-Xcc",
-                            "-fno-implicit-modules")
+                            "-fno-implicit-modules", "-Xcc", "-Xclang", "-Xcc", "-fno-implicit-module-maps")
     var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = []
     var clangDependencyArtifacts: [ClangModuleArtifactInfo] = []
     try addModuleDependencies(moduleId: moduleId, pcmArgs: pcmArgs,
@@ -314,25 +324,28 @@ public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleIn
           let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId)
           let dependencyClangModuleDetails =
             try dependencyGraph.clangModuleDetails(of: dependencyId)
+          let moduleMapPath = dependencyClangModuleDetails.moduleMapPath.path
           let clangModulePath =
             try targetEncodedClangModuleFilePath(for: dependencyInfo,
-                                                 hashParts: getPCMHashParts(pcmArgs: pcmArgs))
+                                                 hashParts: getPCMHashParts(pcmArgs: pcmArgs,
+                                                                            moduleMapPath: moduleMapPath.description))
           // Accumulate the requried information about this dependency
           clangDependencyArtifacts.append(
             ClangModuleArtifactInfo(name: dependencyId.moduleName,
                                     modulePath: TextualVirtualPath(path: clangModulePath),
                                     moduleMapPath: dependencyClangModuleDetails.moduleMapPath))
         case .swiftPrebuiltExternal:
-          let compiledModulePath = try dependencyGraph
-                                         .swiftPrebuiltDetails(of: dependencyId)
-                                         .compiledModulePath
+          let prebuiltModuleDetails = try dependencyGraph.swiftPrebuiltDetails(of: dependencyId)
+          let compiledModulePath = prebuiltModuleDetails.compiledModulePath
+          let isFramework = prebuiltModuleDetails.isFramework
           let swiftModulePath: TypedVirtualPath =
             .init(file: compiledModulePath.path, type: .swiftModule)
           // Accumulate the requried information about this dependency
           // TODO: add .swiftdoc and .swiftsourceinfo for this module.
           swiftDependencyArtifacts.append(
             SwiftModuleArtifactInfo(name: dependencyId.moduleName,
-                                    modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle)))
+                                    modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle),
+                                    isFramework: isFramework))
         case .swiftPlaceholder:
           fatalError("Unresolved placeholder dependencies at planning stage: \(dependencyId) of \(moduleId)")
       }
@@ -380,11 +393,14 @@ public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleIn
     return VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(moduleId.moduleName)-dependencies.json"), contents)
   }
 
-  private func getPCMHashParts(pcmArgs: [String]) -> [String] {
+  private func getPCMHashParts(pcmArgs: [String], moduleMapPath: String) -> [String] {
+    var results: [String] = []
+    results.append(moduleMapPath)
+    results.append(contentsOf: pcmArgs)
     if integratedDriver {
-      return pcmArgs
+      return results
     }
-    var results = pcmArgs
+
     // We need this to enable explict modules in the driver-as-executable mode. For instance,
     // we have two Swift targets A and B, where A depends on X.pcm which in turn depends on Y.pcm,
     // and B only depends on Y.pcm. In the driver-as-executable mode, the build system isn't aware
@@ -436,15 +452,15 @@ extension ExplicitDependencyBuildPlanner {
     #else
     hashedArguments = SHA256().hash(hashInput).hexadecimalRepresentation
     #endif
-    let resultingName = moduleName + hashedArguments
+    let resultingName = moduleName + "-" + hashedArguments
     hashedModuleNameCache[cacheQuery] = resultingName
     return resultingName
   }
 }
 
 /// Encapsulates some of the common queries of the ExplicitDependencyBuildPlanner with error-checking
 /// on the dependency graph's structure.
-internal extension InterModuleDependencyGraph {
+@_spi(Testing) public extension InterModuleDependencyGraph {
   func moduleInfo(of moduleId: ModuleDependencyId) throws -> ModuleInfo {
     guard let moduleInfo = modules[moduleId] else {
       throw Driver.Error.missingModuleDependency(moduleId.moduleName)

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift

@@ -15,11 +15,10 @@ import TSCBasic
   /// For targets that are built alongside the driver's current module, the scanning action will report them as
   /// textual targets to be built from source. Because we can rely on these targets to have been built prior
   /// to the driver's current target, we resolve such external targets as prebuilt binary modules, in the graph.
-  mutating func resolveExternalDependencies(for externalBuildArtifacts: ExternalBuildArtifacts)
+  mutating func resolveExternalDependencies(for externalTargetModuleDetailsMap: ExternalTargetModuleDetailsMap)
   throws {
-    let externalTargetModulePathMap = externalBuildArtifacts.0
-
-     for (externalModuleId, externalModulePath) in externalTargetModulePathMap {
+    for (externalModuleId, externalModuleDetails) in externalTargetModuleDetailsMap {
+      let externalModulePath = externalModuleDetails.path
       // Replace the occurence of a Swift module to-be-built from source-file
       // to an info that describes a pre-built binary module.
       let swiftModuleId: ModuleDependencyId = .swift(externalModuleId.moduleName)
@@ -30,57 +29,22 @@ import TSCBasic
         // a dependency on a target that is not actually used.
         continue
       }
-
-       let newModuleId: ModuleDependencyId = .swiftPrebuiltExternal(externalModuleId.moduleName)
+      
+      let newModuleId: ModuleDependencyId = .swiftPrebuiltExternal(externalModuleId.moduleName)
       let newExternalModuleDetails =
-        try SwiftPrebuiltExternalModuleDetails(compiledModulePath:
-                                                TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()))
+      try SwiftPrebuiltExternalModuleDetails(compiledModulePath:
+                                              TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()),
+                                             isFramework: externalModuleDetails.isFramework)
       let newInfo = ModuleInfo(modulePath: TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()),
                                sourceFiles: [],
                                directDependencies: currentInfo.directDependencies,
                                details: .swiftPrebuiltExternal(newExternalModuleDetails))
-
-       Self.replaceModule(originalId: swiftModuleId, replacementId: newModuleId,
+      Self.replaceModule(originalId: swiftModuleId, replacementId: newModuleId,
                          replacementInfo: newInfo, in: &modules)
     }
   }
 }
 
-@_spi(Testing) public extension InterModuleDependencyOracle {
-  /// An API to allow clients to accumulate InterModuleDependencyGraphs across mutiple main externalModules/targets
-  /// into a single collection of discovered externalModules.
-  func mergeModules(from dependencyGraph: InterModuleDependencyGraph) throws {
-    try queue.sync {
-      for (moduleId, moduleInfo) in dependencyGraph.modules {
-        try InterModuleDependencyGraph.mergeModule(moduleId, moduleInfo, into: &externalModules)
-      }
-    }
-  }
-
-  // This is a backwards-compatibility shim to handle existing ModuleInfoMap-based API
-  // used by SwiftPM
-  func mergeModules(from moduleInfoMap: ModuleInfoMap) throws {
-    try queue.sync {
-      for (moduleId, moduleInfo) in moduleInfoMap {
-        try InterModuleDependencyGraph.mergeModule(moduleId, moduleInfo, into: &externalModules)
-      }
-    }
-  }
-}
-
-public extension InterModuleDependencyGraph {
-  // This is a shim for backwards-compatibility with existing API used by SwiftPM.
-  // TODO: After SwiftPM switches to using the oracle, this should be deleted.
-  static func mergeModules(
-    from dependencyGraph: InterModuleDependencyGraph,
-    into discoveredModules: inout ModuleInfoMap
-  ) throws {
-    for (moduleId, moduleInfo) in dependencyGraph.modules {
-      try mergeModule(moduleId, moduleInfo, into: &discoveredModules)
-    }
-  }
-}
-
 extension InterModuleDependencyGraph {
   /// Compute a set of modules that are "reachable" (form direct or transitive dependency)
   /// from each module in the graph.