[5.5] PrebuiltModuleGen: cherry-pick recent changes

Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift

@@ -24,33 +24,40 @@ public class PrebuitModuleGenerationDelegate: JobExecutionDelegate {
   let diagnosticsEngine: DiagnosticsEngine
   let verbose: Bool
   var failingCriticalOutputs: Set<VirtualPath>
-  public init(_ jobs: [Job], _ diagnosticsEngine: DiagnosticsEngine, _ verbose: Bool) {
+  let logPath: AbsolutePath?
+  public init(_ jobs: [Job], _ diagnosticsEngine: DiagnosticsEngine, _ verbose: Bool, logPath: AbsolutePath?) {
     self.diagnosticsEngine = diagnosticsEngine
     self.verbose = verbose
     self.failingCriticalOutputs = Set<VirtualPath>(jobs.compactMap(PrebuitModuleGenerationDelegate.getCriticalOutput))
+    self.logPath = logPath
   }
 
   /// Dangling jobs are macabi-only modules. We should run those jobs if foundation
   /// is built successfully for macabi.
   public var shouldRunDanglingJobs: Bool {
     return !failingCriticalOutputs.contains(where: isIosMac)
   }
-  func printJobInfo(_ job: Job, _ start: Bool) {
-    guard verbose else {
-      return
-    }
+
+  func getInputInterfacePath(_ job: Job) -> AbsolutePath {
     for arg in job.commandLine {
       if case .path(let p) = arg {
         if p.extension == "swiftinterface" {
-          Driver.stdErrQueue.sync {
-            stderrStream <<< (start ? "started: " : "finished: ")
-            stderrStream <<< p.absolutePath!.pathString <<< "\n"
-            stderrStream.flush()
-          }
-          return
+          return p.absolutePath!
         }
       }
     }
+    fatalError()
+  }
+
+  func printJobInfo(_ job: Job, _ start: Bool) {
+    guard verbose else {
+      return
+    }
+    Driver.stdErrQueue.sync {
+      stderrStream <<< (start ? "started: " : "finished: ")
+      stderrStream <<< getInputInterfacePath(job).pathString <<< "\n"
+      stderrStream.flush()
+    }
   }
 
   static func getCriticalOutput(_ job: Job) -> VirtualPath? {
@@ -66,6 +73,24 @@ public class PrebuitModuleGenerationDelegate: JobExecutionDelegate {
     return !failingCriticalOutputs.isEmpty
   }
 
+  fileprivate func logOutput(_ job: Job, _ result: ProcessResult, _ stdout: Bool) throws {
+    guard let logPath = logPath else {
+      return
+    }
+    let content = stdout ? try result.utf8Output() : try result.utf8stderrOutput()
+    guard !content.isEmpty else {
+      return
+    }
+    if !localFileSystem.exists(logPath) {
+      try localFileSystem.createDirectory(logPath, recursive: true)
+    }
+    let interfaceBase = getInputInterfacePath(job).basenameWithoutExt
+    let fileName = "\(job.moduleName)-\(interfaceBase)-\(stdout ? "out" : "err").txt"
+    try localFileSystem.writeFileContents(logPath.appending(component: fileName)) {
+      $0 <<< content
+    }
+  }
+
   public func jobFinished(job: Job, result: ProcessResult, pid: Int) {
     switch result.exitStatus {
     case .terminated(code: let code):
@@ -86,6 +111,15 @@ public class PrebuitModuleGenerationDelegate: JobExecutionDelegate {
       diagnosticsEngine.emit(.remark("\(job.moduleName) interrupted"))
 #endif
     }
+    do {
+      try logOutput(job, result, true)
+      try logOutput(job, result, false)
+    } catch {
+      Driver.stdErrQueue.sync {
+        stderrStream <<< "Failed to generate log file"
+        stderrStream.flush()
+      }
+    }
   }
 
   public func jobSkipped(job: Job) {
@@ -236,6 +270,57 @@ public struct SDKPrebuiltModuleInputsCollector {
   }
 }
 
+extension InterModuleDependencyGraph {
+  func dumpDotGraph(_ path: AbsolutePath, _ includingPCM: Bool) throws {
+    func isPCM(_ dep: ModuleDependencyId) -> Bool {
+      switch dep {
+      case .clang:
+        return true
+      default:
+        return false
+      }
+    }
+    func dumpModuleName(_ stream: WritableByteStream, _ dep: ModuleDependencyId) {
+      switch dep {
+      case .swift(let name):
+        stream <<< "\"\(name).swiftmodule\""
+      case .clang(let name):
+        stream <<< "\"\(name).pcm\""
+      default:
+        break
+      }
+    }
+    try localFileSystem.writeFileContents(path) {Stream in
+      Stream <<< "digraph {\n"
+      for key in modules.keys {
+        switch key {
+        case .swift(let name):
+          if name == mainModuleName {
+              break
+          }
+          fallthrough
+        case .clang:
+          if !includingPCM && isPCM(key) {
+            break
+          }
+          modules[key]!.directDependencies?.forEach { dep in
+            if !includingPCM && isPCM(dep) {
+              return
+            }
+            dumpModuleName(Stream, key)
+            Stream <<< " -> "
+            dumpModuleName(Stream, dep)
+            Stream <<< ";\n"
+          }
+        default:
+          break
+        }
+      }
+      Stream <<< "}\n"
+    }
+  }
+}
+
 extension Driver {
 
   private mutating func generateSingleModuleBuildingJob(_ moduleName: String,  _ prebuiltModuleDir: AbsolutePath,
@@ -261,7 +346,14 @@ extension Driver {
       commandLine.appendFlag(.Fsystem)
       commandLine.append(.path(iosMacFrameworksSearchPath))
     }
+    // Use the specified module cache dir
+    if let mcp = parsedOptions.getLastArgument(.moduleCachePath)?.asSingle {
+      commandLine.appendFlag(.moduleCachePath)
+      commandLine.append(.path(try VirtualPath(path: mcp)))
+    }
     commandLine.appendFlag(.serializeParseableModuleInterfaceDependencyHashes)
+    commandLine.appendFlag(.badFileDescriptorRetryCount)
+    commandLine.appendFlag("30")
     return Job(
       moduleName: moduleName,
       kind: .compile,
@@ -275,12 +367,16 @@ extension Driver {
 
   public mutating func generatePrebuitModuleGenerationJobs(with inputMap: [String: [PrebuiltModuleInput]],
                                                            into prebuiltModuleDir: AbsolutePath,
-                                                           exhaustive: Bool) throws -> ([Job], [Job]) {
+                                                           exhaustive: Bool,
+                                                           dotGraphPath: AbsolutePath? = nil) throws -> ([Job], [Job]) {
     assert(sdkPath != nil)
     // Run the dependency scanner and update the dependency oracle with the results
     // We only need Swift dependencies here, so we don't need to invoke gatherModuleDependencies,
     // which also resolves versioned clang modules.
     let dependencyGraph = try performDependencyScan()
+    if let dotGraphPath = dotGraphPath {
+      try dependencyGraph.dumpDotGraph(dotGraphPath, false)
+    }
     var jobs: [Job] = []
     var danglingJobs: [Job] = []
     var inputCount = 0

Sources/swift-build-sdk-interfaces/main.swift

@@ -17,19 +17,27 @@ import TSCUtility
 
 let diagnosticsEngine = DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler])
 
-guard let sdkPathRaw = ProcessEnv.vars["SDKROOT"] else {
-  diagnosticsEngine.emit(.error("need to set SDKROOT"))
-  exit(1)
+func getArgument(_ flag: String, _ env: String? = nil) -> String? {
+  if let id = CommandLine.arguments.firstIndex(of: flag) {
+    let nextId = id.advanced(by: 1)
+    if nextId < CommandLine.arguments.count {
+      return CommandLine.arguments[nextId]
+    }
+  }
+  if let env = env {
+    return ProcessEnv.vars[env]
+  }
+  return nil
 }
 
-var rawOutputDir = ""
-if let oid = CommandLine.arguments.firstIndex(of: "-o") {
-  let dirId = oid.advanced(by: 1)
-  if dirId < CommandLine.arguments.count {
-    rawOutputDir = CommandLine.arguments[dirId]
+func getArgumentAsPath(_ flag: String, _ env: String? = nil) throws -> AbsolutePath? {
+  if let raw = getArgument(flag, env) {
+    return try VirtualPath(path: raw).absolutePath
   }
+  return nil
 }
-if rawOutputDir.isEmpty {
+
+guard let rawOutputDir = getArgument("-o") else {
   diagnosticsEngine.emit(.error("need to specify -o"))
   exit(1)
 }
@@ -41,8 +49,15 @@ let coreMode = CommandLine.arguments.contains("-core")
 /// Verbose to print more info
 let verbose = CommandLine.arguments.contains("-v")
 
+/// Skip executing the jobs
+let skipExecution = CommandLine.arguments.contains("-n")
+
 do {
-  let sdkPath = try VirtualPath(path: sdkPathRaw).absolutePath!
+  let sdkPathArg = try getArgumentAsPath("-sdk", "SDKROOT")
+  guard let sdkPath = sdkPathArg else {
+    diagnosticsEngine.emit(.error("need to set SDKROOT"))
+    exit(1)
+  }
   if !localFileSystem.exists(sdkPath) {
     diagnosticsEngine.emit(error: "cannot find sdk: \(sdkPath.pathString)")
     exit(1)
@@ -56,7 +71,7 @@ do {
     outputDir = outputDir.appending(RelativePath(collector.versionString))
   }
   if !localFileSystem.exists(outputDir) {
-    try localFileSystem.createDirectory(outputDir)
+    try localFileSystem.createDirectory(outputDir, recursive: true)
   }
   let swiftcPathRaw = ProcessEnv.vars["SWIFT_EXEC"]
   var swiftcPath: AbsolutePath
@@ -99,15 +114,32 @@ do {
                                            processSet: processSet,
                                            fileSystem: localFileSystem,
                                            env: ProcessEnv.vars)
-    var driver = try Driver(args: ["swiftc",
-                                   "-target", collector.targetTriple,
-                                   tempPath.description,
-                                   "-sdk", sdkPathRaw],
+    var args = ["swiftc",
+                "-target", collector.targetTriple,
+                tempPath.description,
+                "-sdk", sdkPath.pathString]
+    let mcpFlag = "-module-cache-path"
+    // Append module cache path if given by the client
+    if let mcp = getArgument(mcpFlag) {
+      args.append(mcpFlag)
+      args.append(mcp)
+    }
+    var driver = try Driver(args: args,
                             diagnosticsEngine: diagnosticsEngine,
                             executor: executor,
                             compilerExecutableDir: swiftcPath.parentDirectory)
-    let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(with: inputMap, into: outputDir, exhaustive: !coreMode)
-    let delegate = PrebuitModuleGenerationDelegate(jobs, diagnosticsEngine, verbose)
+    let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(with: inputMap,
+      into: outputDir, exhaustive: !coreMode, dotGraphPath: getArgumentAsPath("-dot-graph-path"))
+    if verbose {
+      Driver.stdErrQueue.sync {
+        stderrStream <<< "job count: \(jobs.count + danglingJobs.count)\n"
+        stderrStream.flush()
+      }
+    }
+    if skipExecution {
+      exit(0)
+    }
+    let delegate = PrebuitModuleGenerationDelegate(jobs, diagnosticsEngine, verbose, logPath: try getArgumentAsPath("-log-path"))
     do {
       try executor.execute(workload: DriverExecutorWorkload.init(jobs, nil, continueBuildingAfterErrors: true),
                            delegate: delegate, numParallelJobs: 128)

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -874,8 +874,10 @@ final class ExplicitModuleBuildTests: XCTestCase {
         $0 <<< "import H\n"
         $0 <<< "import Swift\n"
       }
+      let moduleCachePath = "/tmp/module-cache"
       var driver = try Driver(args: ["swiftc", main.pathString,
                                      "-sdk", mockSDKPath,
+                                     "-module-cache-path", moduleCachePath
                                     ])
 
       let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(with: interfaceMap,
@@ -890,6 +892,9 @@ final class ExplicitModuleBuildTests: XCTestCase {
       XCTAssertTrue(jobs.allSatisfy {$0.outputs.count == 1})
       XCTAssertTrue(jobs.allSatisfy {$0.kind == .compile})
       XCTAssertTrue(jobs.allSatisfy {$0.commandLine.contains(.flag("-compile-module-from-interface"))})
+      XCTAssertTrue(jobs.allSatisfy {$0.commandLine.contains(.flag("-module-cache-path"))})
+      XCTAssertTrue(jobs.allSatisfy {$0.commandLine.contains(.flag("-bad-file-descriptor-retry-count"))})
+      XCTAssertTrue(try jobs.allSatisfy {$0.commandLine.contains(.path(try VirtualPath(path: moduleCachePath)))})
       let HJobs = jobs.filter { $0.moduleName == "H"}
       XCTAssertTrue(HJobs.count == 2)
       XCTAssertTrue(getInputModules(HJobs[0]) == ["A", "E", "F", "G", "Swift"])