Propagate explicit module build commands and arguments to compile jobs

artemcm · artemcm · commit 8dadcc77ba4a · 2020-06-01T12:43:00.000-07:00
diff --git a/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift b/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift
@@ -47,7 +47,6 @@ extension Driver {
                                                          moduleName: String,
                                                          dependencyGraph: InterModuleDependencyGraph)
   throws -> Job {
-    // FIXIT: Needs more error handling
     guard case .swift(let swiftModuleDetails) = moduleInfo.details else {
       throw Error.malformedModuleDependency(moduleName, "no `details` object")
     }
@@ -57,7 +56,13 @@ extension Driver {
       TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), type: .swiftModule)
     ]
     var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
-    commandLine.appendFlag("-frontend")
+    // First, take the command line options provided in the dependency information
+    swiftModuleDetails.commandLine?.forEach { commandLine.appendFlags($0) }
+
+    if (swiftModuleDetails.commandLine == nil ||
+          !swiftModuleDetails.commandLine!.contains("-frontend")) {
+      commandLine.appendFlag("-frontend")
+    }
 
     try addModuleDependencies(moduleInfo: moduleInfo,
                               dependencyGraph: dependencyGraph,
@@ -72,7 +77,6 @@ extension Driver {
     inputs.append(TypedVirtualPath(file: try VirtualPath(path: moduleInterfacePath),
                                    type: .swiftInterface))
     try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
-    swiftModuleDetails.commandLine?.forEach { commandLine.appendFlag($0) }
 
     return Job(
       moduleName: moduleName,
@@ -101,7 +105,14 @@ extension Driver {
       TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), type: .pcm)
     ]
     var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
-    commandLine.appendFlag("-frontend")
+
+    // First, take the command line options provided in the dependency information
+    clangModuleDetails.commandLine?.forEach { commandLine.appendFlags($0) }
+
+    if (clangModuleDetails.commandLine == nil ||
+          !clangModuleDetails.commandLine!.contains("-frontend")) {
+      commandLine.appendFlag("-frontend")
+    }
     commandLine.appendFlags("-emit-pcm", "-module-name", moduleName)
 
     try addModuleDependencies(moduleInfo: moduleInfo,
@@ -115,7 +126,6 @@ extension Driver {
     inputs.append(TypedVirtualPath(file: try VirtualPath(path: clangModuleDetails.moduleMapPath),
                                    type: .clangModuleMap))
     try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
-    clangModuleDetails.commandLine?.forEach { commandLine.appendFlags($0) }
 
     return Job(
       moduleName: moduleName,
@@ -134,29 +144,15 @@ extension Driver {
                                      dependencyGraph: InterModuleDependencyGraph,
                                      inputs: inout [TypedVirtualPath],
                                      commandLine: inout [Job.ArgTemplate]) throws {
-    // These options ensure that the frontend only uses explicitly-specified module dependencies
-    // and the frontend errors if it has to perform any implicit module builds.
+    // Prohibit the frontend from implicitly building textual modules into binary modules.
     commandLine.appendFlags("-disable-implicit-swift-modules", "-disable-implicit-pcms")
     for moduleId in moduleInfo.directDependencies {
       guard let dependencyInfo = dependencyGraph.modules[moduleId] else {
         throw Error.missingModuleDependency(moduleId.getName())
       }
-      switch dependencyInfo.details {
-        case .swift:
-          let swiftModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath),
-                                                 type: .swiftModule)
-          commandLine.appendFlag("-swift-module-file=\(swiftModulePath.file.description)")
-          inputs.append(swiftModulePath)
-        case .clang(let clangDependencyDetails):
-          let clangModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath),
-                                                 type: .pcm)
-          let clangModuleMapPath = TypedVirtualPath(file: try VirtualPath(path: clangDependencyDetails.moduleMapPath),
-                                                    type: .pcm)
-          commandLine.appendFlag("-clang-module-file=\(clangModulePath.file.description)")
-          commandLine.appendFlag("-clang-module-map-file=\(clangModuleMapPath.file.description)")
-          inputs.append(clangModulePath)
-          inputs.append(clangModuleMapPath)
-      }
+
+      try addModuleAsExplicitDependency(moduleInfo: dependencyInfo, commandLine: &commandLine,
+                                        inputs: &inputs)
     }
   }
 }
diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift
@@ -50,6 +50,7 @@ public struct Driver {
     case subcommandPassedToDriver
     case integratedReplRemoved
     case conflictingOptions(Option, Option)
+    // Explicit Module Build Failures
     case malformedModuleDependency(String, String)
     case missingModuleDependency(String)
     case dependencyScanningFailure(Int, String)
@@ -71,6 +72,7 @@ public struct Driver {
         return "Compiler-internal integrated REPL has been removed; use the LLDB-enhanced REPL instead."
       case .conflictingOptions(let one, let two):
         return "conflicting options '\(one.spelling)' and '\(two.spelling)'"
+      // Explicit Module Build Failures
       case .malformedModuleDependency(let moduleName, let errorDescription):
         return "Malformed Module Dependency: \(moduleName), \(errorDescription)"
       case .missingModuleDependency(let moduleName):
@@ -206,6 +208,11 @@ public struct Driver {
   /// This will force the driver to first emit the module and then run compile jobs.
   public var forceEmitModuleInSingleInvocation: Bool = false
 
+  /// The module dependency graph, which is populated during the planning phase
+  /// only when all modules will be prebuilt and treated as explicit by the
+  /// various compilation jobs.
+  var interModuleDependencyGraph: InterModuleDependencyGraph? = nil
+
   /// Handler for emitting diagnostics to stderr.
   public static let stderrDiagnosticsHandler: DiagnosticsEngine.DiagnosticsHandler = { diagnostic in
     let stream = stderrStream
diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift
@@ -155,6 +155,10 @@ extension Driver {
     try addCommonFrontendOptions(commandLine: &commandLine)
     // FIXME: MSVC runtime flags
 
+    if parsedOptions.contains(.driverExplicitModuleBuild) {
+      try addExplicitModuleBuildArguments(commandLine: &commandLine, inputs: &inputs)
+    }
+
     if parsedOptions.hasArgument(.parseAsLibrary, .emitLibrary) {
       commandLine.appendFlag(.parseAsLibrary)
     }
diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift
@@ -304,4 +304,49 @@ extension Driver {
 
     return outputs
   }
+
+  /// Adds the specified module as an explicit module dependency to given
+  /// inputs and comman line arguments of a compile job.
+  func addModuleAsExplicitDependency(moduleInfo: ModuleInfo,
+                                     commandLine: inout [Job.ArgTemplate],
+                                     inputs: inout [TypedVirtualPath]) throws {
+    switch moduleInfo.details {
+      case .swift:
+        let swiftModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath),
+                                               type: .swiftModule)
+        commandLine.appendFlag("-swift-module-file=\(swiftModulePath.file.description)")
+        inputs.append(swiftModulePath)
+      case .clang(let clangDependencyDetails):
+        let clangModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath),
+                                               type: .pcm)
+        let clangModuleMapPath = TypedVirtualPath(file: try VirtualPath(path: clangDependencyDetails.moduleMapPath),
+                                                  type: .pcm)
+        commandLine.appendFlag("-clang-module-file=\(clangModulePath.file.description)")
+        commandLine.appendFlag("-clang-module-map-file=\(clangModuleMapPath.file.description)")
+        inputs.append(clangModulePath)
+        inputs.append(clangModuleMapPath)
+    }
+  }
+
+  /// Adds all dependecies required for an explicit module build
+  /// to inputs and comman line arguments of a compile job.
+  func addExplicitModuleBuildArguments(commandLine: inout [Job.ArgTemplate],
+                                       inputs: inout [TypedVirtualPath]) throws {
+    guard let dependencyGraph = interModuleDependencyGraph else {
+      fatalError("Inter Module Dependency Graph does not exist in explicit module build mode.")
+    }
+    // Prohibit the frontend from implicitly building textual modules into binary modules.
+    commandLine.appendFlags("-disable-implicit-swift-modules", "-disable-implicit-pcms")
+
+    // Provide the frontend with a list of explicitly pre-built modules.
+    for (moduleId, moduleInfo) in dependencyGraph.modules {
+      // Skip the main output module as it is not its own dependency
+      guard moduleId.getName() != dependencyGraph.mainModuleName else {
+        continue
+      }
+
+      try addModuleAsExplicitDependency(moduleInfo: moduleInfo, commandLine: &commandLine,
+                                        inputs: &inputs)
+    }
+  }
 }
diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift
@@ -34,17 +34,6 @@ extension Driver {
   private mutating func planStandardCompile() throws -> [Job] {
     var jobs = [Job]()
 
-    // If we've been asked to prebuild module dependencies,
-    // for the time being, just print the jobs' compile commands.
-    if parsedOptions.contains(.driverPrintModuleDependenciesJobs) {
-      let modulePrebuildJobs = try generateExplicitModuleBuildJobs()
-      let forceResponseFiles = parsedOptions.contains(.driverForceResponseFiles)
-      for job in modulePrebuildJobs {
-        try Self.printJob(job, resolver: try ArgsResolver(),
-                          forceResponseFiles: forceResponseFiles)
-      }
-    }
-
     // Keep track of the various outputs we care about from the jobs we build.
     var linkerInputs: [TypedVirtualPath] = []
     var moduleInputs: [TypedVirtualPath] = []
@@ -63,9 +52,24 @@ extension Driver {
       }
     }
 
-    // Precompile module dependencies, if asked.
-    if parsedOptions.contains(.driverExplicitModuleBuild) {
-      jobs.append(contentsOf: try generateExplicitModuleBuildJobs())
+    // If asked, add jobs to precompile module dependencies
+    if parsedOptions.contains(.driverExplicitModuleBuild) ||
+        parsedOptions.contains(.driverPrintModuleDependenciesJobs) {
+      let modulePrebuildJobs = try generateExplicitModuleBuildJobs()
+
+      if parsedOptions.contains(.driverExplicitModuleBuild) {
+        jobs.append(contentsOf: modulePrebuildJobs)
+      }
+
+      // If we've been asked to prebuild module dependencies,
+      // for the time being, just print the jobs' compile commands.
+      if parsedOptions.contains(.driverPrintModuleDependenciesJobs) {
+        let forceResponseFiles = parsedOptions.contains(.driverForceResponseFiles)
+        for job in modulePrebuildJobs {
+          try Self.printJob(job, resolver: try ArgsResolver(),
+                            forceResponseFiles: forceResponseFiles)
+        }
+      }
     }
 
     // Precompile the bridging header if needed.
@@ -218,13 +222,11 @@ extension Driver {
   /// Prescan the source files to produce a module dependency graph and turn it into a set
   /// of jobs required to build all dependencies.
   public mutating func generateExplicitModuleBuildJobs() throws -> [Job] {
-    let moduleDependencyGraph = try computeModuleDependencyGraph()
-    if let dependencyGraph = moduleDependencyGraph {
-      let modulePrebuildJobs =
-            try planExplicitModuleDependenciesCompile(dependencyGraph: dependencyGraph)
-      return modulePrebuildJobs
+    interModuleDependencyGraph = try computeModuleDependencyGraph()
+    guard let dependencyGraph = interModuleDependencyGraph else {
+      fatalError("Attempting to perform Explicit Module Build job generation, but the Inter Module Dependency Graph does not exist.")
     }
-    return []
+    return try planExplicitModuleDependenciesCompile(dependencyGraph: dependencyGraph)
   }
 
   /// Create a job if needed for simple requests that can be immediately
diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift
@@ -38,7 +38,6 @@ private func checkExplicitModuleBuildJob(job: Job,
   }
   try checkExplicitModuleBuildJobDependencies(job: job, moduleInfo: moduleInfo,
                                               moduleDependencyGraph: moduleDependencyGraph)
-
 }
 
 /// Checks that the build job for the specified module contains the required options and inputs
@@ -54,22 +53,22 @@ throws {
     switch dependencyInfo.details {
       case .swift:
         let swiftDependencyModulePath =
-          TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath),
+          TypedVirtualPath(file: try VirtualPath(path: dependencyInfo.modulePath),
                            type: .swiftModule)
         XCTAssertTrue(job.inputs.contains(swiftDependencyModulePath))
         XCTAssertTrue(job.commandLine.contains(
-                        .flag(String("-swift-module-file=\(moduleInfo.modulePath)"))))
+                        .flag(String("-swift-module-file=\(dependencyInfo.modulePath)"))))
       case .clang(let clangDependencyDetails):
         let clangDependencyModulePath =
-          TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath),
+          TypedVirtualPath(file: try VirtualPath(path: dependencyInfo.modulePath),
                            type: .pcm)
         let clangDependencyModuleMapPath =
           TypedVirtualPath(file: try VirtualPath(path: clangDependencyDetails.moduleMapPath),
                            type: .pcm)
         XCTAssertTrue(job.inputs.contains(clangDependencyModulePath))
         XCTAssertTrue(job.inputs.contains(clangDependencyModuleMapPath))
         XCTAssertTrue(job.commandLine.contains(
-                        .flag(String("-clang-module-file=\(moduleInfo.modulePath)"))))
+                        .flag(String("-clang-module-file=\(dependencyInfo.modulePath)"))))
         XCTAssertTrue(job.commandLine.contains(
                         .flag(String("-clang-module-map-file=\(clangDependencyDetails.moduleMapPath)"))))
     }
@@ -116,8 +115,8 @@ final class ExplicitModuleBuildTests: XCTestCase {
   }
 
   /// Test generation of explicit module build jobs for dependency modules when the driver
-  /// is invoked with -driver-print-module-dependencies-jobs
-  func testModuleDependencyBuildEndToEnd() throws {
+  /// is invoked with -experimental-explicit-module-build
+  func testExplicitModuleBuildJobs() throws {
     try withTemporaryDirectory { path in
       let main = path.appending(component: "main.swift")
       try localFileSystem.writeFileContents(main) {
@@ -131,36 +130,64 @@ final class ExplicitModuleBuildTests: XCTestCase {
       let testInputsPath = packageRootPath + "/TestInputs"
       let cHeadersPath : String = testInputsPath + "/ExplicitModuleBuilds/CHeaders"
       let swiftModuleInterfacesPath : String = testInputsPath + "/ExplicitModuleBuilds/Swift"
-      var driver = try Driver(args: ["swift",
+      var driver = try Driver(args: ["swiftc",
                                      "-I", cHeadersPath,
                                      "-I", swiftModuleInterfacesPath,
-                                     "-driver-print-module-dependencies-jobs",
+                                     "-experimental-explicit-module-build",
                                      main.pathString])
-      let jobs = try driver.generateExplicitModuleBuildJobs()
-      XCTAssertEqual(jobs.count, 10)
+      let jobs = try driver.planBuild()
+      XCTAssertTrue(driver.parsedOptions.contains(.driverExplicitModuleBuild))
+      let dependencyGraph = driver.interModuleDependencyGraph!
+      XCTAssertEqual(jobs.count, 12)
       for job in jobs {
         XCTAssertEqual(job.outputs.count, 1)
         switch (job.outputs[0].file) {
           case .relative(RelativePath("A.swiftmodule")):
-            XCTAssertEqual(job.kind, .emitModule)
+            try checkExplicitModuleBuildJob(job: job, moduleName: "A",
+                                            moduleKind: ModuleDependencyId.CodingKeys.swift,
+                                            moduleDependencyGraph: dependencyGraph)
           case .relative(RelativePath("E.swiftmodule")):
-            XCTAssertEqual(job.kind, .emitModule)
+            try checkExplicitModuleBuildJob(job: job, moduleName: "E",
+                                            moduleKind: ModuleDependencyId.CodingKeys.swift,
+                                            moduleDependencyGraph: dependencyGraph)
           case .relative(RelativePath("G.swiftmodule")):
-            XCTAssertEqual(job.kind, .emitModule)
+            try checkExplicitModuleBuildJob(job: job, moduleName: "G",
+                                            moduleKind: ModuleDependencyId.CodingKeys.swift,
+                                            moduleDependencyGraph: dependencyGraph)
           case .relative(RelativePath("A.pcm")):
-            XCTAssertEqual(job.kind, .generatePCM)
+            try checkExplicitModuleBuildJob(job: job, moduleName: "A",
+                                            moduleKind: ModuleDependencyId.CodingKeys.clang,
+                                            moduleDependencyGraph: dependencyGraph)
           case .relative(RelativePath("B.pcm")):
-            XCTAssertEqual(job.kind, .generatePCM)
+            try checkExplicitModuleBuildJob(job: job, moduleName: "B",
+                                            moduleKind: ModuleDependencyId.CodingKeys.clang,
+                                            moduleDependencyGraph: dependencyGraph)
           case .relative(RelativePath("C.pcm")):
-            XCTAssertEqual(job.kind, .generatePCM)
+            try checkExplicitModuleBuildJob(job: job, moduleName: "C",
+                                            moduleKind: ModuleDependencyId.CodingKeys.clang,
+                                            moduleDependencyGraph: dependencyGraph)
           case .relative(RelativePath("G.pcm")):
-            XCTAssertEqual(job.kind, .generatePCM)
+            try checkExplicitModuleBuildJob(job: job, moduleName: "G",
+                                            moduleKind: ModuleDependencyId.CodingKeys.clang,
+                                            moduleDependencyGraph: dependencyGraph)
           case .relative(RelativePath("Swift.swiftmodule")):
-            XCTAssertEqual(job.kind, .emitModule)
+            try checkExplicitModuleBuildJob(job: job, moduleName: "Swift",
+                                            moduleKind: ModuleDependencyId.CodingKeys.swift,
+                                            moduleDependencyGraph: dependencyGraph)
           case .relative(RelativePath("SwiftOnoneSupport.swiftmodule")):
-            XCTAssertEqual(job.kind, .emitModule)
+            try checkExplicitModuleBuildJob(job: job, moduleName: "SwiftOnoneSupport",
+                                            moduleKind: ModuleDependencyId.CodingKeys.swift,
+                                            moduleDependencyGraph: dependencyGraph)
           case .relative(RelativePath("SwiftShims.pcm")):
-            XCTAssertEqual(job.kind, .generatePCM)
+            try checkExplicitModuleBuildJob(job: job, moduleName: "SwiftShims",
+                                            moduleKind: ModuleDependencyId.CodingKeys.clang,
+                                            moduleDependencyGraph: dependencyGraph)
+          case .temporary(RelativePath("main.o")):
+            try checkExplicitModuleBuildJobDependencies(job: job,
+                                                        moduleInfo: dependencyGraph.mainModule,
+                                                        moduleDependencyGraph: dependencyGraph)
+          case .relative(RelativePath("main")):
+            XCTAssertEqual(job.kind, .link)
           default:
             XCTFail("Unexpected module dependency build job output: \(job.outputs[0].file)")
         }
