Specify module dependencies explicitly when pre-building module dependencies.

artemcm · artemcm · commit a1cbb36161e2 · 2020-06-01T11:57:02.000-07:00
diff --git a/Sources/SwiftDriver/Dependency Scanning/InterModuleDependencyGraph.swift b/Sources/SwiftDriver/Dependency Scanning/InterModuleDependencyGraph.swift
@@ -50,6 +50,25 @@ extension ModuleDependencyId: Codable {
         try container.encode(moduleName, forKey: .clang)
     }
   }
+
+  func getName() -> String {
+    switch self {
+      case .swift(let moduleName):
+        return moduleName
+      case .clang(let moduleName):
+        return moduleName
+    }
+  }
+
+  // Used in testing
+  init(name: String, kind: CodingKeys) {
+    switch kind {
+      case .swift:
+        self = .swift(name)
+      case .clang:
+        self = .clang(name)
+    }
+  }
 }
 
 /// Details specific to Swift modules.
diff --git a/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift b/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift
@@ -26,22 +26,27 @@ extension Driver {
       }
       switch id {
         case .swift(let moduleName):
-          let swiftModuleBuildJob = try genSwiftModuleDependencyBuildJob(moduleInfo: moduleInfo,
-                                                                         moduleName: moduleName)
+          let swiftModuleBuildJob =
+            try genSwiftModuleDependencyBuildJob(moduleInfo: moduleInfo,
+                                                 moduleName: moduleName,
+                                                 dependencyGraph: dependencyGraph)
           jobs.append(swiftModuleBuildJob)
         case .clang(let moduleName):
-          let clangModuleBuildJob = try genClangModuleDependencyBuildJob(moduleInfo: moduleInfo,
-                                                                         moduleName: moduleName)
+          let clangModuleBuildJob =
+            try genClangModuleDependencyBuildJob(moduleInfo: moduleInfo,
+                                                 moduleName: moduleName,
+                                                 dependencyGraph: dependencyGraph)
           jobs.append(clangModuleBuildJob)
-
       }
     }
     return jobs
   }
 
   /// For a given swift module dependency, generate a build job
   mutating private func genSwiftModuleDependencyBuildJob(moduleInfo: ModuleInfo,
-                                                         moduleName: String) throws -> Job {
+                                                         moduleName: String,
+                                                         dependencyGraph: InterModuleDependencyGraph)
+  throws -> Job {
     // FIXIT: Needs more error handling
     guard case .swift(let swiftModuleDetails) = moduleInfo.details else {
       throw Error.malformedModuleDependency(moduleName, "no `details` object")
@@ -54,6 +59,11 @@ extension Driver {
     var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
     commandLine.appendFlag("-frontend")
 
+    try addModuleDependencies(moduleInfo: moduleInfo,
+                              dependencyGraph: dependencyGraph,
+                              inputs: &inputs,
+                              commandLine: &commandLine)
+
     // Build the .swiftinterfaces file using a list of command line options specified in the
     // `details` field.
     guard let moduleInterfacePath = swiftModuleDetails.moduleInterfacePath else {
@@ -76,7 +86,9 @@ extension Driver {
 
   /// For a given clang module dependency, generate a build job
   mutating private func genClangModuleDependencyBuildJob(moduleInfo: ModuleInfo,
-                                                         moduleName: String) throws -> Job {
+                                                         moduleName: String,
+                                                         dependencyGraph: InterModuleDependencyGraph)
+  throws -> Job {
     // For clang modules, the Fast Dependency Scanner emits a list of source
     // files (with a .modulemap among them), and a list of compile command
     // options.
@@ -92,12 +104,18 @@ extension Driver {
     commandLine.appendFlag("-frontend")
     commandLine.appendFlags("-emit-pcm", "-module-name", moduleName)
 
+    try addModuleDependencies(moduleInfo: moduleInfo,
+                              dependencyGraph: dependencyGraph,
+                              inputs: &inputs,
+                              commandLine: &commandLine)
+
     // The only required input is the .modulemap for this module.
-    commandLine.append(Job.ArgTemplate.path(try VirtualPath(path: clangModuleDetails.moduleMapPath)))
+    commandLine.append(Job.ArgTemplate.path(
+                        try VirtualPath(path: clangModuleDetails.moduleMapPath)))
     inputs.append(TypedVirtualPath(file: try VirtualPath(path: clangModuleDetails.moduleMapPath),
                                    type: .clangModuleMap))
     try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
-    clangModuleDetails.commandLine?.forEach { commandLine.appendFlags("-Xcc", $0) }
+    clangModuleDetails.commandLine?.forEach { commandLine.appendFlags($0) }
 
     return Job(
       moduleName: moduleName,
@@ -108,4 +126,37 @@ extension Driver {
       outputs: outputs
     )
   }
+
+
+  /// For the specified module, update its command line flags and inputs
+  /// to use explicitly-built module dependencies.
+  private func addModuleDependencies(moduleInfo: ModuleInfo,
+                                     dependencyGraph: InterModuleDependencyGraph,
+                                     inputs: inout [TypedVirtualPath],
+                                     commandLine: inout [Job.ArgTemplate]) throws {
+    // These options ensure that the frontend only uses explicitly-specified module dependencies
+    // and the frontend errors if it has to perform any implicit module builds.
+    commandLine.appendFlags("-disable-implicit-swift-modules", "-disable-implicit-pcms")
+    for moduleId in moduleInfo.directDependencies {
+      guard let dependencyInfo = dependencyGraph.modules[moduleId] else {
+        throw Error.missingModuleDependency(moduleId.getName())
+      }
+      switch dependencyInfo.details {
+        case .swift:
+          let swiftModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath),
+                                                 type: .swiftModule)
+          commandLine.appendFlag("-swift-module-file=\(swiftModulePath.file.description)")
+          inputs.append(swiftModulePath)
+        case .clang(let clangDependencyDetails):
+          let clangModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath),
+                                                 type: .pcm)
+          let clangModuleMapPath = TypedVirtualPath(file: try VirtualPath(path: clangDependencyDetails.moduleMapPath),
+                                                    type: .pcm)
+          commandLine.appendFlag("-clang-module-file=\(clangModulePath.file.description)")
+          commandLine.appendFlag("-clang-module-map-file=\(clangModuleMapPath.file.description)")
+          inputs.append(clangModulePath)
+          inputs.append(clangModuleMapPath)
+      }
+    }
+  }
 }
diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift
@@ -51,6 +51,7 @@ public struct Driver {
     case integratedReplRemoved
     case conflictingOptions(Option, Option)
     case malformedModuleDependency(String, String)
+    case missingModuleDependency(String)
     case dependencyScanningFailure(Int, String)
 
     public var description: String {
@@ -72,6 +73,8 @@ public struct Driver {
         return "conflicting options '\(one.spelling)' and '\(two.spelling)'"
       case .malformedModuleDependency(let moduleName, let errorDescription):
         return "Malformed Module Dependency: \(moduleName), \(errorDescription)"
+      case .missingModuleDependency(let moduleName):
+        return "Missing Module Dependency Info: \(moduleName)"
       case .dependencyScanningFailure(let code, let error):
         return "Module Dependency Scanner returned with non-zero exit status: \(code), \(error)"
       }
diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift
@@ -63,6 +63,11 @@ extension Driver {
       }
     }
 
+    // Precompile module dependencies, if asked.
+    if parsedOptions.contains(.driverExplicitModuleBuild) {
+      jobs.append(contentsOf: try generateExplicitModuleBuildJobs())
+    }
+
     // Precompile the bridging header if needed.
     if let importedObjCHeader = importedObjCHeader,
       let bridgingPrecompiledHeader = bridgingPrecompiledHeader {
diff --git a/Sources/SwiftOptions/ExtraOptions.swift b/Sources/SwiftOptions/ExtraOptions.swift
@@ -11,13 +11,13 @@
 //===----------------------------------------------------------------------===//
 extension Option {
   public static let driverPrintGraphviz: Option = Option("-driver-print-graphviz", .flag, attributes: [.helpHidden, .doesNotAffectIncrementalBuild], helpText: "Write the job graph as a graphviz file", group: .internalDebug)
-  public static let driverPrebuildModuleDependencies: Option = Option("-driver-prebuild-module-dependencies", .flag, attributes: [.helpHidden], helpText: "Prebuild module dependencies to make them explicit")
+  public static let driverExplicitModuleBuild: Option = Option("-experimental-explicit-module-build", .flag, attributes: [.helpHidden], helpText: "Prebuild module dependencies to make them explicit")
   public static let driverPrintModuleDependenciesJobs: Option = Option("-driver-print-module-dependencies-jobs", .flag, attributes: [.helpHidden], helpText: "Print commands to explicitly build module dependencies")
 
   public static var extraOptions: [Option] {
     return [
       Option.driverPrintGraphviz,
-      Option.driverPrebuildModuleDependencies,
+      Option.driverExplicitModuleBuild,
       Option.driverPrintModuleDependenciesJobs
     ]
   }
diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift
@@ -15,11 +15,73 @@ import Foundation
 import TSCBasic
 import XCTest
 
+/// Check that an explicit module build job contains expected inputs and options
+private func checkExplicitModuleBuildJob(job: Job,
+                                         moduleName: String,
+                                         moduleKind: ModuleDependencyId.CodingKeys,
+                                         moduleDependencyGraph: InterModuleDependencyGraph) throws {
+  let moduleId = ModuleDependencyId(name: moduleName, kind: moduleKind)
+  let moduleInfo = moduleDependencyGraph.modules[moduleId]!
+  switch moduleInfo.details {
+    case .swift(let swiftModuleDetails):
+      let moduleInterfacePath =
+        TypedVirtualPath(file: try VirtualPath(path: swiftModuleDetails.moduleInterfacePath!),
+                         type: .swiftInterface)
+      XCTAssertEqual(job.kind, .emitModule)
+      XCTAssertTrue(job.inputs.contains(moduleInterfacePath))
+    case .clang(let clangModuleDetails):
+      let moduleMapPath =
+        TypedVirtualPath(file: try VirtualPath(path: clangModuleDetails.moduleMapPath),
+                         type: .clangModuleMap)
+      XCTAssertEqual(job.kind, .generatePCM)
+      XCTAssertTrue(job.inputs.contains(moduleMapPath))
+  }
+  try checkExplicitModuleBuildJobDependencies(job: job, moduleInfo: moduleInfo,
+                                              moduleDependencyGraph: moduleDependencyGraph)
+
+}
+
+/// Checks that the build job for the specified module contains the required options and inputs
+/// to build all of its dependencies explicitly
+private func checkExplicitModuleBuildJobDependencies(job: Job,
+                                                     moduleInfo : ModuleInfo,
+                                                     moduleDependencyGraph: InterModuleDependencyGraph)
+throws {
+  XCTAssertTrue(job.commandLine.contains(.flag(String("-disable-implicit-swift-modules"))))
+  XCTAssertTrue(job.commandLine.contains(.flag(String("-disable-implicit-pcms"))))
+  for dependencyId in moduleInfo.directDependencies {
+    let dependencyInfo = moduleDependencyGraph.modules[dependencyId]!
+    switch dependencyInfo.details {
+      case .swift:
+        let swiftDependencyModulePath =
+          TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath),
+                           type: .swiftModule)
+        XCTAssertTrue(job.inputs.contains(swiftDependencyModulePath))
+        XCTAssertTrue(job.commandLine.contains(
+                        .flag(String("-swift-module-file=\(moduleInfo.modulePath)"))))
+      case .clang(let clangDependencyDetails):
+        let clangDependencyModulePath =
+          TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath),
+                           type: .pcm)
+        let clangDependencyModuleMapPath =
+          TypedVirtualPath(file: try VirtualPath(path: clangDependencyDetails.moduleMapPath),
+                           type: .pcm)
+        XCTAssertTrue(job.inputs.contains(clangDependencyModulePath))
+        XCTAssertTrue(job.inputs.contains(clangDependencyModuleMapPath))
+        XCTAssertTrue(job.commandLine.contains(
+                        .flag(String("-clang-module-file=\(moduleInfo.modulePath)"))))
+        XCTAssertTrue(job.commandLine.contains(
+                        .flag(String("-clang-module-map-file=\(clangDependencyDetails.moduleMapPath)"))))
+    }
+  }
+}
+
 /// Test that for the given JSON module dependency graph, valid jobs are generated
 final class ExplicitModuleBuildTests: XCTestCase {
   func testModuleDependencyBuildCommandGeneration() throws {
     do {
-      var driver = try Driver(args: ["swiftc", "-driver-print-module-dependencies-jobs", "test.swift"])
+      var driver = try Driver(args: ["swiftc", "-driver-print-module-dependencies-jobs",
+                                     "test.swift"])
       let moduleDependencyGraph =
             try JSONDecoder().decode(
               InterModuleDependencyGraph.self,
@@ -31,21 +93,21 @@ final class ExplicitModuleBuildTests: XCTestCase {
         XCTAssertEqual(job.outputs.count, 1)
         switch (job.outputs[0].file) {
           case .relative(RelativePath("SwiftShims.pcm")):
-            XCTAssertEqual(job.kind, .generatePCM)
-            XCTAssertEqual(job.inputs.count, 1)
-            XCTAssertTrue(job.inputs[0].file.absolutePath!.pathString.contains("swift/shims/module.modulemap"))
+            try checkExplicitModuleBuildJob(job: job, moduleName: "SwiftShims",
+                                            moduleKind: ModuleDependencyId.CodingKeys.clang,
+                                            moduleDependencyGraph: moduleDependencyGraph)
           case .relative(RelativePath("c_simd.pcm")):
-            XCTAssertEqual(job.kind, .generatePCM)
-            XCTAssertEqual(job.inputs.count, 1)
-            XCTAssertTrue(job.inputs[0].file.absolutePath!.pathString.contains("clang-importer-sdk/usr/include/module.map"))
+            try checkExplicitModuleBuildJob(job: job, moduleName: "c_simd",
+                                            moduleKind: ModuleDependencyId.CodingKeys.clang,
+                                            moduleDependencyGraph: moduleDependencyGraph)
           case .relative(RelativePath("Swift.swiftmodule")):
-            XCTAssertEqual(job.kind, .emitModule)
-            XCTAssertEqual(job.inputs.count, 1)
-            XCTAssertTrue(job.inputs[0].file.absolutePath!.pathString.contains("Swift.swiftmodule/x86_64-apple-macos.swiftinterface"))
+            try checkExplicitModuleBuildJob(job: job, moduleName: "Swift",
+                                            moduleKind: ModuleDependencyId.CodingKeys.swift,
+                                            moduleDependencyGraph: moduleDependencyGraph)
           case .relative(RelativePath("SwiftOnoneSupport.swiftmodule")):
-            XCTAssertEqual(job.kind, .emitModule)
-            XCTAssertEqual(job.inputs.count, 1)
-            XCTAssertTrue(job.inputs[0].file.absolutePath!.pathString.contains("SwiftOnoneSupport.swiftmodule/x86_64-apple-macos.swiftinterface"))
+            try checkExplicitModuleBuildJob(job: job, moduleName: "SwiftOnoneSupport",
+                                            moduleKind: ModuleDependencyId.CodingKeys.swift,
+                                            moduleDependencyGraph: moduleDependencyGraph)
           default:
             XCTFail("Unexpected module dependency build job output")
         }

Original file line number	Diff line number	Diff line change
`@@ -63,6 +63,11 @@ extension Driver {`
`63`	`63`	`}`
`64`	`64`	`}`
`65`	`65`
	`66`	`+ // Precompile module dependencies, if asked.`
	`67`	`+ if parsedOptions.contains(.driverExplicitModuleBuild) {`
	`68`	`+ jobs.append(contentsOf: try generateExplicitModuleBuildJobs())`
	`69`	`+ }`
	`70`	`+`
`66`	`71`	`// Precompile the bridging header if needed.`
`67`	`72`	`if let importedObjCHeader = importedObjCHeader,`
`68`	`73`	`let bridgingPrecompiledHeader = bridgingPrecompiledHeader {`