From 65738b50d10639c92cc2dfe711f8e132b4e9cac8 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Tue, 26 May 2020 12:47:56 -0700 Subject: [PATCH 01/10] Add an end-to-end test for the option to print compile jobs for module dependencies discovered by the fast dependency scanner. --- Sources/SwiftDriver/Jobs/Planning.swift | 30 ++++++----- TestInputs/ExplicitModuleBuilds/CHeaders/A.h | 1 + TestInputs/ExplicitModuleBuilds/CHeaders/B.h | 4 ++ .../ExplicitModuleBuilds/CHeaders/Bridging.h | 3 ++ .../CHeaders/BridgingOther.h | 3 ++ TestInputs/ExplicitModuleBuilds/CHeaders/C.h | 3 ++ TestInputs/ExplicitModuleBuilds/CHeaders/D.h | 1 + TestInputs/ExplicitModuleBuilds/CHeaders/F.h | 1 + TestInputs/ExplicitModuleBuilds/CHeaders/G.h | 1 + .../CHeaders/module.modulemap | 29 +++++++++++ .../Swift/A.swiftinterface | 5 ++ .../Swift/E.swiftinterface | 4 ++ .../Swift/F.swiftinterface | 5 ++ .../Swift/G.swiftinterface | 9 ++++ .../ExplicitModuleBuildTests.swift | 51 +++++++++++++++++++ 15 files changed, 138 insertions(+), 12 deletions(-) create mode 100644 TestInputs/ExplicitModuleBuilds/CHeaders/A.h create mode 100644 TestInputs/ExplicitModuleBuilds/CHeaders/B.h create mode 100644 TestInputs/ExplicitModuleBuilds/CHeaders/Bridging.h create mode 100644 TestInputs/ExplicitModuleBuilds/CHeaders/BridgingOther.h create mode 100644 TestInputs/ExplicitModuleBuilds/CHeaders/C.h create mode 100644 TestInputs/ExplicitModuleBuilds/CHeaders/D.h create mode 100644 TestInputs/ExplicitModuleBuilds/CHeaders/F.h create mode 100644 TestInputs/ExplicitModuleBuilds/CHeaders/G.h create mode 100644 TestInputs/ExplicitModuleBuilds/CHeaders/module.modulemap create mode 100644 TestInputs/ExplicitModuleBuilds/Swift/A.swiftinterface create mode 100644 TestInputs/ExplicitModuleBuilds/Swift/E.swiftinterface create mode 100644 TestInputs/ExplicitModuleBuilds/Swift/F.swiftinterface create mode 100644 TestInputs/ExplicitModuleBuilds/Swift/G.swiftinterface diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index 4b2f294e1..6c10adf46 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -34,20 +34,14 @@ extension Driver { private mutating func planStandardCompile() throws -> [Job] { var jobs = [Job]() - // If we've been asked to prebuild module dependencies, prescan the source - // files to produce a module dependency graph and turn it into a set - // of jobs required to build all dependencies. - // For the time being, just prints the jobs' compile commands. + // If we've been asked to prebuild module dependencies, + // for the time being, just print the jobs' compile commands. if parsedOptions.contains(.driverPrintModuleDependenciesJobs) { - let moduleDependencyGraph = try computeModuleDependencyGraph() + let modulePrebuildJobs = try generateExplicitModuleBuildJobs() let forceResponseFiles = parsedOptions.contains(.driverForceResponseFiles) - if let dependencyGraph = moduleDependencyGraph { - let modulePrebuildJobs = - try planExplicitModuleDependenciesCompile(dependencyGraph: dependencyGraph) - for job in modulePrebuildJobs { - try Self.printJob(job, resolver: try ArgsResolver(), - forceResponseFiles: forceResponseFiles) - } + for job in modulePrebuildJobs { + try Self.printJob(job, resolver: try ArgsResolver(), + forceResponseFiles: forceResponseFiles) } } @@ -216,6 +210,18 @@ extension Driver { return jobs } + /// Prescan the source files to produce a module dependency graph and turn it into a set + /// of jobs required to build all dependencies. + public mutating func generateExplicitModuleBuildJobs() throws -> [Job] { + let moduleDependencyGraph = try computeModuleDependencyGraph() + if let dependencyGraph = moduleDependencyGraph { + let modulePrebuildJobs = + try planExplicitModuleDependenciesCompile(dependencyGraph: dependencyGraph) + return modulePrebuildJobs + } + return [] + } + /// Create a job if needed for simple requests that can be immediately /// forwarded to the frontend. public mutating func immediateForwardingJob() throws -> Job? { diff --git a/TestInputs/ExplicitModuleBuilds/CHeaders/A.h b/TestInputs/ExplicitModuleBuilds/CHeaders/A.h new file mode 100644 index 000000000..ea06ce27a --- /dev/null +++ b/TestInputs/ExplicitModuleBuilds/CHeaders/A.h @@ -0,0 +1 @@ +void funcA(void); diff --git a/TestInputs/ExplicitModuleBuilds/CHeaders/B.h b/TestInputs/ExplicitModuleBuilds/CHeaders/B.h new file mode 100644 index 000000000..dedb1b552 --- /dev/null +++ b/TestInputs/ExplicitModuleBuilds/CHeaders/B.h @@ -0,0 +1,4 @@ +#include + +void funcB(void); + diff --git a/TestInputs/ExplicitModuleBuilds/CHeaders/Bridging.h b/TestInputs/ExplicitModuleBuilds/CHeaders/Bridging.h new file mode 100644 index 000000000..c0261c5d9 --- /dev/null +++ b/TestInputs/ExplicitModuleBuilds/CHeaders/Bridging.h @@ -0,0 +1,3 @@ +#include "BridgingOther.h" + +int bridging_other(void); diff --git a/TestInputs/ExplicitModuleBuilds/CHeaders/BridgingOther.h b/TestInputs/ExplicitModuleBuilds/CHeaders/BridgingOther.h new file mode 100644 index 000000000..ee79a8f5b --- /dev/null +++ b/TestInputs/ExplicitModuleBuilds/CHeaders/BridgingOther.h @@ -0,0 +1,3 @@ +#include "F.h" + +int bridging_other(void); diff --git a/TestInputs/ExplicitModuleBuilds/CHeaders/C.h b/TestInputs/ExplicitModuleBuilds/CHeaders/C.h new file mode 100644 index 000000000..8a5a9b119 --- /dev/null +++ b/TestInputs/ExplicitModuleBuilds/CHeaders/C.h @@ -0,0 +1,3 @@ +#include + +void funcC(void); diff --git a/TestInputs/ExplicitModuleBuilds/CHeaders/D.h b/TestInputs/ExplicitModuleBuilds/CHeaders/D.h new file mode 100644 index 000000000..2b6ccbb76 --- /dev/null +++ b/TestInputs/ExplicitModuleBuilds/CHeaders/D.h @@ -0,0 +1 @@ +void funcD(void); diff --git a/TestInputs/ExplicitModuleBuilds/CHeaders/F.h b/TestInputs/ExplicitModuleBuilds/CHeaders/F.h new file mode 100644 index 000000000..c004f15ca --- /dev/null +++ b/TestInputs/ExplicitModuleBuilds/CHeaders/F.h @@ -0,0 +1 @@ +void funcF(void); diff --git a/TestInputs/ExplicitModuleBuilds/CHeaders/G.h b/TestInputs/ExplicitModuleBuilds/CHeaders/G.h new file mode 100644 index 000000000..ef09c49a9 --- /dev/null +++ b/TestInputs/ExplicitModuleBuilds/CHeaders/G.h @@ -0,0 +1 @@ +void funcG(void); diff --git a/TestInputs/ExplicitModuleBuilds/CHeaders/module.modulemap b/TestInputs/ExplicitModuleBuilds/CHeaders/module.modulemap new file mode 100644 index 000000000..dbd29ce9d --- /dev/null +++ b/TestInputs/ExplicitModuleBuilds/CHeaders/module.modulemap @@ -0,0 +1,29 @@ +module A { + header "A.h" + export * +} + +module B { + header "B.h" + export * +} + +module C { + header "C.h" + export * +} + +module D { + header "D.h" + export * +} + +module F { + header "F.h" + export * +} + +module G { + header "G.h" + export * +} diff --git a/TestInputs/ExplicitModuleBuilds/Swift/A.swiftinterface b/TestInputs/ExplicitModuleBuilds/Swift/A.swiftinterface new file mode 100644 index 000000000..1b6292940 --- /dev/null +++ b/TestInputs/ExplicitModuleBuilds/Swift/A.swiftinterface @@ -0,0 +1,5 @@ +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name A +@_exported import A +public func overlayFuncA() { } + diff --git a/TestInputs/ExplicitModuleBuilds/Swift/E.swiftinterface b/TestInputs/ExplicitModuleBuilds/Swift/E.swiftinterface new file mode 100644 index 000000000..824fe883f --- /dev/null +++ b/TestInputs/ExplicitModuleBuilds/Swift/E.swiftinterface @@ -0,0 +1,4 @@ +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name E +import Swift +public func funcE() { } \ No newline at end of file diff --git a/TestInputs/ExplicitModuleBuilds/Swift/F.swiftinterface b/TestInputs/ExplicitModuleBuilds/Swift/F.swiftinterface new file mode 100644 index 000000000..380d87930 --- /dev/null +++ b/TestInputs/ExplicitModuleBuilds/Swift/F.swiftinterface @@ -0,0 +1,5 @@ +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name F +import Swift +@_exported import F +public func funcF() { } \ No newline at end of file diff --git a/TestInputs/ExplicitModuleBuilds/Swift/G.swiftinterface b/TestInputs/ExplicitModuleBuilds/Swift/G.swiftinterface new file mode 100644 index 000000000..ae9d32b2c --- /dev/null +++ b/TestInputs/ExplicitModuleBuilds/Swift/G.swiftinterface @@ -0,0 +1,9 @@ +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name G -swift-version 5 + +#if swift(>=5.0) + +@_exported import G +public func overlayFuncG() { } + +#endif diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index 987bdbc10..2fcb2d373 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -53,5 +53,56 @@ final class ExplicitModuleBuildTests: XCTestCase { } } + /// Test generation of explicit module build jobs for dependency modules when the driver + /// is invoked with -driver-print-module-dependencies-jobs + func testModuleDependencyBuildEndToEnd() throws { + try withTemporaryDirectory { path in + let main = path.appending(component: "main.swift") + try localFileSystem.writeFileContents(main) { + $0 <<< "import C;" + $0 <<< "import E;" + $0 <<< "import G;" + } + let packageRootPath = URL(fileURLWithPath: #file).pathComponents + .prefix(while: { $0 != "Tests" }).joined(separator: "/").dropFirst() + let testInputsPath = packageRootPath + "/TestInputs" + let cHeadersPath : String = testInputsPath + "/ExplicitModuleBuilds/CHeaders" + let swiftModuleInterfacesPath : String = testInputsPath + "/ExplicitModuleBuilds/Swift" + var driver = try Driver(args: ["swift", + "-I", cHeadersPath, + "-I", swiftModuleInterfacesPath, + "-driver-print-module-dependencies-jobs", + main.pathString]) + let jobs = try driver.generateExplicitModuleBuildJobs() + XCTAssertEqual(jobs.count, 10) + for job in jobs { + XCTAssertEqual(job.outputs.count, 1) + switch (job.outputs[0].file) { + case .relative(RelativePath("A.swiftmodule")): + XCTAssertEqual(job.kind, .emitModule) + case .relative(RelativePath("E.swiftmodule")): + XCTAssertEqual(job.kind, .emitModule) + case .relative(RelativePath("G.swiftmodule")): + XCTAssertEqual(job.kind, .emitModule) + case .relative(RelativePath("A.pcm")): + XCTAssertEqual(job.kind, .generatePCM) + case .relative(RelativePath("B.pcm")): + XCTAssertEqual(job.kind, .generatePCM) + case .relative(RelativePath("C.pcm")): + XCTAssertEqual(job.kind, .generatePCM) + case .relative(RelativePath("G.pcm")): + XCTAssertEqual(job.kind, .generatePCM) + case .relative(RelativePath("Swift.swiftmodule")): + XCTAssertEqual(job.kind, .emitModule) + case .relative(RelativePath("SwiftOnoneSupport.swiftmodule")): + XCTAssertEqual(job.kind, .emitModule) + case .relative(RelativePath("SwiftShims.pcm")): + XCTAssertEqual(job.kind, .generatePCM) + default: + XCTFail("Unexpected module dependency build job output: \(job.outputs[0].file)") + } + } + } + } } From c8ff1e4d5856e4bb0131e0b00ced415bd4486024 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Thu, 28 May 2020 09:36:00 -0700 Subject: [PATCH 02/10] Add a more-informative Driver error on Fast Dependency Scanner failure. --- Sources/SwiftDriver/Driver/Driver.swift | 3 +++ .../SwiftDriver/Driver/ModuleDependencyScanning.swift | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index 8260004f2..dcbafbd5b 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -51,6 +51,7 @@ public struct Driver { case integratedReplRemoved case conflictingOptions(Option, Option) case malformedModuleDependency(String, String) + case dependencyScanningFailure(Int, String) public var description: String { switch self { @@ -71,6 +72,8 @@ public struct Driver { return "conflicting options '\(one.spelling)' and '\(two.spelling)'" case .malformedModuleDependency(let moduleName, let errorDescription): return "Malformed Module Dependency: \(moduleName), \(errorDescription)" + case .dependencyScanningFailure(let code, let error): + return "Module Dependency Scanner returned with non-zero exit status: \(code), \(error)" } } } diff --git a/Sources/SwiftDriver/Driver/ModuleDependencyScanning.swift b/Sources/SwiftDriver/Driver/ModuleDependencyScanning.swift index 97b5a36f6..340ab2d74 100644 --- a/Sources/SwiftDriver/Driver/ModuleDependencyScanning.swift +++ b/Sources/SwiftDriver/Driver/ModuleDependencyScanning.swift @@ -41,6 +41,17 @@ extension Driver { let arguments = [tool] + (try commandLine.map { try resolver.resolve($0) }) let scanProcess = try Process.launchProcess(arguments: arguments, env: env) let result = try scanProcess.waitUntilExit() + // Error on dependency scanning failure + if (result.exitStatus != .terminated(code: 0)) { + var returnCode = 0 + switch result.exitStatus { + case .terminated(let code): + returnCode = Int(code) + case .signalled(let signal): + returnCode = Int(signal) + } + throw Error.dependencyScanningFailure(returnCode, try result.utf8stderrOutput()) + } guard let outputData = try? Data(result.utf8Output().utf8) else { return nil } From 1ba27a7c9a2f6f0fb8ffcd64d1e65a683bebea39 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Thu, 28 May 2020 13:10:23 -0700 Subject: [PATCH 03/10] Specify module dependencies explicitly when pre-building module dependencies. --- .../InterModuleDependencyGraph.swift | 19 ++++ .../ModuleDependencyBuildGeneration.swift | 69 +++++++++++++-- Sources/SwiftDriver/Driver/Driver.swift | 3 + Sources/SwiftDriver/Jobs/Planning.swift | 5 ++ Sources/SwiftOptions/ExtraOptions.swift | 4 +- .../ExplicitModuleBuildTests.swift | 88 ++++++++++++++++--- 6 files changed, 164 insertions(+), 24 deletions(-) diff --git a/Sources/SwiftDriver/Dependency Scanning/InterModuleDependencyGraph.swift b/Sources/SwiftDriver/Dependency Scanning/InterModuleDependencyGraph.swift index 10abd87d4..ef3349331 100644 --- a/Sources/SwiftDriver/Dependency Scanning/InterModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/Dependency Scanning/InterModuleDependencyGraph.swift @@ -50,6 +50,25 @@ extension ModuleDependencyId: Codable { try container.encode(moduleName, forKey: .clang) } } + + func getName() -> String { + switch self { + case .swift(let moduleName): + return moduleName + case .clang(let moduleName): + return moduleName + } + } + + // Used in testing + init(name: String, kind: CodingKeys) { + switch kind { + case .swift: + self = .swift(name) + case .clang: + self = .clang(name) + } + } } /// Details specific to Swift modules. diff --git a/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift b/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift index f6be13683..abba3f96a 100644 --- a/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift +++ b/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift @@ -26,14 +26,17 @@ extension Driver { } switch id { case .swift(let moduleName): - let swiftModuleBuildJob = try genSwiftModuleDependencyBuildJob(moduleInfo: moduleInfo, - moduleName: moduleName) + let swiftModuleBuildJob = + try genSwiftModuleDependencyBuildJob(moduleInfo: moduleInfo, + moduleName: moduleName, + dependencyGraph: dependencyGraph) jobs.append(swiftModuleBuildJob) case .clang(let moduleName): - let clangModuleBuildJob = try genClangModuleDependencyBuildJob(moduleInfo: moduleInfo, - moduleName: moduleName) + let clangModuleBuildJob = + try genClangModuleDependencyBuildJob(moduleInfo: moduleInfo, + moduleName: moduleName, + dependencyGraph: dependencyGraph) jobs.append(clangModuleBuildJob) - } } return jobs @@ -41,7 +44,9 @@ extension Driver { /// For a given swift module dependency, generate a build job mutating private func genSwiftModuleDependencyBuildJob(moduleInfo: ModuleInfo, - moduleName: String) throws -> Job { + moduleName: String, + dependencyGraph: InterModuleDependencyGraph) + throws -> Job { // FIXIT: Needs more error handling guard case .swift(let swiftModuleDetails) = moduleInfo.details else { throw Error.malformedModuleDependency(moduleName, "no `details` object") @@ -54,6 +59,11 @@ extension Driver { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } commandLine.appendFlag("-frontend") + try addModuleDependencies(moduleInfo: moduleInfo, + dependencyGraph: dependencyGraph, + inputs: &inputs, + commandLine: &commandLine) + // Build the .swiftinterfaces file using a list of command line options specified in the // `details` field. guard let moduleInterfacePath = swiftModuleDetails.moduleInterfacePath else { @@ -76,7 +86,9 @@ extension Driver { /// For a given clang module dependency, generate a build job mutating private func genClangModuleDependencyBuildJob(moduleInfo: ModuleInfo, - moduleName: String) throws -> Job { + moduleName: String, + dependencyGraph: InterModuleDependencyGraph) + throws -> Job { // For clang modules, the Fast Dependency Scanner emits a list of source // files (with a .modulemap among them), and a list of compile command // options. @@ -92,12 +104,18 @@ extension Driver { commandLine.appendFlag("-frontend") commandLine.appendFlags("-emit-pcm", "-module-name", moduleName) + try addModuleDependencies(moduleInfo: moduleInfo, + dependencyGraph: dependencyGraph, + inputs: &inputs, + commandLine: &commandLine) + // The only required input is the .modulemap for this module. - commandLine.append(Job.ArgTemplate.path(try VirtualPath(path: clangModuleDetails.moduleMapPath))) + commandLine.append(Job.ArgTemplate.path( + try VirtualPath(path: clangModuleDetails.moduleMapPath))) inputs.append(TypedVirtualPath(file: try VirtualPath(path: clangModuleDetails.moduleMapPath), type: .clangModuleMap)) try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs) - clangModuleDetails.commandLine?.forEach { commandLine.appendFlags("-Xcc", $0) } + clangModuleDetails.commandLine?.forEach { commandLine.appendFlags($0) } return Job( moduleName: moduleName, @@ -108,4 +126,37 @@ extension Driver { outputs: outputs ) } + + + /// For the specified module, update its command line flags and inputs + /// to use explicitly-built module dependencies. + private func addModuleDependencies(moduleInfo: ModuleInfo, + dependencyGraph: InterModuleDependencyGraph, + inputs: inout [TypedVirtualPath], + commandLine: inout [Job.ArgTemplate]) throws { + // These options ensure that the frontend only uses explicitly-specified module dependencies + // and the frontend errors if it has to perform any implicit module builds. + commandLine.appendFlags("-disable-implicit-swift-modules", "-disable-implicit-pcms") + for moduleId in moduleInfo.directDependencies { + guard let dependencyInfo = dependencyGraph.modules[moduleId] else { + throw Error.missingModuleDependency(moduleId.getName()) + } + switch dependencyInfo.details { + case .swift: + let swiftModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), + type: .swiftModule) + commandLine.appendFlag("-swift-module-file=\(swiftModulePath.file.description)") + inputs.append(swiftModulePath) + case .clang(let clangDependencyDetails): + let clangModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), + type: .pcm) + let clangModuleMapPath = TypedVirtualPath(file: try VirtualPath(path: clangDependencyDetails.moduleMapPath), + type: .pcm) + commandLine.appendFlag("-clang-module-file=\(clangModulePath.file.description)") + commandLine.appendFlag("-clang-module-map-file=\(clangModuleMapPath.file.description)") + inputs.append(clangModulePath) + inputs.append(clangModuleMapPath) + } + } + } } diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index dcbafbd5b..1eae16f5d 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -51,6 +51,7 @@ public struct Driver { case integratedReplRemoved case conflictingOptions(Option, Option) case malformedModuleDependency(String, String) + case missingModuleDependency(String) case dependencyScanningFailure(Int, String) public var description: String { @@ -72,6 +73,8 @@ public struct Driver { return "conflicting options '\(one.spelling)' and '\(two.spelling)'" case .malformedModuleDependency(let moduleName, let errorDescription): return "Malformed Module Dependency: \(moduleName), \(errorDescription)" + case .missingModuleDependency(let moduleName): + return "Missing Module Dependency Info: \(moduleName)" case .dependencyScanningFailure(let code, let error): return "Module Dependency Scanner returned with non-zero exit status: \(code), \(error)" } diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index 6c10adf46..46fbaa514 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -63,6 +63,11 @@ extension Driver { } } + // Precompile module dependencies, if asked. + if parsedOptions.contains(.driverExplicitModuleBuild) { + jobs.append(contentsOf: try generateExplicitModuleBuildJobs()) + } + // Precompile the bridging header if needed. if let importedObjCHeader = importedObjCHeader, let bridgingPrecompiledHeader = bridgingPrecompiledHeader { diff --git a/Sources/SwiftOptions/ExtraOptions.swift b/Sources/SwiftOptions/ExtraOptions.swift index e532c7f29..67e4ffa93 100644 --- a/Sources/SwiftOptions/ExtraOptions.swift +++ b/Sources/SwiftOptions/ExtraOptions.swift @@ -11,13 +11,13 @@ //===----------------------------------------------------------------------===// extension Option { public static let driverPrintGraphviz: Option = Option("-driver-print-graphviz", .flag, attributes: [.helpHidden, .doesNotAffectIncrementalBuild], helpText: "Write the job graph as a graphviz file", group: .internalDebug) - public static let driverPrebuildModuleDependencies: Option = Option("-driver-prebuild-module-dependencies", .flag, attributes: [.helpHidden], helpText: "Prebuild module dependencies to make them explicit") + public static let driverExplicitModuleBuild: Option = Option("-experimental-explicit-module-build", .flag, attributes: [.helpHidden], helpText: "Prebuild module dependencies to make them explicit") public static let driverPrintModuleDependenciesJobs: Option = Option("-driver-print-module-dependencies-jobs", .flag, attributes: [.helpHidden], helpText: "Print commands to explicitly build module dependencies") public static var extraOptions: [Option] { return [ Option.driverPrintGraphviz, - Option.driverPrebuildModuleDependencies, + Option.driverExplicitModuleBuild, Option.driverPrintModuleDependenciesJobs ] } diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index 2fcb2d373..3538a4b75 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -15,11 +15,73 @@ import Foundation import TSCBasic import XCTest +/// Check that an explicit module build job contains expected inputs and options +private func checkExplicitModuleBuildJob(job: Job, + moduleName: String, + moduleKind: ModuleDependencyId.CodingKeys, + moduleDependencyGraph: InterModuleDependencyGraph) throws { + let moduleId = ModuleDependencyId(name: moduleName, kind: moduleKind) + let moduleInfo = moduleDependencyGraph.modules[moduleId]! + switch moduleInfo.details { + case .swift(let swiftModuleDetails): + let moduleInterfacePath = + TypedVirtualPath(file: try VirtualPath(path: swiftModuleDetails.moduleInterfacePath!), + type: .swiftInterface) + XCTAssertEqual(job.kind, .emitModule) + XCTAssertTrue(job.inputs.contains(moduleInterfacePath)) + case .clang(let clangModuleDetails): + let moduleMapPath = + TypedVirtualPath(file: try VirtualPath(path: clangModuleDetails.moduleMapPath), + type: .clangModuleMap) + XCTAssertEqual(job.kind, .generatePCM) + XCTAssertTrue(job.inputs.contains(moduleMapPath)) + } + try checkExplicitModuleBuildJobDependencies(job: job, moduleInfo: moduleInfo, + moduleDependencyGraph: moduleDependencyGraph) + +} + +/// Checks that the build job for the specified module contains the required options and inputs +/// to build all of its dependencies explicitly +private func checkExplicitModuleBuildJobDependencies(job: Job, + moduleInfo : ModuleInfo, + moduleDependencyGraph: InterModuleDependencyGraph) +throws { + XCTAssertTrue(job.commandLine.contains(.flag(String("-disable-implicit-swift-modules")))) + XCTAssertTrue(job.commandLine.contains(.flag(String("-disable-implicit-pcms")))) + for dependencyId in moduleInfo.directDependencies { + let dependencyInfo = moduleDependencyGraph.modules[dependencyId]! + switch dependencyInfo.details { + case .swift: + let swiftDependencyModulePath = + TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), + type: .swiftModule) + XCTAssertTrue(job.inputs.contains(swiftDependencyModulePath)) + XCTAssertTrue(job.commandLine.contains( + .flag(String("-swift-module-file=\(moduleInfo.modulePath)")))) + case .clang(let clangDependencyDetails): + let clangDependencyModulePath = + TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), + type: .pcm) + let clangDependencyModuleMapPath = + TypedVirtualPath(file: try VirtualPath(path: clangDependencyDetails.moduleMapPath), + type: .pcm) + XCTAssertTrue(job.inputs.contains(clangDependencyModulePath)) + XCTAssertTrue(job.inputs.contains(clangDependencyModuleMapPath)) + XCTAssertTrue(job.commandLine.contains( + .flag(String("-clang-module-file=\(moduleInfo.modulePath)")))) + XCTAssertTrue(job.commandLine.contains( + .flag(String("-clang-module-map-file=\(clangDependencyDetails.moduleMapPath)")))) + } + } +} + /// Test that for the given JSON module dependency graph, valid jobs are generated final class ExplicitModuleBuildTests: XCTestCase { func testModuleDependencyBuildCommandGeneration() throws { do { - var driver = try Driver(args: ["swiftc", "-driver-print-module-dependencies-jobs", "test.swift"]) + var driver = try Driver(args: ["swiftc", "-driver-print-module-dependencies-jobs", + "test.swift"]) let moduleDependencyGraph = try JSONDecoder().decode( InterModuleDependencyGraph.self, @@ -31,21 +93,21 @@ final class ExplicitModuleBuildTests: XCTestCase { XCTAssertEqual(job.outputs.count, 1) switch (job.outputs[0].file) { case .relative(RelativePath("SwiftShims.pcm")): - XCTAssertEqual(job.kind, .generatePCM) - XCTAssertEqual(job.inputs.count, 1) - XCTAssertTrue(job.inputs[0].file.absolutePath!.pathString.contains("swift/shims/module.modulemap")) + try checkExplicitModuleBuildJob(job: job, moduleName: "SwiftShims", + moduleKind: ModuleDependencyId.CodingKeys.clang, + moduleDependencyGraph: moduleDependencyGraph) case .relative(RelativePath("c_simd.pcm")): - XCTAssertEqual(job.kind, .generatePCM) - XCTAssertEqual(job.inputs.count, 1) - XCTAssertTrue(job.inputs[0].file.absolutePath!.pathString.contains("clang-importer-sdk/usr/include/module.map")) + try checkExplicitModuleBuildJob(job: job, moduleName: "c_simd", + moduleKind: ModuleDependencyId.CodingKeys.clang, + moduleDependencyGraph: moduleDependencyGraph) case .relative(RelativePath("Swift.swiftmodule")): - XCTAssertEqual(job.kind, .emitModule) - XCTAssertEqual(job.inputs.count, 1) - XCTAssertTrue(job.inputs[0].file.absolutePath!.pathString.contains("Swift.swiftmodule/x86_64-apple-macos.swiftinterface")) + try checkExplicitModuleBuildJob(job: job, moduleName: "Swift", + moduleKind: ModuleDependencyId.CodingKeys.swift, + moduleDependencyGraph: moduleDependencyGraph) case .relative(RelativePath("SwiftOnoneSupport.swiftmodule")): - XCTAssertEqual(job.kind, .emitModule) - XCTAssertEqual(job.inputs.count, 1) - XCTAssertTrue(job.inputs[0].file.absolutePath!.pathString.contains("SwiftOnoneSupport.swiftmodule/x86_64-apple-macos.swiftinterface")) + try checkExplicitModuleBuildJob(job: job, moduleName: "SwiftOnoneSupport", + moduleKind: ModuleDependencyId.CodingKeys.swift, + moduleDependencyGraph: moduleDependencyGraph) default: XCTFail("Unexpected module dependency build job output") } From 733b083d3a787ec5321b9df3129254b542e2c2d4 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Fri, 29 May 2020 15:58:15 -0700 Subject: [PATCH 04/10] Propagate explicit module build commands and arguments to compile jobs --- .../ModuleDependencyBuildGeneration.swift | 42 +++++------ Sources/SwiftDriver/Driver/Driver.swift | 7 ++ Sources/SwiftDriver/Jobs/CompileJob.swift | 4 ++ .../SwiftDriver/Jobs/FrontendJobHelpers.swift | 45 ++++++++++++ Sources/SwiftDriver/Jobs/Planning.swift | 42 +++++------ .../ExplicitModuleBuildTests.swift | 69 +++++++++++++------ 6 files changed, 145 insertions(+), 64 deletions(-) diff --git a/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift b/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift index abba3f96a..b79a70ed4 100644 --- a/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift +++ b/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift @@ -47,7 +47,6 @@ extension Driver { moduleName: String, dependencyGraph: InterModuleDependencyGraph) throws -> Job { - // FIXIT: Needs more error handling guard case .swift(let swiftModuleDetails) = moduleInfo.details else { throw Error.malformedModuleDependency(moduleName, "no `details` object") } @@ -57,7 +56,13 @@ extension Driver { TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), type: .swiftModule) ] var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } - commandLine.appendFlag("-frontend") + // First, take the command line options provided in the dependency information + swiftModuleDetails.commandLine?.forEach { commandLine.appendFlags($0) } + + if (swiftModuleDetails.commandLine == nil || + !swiftModuleDetails.commandLine!.contains("-frontend")) { + commandLine.appendFlag("-frontend") + } try addModuleDependencies(moduleInfo: moduleInfo, dependencyGraph: dependencyGraph, @@ -72,7 +77,6 @@ extension Driver { inputs.append(TypedVirtualPath(file: try VirtualPath(path: moduleInterfacePath), type: .swiftInterface)) try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs) - swiftModuleDetails.commandLine?.forEach { commandLine.appendFlag($0) } return Job( moduleName: moduleName, @@ -101,7 +105,14 @@ extension Driver { TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), type: .pcm) ] var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } - commandLine.appendFlag("-frontend") + + // First, take the command line options provided in the dependency information + clangModuleDetails.commandLine?.forEach { commandLine.appendFlags($0) } + + if (clangModuleDetails.commandLine == nil || + !clangModuleDetails.commandLine!.contains("-frontend")) { + commandLine.appendFlag("-frontend") + } commandLine.appendFlags("-emit-pcm", "-module-name", moduleName) try addModuleDependencies(moduleInfo: moduleInfo, @@ -115,7 +126,6 @@ extension Driver { inputs.append(TypedVirtualPath(file: try VirtualPath(path: clangModuleDetails.moduleMapPath), type: .clangModuleMap)) try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs) - clangModuleDetails.commandLine?.forEach { commandLine.appendFlags($0) } return Job( moduleName: moduleName, @@ -134,29 +144,15 @@ extension Driver { dependencyGraph: InterModuleDependencyGraph, inputs: inout [TypedVirtualPath], commandLine: inout [Job.ArgTemplate]) throws { - // These options ensure that the frontend only uses explicitly-specified module dependencies - // and the frontend errors if it has to perform any implicit module builds. + // Prohibit the frontend from implicitly building textual modules into binary modules. commandLine.appendFlags("-disable-implicit-swift-modules", "-disable-implicit-pcms") for moduleId in moduleInfo.directDependencies { guard let dependencyInfo = dependencyGraph.modules[moduleId] else { throw Error.missingModuleDependency(moduleId.getName()) } - switch dependencyInfo.details { - case .swift: - let swiftModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), - type: .swiftModule) - commandLine.appendFlag("-swift-module-file=\(swiftModulePath.file.description)") - inputs.append(swiftModulePath) - case .clang(let clangDependencyDetails): - let clangModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), - type: .pcm) - let clangModuleMapPath = TypedVirtualPath(file: try VirtualPath(path: clangDependencyDetails.moduleMapPath), - type: .pcm) - commandLine.appendFlag("-clang-module-file=\(clangModulePath.file.description)") - commandLine.appendFlag("-clang-module-map-file=\(clangModuleMapPath.file.description)") - inputs.append(clangModulePath) - inputs.append(clangModuleMapPath) - } + + try addModuleAsExplicitDependency(moduleInfo: dependencyInfo, commandLine: &commandLine, + inputs: &inputs) } } } diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index 1eae16f5d..c7c21ed07 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -50,6 +50,7 @@ public struct Driver { case subcommandPassedToDriver case integratedReplRemoved case conflictingOptions(Option, Option) + // Explicit Module Build Failures case malformedModuleDependency(String, String) case missingModuleDependency(String) case dependencyScanningFailure(Int, String) @@ -71,6 +72,7 @@ public struct Driver { return "Compiler-internal integrated REPL has been removed; use the LLDB-enhanced REPL instead." case .conflictingOptions(let one, let two): return "conflicting options '\(one.spelling)' and '\(two.spelling)'" + // Explicit Module Build Failures case .malformedModuleDependency(let moduleName, let errorDescription): return "Malformed Module Dependency: \(moduleName), \(errorDescription)" case .missingModuleDependency(let moduleName): @@ -206,6 +208,11 @@ public struct Driver { /// This will force the driver to first emit the module and then run compile jobs. public var forceEmitModuleInSingleInvocation: Bool = false + /// The module dependency graph, which is populated during the planning phase + /// only when all modules will be prebuilt and treated as explicit by the + /// various compilation jobs. + var interModuleDependencyGraph: InterModuleDependencyGraph? = nil + /// Handler for emitting diagnostics to stderr. public static let stderrDiagnosticsHandler: DiagnosticsEngine.DiagnosticsHandler = { diagnostic in let stream = stderrStream diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift index d3935f1f1..3106c3bc8 100644 --- a/Sources/SwiftDriver/Jobs/CompileJob.swift +++ b/Sources/SwiftDriver/Jobs/CompileJob.swift @@ -155,6 +155,10 @@ extension Driver { try addCommonFrontendOptions(commandLine: &commandLine) // FIXME: MSVC runtime flags + if parsedOptions.contains(.driverExplicitModuleBuild) { + try addExplicitModuleBuildArguments(commandLine: &commandLine, inputs: &inputs) + } + if parsedOptions.hasArgument(.parseAsLibrary, .emitLibrary) { commandLine.appendFlag(.parseAsLibrary) } diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index 1e68023c3..61a501c5b 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -304,4 +304,49 @@ extension Driver { return outputs } + + /// Adds the specified module as an explicit module dependency to given + /// inputs and comman line arguments of a compile job. + func addModuleAsExplicitDependency(moduleInfo: ModuleInfo, + commandLine: inout [Job.ArgTemplate], + inputs: inout [TypedVirtualPath]) throws { + switch moduleInfo.details { + case .swift: + let swiftModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), + type: .swiftModule) + commandLine.appendFlag("-swift-module-file=\(swiftModulePath.file.description)") + inputs.append(swiftModulePath) + case .clang(let clangDependencyDetails): + let clangModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), + type: .pcm) + let clangModuleMapPath = TypedVirtualPath(file: try VirtualPath(path: clangDependencyDetails.moduleMapPath), + type: .pcm) + commandLine.appendFlag("-clang-module-file=\(clangModulePath.file.description)") + commandLine.appendFlag("-clang-module-map-file=\(clangModuleMapPath.file.description)") + inputs.append(clangModulePath) + inputs.append(clangModuleMapPath) + } + } + + /// Adds all dependecies required for an explicit module build + /// to inputs and comman line arguments of a compile job. + func addExplicitModuleBuildArguments(commandLine: inout [Job.ArgTemplate], + inputs: inout [TypedVirtualPath]) throws { + guard let dependencyGraph = interModuleDependencyGraph else { + fatalError("Inter Module Dependency Graph does not exist in explicit module build mode.") + } + // Prohibit the frontend from implicitly building textual modules into binary modules. + commandLine.appendFlags("-disable-implicit-swift-modules", "-disable-implicit-pcms") + + // Provide the frontend with a list of explicitly pre-built modules. + for (moduleId, moduleInfo) in dependencyGraph.modules { + // Skip the main output module as it is not its own dependency + guard moduleId.getName() != dependencyGraph.mainModuleName else { + continue + } + + try addModuleAsExplicitDependency(moduleInfo: moduleInfo, commandLine: &commandLine, + inputs: &inputs) + } + } } diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index 46fbaa514..fa695ce19 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -34,17 +34,6 @@ extension Driver { private mutating func planStandardCompile() throws -> [Job] { var jobs = [Job]() - // If we've been asked to prebuild module dependencies, - // for the time being, just print the jobs' compile commands. - if parsedOptions.contains(.driverPrintModuleDependenciesJobs) { - let modulePrebuildJobs = try generateExplicitModuleBuildJobs() - let forceResponseFiles = parsedOptions.contains(.driverForceResponseFiles) - for job in modulePrebuildJobs { - try Self.printJob(job, resolver: try ArgsResolver(), - forceResponseFiles: forceResponseFiles) - } - } - // Keep track of the various outputs we care about from the jobs we build. var linkerInputs: [TypedVirtualPath] = [] var moduleInputs: [TypedVirtualPath] = [] @@ -63,9 +52,24 @@ extension Driver { } } - // Precompile module dependencies, if asked. - if parsedOptions.contains(.driverExplicitModuleBuild) { - jobs.append(contentsOf: try generateExplicitModuleBuildJobs()) + // If asked, add jobs to precompile module dependencies + if parsedOptions.contains(.driverExplicitModuleBuild) || + parsedOptions.contains(.driverPrintModuleDependenciesJobs) { + let modulePrebuildJobs = try generateExplicitModuleBuildJobs() + + if parsedOptions.contains(.driverExplicitModuleBuild) { + jobs.append(contentsOf: modulePrebuildJobs) + } + + // If we've been asked to prebuild module dependencies, + // for the time being, just print the jobs' compile commands. + if parsedOptions.contains(.driverPrintModuleDependenciesJobs) { + let forceResponseFiles = parsedOptions.contains(.driverForceResponseFiles) + for job in modulePrebuildJobs { + try Self.printJob(job, resolver: try ArgsResolver(), + forceResponseFiles: forceResponseFiles) + } + } } // Precompile the bridging header if needed. @@ -218,13 +222,11 @@ extension Driver { /// Prescan the source files to produce a module dependency graph and turn it into a set /// of jobs required to build all dependencies. public mutating func generateExplicitModuleBuildJobs() throws -> [Job] { - let moduleDependencyGraph = try computeModuleDependencyGraph() - if let dependencyGraph = moduleDependencyGraph { - let modulePrebuildJobs = - try planExplicitModuleDependenciesCompile(dependencyGraph: dependencyGraph) - return modulePrebuildJobs + interModuleDependencyGraph = try computeModuleDependencyGraph() + guard let dependencyGraph = interModuleDependencyGraph else { + fatalError("Attempting to perform Explicit Module Build job generation, but the Inter Module Dependency Graph does not exist.") } - return [] + return try planExplicitModuleDependenciesCompile(dependencyGraph: dependencyGraph) } /// Create a job if needed for simple requests that can be immediately diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index 3538a4b75..20cd19d4e 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -38,7 +38,6 @@ private func checkExplicitModuleBuildJob(job: Job, } try checkExplicitModuleBuildJobDependencies(job: job, moduleInfo: moduleInfo, moduleDependencyGraph: moduleDependencyGraph) - } /// Checks that the build job for the specified module contains the required options and inputs @@ -54,14 +53,14 @@ throws { switch dependencyInfo.details { case .swift: let swiftDependencyModulePath = - TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), + TypedVirtualPath(file: try VirtualPath(path: dependencyInfo.modulePath), type: .swiftModule) XCTAssertTrue(job.inputs.contains(swiftDependencyModulePath)) XCTAssertTrue(job.commandLine.contains( - .flag(String("-swift-module-file=\(moduleInfo.modulePath)")))) + .flag(String("-swift-module-file=\(dependencyInfo.modulePath)")))) case .clang(let clangDependencyDetails): let clangDependencyModulePath = - TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), + TypedVirtualPath(file: try VirtualPath(path: dependencyInfo.modulePath), type: .pcm) let clangDependencyModuleMapPath = TypedVirtualPath(file: try VirtualPath(path: clangDependencyDetails.moduleMapPath), @@ -69,7 +68,7 @@ throws { XCTAssertTrue(job.inputs.contains(clangDependencyModulePath)) XCTAssertTrue(job.inputs.contains(clangDependencyModuleMapPath)) XCTAssertTrue(job.commandLine.contains( - .flag(String("-clang-module-file=\(moduleInfo.modulePath)")))) + .flag(String("-clang-module-file=\(dependencyInfo.modulePath)")))) XCTAssertTrue(job.commandLine.contains( .flag(String("-clang-module-map-file=\(clangDependencyDetails.moduleMapPath)")))) } @@ -116,8 +115,8 @@ final class ExplicitModuleBuildTests: XCTestCase { } /// Test generation of explicit module build jobs for dependency modules when the driver - /// is invoked with -driver-print-module-dependencies-jobs - func testModuleDependencyBuildEndToEnd() throws { + /// is invoked with -experimental-explicit-module-build + func testExplicitModuleBuildJobs() throws { try withTemporaryDirectory { path in let main = path.appending(component: "main.swift") try localFileSystem.writeFileContents(main) { @@ -131,36 +130,64 @@ final class ExplicitModuleBuildTests: XCTestCase { let testInputsPath = packageRootPath + "/TestInputs" let cHeadersPath : String = testInputsPath + "/ExplicitModuleBuilds/CHeaders" let swiftModuleInterfacesPath : String = testInputsPath + "/ExplicitModuleBuilds/Swift" - var driver = try Driver(args: ["swift", + var driver = try Driver(args: ["swiftc", "-I", cHeadersPath, "-I", swiftModuleInterfacesPath, - "-driver-print-module-dependencies-jobs", + "-experimental-explicit-module-build", main.pathString]) - let jobs = try driver.generateExplicitModuleBuildJobs() - XCTAssertEqual(jobs.count, 10) + let jobs = try driver.planBuild() + XCTAssertTrue(driver.parsedOptions.contains(.driverExplicitModuleBuild)) + let dependencyGraph = driver.interModuleDependencyGraph! + XCTAssertEqual(jobs.count, 12) for job in jobs { XCTAssertEqual(job.outputs.count, 1) switch (job.outputs[0].file) { case .relative(RelativePath("A.swiftmodule")): - XCTAssertEqual(job.kind, .emitModule) + try checkExplicitModuleBuildJob(job: job, moduleName: "A", + moduleKind: ModuleDependencyId.CodingKeys.swift, + moduleDependencyGraph: dependencyGraph) case .relative(RelativePath("E.swiftmodule")): - XCTAssertEqual(job.kind, .emitModule) + try checkExplicitModuleBuildJob(job: job, moduleName: "E", + moduleKind: ModuleDependencyId.CodingKeys.swift, + moduleDependencyGraph: dependencyGraph) case .relative(RelativePath("G.swiftmodule")): - XCTAssertEqual(job.kind, .emitModule) + try checkExplicitModuleBuildJob(job: job, moduleName: "G", + moduleKind: ModuleDependencyId.CodingKeys.swift, + moduleDependencyGraph: dependencyGraph) case .relative(RelativePath("A.pcm")): - XCTAssertEqual(job.kind, .generatePCM) + try checkExplicitModuleBuildJob(job: job, moduleName: "A", + moduleKind: ModuleDependencyId.CodingKeys.clang, + moduleDependencyGraph: dependencyGraph) case .relative(RelativePath("B.pcm")): - XCTAssertEqual(job.kind, .generatePCM) + try checkExplicitModuleBuildJob(job: job, moduleName: "B", + moduleKind: ModuleDependencyId.CodingKeys.clang, + moduleDependencyGraph: dependencyGraph) case .relative(RelativePath("C.pcm")): - XCTAssertEqual(job.kind, .generatePCM) + try checkExplicitModuleBuildJob(job: job, moduleName: "C", + moduleKind: ModuleDependencyId.CodingKeys.clang, + moduleDependencyGraph: dependencyGraph) case .relative(RelativePath("G.pcm")): - XCTAssertEqual(job.kind, .generatePCM) + try checkExplicitModuleBuildJob(job: job, moduleName: "G", + moduleKind: ModuleDependencyId.CodingKeys.clang, + moduleDependencyGraph: dependencyGraph) case .relative(RelativePath("Swift.swiftmodule")): - XCTAssertEqual(job.kind, .emitModule) + try checkExplicitModuleBuildJob(job: job, moduleName: "Swift", + moduleKind: ModuleDependencyId.CodingKeys.swift, + moduleDependencyGraph: dependencyGraph) case .relative(RelativePath("SwiftOnoneSupport.swiftmodule")): - XCTAssertEqual(job.kind, .emitModule) + try checkExplicitModuleBuildJob(job: job, moduleName: "SwiftOnoneSupport", + moduleKind: ModuleDependencyId.CodingKeys.swift, + moduleDependencyGraph: dependencyGraph) case .relative(RelativePath("SwiftShims.pcm")): - XCTAssertEqual(job.kind, .generatePCM) + try checkExplicitModuleBuildJob(job: job, moduleName: "SwiftShims", + moduleKind: ModuleDependencyId.CodingKeys.clang, + moduleDependencyGraph: dependencyGraph) + case .temporary(RelativePath("main.o")): + try checkExplicitModuleBuildJobDependencies(job: job, + moduleInfo: dependencyGraph.mainModule, + moduleDependencyGraph: dependencyGraph) + case .relative(RelativePath("main")): + XCTAssertEqual(job.kind, .link) default: XCTFail("Unexpected module dependency build job output: \(job.outputs[0].file)") } From 360db0e628bf4d54f880684af4d3f4eb8cef14d1 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Mon, 1 Jun 2020 13:41:58 -0700 Subject: [PATCH 05/10] Adjust the clang module-file and disable-explicit-pcm interface to match the frontend --- .../ModuleDependencyBuildGeneration.swift | 3 ++- Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift | 9 ++++++--- Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift | 6 +++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift b/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift index b79a70ed4..37477aa64 100644 --- a/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift +++ b/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift @@ -145,7 +145,8 @@ extension Driver { inputs: inout [TypedVirtualPath], commandLine: inout [Job.ArgTemplate]) throws { // Prohibit the frontend from implicitly building textual modules into binary modules. - commandLine.appendFlags("-disable-implicit-swift-modules", "-disable-implicit-pcms") + commandLine.appendFlags("-disable-implicit-swift-modules", "-Xcc", "-Xclang", "-Xcc", + "-fno-implicit-modules") for moduleId in moduleInfo.directDependencies { guard let dependencyInfo = dependencyGraph.modules[moduleId] else { throw Error.missingModuleDependency(moduleId.getName()) diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index 61a501c5b..d82b5e827 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -321,8 +321,10 @@ extension Driver { type: .pcm) let clangModuleMapPath = TypedVirtualPath(file: try VirtualPath(path: clangDependencyDetails.moduleMapPath), type: .pcm) - commandLine.appendFlag("-clang-module-file=\(clangModulePath.file.description)") - commandLine.appendFlag("-clang-module-map-file=\(clangModuleMapPath.file.description)") + commandLine.appendFlags("-Xcc", "-Xclang", "-Xcc", + "-fmodule-map-file=\(clangModuleMapPath.file.description)") + commandLine.appendFlags("-Xcc", "-Xclang", "-Xcc", + "-fmodule-file=\(clangModulePath.file.description)") inputs.append(clangModulePath) inputs.append(clangModuleMapPath) } @@ -336,7 +338,8 @@ extension Driver { fatalError("Inter Module Dependency Graph does not exist in explicit module build mode.") } // Prohibit the frontend from implicitly building textual modules into binary modules. - commandLine.appendFlags("-disable-implicit-swift-modules", "-disable-implicit-pcms") + commandLine.appendFlags("-disable-implicit-swift-modules", "-Xcc", "-Xclang", "-Xcc", + "-fno-implicit-modules") // Provide the frontend with a list of explicitly pre-built modules. for (moduleId, moduleInfo) in dependencyGraph.modules { diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index 20cd19d4e..b1be1b95e 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -47,7 +47,7 @@ private func checkExplicitModuleBuildJobDependencies(job: Job, moduleDependencyGraph: InterModuleDependencyGraph) throws { XCTAssertTrue(job.commandLine.contains(.flag(String("-disable-implicit-swift-modules")))) - XCTAssertTrue(job.commandLine.contains(.flag(String("-disable-implicit-pcms")))) + XCTAssertTrue(job.commandLine.contains(.flag(String("-fno-implicit-modules")))) for dependencyId in moduleInfo.directDependencies { let dependencyInfo = moduleDependencyGraph.modules[dependencyId]! switch dependencyInfo.details { @@ -68,9 +68,9 @@ throws { XCTAssertTrue(job.inputs.contains(clangDependencyModulePath)) XCTAssertTrue(job.inputs.contains(clangDependencyModuleMapPath)) XCTAssertTrue(job.commandLine.contains( - .flag(String("-clang-module-file=\(dependencyInfo.modulePath)")))) + .flag(String("-fmodule-file=\(dependencyInfo.modulePath)")))) XCTAssertTrue(job.commandLine.contains( - .flag(String("-clang-module-map-file=\(clangDependencyDetails.moduleMapPath)")))) + .flag(String("-fmodule-map-file=\(clangDependencyDetails.moduleMapPath)")))) } } } From 4cdd2d1a8958b645d3c8fd2b1603604d3986fd55 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Tue, 2 Jun 2020 09:03:33 -0700 Subject: [PATCH 06/10] Add transitive module dependencies as explicit module arguments to compile jobs --- .../InterModuleDependencyGraph.swift | 19 -------------- .../ModuleDependencyBuildGeneration.swift | 20 +++++++-------- Sources/SwiftDriver/Jobs/CompileJob.swift | 6 ++++- .../SwiftDriver/Jobs/FrontendJobHelpers.swift | 25 +++++++++++++------ .../ExplicitModuleBuildTests.swift | 23 +++++++++++++---- 5 files changed, 50 insertions(+), 43 deletions(-) diff --git a/Sources/SwiftDriver/Dependency Scanning/InterModuleDependencyGraph.swift b/Sources/SwiftDriver/Dependency Scanning/InterModuleDependencyGraph.swift index ef3349331..10abd87d4 100644 --- a/Sources/SwiftDriver/Dependency Scanning/InterModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/Dependency Scanning/InterModuleDependencyGraph.swift @@ -50,25 +50,6 @@ extension ModuleDependencyId: Codable { try container.encode(moduleName, forKey: .clang) } } - - func getName() -> String { - switch self { - case .swift(let moduleName): - return moduleName - case .clang(let moduleName): - return moduleName - } - } - - // Used in testing - init(name: String, kind: CodingKeys) { - switch kind { - case .swift: - self = .swift(name) - case .clang: - self = .clang(name) - } - } } /// Details specific to Swift modules. diff --git a/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift b/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift index 37477aa64..ec7cad370 100644 --- a/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift +++ b/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift @@ -16,8 +16,8 @@ import Foundation extension Driver { /// For the current moduleDependencyGraph, plan the order and generate jobs /// for explicitly building all dependency modules. - mutating func planExplicitModuleDependenciesCompile(dependencyGraph: InterModuleDependencyGraph) - throws -> [Job] { + mutating func planExplicitModuleDependenciesCompile(dependencyGraph: InterModuleDependencyGraph + ) throws -> [Job] { var jobs: [Job] = [] for (id, moduleInfo) in dependencyGraph.modules { // The generation of the main module file will be handled elsewhere in the driver. @@ -45,8 +45,8 @@ extension Driver { /// For a given swift module dependency, generate a build job mutating private func genSwiftModuleDependencyBuildJob(moduleInfo: ModuleInfo, moduleName: String, - dependencyGraph: InterModuleDependencyGraph) - throws -> Job { + dependencyGraph: InterModuleDependencyGraph + ) throws -> Job { guard case .swift(let swiftModuleDetails) = moduleInfo.details else { throw Error.malformedModuleDependency(moduleName, "no `details` object") } @@ -91,8 +91,8 @@ extension Driver { /// For a given clang module dependency, generate a build job mutating private func genClangModuleDependencyBuildJob(moduleInfo: ModuleInfo, moduleName: String, - dependencyGraph: InterModuleDependencyGraph) - throws -> Job { + dependencyGraph: InterModuleDependencyGraph + ) throws -> Job { // For clang modules, the Fast Dependency Scanner emits a list of source // files (with a .modulemap among them), and a list of compile command // options. @@ -149,11 +149,11 @@ extension Driver { "-fno-implicit-modules") for moduleId in moduleInfo.directDependencies { guard let dependencyInfo = dependencyGraph.modules[moduleId] else { - throw Error.missingModuleDependency(moduleId.getName()) + throw Error.missingModuleDependency(moduleId.moduleName) } - - try addModuleAsExplicitDependency(moduleInfo: dependencyInfo, commandLine: &commandLine, - inputs: &inputs) + try addModuleAsExplicitDependency(moduleInfo: dependencyInfo, + dependencyGraph: dependencyGraph, + commandLine: &commandLine, inputs: &inputs) } } } diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift index 3106c3bc8..34247a107 100644 --- a/Sources/SwiftDriver/Jobs/CompileJob.swift +++ b/Sources/SwiftDriver/Jobs/CompileJob.swift @@ -156,7 +156,11 @@ extension Driver { // FIXME: MSVC runtime flags if parsedOptions.contains(.driverExplicitModuleBuild) { - try addExplicitModuleBuildArguments(commandLine: &commandLine, inputs: &inputs) + guard let dependencyGraph = interModuleDependencyGraph else { + fatalError("Inter Module Dependency Graph does not exist in explicit module build mode.") + } + try addExplicitModuleBuildArguments(dependencyGraph: dependencyGraph, + commandLine: &commandLine, inputs: &inputs) } if parsedOptions.hasArgument(.parseAsLibrary, .emitLibrary) { diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index d82b5e827..94783fa49 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -307,7 +307,9 @@ extension Driver { /// Adds the specified module as an explicit module dependency to given /// inputs and comman line arguments of a compile job. + /// Also add transitive dependencies that arise from dependencies of this module. func addModuleAsExplicitDependency(moduleInfo: ModuleInfo, + dependencyGraph: InterModuleDependencyGraph, commandLine: inout [Job.ArgTemplate], inputs: inout [TypedVirtualPath]) throws { switch moduleInfo.details { @@ -328,15 +330,23 @@ extension Driver { inputs.append(clangModulePath) inputs.append(clangModuleMapPath) } + // Add transitive dependencies to the command line as well + for transitiveDependencyId in moduleInfo.directDependencies { + guard let transitiveDependencyInfo = dependencyGraph.modules[transitiveDependencyId] else { + throw Error.missingModuleDependency(transitiveDependencyId.moduleName) + } + try addModuleAsExplicitDependency(moduleInfo: transitiveDependencyInfo, + dependencyGraph: dependencyGraph, + commandLine: &commandLine, + inputs: &inputs) + } } /// Adds all dependecies required for an explicit module build /// to inputs and comman line arguments of a compile job. - func addExplicitModuleBuildArguments(commandLine: inout [Job.ArgTemplate], + func addExplicitModuleBuildArguments(dependencyGraph: InterModuleDependencyGraph, + commandLine: inout [Job.ArgTemplate], inputs: inout [TypedVirtualPath]) throws { - guard let dependencyGraph = interModuleDependencyGraph else { - fatalError("Inter Module Dependency Graph does not exist in explicit module build mode.") - } // Prohibit the frontend from implicitly building textual modules into binary modules. commandLine.appendFlags("-disable-implicit-swift-modules", "-Xcc", "-Xclang", "-Xcc", "-fno-implicit-modules") @@ -344,12 +354,11 @@ extension Driver { // Provide the frontend with a list of explicitly pre-built modules. for (moduleId, moduleInfo) in dependencyGraph.modules { // Skip the main output module as it is not its own dependency - guard moduleId.getName() != dependencyGraph.mainModuleName else { + guard moduleId.moduleName != dependencyGraph.mainModuleName else { continue } - - try addModuleAsExplicitDependency(moduleInfo: moduleInfo, commandLine: &commandLine, - inputs: &inputs) + try addModuleAsExplicitDependency(moduleInfo: moduleInfo, dependencyGraph: dependencyGraph, + commandLine: &commandLine, inputs: &inputs) } } } diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index b1be1b95e..6a00fcdfd 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -20,7 +20,11 @@ private func checkExplicitModuleBuildJob(job: Job, moduleName: String, moduleKind: ModuleDependencyId.CodingKeys, moduleDependencyGraph: InterModuleDependencyGraph) throws { - let moduleId = ModuleDependencyId(name: moduleName, kind: moduleKind) + var moduleId : ModuleDependencyId + switch moduleKind { + case .swift: moduleId = .swift(moduleName) + case .clang: moduleId = .clang(moduleName) + } let moduleInfo = moduleDependencyGraph.modules[moduleId]! switch moduleInfo.details { case .swift(let swiftModuleDetails): @@ -36,6 +40,9 @@ private func checkExplicitModuleBuildJob(job: Job, XCTAssertEqual(job.kind, .generatePCM) XCTAssertTrue(job.inputs.contains(moduleMapPath)) } + // Ensure the frontend was prohibited from doing implicit module builds + XCTAssertTrue(job.commandLine.contains(.flag(String("-disable-implicit-swift-modules")))) + XCTAssertTrue(job.commandLine.contains(.flag(String("-fno-implicit-modules")))) try checkExplicitModuleBuildJobDependencies(job: job, moduleInfo: moduleInfo, moduleDependencyGraph: moduleDependencyGraph) } @@ -44,10 +51,8 @@ private func checkExplicitModuleBuildJob(job: Job, /// to build all of its dependencies explicitly private func checkExplicitModuleBuildJobDependencies(job: Job, moduleInfo : ModuleInfo, - moduleDependencyGraph: InterModuleDependencyGraph) -throws { - XCTAssertTrue(job.commandLine.contains(.flag(String("-disable-implicit-swift-modules")))) - XCTAssertTrue(job.commandLine.contains(.flag(String("-fno-implicit-modules")))) + moduleDependencyGraph: InterModuleDependencyGraph +) throws { for dependencyId in moduleInfo.directDependencies { let dependencyInfo = moduleDependencyGraph.modules[dependencyId]! switch dependencyInfo.details { @@ -72,6 +77,14 @@ throws { XCTAssertTrue(job.commandLine.contains( .flag(String("-fmodule-map-file=\(clangDependencyDetails.moduleMapPath)")))) } + + // Ensure all transitive dependencies got added as well. + for transitiveDependencyId in dependencyInfo.directDependencies { + try checkExplicitModuleBuildJobDependencies(job: job, + moduleInfo: moduleDependencyGraph.modules[transitiveDependencyId]!, + moduleDependencyGraph: moduleDependencyGraph) + + } } } From de75d02fe01a0380adff1a4b2814b16084057412 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Tue, 2 Jun 2020 09:30:58 -0700 Subject: [PATCH 07/10] Move addition of explicit module build arguments and inputs to addCommonFrontendOptions --- .../Driver/ModuleDependencyScanning.swift | 5 +++- Sources/SwiftDriver/Jobs/CompileJob.swift | 10 +------- Sources/SwiftDriver/Jobs/EmitModuleJob.swift | 2 +- .../SwiftDriver/Jobs/FrontendJobHelpers.swift | 25 ++++++++++++++++++- Sources/SwiftDriver/Jobs/GeneratePCHJob.swift | 2 +- Sources/SwiftDriver/Jobs/GeneratePCMJob.swift | 2 +- Sources/SwiftDriver/Jobs/InterpretJob.swift | 2 +- Sources/SwiftDriver/Jobs/MergeModuleJob.swift | 2 +- Sources/SwiftDriver/Jobs/ReplJob.swift | 5 ++-- 9 files changed, 37 insertions(+), 18 deletions(-) diff --git a/Sources/SwiftDriver/Driver/ModuleDependencyScanning.swift b/Sources/SwiftDriver/Driver/ModuleDependencyScanning.swift index 340ab2d74..a2159a8f5 100644 --- a/Sources/SwiftDriver/Driver/ModuleDependencyScanning.swift +++ b/Sources/SwiftDriver/Driver/ModuleDependencyScanning.swift @@ -23,6 +23,7 @@ extension Driver { let resolver = try ArgsResolver() let compilerPath = VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler)) let tool = try resolver.resolve(.path(compilerPath)) + var inputs: [TypedVirtualPath] = [] // Aggregate the fast dependency scanner arguments var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } @@ -31,7 +32,9 @@ extension Driver { if parsedOptions.hasArgument(.parseStdlib) { commandLine.appendFlag(.disableObjcAttrRequiresFoundationModule) } - try addCommonFrontendOptions(commandLine: &commandLine) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, + bridgingHeaderHandling: .precompiled, + moduleDependencyGraphUse: .dependencyScan) // FIXME: MSVC runtime flags // Pass on the input files diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift index 34247a107..c64d836df 100644 --- a/Sources/SwiftDriver/Jobs/CompileJob.swift +++ b/Sources/SwiftDriver/Jobs/CompileJob.swift @@ -152,17 +152,9 @@ extension Driver { commandLine.appendFlag(.disableObjcAttrRequiresFoundationModule) } - try addCommonFrontendOptions(commandLine: &commandLine) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) // FIXME: MSVC runtime flags - if parsedOptions.contains(.driverExplicitModuleBuild) { - guard let dependencyGraph = interModuleDependencyGraph else { - fatalError("Inter Module Dependency Graph does not exist in explicit module build mode.") - } - try addExplicitModuleBuildArguments(dependencyGraph: dependencyGraph, - commandLine: &commandLine, inputs: &inputs) - } - if parsedOptions.hasArgument(.parseAsLibrary, .emitLibrary) { commandLine.appendFlag(.parseAsLibrary) } diff --git a/Sources/SwiftDriver/Jobs/EmitModuleJob.swift b/Sources/SwiftDriver/Jobs/EmitModuleJob.swift index 04c735414..1b88dab72 100644 --- a/Sources/SwiftDriver/Jobs/EmitModuleJob.swift +++ b/Sources/SwiftDriver/Jobs/EmitModuleJob.swift @@ -60,7 +60,7 @@ extension Driver { inputs.append(input) } - try addCommonFrontendOptions(commandLine: &commandLine) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) // FIXME: Add MSVC runtime library flags try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs) diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index 94783fa49..6182495e9 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -35,10 +35,21 @@ extension Driver { /// Use the precompiled bridging header. case precompiled } + /// Whether the driver has already constructed a module dependency graph or is in the process + /// of doing so + enum ModuleDependencyGraphUse { + /// Even though the driver may be in ExplicitModuleBuild mode, the dependency graph has not yet + /// been constructed, omit processing module dependencies + case dependencyScan + /// If the driver is in Explicit Module Build mode, the dependency graph has been computed + case computed + } /// Add frontend options that are common to different frontend invocations. mutating func addCommonFrontendOptions( commandLine: inout [Job.ArgTemplate], - bridgingHeaderHandling: BridgingHeaderHandling = .precompiled + inputs: inout [TypedVirtualPath], + bridgingHeaderHandling: BridgingHeaderHandling = .precompiled, + moduleDependencyGraphUse: ModuleDependencyGraphUse = .computed ) throws { // Only pass -target to the REPL or immediate modes if it was explicitly // specified on the command line. @@ -54,6 +65,18 @@ extension Driver { } } + // If in ExplicitModuleBuild mode and the dependency graph has been computed, add module + // dependencies. + // May also be used for generation of the dependency graph itself in ExplicitModuleBuild mode. + if (parsedOptions.contains(.driverExplicitModuleBuild) && + moduleDependencyGraphUse == .computed) { + guard let dependencyGraph = interModuleDependencyGraph else { + fatalError("Attempting to add Explicit Module job dependencies, but the Inter Module Dependency Graph does not exist.") + } + try addExplicitModuleBuildArguments(dependencyGraph: dependencyGraph, + commandLine: &commandLine, inputs: &inputs) + } + if let variant = parsedOptions.getLastArgument(.targetVariant)?.asSingle { commandLine.appendFlag(.targetVariant) commandLine.appendFlag(Triple(variant, normalizing: true).triple) diff --git a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift index 103e3e738..d516ae607 100644 --- a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift +++ b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift @@ -23,7 +23,7 @@ extension Driver { commandLine.appendFlag("-frontend") try addCommonFrontendOptions( - commandLine: &commandLine, bridgingHeaderHandling: .parsed) + commandLine: &commandLine, inputs: &inputs, bridgingHeaderHandling: .parsed) try commandLine.appendLast(.indexStorePath, from: &parsedOptions) diff --git a/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift index f433ba04e..1a7c15926 100644 --- a/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift +++ b/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift @@ -48,7 +48,7 @@ extension Driver { commandLine.appendPath(output.file) try addCommonFrontendOptions( - commandLine: &commandLine, bridgingHeaderHandling: .ignored) + commandLine: &commandLine, inputs: &inputs, bridgingHeaderHandling: .ignored) try commandLine.appendLast(.indexStorePath, from: &parsedOptions) diff --git a/Sources/SwiftDriver/Jobs/InterpretJob.swift b/Sources/SwiftDriver/Jobs/InterpretJob.swift index b82e7c3be..9cc4a5a8c 100644 --- a/Sources/SwiftDriver/Jobs/InterpretJob.swift +++ b/Sources/SwiftDriver/Jobs/InterpretJob.swift @@ -27,7 +27,7 @@ extension Driver { commandLine.appendFlag(.disableObjcAttrRequiresFoundationModule) } - try addCommonFrontendOptions(commandLine: &commandLine) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) // FIXME: MSVC runtime flags try commandLine.appendLast(.parseSil, from: &parsedOptions) diff --git a/Sources/SwiftDriver/Jobs/MergeModuleJob.swift b/Sources/SwiftDriver/Jobs/MergeModuleJob.swift index 8938ce8dd..2a92894d9 100644 --- a/Sources/SwiftDriver/Jobs/MergeModuleJob.swift +++ b/Sources/SwiftDriver/Jobs/MergeModuleJob.swift @@ -41,7 +41,7 @@ extension Driver { commandLine.appendFlag(.disableDiagnosticPasses) commandLine.appendFlag(.disableSilPerfOptzns) - try addCommonFrontendOptions(commandLine: &commandLine) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) // FIXME: Add MSVC runtime library flags try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs) diff --git a/Sources/SwiftDriver/Jobs/ReplJob.swift b/Sources/SwiftDriver/Jobs/ReplJob.swift index 3feb80d48..d90069ed3 100644 --- a/Sources/SwiftDriver/Jobs/ReplJob.swift +++ b/Sources/SwiftDriver/Jobs/ReplJob.swift @@ -13,8 +13,9 @@ extension Driver { mutating func replJob() throws -> Job { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } + var inputs: [TypedVirtualPath] = [] - try addCommonFrontendOptions(commandLine: &commandLine) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) // FIXME: MSVC runtime flags try commandLine.appendLast(.importObjcHeader, from: &parsedOptions) @@ -27,7 +28,7 @@ extension Driver { kind: .repl, tool: .absolute(try toolchain.getToolPath(.lldb)), commandLine: [Job.ArgTemplate.flag(lldbArg)], - inputs: [], + inputs: inputs, outputs: [], requiresInPlaceExecution: true ) From 497e0293880230c0442a2e788dbc04eb6b1ba9fe Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Tue, 2 Jun 2020 13:56:22 -0700 Subject: [PATCH 08/10] Use separate flag for -swift-module-file instead of EQ --- Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift | 7 ++++--- Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index 6182495e9..5dc116537 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -329,8 +329,8 @@ extension Driver { } /// Adds the specified module as an explicit module dependency to given - /// inputs and comman line arguments of a compile job. - /// Also add transitive dependencies that arise from dependencies of this module. + /// inputs and command line arguments of a compile job. + /// Also adds transitive dependencies that arise from dependencies of this module. func addModuleAsExplicitDependency(moduleInfo: ModuleInfo, dependencyGraph: InterModuleDependencyGraph, commandLine: inout [Job.ArgTemplate], @@ -339,7 +339,8 @@ extension Driver { case .swift: let swiftModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), type: .swiftModule) - commandLine.appendFlag("-swift-module-file=\(swiftModulePath.file.description)") + commandLine.appendFlags("-swift-module-file") + commandLine.appendPath(swiftModulePath.file) inputs.append(swiftModulePath) case .clang(let clangDependencyDetails): let clangModulePath = TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index 6a00fcdfd..e8b395c46 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -62,7 +62,9 @@ private func checkExplicitModuleBuildJobDependencies(job: Job, type: .swiftModule) XCTAssertTrue(job.inputs.contains(swiftDependencyModulePath)) XCTAssertTrue(job.commandLine.contains( - .flag(String("-swift-module-file=\(dependencyInfo.modulePath)")))) + .flag(String("-swift-module-file")))) + XCTAssertTrue( + job.commandLine.contains(.path(try VirtualPath(path: dependencyInfo.modulePath)))) case .clang(let clangDependencyDetails): let clangDependencyModulePath = TypedVirtualPath(file: try VirtualPath(path: dependencyInfo.modulePath), From 50ff7fb72ed492c34980a0377617973f6c8e3c95 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Wed, 3 Jun 2020 14:28:27 -0700 Subject: [PATCH 09/10] Fix small explicit module build test file issues - Explicitly import `Swift` in swiftinterface files - Use quote header file imports instead of <> --- TestInputs/ExplicitModuleBuilds/CHeaders/B.h | 2 +- TestInputs/ExplicitModuleBuilds/CHeaders/C.h | 2 +- TestInputs/ExplicitModuleBuilds/Swift/A.swiftinterface | 1 + TestInputs/ExplicitModuleBuilds/Swift/G.swiftinterface | 2 +- Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift | 1 - 5 files changed, 4 insertions(+), 4 deletions(-) diff --git a/TestInputs/ExplicitModuleBuilds/CHeaders/B.h b/TestInputs/ExplicitModuleBuilds/CHeaders/B.h index dedb1b552..f065892fd 100644 --- a/TestInputs/ExplicitModuleBuilds/CHeaders/B.h +++ b/TestInputs/ExplicitModuleBuilds/CHeaders/B.h @@ -1,4 +1,4 @@ -#include +#include "A.h" void funcB(void); diff --git a/TestInputs/ExplicitModuleBuilds/CHeaders/C.h b/TestInputs/ExplicitModuleBuilds/CHeaders/C.h index 8a5a9b119..ec830425f 100644 --- a/TestInputs/ExplicitModuleBuilds/CHeaders/C.h +++ b/TestInputs/ExplicitModuleBuilds/CHeaders/C.h @@ -1,3 +1,3 @@ -#include +#include "B.h" void funcC(void); diff --git a/TestInputs/ExplicitModuleBuilds/Swift/A.swiftinterface b/TestInputs/ExplicitModuleBuilds/Swift/A.swiftinterface index 1b6292940..d284ad329 100644 --- a/TestInputs/ExplicitModuleBuilds/Swift/A.swiftinterface +++ b/TestInputs/ExplicitModuleBuilds/Swift/A.swiftinterface @@ -1,5 +1,6 @@ // swift-interface-format-version: 1.0 // swift-module-flags: -module-name A +import Swift @_exported import A public func overlayFuncA() { } diff --git a/TestInputs/ExplicitModuleBuilds/Swift/G.swiftinterface b/TestInputs/ExplicitModuleBuilds/Swift/G.swiftinterface index ae9d32b2c..e6fea1948 100644 --- a/TestInputs/ExplicitModuleBuilds/Swift/G.swiftinterface +++ b/TestInputs/ExplicitModuleBuilds/Swift/G.swiftinterface @@ -2,7 +2,7 @@ // swift-module-flags: -module-name G -swift-version 5 #if swift(>=5.0) - +import Swift @_exported import G public func overlayFuncG() { } diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index e8b395c46..4e818969f 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -153,7 +153,6 @@ final class ExplicitModuleBuildTests: XCTestCase { let jobs = try driver.planBuild() XCTAssertTrue(driver.parsedOptions.contains(.driverExplicitModuleBuild)) let dependencyGraph = driver.interModuleDependencyGraph! - XCTAssertEqual(jobs.count, 12) for job in jobs { XCTAssertEqual(job.outputs.count, 1) switch (job.outputs[0].file) { From 5092bdbf2f89c45851bd08fef3b4bce6de9ca3dc Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Fri, 5 Jun 2020 10:47:41 -0700 Subject: [PATCH 10/10] Incorporate code-review formatting feedback. --- .../Dependency Scanning/ModuleDependencyBuildGeneration.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift b/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift index ec7cad370..ec4b27d10 100644 --- a/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift +++ b/Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift @@ -16,7 +16,8 @@ import Foundation extension Driver { /// For the current moduleDependencyGraph, plan the order and generate jobs /// for explicitly building all dependency modules. - mutating func planExplicitModuleDependenciesCompile(dependencyGraph: InterModuleDependencyGraph + mutating func planExplicitModuleDependenciesCompile( + dependencyGraph: InterModuleDependencyGraph ) throws -> [Job] { var jobs: [Job] = [] for (id, moduleInfo) in dependencyGraph.modules {