From e84e9aa9bf0568990d2b250d65018227d6be9d59 Mon Sep 17 00:00:00 2001 From: Xi Ge Date: Sun, 18 Apr 2021 19:23:45 -0700 Subject: [PATCH 1/8] PrebuiltModuleGen: add support for -module-cache-path --- .../SwiftDriver/Jobs/PrebuiltModulesJob.swift | 5 ++++ Sources/swift-build-sdk-interfaces/main.swift | 30 ++++++++++++------- .../ExplicitModuleBuildTests.swift | 4 +++ 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift b/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift index ae7122823..bbe6ba00e 100644 --- a/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift +++ b/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift @@ -261,6 +261,11 @@ extension Driver { commandLine.appendFlag(.Fsystem) commandLine.append(.path(iosMacFrameworksSearchPath)) } + // Use the specified module cache dir + if let mcp = parsedOptions.getLastArgument(.moduleCachePath)?.asSingle { + commandLine.appendFlag(.moduleCachePath) + commandLine.append(.path(try VirtualPath(path: mcp))) + } commandLine.appendFlag(.serializeParseableModuleInterfaceDependencyHashes) return Job( moduleName: moduleName, diff --git a/Sources/swift-build-sdk-interfaces/main.swift b/Sources/swift-build-sdk-interfaces/main.swift index 5fc343645..f17c083bc 100644 --- a/Sources/swift-build-sdk-interfaces/main.swift +++ b/Sources/swift-build-sdk-interfaces/main.swift @@ -22,14 +22,17 @@ guard let sdkPathRaw = ProcessEnv.vars["SDKROOT"] else { exit(1) } -var rawOutputDir = "" -if let oid = CommandLine.arguments.firstIndex(of: "-o") { - let dirId = oid.advanced(by: 1) - if dirId < CommandLine.arguments.count { - rawOutputDir = CommandLine.arguments[dirId] +func getArgument(_ flag: String) -> String? { + if let id = CommandLine.arguments.firstIndex(of: flag) { + let nextId = id.advanced(by: 1) + if nextId < CommandLine.arguments.count { + return CommandLine.arguments[nextId] + } } + return nil } -if rawOutputDir.isEmpty { + +guard let rawOutputDir = getArgument("-o") else { diagnosticsEngine.emit(.error("need to specify -o")) exit(1) } @@ -99,10 +102,17 @@ do { processSet: processSet, fileSystem: localFileSystem, env: ProcessEnv.vars) - var driver = try Driver(args: ["swiftc", - "-target", collector.targetTriple, - tempPath.description, - "-sdk", sdkPathRaw], + var args = ["swiftc", + "-target", collector.targetTriple, + tempPath.description, + "-sdk", sdkPathRaw] + let mcpFlag = "-module-cache-path" + // Append module cache path if given by the client + if let mcp = getArgument(mcpFlag) { + args.append(mcpFlag) + args.append(mcp) + } + var driver = try Driver(args: args, diagnosticsEngine: diagnosticsEngine, executor: executor, compilerExecutableDir: swiftcPath.parentDirectory) diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index 9ab056d2c..c076bcb0d 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -874,8 +874,10 @@ final class ExplicitModuleBuildTests: XCTestCase { $0 <<< "import H\n" $0 <<< "import Swift\n" } + let moduleCachePath = "/tmp/module-cache" var driver = try Driver(args: ["swiftc", main.pathString, "-sdk", mockSDKPath, + "-module-cache-path", moduleCachePath ]) let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(with: interfaceMap, @@ -890,6 +892,8 @@ final class ExplicitModuleBuildTests: XCTestCase { XCTAssertTrue(jobs.allSatisfy {$0.outputs.count == 1}) XCTAssertTrue(jobs.allSatisfy {$0.kind == .compile}) XCTAssertTrue(jobs.allSatisfy {$0.commandLine.contains(.flag("-compile-module-from-interface"))}) + XCTAssertTrue(jobs.allSatisfy {$0.commandLine.contains(.flag("-module-cache-path"))}) + XCTAssertTrue(try jobs.allSatisfy {$0.commandLine.contains(.path(try VirtualPath(path: moduleCachePath)))}) let HJobs = jobs.filter { $0.moduleName == "H"} XCTAssertTrue(HJobs.count == 2) XCTAssertTrue(getInputModules(HJobs[0]) == ["A", "E", "F", "G", "Swift"]) From c3b448c2432fc9dc618eceb049f06b1c2d0b4e45 Mon Sep 17 00:00:00 2001 From: Xi Ge Date: Mon, 19 Apr 2021 22:47:33 -0700 Subject: [PATCH 2/8] PrebuiltModuleGen: add an option for skipping the job execution --- Sources/swift-build-sdk-interfaces/main.swift | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Sources/swift-build-sdk-interfaces/main.swift b/Sources/swift-build-sdk-interfaces/main.swift index f17c083bc..8d61815f9 100644 --- a/Sources/swift-build-sdk-interfaces/main.swift +++ b/Sources/swift-build-sdk-interfaces/main.swift @@ -44,6 +44,9 @@ let coreMode = CommandLine.arguments.contains("-core") /// Verbose to print more info let verbose = CommandLine.arguments.contains("-v") +/// Skip executing the jobs +let skipExecution = CommandLine.arguments.contains("-n") + do { let sdkPath = try VirtualPath(path: sdkPathRaw).absolutePath! if !localFileSystem.exists(sdkPath) { @@ -117,6 +120,15 @@ do { executor: executor, compilerExecutableDir: swiftcPath.parentDirectory) let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(with: inputMap, into: outputDir, exhaustive: !coreMode) + if verbose { + Driver.stdErrQueue.sync { + stderrStream <<< "job count: \(jobs.count + danglingJobs.count)\n" + stderrStream.flush() + } + } + if skipExecution { + exit(0) + } let delegate = PrebuitModuleGenerationDelegate(jobs, diagnosticsEngine, verbose) do { try executor.execute(workload: DriverExecutorWorkload.init(jobs, nil, continueBuildingAfterErrors: true), From c348d5aeb67b8eeb9d1ddaec5b862d98c36a9c7a Mon Sep 17 00:00:00 2001 From: Xi Ge Date: Wed, 5 May 2021 14:31:53 -0700 Subject: [PATCH 3/8] Recursively create non-existing output dir --- Sources/swift-build-sdk-interfaces/main.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/swift-build-sdk-interfaces/main.swift b/Sources/swift-build-sdk-interfaces/main.swift index 8d61815f9..3b72f0b8a 100644 --- a/Sources/swift-build-sdk-interfaces/main.swift +++ b/Sources/swift-build-sdk-interfaces/main.swift @@ -62,7 +62,7 @@ do { outputDir = outputDir.appending(RelativePath(collector.versionString)) } if !localFileSystem.exists(outputDir) { - try localFileSystem.createDirectory(outputDir) + try localFileSystem.createDirectory(outputDir, recursive: true) } let swiftcPathRaw = ProcessEnv.vars["SWIFT_EXEC"] var swiftcPath: AbsolutePath From 68bd331a0656bace32ce8b7d9b47f275b9ae0959 Mon Sep 17 00:00:00 2001 From: Xi Ge Date: Fri, 7 May 2021 10:52:35 -0700 Subject: [PATCH 4/8] PrebuiltModuleGen: allow retrying after hitting bad file descriptor error --- Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift | 2 ++ Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift | 1 + 2 files changed, 3 insertions(+) diff --git a/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift b/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift index bbe6ba00e..661ebcd83 100644 --- a/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift +++ b/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift @@ -267,6 +267,8 @@ extension Driver { commandLine.append(.path(try VirtualPath(path: mcp))) } commandLine.appendFlag(.serializeParseableModuleInterfaceDependencyHashes) + commandLine.appendFlag(.badFileDescriptorRetryCount) + commandLine.appendFlag("30") return Job( moduleName: moduleName, kind: .compile, diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index c076bcb0d..cadad0f6f 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -893,6 +893,7 @@ final class ExplicitModuleBuildTests: XCTestCase { XCTAssertTrue(jobs.allSatisfy {$0.kind == .compile}) XCTAssertTrue(jobs.allSatisfy {$0.commandLine.contains(.flag("-compile-module-from-interface"))}) XCTAssertTrue(jobs.allSatisfy {$0.commandLine.contains(.flag("-module-cache-path"))}) + XCTAssertTrue(jobs.allSatisfy {$0.commandLine.contains(.flag("-bad-file-descriptor-retry-count"))}) XCTAssertTrue(try jobs.allSatisfy {$0.commandLine.contains(.path(try VirtualPath(path: moduleCachePath)))}) let HJobs = jobs.filter { $0.moduleName == "H"} XCTAssertTrue(HJobs.count == 2) From ed34738b4bab11ce924137ba7033aa18b5a30eff Mon Sep 17 00:00:00 2001 From: Xi Ge Date: Thu, 10 Jun 2021 16:41:17 -0700 Subject: [PATCH 5/8] PrebuiltModuleGen: teach the tool to dump a .dot file for module dependency visualization --- .../SwiftDriver/Jobs/PrebuiltModulesJob.swift | 57 ++++++++++++++++++- Sources/swift-build-sdk-interfaces/main.swift | 10 +++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift b/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift index 661ebcd83..72baf3826 100644 --- a/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift +++ b/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift @@ -236,6 +236,57 @@ public struct SDKPrebuiltModuleInputsCollector { } } +extension InterModuleDependencyGraph { + func dumpDotGraph(_ path: AbsolutePath, _ includingPCM: Bool) throws { + func isPCM(_ dep: ModuleDependencyId) -> Bool { + switch dep { + case .clang: + return true + default: + return false + } + } + func dumpModuleName(_ stream: WritableByteStream, _ dep: ModuleDependencyId) { + switch dep { + case .swift(let name): + stream <<< "\"\(name).swiftmodule\"" + case .clang(let name): + stream <<< "\"\(name).pcm\"" + default: + break + } + } + try localFileSystem.writeFileContents(path) {Stream in + Stream <<< "digraph {\n" + for key in modules.keys { + switch key { + case .swift(let name): + if name == mainModuleName { + break + } + fallthrough + case .clang: + if !includingPCM && isPCM(key) { + break + } + modules[key]!.directDependencies?.forEach { dep in + if !includingPCM && isPCM(dep) { + return + } + dumpModuleName(Stream, key) + Stream <<< " -> " + dumpModuleName(Stream, dep) + Stream <<< ";\n" + } + default: + break + } + } + Stream <<< "}\n" + } + } +} + extension Driver { private mutating func generateSingleModuleBuildingJob(_ moduleName: String, _ prebuiltModuleDir: AbsolutePath, @@ -282,12 +333,16 @@ extension Driver { public mutating func generatePrebuitModuleGenerationJobs(with inputMap: [String: [PrebuiltModuleInput]], into prebuiltModuleDir: AbsolutePath, - exhaustive: Bool) throws -> ([Job], [Job]) { + exhaustive: Bool, + dotGraphPath: AbsolutePath? = nil) throws -> ([Job], [Job]) { assert(sdkPath != nil) // Run the dependency scanner and update the dependency oracle with the results // We only need Swift dependencies here, so we don't need to invoke gatherModuleDependencies, // which also resolves versioned clang modules. let dependencyGraph = try performDependencyScan() + if let dotGraphPath = dotGraphPath { + try dependencyGraph.dumpDotGraph(dotGraphPath, false) + } var jobs: [Job] = [] var danglingJobs: [Job] = [] var inputCount = 0 diff --git a/Sources/swift-build-sdk-interfaces/main.swift b/Sources/swift-build-sdk-interfaces/main.swift index 3b72f0b8a..6cb8418df 100644 --- a/Sources/swift-build-sdk-interfaces/main.swift +++ b/Sources/swift-build-sdk-interfaces/main.swift @@ -32,6 +32,13 @@ func getArgument(_ flag: String) -> String? { return nil } +func getArgumentAsPath(_ flag: String) throws -> AbsolutePath? { + if let raw = getArgument(flag) { + return try VirtualPath(path: raw).absolutePath + } + return nil +} + guard let rawOutputDir = getArgument("-o") else { diagnosticsEngine.emit(.error("need to specify -o")) exit(1) @@ -119,7 +126,8 @@ do { diagnosticsEngine: diagnosticsEngine, executor: executor, compilerExecutableDir: swiftcPath.parentDirectory) - let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(with: inputMap, into: outputDir, exhaustive: !coreMode) + let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(with: inputMap, + into: outputDir, exhaustive: !coreMode, dotGraphPath: getArgumentAsPath("-dot-graph-path")) if verbose { Driver.stdErrQueue.sync { stderrStream <<< "job count: \(jobs.count + danglingJobs.count)\n" From 971d68282d9355db56e64441ddfe91c3ee60bca0 Mon Sep 17 00:00:00 2001 From: Xi Ge Date: Thu, 12 Aug 2021 10:49:39 -0700 Subject: [PATCH 6/8] PrebuiltModuleGen: generate per-interface log files This is the behavior of swift_build_sdk_interfaces.py and we should do the same in the new implementation. --- .../SwiftDriver/Jobs/PrebuiltModulesJob.swift | 56 +++++++++++++++---- Sources/swift-build-sdk-interfaces/main.swift | 2 +- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift b/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift index 72baf3826..2070e269a 100644 --- a/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift +++ b/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift @@ -24,10 +24,12 @@ public class PrebuitModuleGenerationDelegate: JobExecutionDelegate { let diagnosticsEngine: DiagnosticsEngine let verbose: Bool var failingCriticalOutputs: Set - public init(_ jobs: [Job], _ diagnosticsEngine: DiagnosticsEngine, _ verbose: Bool) { + let logPath: AbsolutePath? + public init(_ jobs: [Job], _ diagnosticsEngine: DiagnosticsEngine, _ verbose: Bool, logPath: AbsolutePath?) { self.diagnosticsEngine = diagnosticsEngine self.verbose = verbose self.failingCriticalOutputs = Set(jobs.compactMap(PrebuitModuleGenerationDelegate.getCriticalOutput)) + self.logPath = logPath } /// Dangling jobs are macabi-only modules. We should run those jobs if foundation @@ -35,22 +37,27 @@ public class PrebuitModuleGenerationDelegate: JobExecutionDelegate { public var shouldRunDanglingJobs: Bool { return !failingCriticalOutputs.contains(where: isIosMac) } - func printJobInfo(_ job: Job, _ start: Bool) { - guard verbose else { - return - } + + func getInputInterfacePath(_ job: Job) -> AbsolutePath { for arg in job.commandLine { if case .path(let p) = arg { if p.extension == "swiftinterface" { - Driver.stdErrQueue.sync { - stderrStream <<< (start ? "started: " : "finished: ") - stderrStream <<< p.absolutePath!.pathString <<< "\n" - stderrStream.flush() - } - return + return p.absolutePath! } } } + fatalError() + } + + func printJobInfo(_ job: Job, _ start: Bool) { + guard verbose else { + return + } + Driver.stdErrQueue.sync { + stderrStream <<< (start ? "started: " : "finished: ") + stderrStream <<< getInputInterfacePath(job).pathString <<< "\n" + stderrStream.flush() + } } static func getCriticalOutput(_ job: Job) -> VirtualPath? { @@ -66,6 +73,24 @@ public class PrebuitModuleGenerationDelegate: JobExecutionDelegate { return !failingCriticalOutputs.isEmpty } + fileprivate func logOutput(_ job: Job, _ result: ProcessResult, _ stdout: Bool) throws { + guard let logPath = logPath else { + return + } + let content = stdout ? try result.utf8Output() : try result.utf8stderrOutput() + guard !content.isEmpty else { + return + } + if !localFileSystem.exists(logPath) { + try localFileSystem.createDirectory(logPath, recursive: true) + } + let interfaceBase = getInputInterfacePath(job).basenameWithoutExt + let fileName = "\(job.moduleName)-\(interfaceBase)-\(stdout ? "out" : "err").txt" + try localFileSystem.writeFileContents(logPath.appending(component: fileName)) { + $0 <<< content + } + } + public func jobFinished(job: Job, result: ProcessResult, pid: Int) { switch result.exitStatus { case .terminated(code: let code): @@ -86,6 +111,15 @@ public class PrebuitModuleGenerationDelegate: JobExecutionDelegate { diagnosticsEngine.emit(.remark("\(job.moduleName) interrupted")) #endif } + do { + try logOutput(job, result, true) + try logOutput(job, result, false) + } catch { + Driver.stdErrQueue.sync { + stderrStream <<< "Failed to generate log file" + stderrStream.flush() + } + } } public func jobSkipped(job: Job) { diff --git a/Sources/swift-build-sdk-interfaces/main.swift b/Sources/swift-build-sdk-interfaces/main.swift index 6cb8418df..3521a9863 100644 --- a/Sources/swift-build-sdk-interfaces/main.swift +++ b/Sources/swift-build-sdk-interfaces/main.swift @@ -137,7 +137,7 @@ do { if skipExecution { exit(0) } - let delegate = PrebuitModuleGenerationDelegate(jobs, diagnosticsEngine, verbose) + let delegate = PrebuitModuleGenerationDelegate(jobs, diagnosticsEngine, verbose, logPath: try getArgumentAsPath("-log-path")) do { try executor.execute(workload: DriverExecutorWorkload.init(jobs, nil, continueBuildingAfterErrors: true), delegate: delegate, numParallelJobs: 128) From 583b1cd0927779ca8e4b2f9e82087b3c06eb54e4 Mon Sep 17 00:00:00 2001 From: Xi Ge Date: Mon, 16 Aug 2021 10:32:39 -0700 Subject: [PATCH 7/8] PrebuiltModuleGen: allow passing down SDK path via -sdk --- Sources/swift-build-sdk-interfaces/main.swift | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Sources/swift-build-sdk-interfaces/main.swift b/Sources/swift-build-sdk-interfaces/main.swift index 3521a9863..a3b19cbaf 100644 --- a/Sources/swift-build-sdk-interfaces/main.swift +++ b/Sources/swift-build-sdk-interfaces/main.swift @@ -17,23 +17,21 @@ import TSCUtility let diagnosticsEngine = DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]) -guard let sdkPathRaw = ProcessEnv.vars["SDKROOT"] else { - diagnosticsEngine.emit(.error("need to set SDKROOT")) - exit(1) -} - -func getArgument(_ flag: String) -> String? { +func getArgument(_ flag: String, _ env: String? = nil) -> String? { if let id = CommandLine.arguments.firstIndex(of: flag) { let nextId = id.advanced(by: 1) if nextId < CommandLine.arguments.count { return CommandLine.arguments[nextId] } } + if let env = env { + return ProcessEnv.vars[env] + } return nil } -func getArgumentAsPath(_ flag: String) throws -> AbsolutePath? { - if let raw = getArgument(flag) { +func getArgumentAsPath(_ flag: String, _ env: String? = nil) throws -> AbsolutePath? { + if let raw = getArgument(flag, env) { return try VirtualPath(path: raw).absolutePath } return nil @@ -55,7 +53,11 @@ let verbose = CommandLine.arguments.contains("-v") let skipExecution = CommandLine.arguments.contains("-n") do { - let sdkPath = try VirtualPath(path: sdkPathRaw).absolutePath! + let sdkPath = try getArgumentAsPath("-sdk", "SDKROOT") + guard let sdkPath = sdkPath else { + diagnosticsEngine.emit(.error("need to set SDKROOT")) + exit(1) + } if !localFileSystem.exists(sdkPath) { diagnosticsEngine.emit(error: "cannot find sdk: \(sdkPath.pathString)") exit(1) @@ -115,7 +117,7 @@ do { var args = ["swiftc", "-target", collector.targetTriple, tempPath.description, - "-sdk", sdkPathRaw] + "-sdk", sdkPath.pathString] let mcpFlag = "-module-cache-path" // Append module cache path if given by the client if let mcp = getArgument(mcpFlag) { From 926e6303452b01568c89e45a2b8dc8a8e97ad762 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Mon, 16 Aug 2021 11:28:57 -0700 Subject: [PATCH 8/8] Fix `guard let sdkPath = sdkPath` pattern for compatiblity with older compilers Introduced in https://github.com/apple/swift-driver/pull/796, this passed CI because swift-driver CI uses a recent compiler. This broke SwiftPM CI because it uses a 5.3 compiler which contains a bug which makes such patterns fail to build. --- Sources/swift-build-sdk-interfaces/main.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/swift-build-sdk-interfaces/main.swift b/Sources/swift-build-sdk-interfaces/main.swift index a3b19cbaf..b67d885a5 100644 --- a/Sources/swift-build-sdk-interfaces/main.swift +++ b/Sources/swift-build-sdk-interfaces/main.swift @@ -53,8 +53,8 @@ let verbose = CommandLine.arguments.contains("-v") let skipExecution = CommandLine.arguments.contains("-n") do { - let sdkPath = try getArgumentAsPath("-sdk", "SDKROOT") - guard let sdkPath = sdkPath else { + let sdkPathArg = try getArgumentAsPath("-sdk", "SDKROOT") + guard let sdkPath = sdkPathArg else { diagnosticsEngine.emit(.error("need to set SDKROOT")) exit(1) }