From 9a0976d4e223dc51aac49273ec7749eb364c1abd Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Thu, 3 Jun 2021 15:43:03 -0700 Subject: [PATCH] [Explicit Module Builds] Add libSwiftScan API for scanner cache serialization/deserialization https://github.com/apple/swift/pull/37723 added API to libSwiftScan to save and restore the dependency scanner's state on the filesystem. This PR adds this API to the corresponding client code in the driver. --- Package.resolved | 2 +- Sources/CSwiftScan/include/swiftscan_header.h | 9 ++- .../InterModuleDependencyOracle.swift | 28 +++++++ Sources/SwiftDriver/Jobs/CompileJob.swift | 4 +- Sources/SwiftDriver/SwiftScan/SwiftScan.swift | 29 +++++++ Sources/SwiftDriver/Utilities/FileType.swift | 22 ++++-- .../ExplicitModuleBuildTests.swift | 79 +++++++++++++++++-- 7 files changed, 153 insertions(+), 20 deletions(-) diff --git a/Package.resolved b/Package.resolved index 7de9acd16..de7ee6f55 100644 --- a/Package.resolved +++ b/Package.resolved @@ -11,7 +11,7 @@ } }, { - "package": "llbuild", + "package": "swift-llbuild", "repositoryURL": "https://github.com/apple/swift-llbuild.git", "state": { "branch": "main", diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h index 27f391859..30381c176 100644 --- a/Sources/CSwiftScan/include/swiftscan_header.h +++ b/Sources/CSwiftScan/include/swiftscan_header.h @@ -200,17 +200,20 @@ typedef struct { //=== Scanner Functions ---------------------------------------------------===// swiftscan_scanner_t (*swiftscan_scanner_create)(void); void (*swiftscan_scanner_dispose)(swiftscan_scanner_t); - swiftscan_dependency_graph_t (*swiftscan_dependency_graph_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t); - swiftscan_batch_scan_result_t * (*swiftscan_batch_scan_result_create)(swiftscan_scanner_t, swiftscan_batch_scan_input_t *, swiftscan_scan_invocation_t); - swiftscan_import_set_t (*swiftscan_import_set_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t); + + //=== Scanner Cache Functions ---------------------------------------------===// + void (*swiftscan_scanner_cache_serialize)(swiftscan_scanner_t scanner, const char * path); + bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path); + void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner); + } swiftscan_functions_t; #endif // SWIFT_C_DEPENDENCY_SCAN_H diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift index e468da237..66abc8741 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift @@ -73,6 +73,34 @@ public class InterModuleDependencyOracle { } } + @_spi(Testing) public func serializeScannerCache(to path: AbsolutePath) { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to serialize scanner cache with no scanner instance.") + } + if swiftScan.canLoadStoreScannerCache() { + swiftScan.serializeScannerCache(to: path) + } + } + + @_spi(Testing) public func loadScannerCache(from path: AbsolutePath) -> Bool { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to load scanner cache with no scanner instance.") + } + if swiftScan.canLoadStoreScannerCache() { + return swiftScan.loadScannerCache(from: path) + } + return false + } + + @_spi(Testing) public func resetScannerCache() { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to reset scanner cache with no scanner instance.") + } + if swiftScan.canLoadStoreScannerCache() { + swiftScan.resetScannerCache() + } + } + private var hasScannerInstance: Bool { self.swiftScanLibInstance != nil } /// Queue to sunchronize accesses to the scanner diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift index 894b32b0a..01bdc3883 100644 --- a/Sources/SwiftDriver/Jobs/CompileJob.swift +++ b/Sources/SwiftDriver/Jobs/CompileJob.swift @@ -93,7 +93,7 @@ extension Driver { .privateSwiftInterface, .swiftSourceInfoFile, .diagnostics, .objcHeader, .swiftDeps, .remap, .tbd, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm, .pch, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, - .indexUnitOutputPath, nil: + .indexUnitOutputPath, .modDepCache, nil: return false } } @@ -444,7 +444,7 @@ extension FileType { .diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts, - .indexUnitOutputPath: + .indexUnitOutputPath, .modDepCache: fatalError("Output type can never be a primary output") } } diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index 2b03741b6..b58fe3b15 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -181,6 +181,26 @@ internal final class SwiftScan { return resultGraphMap } + @_spi(Testing) public func canLoadStoreScannerCache() -> Bool { + return api.swiftscan_scanner_cache_load != nil && + api.swiftscan_scanner_cache_serialize != nil && + api.swiftscan_scanner_cache_reset != nil + } + + func serializeScannerCache(to path: AbsolutePath) { + api.swiftscan_scanner_cache_serialize(scanner, + path.description.cString(using: String.Encoding.utf8)) + } + + func loadScannerCache(from path: AbsolutePath) -> Bool { + return api.swiftscan_scanner_cache_load(scanner, + path.description.cString(using: String.Encoding.utf8)) + } + + func resetScannerCache() { + api.swiftscan_scanner_cache_reset(scanner) + } + @_spi(Testing) public func canQuerySupportedArguments() -> Bool { return api.swiftscan_compiler_supported_arguments_query != nil && api.swiftscan_string_set_dispose != nil @@ -226,6 +246,7 @@ private extension swiftscan_functions_t { } return sym } + // Supported features/flags query self.swiftscan_string_set_dispose = try loadOptional("swiftscan_string_set_dispose") self.swiftscan_compiler_supported_arguments_query = @@ -233,6 +254,14 @@ private extension swiftscan_functions_t { self.swiftscan_compiler_supported_features_query = try loadOptional("swiftscan_compiler_supported_features_query") + // Dependency scanner serialization/deserialization features + self.swiftscan_scanner_cache_serialize = + try loadOptional("swiftscan_scanner_cache_serialize") + self.swiftscan_scanner_cache_load = + try loadOptional("swiftscan_scanner_cache_load") + self.swiftscan_scanner_cache_reset = + try loadOptional("swiftscan_scanner_cache_reset") + // MARK: Required Methods func loadRequired(_ symbol: String) throws -> T { guard let sym: T = dlsym(swiftscan, symbol: symbol) else { diff --git a/Sources/SwiftDriver/Utilities/FileType.swift b/Sources/SwiftDriver/Utilities/FileType.swift index 2cac81fc4..fd84698da 100644 --- a/Sources/SwiftDriver/Utilities/FileType.swift +++ b/Sources/SwiftDriver/Utilities/FileType.swift @@ -81,6 +81,9 @@ public enum FileType: String, Hashable, CaseIterable, Codable { /// Swift dependencies file. case swiftDeps = "swiftdeps" + /// Serialized dependency scanner state + case modDepCache = "moddepcache" + /// Remapping file case remap @@ -109,9 +112,9 @@ public enum FileType: String, Hashable, CaseIterable, Codable { /// Swift section of the internal wiki. case moduleTrace = "trace.json" - /// Indexing data directory. + /// Indexing data directory /// - /// The extension isn't real. + /// The extension isn't real, rather this FileType specifies a directory path. case indexData /// Output path to record in the indexing data store @@ -119,10 +122,10 @@ public enum FileType: String, Hashable, CaseIterable, Codable { /// This is only needed for use as a key in the output file map. case indexUnitOutputPath - /// Optimization record. + /// Optimization record case yamlOptimizationRecord = "opt.yaml" - /// Bitstream optimization record. + /// Bitstream optimization record case bitstreamOptimizationRecord = "opt.bitstream" /// Clang compiler module file @@ -169,6 +172,9 @@ extension FileType: CustomStringConvertible { case .swiftDeps: return "swift-dependencies" + case .modDepCache: + return "dependency-scanner-cache" + case .jsonDependencies: return "json-dependencies" @@ -218,7 +224,7 @@ extension FileType { .swiftDeps, .moduleTrace, .tbd, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, .jsonDependencies, .clangModuleMap, .jsonTargetInfo, .jsonCompilerFeatures, .jsonSwiftArtifacts, - .indexUnitOutputPath: + .indexUnitOutputPath, .modDepCache: return false } } @@ -289,6 +295,8 @@ extension FileType { return "objc-header" case .swiftDeps: return "swift-dependencies" + case .modDepCache: + return "dependency-scanner-cache" case .jsonDependencies: return "json-dependencies" case .jsonTargetInfo: @@ -327,7 +335,7 @@ extension FileType { case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule, .swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics, .pcm, .swiftDeps, .remap, .indexData, .bitstreamOptimizationRecord, - .indexUnitOutputPath: + .indexUnitOutputPath, .modDepCache: return false } } @@ -341,7 +349,7 @@ extension FileType { case .swift, .sil, .sib, .ast, .image, .dSYM, .dependencies, .autolink, .swiftModule, .swiftDocumentation, .swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, .raw_sil, .raw_sib, .diagnostics, .objcHeader, .swiftDeps, .remap, - .importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord, + .importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord, .modDepCache, .bitstreamOptimizationRecord, .pcm, .pch, .jsonDependencies, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, .indexUnitOutputPath: return false diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index 7dc3f2a00..c4c438c5a 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -601,24 +601,31 @@ final class ExplicitModuleBuildTests: XCTestCase { } } - /// Test the libSwiftScan dependency scanning. - func testDependencyScanning() throws { + private func getDriverArtifactsForScanning() throws -> (stdLibPath: AbsolutePath, + shimsPath: AbsolutePath, + toolchain: Toolchain, + hostTriple: Triple) { // Just instantiating to get at the toolchain path let driver = try Driver(args: ["swiftc", "-experimental-explicit-module-build", "-module-name", "testDependencyScanning", "test.swift"]) let (stdLibPath, shimsPath) = try getStdlibShimsPaths(driver) - XCTAssertTrue(localFileSystem.exists(stdLibPath), "expected Swift StdLib at: \(stdLibPath.description)") XCTAssertTrue(localFileSystem.exists(shimsPath), "expected Swift Shims at: \(shimsPath.description)") + return (stdLibPath, shimsPath, driver.toolchain, driver.hostTriple) + } + + /// Test the libSwiftScan dependency scanning. + func testDependencyScanning() throws { + let (stdLibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning() // The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across // queries. let dependencyOracle = InterModuleDependencyOracle() - let scanLibPath = try Driver.getScanLibPath(of: driver.toolchain, - hostTriple: driver.hostTriple, + let scanLibPath = try Driver.getScanLibPath(of: toolchain, + hostTriple: hostTriple, env: ProcessEnv.vars) guard try dependencyOracle .verifyOrCreateScannerInstance(fileSystem: localFileSystem, @@ -653,8 +660,8 @@ final class ExplicitModuleBuildTests: XCTestCase { // Module `X` is only imported on Darwin when: // #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 110000 let expectedNumberOfDependencies: Int - if driver.targetTriple.isMacOSX, - driver.targetTriple.version(for: .macOS) >= Triple.Version(11, 0, 0) { + if hostTriple.isMacOSX, + hostTriple.version(for: .macOS) >= Triple.Version(11, 0, 0) { expectedNumberOfDependencies = 11 } else { expectedNumberOfDependencies = 12 @@ -692,6 +699,64 @@ final class ExplicitModuleBuildTests: XCTestCase { } } + + /// Test the libSwiftScan dependency scanning. + func testDependencyScanReuseCache() throws { + let (stdLibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning() + try withTemporaryDirectory { path in + let cacheSavePath = path.appending(component: "saved.moddepcache") + let main = path.appending(component: "testDependencyScanning.swift") + try localFileSystem.writeFileContents(main) { + $0 <<< "import C;" + $0 <<< "import E;" + $0 <<< "import G;" + } + let packageRootPath = URL(fileURLWithPath: #file).pathComponents + .prefix(while: { $0 != "Tests" }).joined(separator: "/").dropFirst() + let testInputsPath = packageRootPath + "/TestInputs" + let cHeadersPath : String = testInputsPath + "/ExplicitModuleBuilds/CHeaders" + let swiftModuleInterfacesPath : String = testInputsPath + "/ExplicitModuleBuilds/Swift" + let scannerCommand = ["-scan-dependencies", + "-I", cHeadersPath, + "-I", swiftModuleInterfacesPath, + "-I", stdLibPath.description, + "-I", shimsPath.description, + main.pathString] + + let scanLibPath = try Driver.getScanLibPath(of: toolchain, + hostTriple: hostTriple, + env: ProcessEnv.vars) + // Run the first scan and serialize the cache contents. + let firstDependencyOracle = InterModuleDependencyOracle() + guard try firstDependencyOracle + .verifyOrCreateScannerInstance(fileSystem: localFileSystem, + swiftScanLibPath: scanLibPath) else { + XCTFail("Dependency scanner library not found") + return + } + + let firstScanGraph = + try! firstDependencyOracle.getDependencies(workingDirectory: path, + commandLine: scannerCommand) + firstDependencyOracle.serializeScannerCache(to: cacheSavePath) + + // Run the second scan, re-using the serialized cache contents. + let secondDependencyOracle = InterModuleDependencyOracle() + guard try secondDependencyOracle + .verifyOrCreateScannerInstance(fileSystem: localFileSystem, + swiftScanLibPath: scanLibPath) else { + XCTFail("Dependency scanner library not found") + return + } + XCTAssertFalse(secondDependencyOracle.loadScannerCache(from: cacheSavePath)) + let secondScanGraph = + try! secondDependencyOracle.getDependencies(workingDirectory: path, + commandLine: scannerCommand) + + XCTAssertTrue(firstScanGraph.modules.count == secondScanGraph.modules.count) + } + } + func testDependencyGraphMerge() throws { let moduleDependencyGraph1 = try JSONDecoder().decode(