From a4a1b96a71f4f5df5f4341bbe26845465ea0e4e1 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 11 Dec 2023 14:20:21 -0800 Subject: [PATCH] [Caching] Adopt new APIs from libSwiftScan for key computation Using the new APIs from libSwiftScan for cache key computation. The new API doesn't require swift-driver and scanner to have the same file system view for the path of the input files, which eliminates one failure point. rdar://119517627 --- Sources/CSwiftScan/include/swiftscan_header.h | 3 +++ Sources/SwiftDriver/Driver/Driver.swift | 4 ++-- Sources/SwiftDriver/Jobs/CompileJob.swift | 6 +++++- Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift | 10 ++++------ Sources/SwiftDriver/Jobs/GeneratePCHJob.swift | 2 +- Sources/SwiftDriver/Jobs/GeneratePCMJob.swift | 2 +- .../Jobs/VerifyModuleInterfaceJob.swift | 4 ++-- Sources/SwiftDriver/SwiftScan/SwiftScan.swift | 2 ++ Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift | 14 ++++++++++++++ Tests/SwiftDriverTests/CachingBuildTests.swift | 2 +- 10 files changed, 35 insertions(+), 14 deletions(-) diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h index 422ee8a84..377de2bec 100644 --- a/Sources/CSwiftScan/include/swiftscan_header.h +++ b/Sources/CSwiftScan/include/swiftscan_header.h @@ -291,6 +291,9 @@ typedef struct { swiftscan_string_ref_t (*swiftscan_cache_compute_key)( swiftscan_cas_t cas, int argc, const char **argv, const char *input, swiftscan_string_ref_t *error); + swiftscan_string_ref_t (*swiftscan_cache_compute_key_from_input_index)( + swiftscan_cas_t cas, int argc, const char **argv, unsigned input_index, + swiftscan_string_ref_t *error); //=== Scanner Caching Query/Replay Operations -----------------------------===// swiftscan_cached_compilation_t (*swiftscan_cache_query)( diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index 323578c6b..da55c1480 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -279,7 +279,7 @@ public struct Driver { /// Is swift caching enabled. lazy var isCachingEnabled: Bool = { - return enableCaching && isFeatureSupported(.cache_compile_job) + return enableCaching && isFeatureSupported(.compilation_caching) }() /// Scanner prefix mapping. @@ -427,7 +427,7 @@ public struct Driver { @_spi(Testing) public enum KnownCompilerFeature: String { case emit_abi_descriptor = "emit-abi-descriptor" - case cache_compile_job = "cache-compile-job" + case compilation_caching = "compilation-caching" } lazy var sdkPath: VirtualPath? = { diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift index 54540eed1..bb76e90e9 100644 --- a/Sources/SwiftDriver/Jobs/CompileJob.swift +++ b/Sources/SwiftDriver/Jobs/CompileJob.swift @@ -398,7 +398,11 @@ extension Driver { displayInputs = primaryInputs } // Only swift input files are contributing to the cache keys. - let cacheContributingInputs = displayInputs.filter() { $0.type == .swift } + let cacheContributingInputs = inputs.enumerated().reduce(into: [(TypedVirtualPath, Int)]()) { result, input in + if input.element.type == .swift, displayInputs.contains(input.element) { + result.append((input.element, input.offset)) + } + } let cacheKeys = try computeOutputCacheKeyForJob(commandLine: commandLine, inputs: cacheContributingInputs) return Job( diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index e872f0e5a..18c534fbd 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -853,7 +853,7 @@ extension Driver { extension Driver { public mutating func computeOutputCacheKeyForJob(commandLine: [Job.ArgTemplate], - inputs: [TypedVirtualPath]) throws -> [TypedVirtualPath: String] { + inputs: [(TypedVirtualPath, Int)]) throws -> [TypedVirtualPath: String] { // No caching setup, return empty dictionary. guard let cas = self.cas else { return [:] @@ -863,13 +863,12 @@ extension Driver { let arguments: [String] = try resolver.resolveArgumentList(for: commandLine) return try inputs.reduce(into: [:]) { keys, input in - let remappedPath = remapPath(input.file) - keys[input] = try cas.computeCacheKey(commandLine: arguments, input: remappedPath.name) + keys[input.0] = try cas.computeCacheKey(commandLine: arguments, index: input.1) } } public mutating func computeOutputCacheKey(commandLine: [Job.ArgTemplate], - input: TypedVirtualPath) throws -> String? { + index: Int) throws -> String? { // No caching setup, return empty dictionary. guard let cas = self.cas else { return nil @@ -877,8 +876,7 @@ extension Driver { // Resolve command-line first. let resolver = try ArgsResolver(fileSystem: fileSystem) let arguments: [String] = try resolver.resolveArgumentList(for: commandLine) - let remappedPath = remapPath(input.file) - return try cas.computeCacheKey(commandLine: arguments, input: remappedPath.name) + return try cas.computeCacheKey(commandLine: arguments, index: index) } } diff --git a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift index 9628c1fa7..29c58d00d 100644 --- a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift +++ b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift @@ -70,7 +70,7 @@ extension Driver { inputs.append(input) try addPathArgument(input.file, to: &commandLine) - let cacheKeys = try computeOutputCacheKeyForJob(commandLine: commandLine, inputs: [input]) + let cacheKeys = try computeOutputCacheKeyForJob(commandLine: commandLine, inputs: [(input, 0)]) return Job( moduleName: moduleOutputInfo.name, diff --git a/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift index 225a4bffe..6e25aa72b 100644 --- a/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift +++ b/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift @@ -51,7 +51,7 @@ extension Driver { commandLine: &commandLine, inputs: &inputs, kind: .generatePCM, bridgingHeaderHandling: .ignored) try commandLine.appendLast(.indexStorePath, from: &parsedOptions) - let cacheKeys = try computeOutputCacheKeyForJob(commandLine: commandLine, inputs: [input]) + let cacheKeys = try computeOutputCacheKeyForJob(commandLine: commandLine, inputs: [(input, 0)]) return Job( moduleName: moduleOutputInfo.name, diff --git a/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift b/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift index 896631d25..303892665 100644 --- a/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift +++ b/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift @@ -20,7 +20,7 @@ extension Driver { // Assume swiftinterface file is always the supplementary output for first input file. let key = try computeOutputCacheKey(commandLine: emitModuleJob.commandLine, - input: emitModuleJob.inputs[0]) + index: 0) return key } @@ -61,7 +61,7 @@ extension Driver { commandLine.appendFlag(.downgradeTypecheckInterfaceError) } - let cacheKeys = try computeOutputCacheKeyForJob(commandLine: commandLine, inputs: [interfaceInput]) + let cacheKeys = try computeOutputCacheKeyForJob(commandLine: commandLine, inputs: [(interfaceInput, 0)]) return Job( moduleName: moduleOutputInfo.name, kind: .verifyModuleInterface, diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index 86463aa66..3d4dee314 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -287,6 +287,7 @@ internal extension swiftscan_diagnostic_severity_t { api.swiftscan_cas_create_from_options != nil && api.swiftscan_cas_dispose != nil && api.swiftscan_cache_compute_key != nil && + api.swiftscan_cache_compute_key_from_input_index != nil && api.swiftscan_cas_store != nil && api.swiftscan_swift_textual_detail_get_module_cache_key != nil && api.swiftscan_swift_binary_detail_get_module_cache_key != nil && @@ -507,6 +508,7 @@ private extension swiftscan_functions_t { self.swiftscan_cas_create_from_options = try loadOptional("swiftscan_cas_create_from_options") self.swiftscan_cas_dispose = try loadOptional("swiftscan_cas_dispose") self.swiftscan_cache_compute_key = try loadOptional("swiftscan_cache_compute_key") + self.swiftscan_cache_compute_key_from_input_index = try loadOptional("swiftscan_cache_compute_key_from_input_index") self.swiftscan_cas_store = try loadOptional("swiftscan_cas_store") self.swiftscan_cache_query = try loadOptional("swiftscan_cache_query") diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift b/Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift index 691d41ed5..668e269a3 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift @@ -198,6 +198,7 @@ public final class SwiftScanCAS { return try scanner.toSwiftString(casid) } + @available(*, deprecated) public func computeCacheKey(commandLine: [String], input: String) throws -> String { let casid = try scanner.handleCASError { err_msg in withArrayOfCStrings(commandLine) { commandArray in @@ -211,6 +212,19 @@ public final class SwiftScanCAS { return try scanner.toSwiftString(casid) } + public func computeCacheKey(commandLine: [String], index: Int) throws -> String { + let casid = try scanner.handleCASError { err_msg in + withArrayOfCStrings(commandLine) { commandArray in + scanner.api.swiftscan_cache_compute_key_from_input_index(cas, + Int32(commandLine.count), + commandArray, + UInt32(index), + &err_msg) + } + } + return try scanner.toSwiftString(casid) + } + public func createReplayInstance(commandLine: [String]) throws -> CacheReplayInstance { let instance = try scanner.handleCASError { err_msg in withArrayOfCStrings(commandLine) { commandArray in diff --git a/Tests/SwiftDriverTests/CachingBuildTests.swift b/Tests/SwiftDriverTests/CachingBuildTests.swift index 177080282..ea75a309c 100644 --- a/Tests/SwiftDriverTests/CachingBuildTests.swift +++ b/Tests/SwiftDriverTests/CachingBuildTests.swift @@ -215,7 +215,7 @@ final class CachingBuildTests: XCTestCase { #if os(Windows) throw XCTSkip("caching not supported on windows") #else - guard driver.isFeatureSupported(.cache_compile_job) else { + guard driver.isFeatureSupported(.compilation_caching) else { throw XCTSkip("caching not supported") } #endif