From 96f2b42f47a7e54e8492c7d9941a89c554e3bcb1 Mon Sep 17 00:00:00 2001 From: Ben Blackburne Date: Tue, 24 Jun 2025 20:08:37 +0100 Subject: [PATCH 1/5] Fix test build --- Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index fee8db419..b45e43607 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -536,7 +536,7 @@ final class ExplicitModuleBuildTests: XCTestCase { main.nativePathString(escaped: true)] + sdkArgumentsForTesting var driver = try Driver(args: args) let _ = try driver.planBuild() - let dependencyGraph = try XCTUnwrap(driver.explicitDependencyBuildPlanner?.dependencyGraph) + let dependencyGraph = try XCTUnwrap(driver.intermoduleDependencyGraph) let mainModuleImports = try XCTUnwrap(dependencyGraph.mainModule.importInfos) XCTAssertEqual(mainModuleImports.count, 5) XCTAssertTrue(mainModuleImports.contains(ImportInfo(importIdentifier: "Swift", From 87c242c9b3e55bf78e1c25693e9e344c3110eadb Mon Sep 17 00:00:00 2001 From: Ben Blackburne Date: Thu, 29 May 2025 16:43:05 +0100 Subject: [PATCH 2/5] Add incremental hash option to swift-driver --- .../IncrementalCompilationState+Extensions.swift | 4 ++++ .../IncrementalDependencyAndInputSetup.swift | 9 +++++++++ Sources/SwiftOptions/Options.swift | 4 ++++ 3 files changed, 17 insertions(+) diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift index d8cecc457..4fcc340ac 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift @@ -373,6 +373,10 @@ extension IncrementalCompilationState { /// Enables additional handling of explicit module build artifacts: /// Additional reading and writing of the inter-module dependency graph. public static let explicitModuleBuild = Options(rawValue: 1 << 6) + + /// Enables use of file hashes as a fallback in the case that a timestamp + /// change might invalidate a node + public static let useFileHashesInModuleDependencyGraph = Options(rawValue: 1 << 7) } } diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift index 2ee0498ff..6b4a10606 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift @@ -89,6 +89,12 @@ extension IncrementalCompilationState { if driver.parsedOptions.contains(.driverExplicitModuleBuild) { options.formUnion(.explicitModuleBuild) } + if driver.parsedOptions.hasFlag(positive: .enableIncrementalFileHashing, + negative: .disableIncrementalFileHashing, + default: false) { + options.formUnion(.useFileHashesInModuleDependencyGraph) + } + return options } } @@ -128,6 +134,9 @@ extension IncrementalCompilationState { @_spi(Testing) public var emitDependencyDotFileAfterEveryImport: Bool { options.contains(.emitDependencyDotFileAfterEveryImport) } + @_spi(Testing) public var useFileHashesInModuleDependencyGraph: Bool { + options.contains(.useFileHashesInModuleDependencyGraph) + } @_spi(Testing) public init( _ options: Options, diff --git a/Sources/SwiftOptions/Options.swift b/Sources/SwiftOptions/Options.swift index a3e4576ff..bd4fea1ae 100644 --- a/Sources/SwiftOptions/Options.swift +++ b/Sources/SwiftOptions/Options.swift @@ -213,6 +213,7 @@ extension Option { public static let disableImplicitStringProcessingModuleImport: Option = Option("-disable-implicit-string-processing-module-import", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Disable the implicit import of the _StringProcessing module.") public static let disableImplicitSwiftModules: Option = Option("-disable-implicit-swift-modules", .flag, attributes: [.frontend, .noDriver], helpText: "Disable building Swift modules implicitly by the compiler") public static let disableImportPtrauthFieldFunctionPointers: Option = Option("-disable-import-ptrauth-field-function-pointers", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Disable import of custom ptrauth qualified field function pointers") + public static let disableIncrementalFileHashing: Option = Option("-disable-incremental-file-hashing", .flag, helpText: "Disable hashing of input and dependency file data that can prevent unnecessary invalidation") public static let disableIncrementalImports: Option = Option("-disable-incremental-imports", .flag, attributes: [.frontend], helpText: "Disable cross-module incremental build metadata and driver scheduling for Swift modules") public static let disableIncrementalLlvmCodegeneration: Option = Option("-disable-incremental-llvm-codegen", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Disable incremental llvm code generation.") public static let disableInferPublicConcurrentValue: Option = Option("-disable-infer-public-sendable", .flag, attributes: [.frontend, .noDriver], helpText: "Disable inference of Sendable conformances for public structs and enums") @@ -468,6 +469,7 @@ extension Option { public static let enableFragileResilientProtocolWitnesses: Option = Option("-enable-fragile-relative-protocol-tables", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Enable relative protocol witness tables") public static let enableImplicitDynamic: Option = Option("-enable-implicit-dynamic", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Add 'dynamic' to all declarations") public static let enableImportPtrauthFieldFunctionPointers: Option = Option("-enable-import-ptrauth-field-function-pointers", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Enable import of custom ptrauth qualified field function pointers. This is on by default.") + public static let enableIncrementalFileHashing: Option = Option("-enable-incremental-file-hashing", .flag, helpText: "Enable hashing of input and dependency file data that can prevent unnecessary invalidation") public static let enableIncrementalImports: Option = Option("-enable-incremental-imports", .flag, attributes: [.frontend], helpText: "Enable cross-module incremental build metadata and driver scheduling for Swift modules") public static let enableInvalidEphemeralnessAsError: Option = Option("-enable-invalid-ephemeralness-as-error", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Diagnose invalid ephemeral to non-ephemeral conversions as errors") public static let enableLargeLoadableTypesReg2mem: Option = Option("-enable-large-loadable-types-reg2mem", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Enable large loadable types register to memory pass") @@ -1177,6 +1179,7 @@ extension Option { Option.disableImplicitStringProcessingModuleImport, Option.disableImplicitSwiftModules, Option.disableImportPtrauthFieldFunctionPointers, + Option.disableIncrementalFileHashing, Option.disableIncrementalImports, Option.disableIncrementalLlvmCodegeneration, Option.disableInferPublicConcurrentValue, @@ -1432,6 +1435,7 @@ extension Option { Option.enableFragileResilientProtocolWitnesses, Option.enableImplicitDynamic, Option.enableImportPtrauthFieldFunctionPointers, + Option.enableIncrementalFileHashing, Option.enableIncrementalImports, Option.enableInvalidEphemeralnessAsError, Option.enableLargeLoadableTypesReg2mem, From 87d7e501a4e57ce7bb8179cde115de6df575fca6 Mon Sep 17 00:00:00 2001 From: Ben Blackburne Date: Tue, 22 Apr 2025 17:39:54 +0100 Subject: [PATCH 3/5] source file hashing for incremental --- Sources/SwiftDriver/Driver/Driver.swift | 38 ++++++++++++++----- .../Execution/DriverExecutor.swift | 14 +++---- .../ModuleDependencyScanning.swift | 4 +- .../IncrementalCompilation/BuildRecord.swift | 6 +-- .../BuildRecordInfo.swift | 8 ++-- .../FirstWaveComputer.swift | 21 ++++++---- .../IncrementalCompilation/InputInfo.swift | 6 ++- .../ModuleDependencyGraph.swift | 11 ++++-- .../Jobs/EmitSupportedFeaturesJob.swift | 2 +- Sources/SwiftDriver/Jobs/Job.swift | 4 +- .../SwiftDriver/Jobs/PrintTargetInfoJob.swift | 2 +- .../ToolingInterface/SimpleExecutor.swift | 8 ++-- .../MultiJobExecutor.swift | 18 ++++----- .../SwiftDriverExecutor.swift | 8 ++-- Tests/SwiftDriverTests/JobExecutorTests.swift | 2 +- Tests/SwiftDriverTests/SwiftDriverTests.swift | 6 +-- 16 files changed, 96 insertions(+), 62 deletions(-) diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index de8ffc967..08c6ce51e 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -24,6 +24,7 @@ import struct TSCBasic.ByteString import struct TSCBasic.Diagnostic import struct TSCBasic.FileInfo import struct TSCBasic.RelativePath +import struct TSCBasic.SHA256 import var TSCBasic.localFileSystem import var TSCBasic.stderrStream import var TSCBasic.stdoutStream @@ -44,6 +45,14 @@ extension Driver.ErrorDiagnostics: CustomStringConvertible { } } +public struct FileMetadata { + public let mTime: TimePoint + public let hash: String + init(mTime: TimePoint, hash: String = "") { + self.mTime = mTime + self.hash = hash + } +} /// The Swift driver. public struct Driver { @@ -212,8 +221,8 @@ public struct Driver { /// The set of input files @_spi(Testing) public let inputFiles: [TypedVirtualPath] - /// The last time each input file was modified, recorded at the start of the build. - @_spi(Testing) public let recordedInputModificationDates: [TypedVirtualPath: TimePoint] + /// The last time each input file was modified, and the file's SHA256 hash, recorded at the start of the build. + @_spi(Testing) public let recordedInputMetadata: [TypedVirtualPath: FileMetadata] /// The mapping from input files to output files for each kind. let outputFileMap: OutputFileMap? @@ -950,11 +959,20 @@ public struct Driver { // Classify and collect all of the input files. let inputFiles = try Self.collectInputFiles(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine, fileSystem: self.fileSystem) self.inputFiles = inputFiles - self.recordedInputModificationDates = .init(uniqueKeysWithValues: - Set(inputFiles).compactMap { - guard let modTime = try? fileSystem - .lastModificationTime(for: $0.file) else { return nil } - return ($0, modTime) + + let incrementalFileHashes = parsedOptions.hasFlag(positive: .enableIncrementalFileHashing, + negative: .disableIncrementalFileHashing, + default: false) + self.recordedInputMetadata = .init(uniqueKeysWithValues: + Set(inputFiles).compactMap { inputFile -> (TypedVirtualPath, FileMetadata)? in + guard let modTime = try? fileSystem.lastModificationTime(for: inputFile.file) else { return nil } + guard let data = try? fileSystem.readFileContents(inputFile.file) else { return nil } + if incrementalFileHashes { + let hash = SHA256().hash(data).hexadecimalRepresentation + return (inputFile, FileMetadata(mTime: modTime, hash: hash)) + } else { + return (inputFile, FileMetadata(mTime: modTime)) + } }) do { @@ -1073,7 +1091,7 @@ public struct Driver { outputFileMap: outputFileMap, incremental: self.shouldAttemptIncrementalCompilation, parsedOptions: parsedOptions, - recordedInputModificationDates: recordedInputModificationDates) + recordedInputMetadata: recordedInputMetadata) self.supportedFrontendFlags = try Self.computeSupportedCompilerArgs(of: self.toolchain, @@ -1912,7 +1930,7 @@ extension Driver { } try executor.execute(job: inPlaceJob, forceResponseFiles: forceResponseFiles, - recordedInputModificationDates: recordedInputModificationDates) + recordedInputMetadata: recordedInputMetadata) } // If requested, warn for options that weren't used by the driver after the build is finished. @@ -1957,7 +1975,7 @@ extension Driver { delegate: jobExecutionDelegate, numParallelJobs: numParallelJobs ?? 1, forceResponseFiles: forceResponseFiles, - recordedInputModificationDates: recordedInputModificationDates) + recordedInputMetadata: recordedInputMetadata) } public func writeIncrementalBuildInformation(_ jobs: [Job]) { diff --git a/Sources/SwiftDriver/Execution/DriverExecutor.swift b/Sources/SwiftDriver/Execution/DriverExecutor.swift index 804119003..7fe4e800a 100644 --- a/Sources/SwiftDriver/Execution/DriverExecutor.swift +++ b/Sources/SwiftDriver/Execution/DriverExecutor.swift @@ -25,7 +25,7 @@ public protocol DriverExecutor { @discardableResult func execute(job: Job, forceResponseFiles: Bool, - recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws -> ProcessResult + recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> ProcessResult /// Execute multiple jobs, tracking job status using the provided execution delegate. /// Pass in the `IncrementalCompilationState` to allow for incremental compilation. @@ -34,7 +34,7 @@ public protocol DriverExecutor { delegate: JobExecutionDelegate, numParallelJobs: Int, forceResponseFiles: Bool, - recordedInputModificationDates: [TypedVirtualPath: TimePoint] + recordedInputMetadata: [TypedVirtualPath: FileMetadata] ) throws /// Execute multiple jobs, tracking job status using the provided execution delegate. @@ -42,7 +42,7 @@ public protocol DriverExecutor { delegate: JobExecutionDelegate, numParallelJobs: Int, forceResponseFiles: Bool, - recordedInputModificationDates: [TypedVirtualPath: TimePoint] + recordedInputMetadata: [TypedVirtualPath: FileMetadata] ) throws /// Launch a process with the given command line and report the result. @@ -96,10 +96,10 @@ extension DriverExecutor { func execute(job: Job, capturingJSONOutputAs outputType: T.Type, forceResponseFiles: Bool, - recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws -> T { + recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> T { let result = try execute(job: job, forceResponseFiles: forceResponseFiles, - recordedInputModificationDates: recordedInputModificationDates) + recordedInputMetadata: recordedInputMetadata) if (result.exitStatus != .terminated(code: EXIT_SUCCESS)) { let returnCode = Self.computeReturnCode(exitStatus: result.exitStatus) @@ -121,14 +121,14 @@ extension DriverExecutor { delegate: JobExecutionDelegate, numParallelJobs: Int, forceResponseFiles: Bool, - recordedInputModificationDates: [TypedVirtualPath: TimePoint] + recordedInputMetadata: [TypedVirtualPath: FileMetadata] ) throws { try execute( workload: .all(jobs), delegate: delegate, numParallelJobs: numParallelJobs, forceResponseFiles: forceResponseFiles, - recordedInputModificationDates: recordedInputModificationDates) + recordedInputMetadata: recordedInputMetadata) } static func computeReturnCode(exitStatus: ProcessResult.ExitStatus) -> Int { diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift index 9f1482ca8..5cc9d6d09 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift @@ -235,7 +235,7 @@ public extension Driver { try self.executor.execute(job: preScanJob, capturingJSONOutputAs: InterModuleDependencyImports.self, forceResponseFiles: forceResponseFiles, - recordedInputModificationDates: recordedInputModificationDates) + recordedInputMetadata: recordedInputMetadata) } return imports } @@ -312,7 +312,7 @@ public extension Driver { try self.executor.execute(job: scannerJob, capturingJSONOutputAs: InterModuleDependencyGraph.self, forceResponseFiles: forceResponseFiles, - recordedInputModificationDates: recordedInputModificationDates) + recordedInputMetadata: recordedInputMetadata) } return dependencyGraph } diff --git a/Sources/SwiftDriver/IncrementalCompilation/BuildRecord.swift b/Sources/SwiftDriver/IncrementalCompilation/BuildRecord.swift index a379c6abc..ecab08a3d 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/BuildRecord.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/BuildRecord.swift @@ -46,7 +46,7 @@ extension BuildRecord { init(jobs: [Job], finishedJobResults: [BuildRecordInfo.JobResult], skippedInputs: Set?, - compilationInputModificationDates: [TypedVirtualPath: TimePoint], + compilationInputModificationDates: [TypedVirtualPath: FileMetadata], actualSwiftVersion: String, argsHash: String, timeBeforeFirstJob: TimePoint, @@ -57,10 +57,10 @@ extension BuildRecord { entry.job.inputsGeneratingCode.map { ($0, entry.result) } }) let inputInfosArray = compilationInputModificationDates - .map { input, modDate -> (VirtualPath, InputInfo) in + .map { input, metadata -> (VirtualPath, InputInfo) in let status = InputInfo.Status( wasSkipped: skippedInputs?.contains(input), jobResult: jobResultsByInput[input]) - return (input.file, InputInfo(status: status, previousModTime: modDate)) + return (input.file, InputInfo(status: status, previousModTime: metadata.mTime, hash: metadata.hash)) } self.init( diff --git a/Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift b/Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift index 126b680d3..7429ea16a 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift @@ -48,7 +48,7 @@ import class Dispatch.DispatchQueue @_spi(Testing) public let actualSwiftVersion: String @_spi(Testing) public let timeBeforeFirstJob: TimePoint let diagnosticEngine: DiagnosticsEngine - let compilationInputModificationDates: [TypedVirtualPath: TimePoint] + let compilationInputModificationDates: [TypedVirtualPath: FileMetadata] private var explicitModuleDependencyGraph: InterModuleDependencyGraph? = nil private var finishedJobResults = [JobResult]() @@ -64,7 +64,7 @@ import class Dispatch.DispatchQueue actualSwiftVersion: String, timeBeforeFirstJob: TimePoint, diagnosticEngine: DiagnosticsEngine, - compilationInputModificationDates: [TypedVirtualPath: TimePoint]) + compilationInputModificationDates: [TypedVirtualPath: FileMetadata]) { self.buildRecordPath = buildRecordPath self.fileSystem = fileSystem @@ -85,7 +85,7 @@ import class Dispatch.DispatchQueue outputFileMap: OutputFileMap?, incremental: Bool, parsedOptions: ParsedOptions, - recordedInputModificationDates: [TypedVirtualPath: TimePoint] + recordedInputMetadata: [TypedVirtualPath: FileMetadata] ) { // Cannot write a buildRecord without a path. guard let buildRecordPath = try? Self.computeBuildRecordPath( @@ -99,7 +99,7 @@ import class Dispatch.DispatchQueue } let currentArgsHash = BuildRecordArguments.computeHash(parsedOptions) let compilationInputModificationDates = - recordedInputModificationDates.filter { input, _ in + recordedInputMetadata.filter { input, _ in input.type.isPartOfSwiftCompilation } diff --git a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift index 4cc0b11cc..45d136259 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift @@ -26,6 +26,7 @@ extension IncrementalCompilationState { let showJobLifecycle: Bool let alwaysRebuildDependents: Bool let explicitModulePlanner: ExplicitDependencyBuildPlanner? + let useHashes: Bool /// If non-null outputs information for `-driver-show-incremental` for input path private let reporter: Reporter? @@ -46,6 +47,7 @@ extension IncrementalCompilationState { self.alwaysRebuildDependents = initialState.incrementalOptions.contains( .alwaysRebuildDependents) self.explicitModulePlanner = explicitModulePlanner + self.useHashes = initialState.incrementalOptions.contains(.useFileHashesInModuleDependencyGraph) self.reporter = reporter } @@ -303,8 +305,9 @@ extension IncrementalCompilationState.FirstWaveComputer { /// The status of the input file. let status: InputInfo.Status /// If `true`, the modification time of this input matches the modification - /// time recorded from the prior build in the build record. - let datesMatch: Bool + /// time recorded from the prior build in the build record, or the hash of + /// its contents match. + let metadataMatch: Bool } // Find the inputs that have changed since last compilation, or were marked as needed a build @@ -313,16 +316,19 @@ extension IncrementalCompilationState.FirstWaveComputer { ) -> [ChangedInput] { jobsInPhases.compileJobs.compactMap { job in let input = job.primaryInputs[0] - let modDate = buildRecordInfo.compilationInputModificationDates[input] ?? .distantFuture + let metadata = buildRecordInfo.compilationInputModificationDates[input] ?? FileMetadata(mTime: .distantFuture) let inputInfo = outOfDateBuildRecord.inputInfos[input.file] let previousCompilationStatus = inputInfo?.status ?? .newlyAdded let previousModTime = inputInfo?.previousModTime + let previousHash = inputInfo?.hash switch previousCompilationStatus { - case .upToDate where modDate == previousModTime: + case .upToDate where metadata.mTime == previousModTime: reporter?.report("May skip current input:", input) return nil - + case .upToDate where useHashes && (metadata.hash == previousHash): + reporter?.report("May skip current input (identical hash):", input) + return nil case .upToDate: reporter?.report("Scheduling changed input", input) case .newlyAdded: @@ -332,9 +338,10 @@ extension IncrementalCompilationState.FirstWaveComputer { case .needsNonCascadingBuild: reporter?.report("Scheduling noncascading build", input) } + let metadataMatch = metadata.mTime == previousModTime || (useHashes && metadata.hash == previousHash) return ChangedInput(typedFile: input, status: previousCompilationStatus, - datesMatch: modDate == previousModTime) + metadataMatch: metadataMatch ) } } @@ -383,7 +390,7 @@ extension IncrementalCompilationState.FirstWaveComputer { ) -> [TypedVirtualPath] { changedInputs.compactMap { changedInput in let inputIsUpToDate = - changedInput.datesMatch && !inputsMissingOutputs.contains(changedInput.typedFile) + changedInput.metadataMatch && !inputsMissingOutputs.contains(changedInput.typedFile) let basename = changedInput.typedFile.file.basename // If we're asked to always rebuild dependents, all we need to do is diff --git a/Sources/SwiftDriver/IncrementalCompilation/InputInfo.swift b/Sources/SwiftDriver/IncrementalCompilation/InputInfo.swift index dc4f479fd..dcce647fb 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/InputInfo.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/InputInfo.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import struct TSCBasic.ProcessResult +import struct TSCBasic.SHA256 /// Contains information about the current status of an input to the incremental /// build. @@ -25,9 +26,12 @@ import struct TSCBasic.ProcessResult /// The last known modification time of this input. /*@_spi(Testing)*/ public let previousModTime: TimePoint - /*@_spi(Testing)*/ public init(status: Status, previousModTime: TimePoint) { + /*@_spi(Testing)*/ public let hash: String + + /*@_spi(Testing)*/ public init(status: Status, previousModTime: TimePoint, hash: String) { self.status = status self.previousModTime = previousModTime + self.hash = hash } } diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift index 621d9778a..0abeaaa03 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift @@ -923,6 +923,7 @@ extension ModuleDependencyGraph { case .inputInfo: guard record.fields.count == 5, + case .blob(let hashBlob) = record.payload, let path = try nonemptyInternedString(field: 4) else { throw malformedError @@ -931,12 +932,14 @@ extension ModuleDependencyGraph { lower: UInt32(record.fields[0]), upper: UInt32(record.fields[1]), nanoseconds: UInt32(record.fields[2])) + let status = try InputInfo.Status(code: UInt32(record.fields[3])) let pathString = path.lookup(in: internedStringTable) let pathHandle = try VirtualPath.intern(path: pathString) + let hash = String(decoding: hashBlob, as: UTF8.self) self.inputInfos[VirtualPath.lookup(pathHandle)] = InputInfo( status: status, - previousModTime: modTime) + previousModTime: modTime, hash: hash) case .moduleDepGraphNode: guard record.fields.count == 6 else { throw malformedError @@ -1162,12 +1165,12 @@ extension ModuleDependencyGraph { let inputID = input.name.intern(in: self.internedStringTable) let pathID = self.lookupIdentifierCode(for: inputID) - self.stream.writeRecord(self.abbreviations[.inputInfo]!) { + self.stream.writeRecord(self.abbreviations[.inputInfo]!, { $0.append(RecordID.inputInfo) $0.append(inputInfo.previousModTime) $0.append(inputInfo.status.code) $0.append(pathID) - } + }, blob: inputInfo.hash) } } @@ -1254,6 +1257,8 @@ extension ModuleDependencyGraph { .fixed(bitWidth: 3), // path ID .vbr(chunkBitWidth: 13), + // file hash + .blob, ]) self.abbreviate(.moduleDepGraphNode, [Bitstream.Abbreviation.Operand.literal(RecordID.moduleDepGraphNode.rawValue)] + diff --git a/Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift b/Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift index 06fce203d..4a06478b7 100644 --- a/Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift +++ b/Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift @@ -85,7 +85,7 @@ extension Driver { job: frontendFeaturesJob, capturingJSONOutputAs: SupportedCompilerFeatures.self, forceResponseFiles: false, - recordedInputModificationDates: [:]).SupportedArguments + recordedInputMetadata: [:]).SupportedArguments return Set(decodedSupportedFlagList) } diff --git a/Sources/SwiftDriver/Jobs/Job.swift b/Sources/SwiftDriver/Jobs/Job.swift index bf141db16..d77d0b9a5 100644 --- a/Sources/SwiftDriver/Jobs/Job.swift +++ b/Sources/SwiftDriver/Jobs/Job.swift @@ -151,9 +151,9 @@ extension Job { } } - public func verifyInputsNotModified(since recordedInputModificationDates: [TypedVirtualPath: TimePoint], fileSystem: FileSystem) throws { + public func verifyInputsNotModified(since recordedInputMetadata: [TypedVirtualPath: TimePoint], fileSystem: FileSystem) throws { for input in inputs { - if let recordedModificationTime = recordedInputModificationDates[input], + if let recordedModificationTime = recordedInputMetadata[input], try fileSystem.lastModificationTime(for: input.file) != recordedModificationTime { throw InputError.inputUnexpectedlyModified(input) } diff --git a/Sources/SwiftDriver/Jobs/PrintTargetInfoJob.swift b/Sources/SwiftDriver/Jobs/PrintTargetInfoJob.swift index f4bfb2d15..4920f691f 100644 --- a/Sources/SwiftDriver/Jobs/PrintTargetInfoJob.swift +++ b/Sources/SwiftDriver/Jobs/PrintTargetInfoJob.swift @@ -259,6 +259,6 @@ extension Driver { job: frontendTargetInfoJob, capturingJSONOutputAs: FrontendTargetInfo.self, forceResponseFiles: false, - recordedInputModificationDates: [:]) + recordedInputMetadata: [:]) } } diff --git a/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift b/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift index 8196c47d8..4d6330a9c 100644 --- a/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift +++ b/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift @@ -32,8 +32,8 @@ import class TSCBasic.Process } public func execute(job: Job, - forceResponseFiles: Bool, - recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws -> ProcessResult { + forceResponseFiles: Bool, + recordedInputMetadata: [TypedVirtualPath : FileMetadata]) throws -> ProcessResult { let arguments: [String] = try resolver.resolveArgumentList(for: job, useResponseFiles: .heuristic) var childEnv = env @@ -43,8 +43,8 @@ import class TSCBasic.Process } public func execute(workload: DriverExecutorWorkload, delegate: JobExecutionDelegate, - numParallelJobs: Int, forceResponseFiles: Bool, - recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws { + numParallelJobs: Int, forceResponseFiles: Bool, + recordedInputMetadata: [TypedVirtualPath : FileMetadata]) throws { fatalError("Unsupported operation on current executor") } diff --git a/Sources/SwiftDriverExecution/MultiJobExecutor.swift b/Sources/SwiftDriverExecution/MultiJobExecutor.swift index e042ecc62..b1d0f0faa 100644 --- a/Sources/SwiftDriverExecution/MultiJobExecutor.swift +++ b/Sources/SwiftDriverExecution/MultiJobExecutor.swift @@ -85,7 +85,7 @@ public final class MultiJobExecutor { let forceResponseFiles: Bool /// The last time each input file was modified, recorded at the start of the build. - public let recordedInputModificationDates: [TypedVirtualPath: TimePoint] + public let recordedInputMetadata: [TypedVirtualPath: FileMetadata] /// The diagnostics engine to use when reporting errors. let diagnosticsEngine: DiagnosticsEngine @@ -112,7 +112,7 @@ public final class MultiJobExecutor { jobQueue: OperationQueue, processSet: ProcessSet?, forceResponseFiles: Bool, - recordedInputModificationDates: [TypedVirtualPath: TimePoint], + recordedInputMetadata: [TypedVirtualPath: FileMetadata], diagnosticsEngine: DiagnosticsEngine, processType: ProcessProtocol.Type = Process.self, inputHandleOverride: FileHandle? = nil @@ -133,7 +133,7 @@ public final class MultiJobExecutor { self.jobQueue = jobQueue self.processSet = processSet self.forceResponseFiles = forceResponseFiles - self.recordedInputModificationDates = recordedInputModificationDates + self.recordedInputMetadata = recordedInputMetadata self.diagnosticsEngine = diagnosticsEngine self.processType = processType self.testInputHandle = inputHandleOverride @@ -260,7 +260,7 @@ public final class MultiJobExecutor { private let forceResponseFiles: Bool /// The last time each input file was modified, recorded at the start of the build. - private let recordedInputModificationDates: [TypedVirtualPath: TimePoint] + private let recordedInputMetadata: [TypedVirtualPath: FileMetadata] /// The diagnostics engine to use when reporting errors. private let diagnosticsEngine: DiagnosticsEngine @@ -279,7 +279,7 @@ public final class MultiJobExecutor { numParallelJobs: Int? = nil, processSet: ProcessSet? = nil, forceResponseFiles: Bool = false, - recordedInputModificationDates: [TypedVirtualPath: TimePoint] = [:], + recordedInputMetadata: [TypedVirtualPath: FileMetadata] = [:], processType: ProcessProtocol.Type = Process.self, inputHandleOverride: FileHandle? = nil ) { @@ -290,7 +290,7 @@ public final class MultiJobExecutor { self.numParallelJobs = numParallelJobs ?? 1 self.processSet = processSet self.forceResponseFiles = forceResponseFiles - self.recordedInputModificationDates = recordedInputModificationDates + self.recordedInputMetadata = recordedInputMetadata self.processType = processType self.testInputHandle = inputHandleOverride } @@ -308,8 +308,8 @@ public final class MultiJobExecutor { // Check for any inputs that were modified during the build. Report these // as errors so we don't e.g. reuse corrupted incremental build state. - for (input, recordedModTime) in context.recordedInputModificationDates { - guard try fileSystem.lastModificationTime(for: input.file) == recordedModTime else { + for (input, metadata) in context.recordedInputMetadata { + guard try fileSystem.lastModificationTime(for: input.file) == metadata.mTime else { let err = Job.InputError.inputUnexpectedlyModified(input) context.diagnosticsEngine.emit(err) throw err @@ -337,7 +337,7 @@ public final class MultiJobExecutor { jobQueue: jobQueue, processSet: processSet, forceResponseFiles: forceResponseFiles, - recordedInputModificationDates: recordedInputModificationDates, + recordedInputMetadata: recordedInputMetadata, diagnosticsEngine: diagnosticsEngine, processType: processType, inputHandleOverride: testInputHandle diff --git a/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift b/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift index 2c7478964..cf2ea16c7 100644 --- a/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift +++ b/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift @@ -41,12 +41,12 @@ public final class SwiftDriverExecutor: DriverExecutor { public func execute(job: Job, forceResponseFiles: Bool = false, - recordedInputModificationDates: [TypedVirtualPath: TimePoint] = [:]) throws -> ProcessResult { + recordedInputMetadata: [TypedVirtualPath: FileMetadata] = [:]) throws -> ProcessResult { let useResponseFiles : ResponseFileHandling = forceResponseFiles ? .forced : .heuristic let arguments: [String] = try resolver.resolveArgumentList(for: job, useResponseFiles: useResponseFiles) - try job.verifyInputsNotModified(since: recordedInputModificationDates, + try job.verifyInputsNotModified(since: recordedInputMetadata.mapValues{metadata in metadata.mTime}, fileSystem: fileSystem) if job.requiresInPlaceExecution { @@ -74,7 +74,7 @@ public final class SwiftDriverExecutor: DriverExecutor { delegate: JobExecutionDelegate, numParallelJobs: Int = 1, forceResponseFiles: Bool = false, - recordedInputModificationDates: [TypedVirtualPath: TimePoint] = [:] + recordedInputMetadata: [TypedVirtualPath: FileMetadata] = [:] ) throws { let llbuildExecutor = MultiJobExecutor( workload: workload, @@ -84,7 +84,7 @@ public final class SwiftDriverExecutor: DriverExecutor { numParallelJobs: numParallelJobs, processSet: processSet, forceResponseFiles: forceResponseFiles, - recordedInputModificationDates: recordedInputModificationDates) + recordedInputMetadata: recordedInputMetadata) try llbuildExecutor.execute(env: env, fileSystem: fileSystem) } diff --git a/Tests/SwiftDriverTests/JobExecutorTests.swift b/Tests/SwiftDriverTests/JobExecutorTests.swift index 0f51b5d70..a8e8dd56b 100644 --- a/Tests/SwiftDriverTests/JobExecutorTests.swift +++ b/Tests/SwiftDriverTests/JobExecutorTests.swift @@ -365,7 +365,7 @@ final class JobExecutorTests: XCTestCase { try localFileSystem.writeFileContents(main, bytes: "let foo = 1") // Ensure that the file modification since the start of the build planning process // results in a corresponding error. - XCTAssertThrowsError(try soleJob.verifyInputsNotModified(since: driver.recordedInputModificationDates, fileSystem: localFileSystem)) { + XCTAssertThrowsError(try soleJob.verifyInputsNotModified(since: driver.recordedInputMetadata.mapValues{$0.mTime}, fileSystem: localFileSystem)) { XCTAssertEqual($0 as? Job.InputError, .inputUnexpectedlyModified(TypedVirtualPath(file: VirtualPath.absolute(main).intern(), type: .swift))) } diff --git a/Tests/SwiftDriverTests/SwiftDriverTests.swift b/Tests/SwiftDriverTests/SwiftDriverTests.swift index 7350e5a0f..25a6099b3 100644 --- a/Tests/SwiftDriverTests/SwiftDriverTests.swift +++ b/Tests/SwiftDriverTests/SwiftDriverTests.swift @@ -367,7 +367,7 @@ final class SwiftDriverTests: XCTestCase { let driver = try Driver(args: [ "swiftc", main.pathString, utilRelative.pathString, ]) - XCTAssertEqual(driver.recordedInputModificationDates, [ + XCTAssertEqual(driver.recordedInputMetadata.mapValues{$0.mTime}, [ .init(file: VirtualPath.absolute(main).intern(), type: .swift) : mainMDate, .init(file: VirtualPath.relative(utilRelative).intern(), type: .swift) : utilMDate, ]) @@ -5724,14 +5724,14 @@ final class SwiftDriverTests: XCTestCase { struct MockExecutor: DriverExecutor { let resolver: ArgsResolver - func execute(job: Job, forceResponseFiles: Bool, recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws -> ProcessResult { + func execute(job: Job, forceResponseFiles: Bool, recordedInputMetadata: [TypedVirtualPath : FileMetadata]) throws -> ProcessResult { return ProcessResult(arguments: [], environment: [:], exitStatus: .terminated(code: 0), output: .success(Array("bad JSON".utf8)), stderrOutput: .success([])) } func execute(workload: DriverExecutorWorkload, delegate: JobExecutionDelegate, numParallelJobs: Int, forceResponseFiles: Bool, - recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws { + recordedInputMetadata: [TypedVirtualPath : FileMetadata]) throws { fatalError() } func checkNonZeroExit(args: String..., environment: [String : String]) throws -> String { From 3e2f3ca4587345edc7003389e43d1054f35f3571 Mon Sep 17 00:00:00 2001 From: Ben Blackburne Date: Fri, 25 Apr 2025 12:12:22 +0100 Subject: [PATCH 4/5] external dependency hashing for incremental --- Sources/SwiftDriver/CMakeLists.txt | 1 + Sources/SwiftDriver/Driver/Driver.swift | 11 +-- .../Execution/DriverExecutor.swift | 51 +++++++++++- .../CommonDependencyOperations.swift | 2 + .../FirstWaveComputer.swift | 2 + .../IncrementalCompilation/InputInfo.swift | 4 +- .../ModuleDependencyGraph.swift | 77 +++++++++++++++---- .../ToolingInterface/SimpleExecutor.swift | 38 ++++++++- .../SwiftDriver/Utilities/FileMetadata.swift | 25 ++++++ .../SwiftDriverExecutor.swift | 14 ++++ .../IncrementalCompilationTests.swift | 38 +++++++++ Tests/SwiftDriverTests/SwiftDriverTests.swift | 24 +++++- 12 files changed, 256 insertions(+), 31 deletions(-) create mode 100644 Sources/SwiftDriver/Utilities/FileMetadata.swift diff --git a/Sources/SwiftDriver/CMakeLists.txt b/Sources/SwiftDriver/CMakeLists.txt index 7ee618e14..9f4356841 100644 --- a/Sources/SwiftDriver/CMakeLists.txt +++ b/Sources/SwiftDriver/CMakeLists.txt @@ -113,6 +113,7 @@ add_library(SwiftDriver Utilities/DateAdditions.swift Utilities/Diagnostics.swift Utilities/FileList.swift + Utilities/FileMetadata.swift Utilities/FileType.swift Utilities/PredictableRandomNumberGenerator.swift Utilities/RelativePathAdditions.swift diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index 08c6ce51e..6f7c74ba0 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -45,15 +45,6 @@ extension Driver.ErrorDiagnostics: CustomStringConvertible { } } -public struct FileMetadata { - public let mTime: TimePoint - public let hash: String - init(mTime: TimePoint, hash: String = "") { - self.mTime = mTime - self.hash = hash - } -} - /// The Swift driver. public struct Driver { public enum Error: Swift.Error, Equatable, DiagnosticData { @@ -966,8 +957,8 @@ public struct Driver { self.recordedInputMetadata = .init(uniqueKeysWithValues: Set(inputFiles).compactMap { inputFile -> (TypedVirtualPath, FileMetadata)? in guard let modTime = try? fileSystem.lastModificationTime(for: inputFile.file) else { return nil } - guard let data = try? fileSystem.readFileContents(inputFile.file) else { return nil } if incrementalFileHashes { + guard let data = try? fileSystem.readFileContents(inputFile.file) else { return nil } let hash = SHA256().hash(data).hexadecimalRepresentation return (inputFile, FileMetadata(mTime: modTime, hash: hash)) } else { diff --git a/Sources/SwiftDriver/Execution/DriverExecutor.swift b/Sources/SwiftDriver/Execution/DriverExecutor.swift index 7fe4e800a..3b097287f 100644 --- a/Sources/SwiftDriver/Execution/DriverExecutor.swift +++ b/Sources/SwiftDriver/Execution/DriverExecutor.swift @@ -27,6 +27,11 @@ public protocol DriverExecutor { forceResponseFiles: Bool, recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> ProcessResult + func execute(job: Job, + forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws -> ProcessResult + + /// Execute multiple jobs, tracking job status using the provided execution delegate. /// Pass in the `IncrementalCompilationState` to allow for incremental compilation. /// Pass in the `InterModuleDependencyGraph` to allow for module dependency tracking. @@ -37,6 +42,13 @@ public protocol DriverExecutor { recordedInputMetadata: [TypedVirtualPath: FileMetadata] ) throws + func execute(workload: DriverExecutorWorkload, + delegate: JobExecutionDelegate, + numParallelJobs: Int, + forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath: TimePoint] + ) throws + /// Execute multiple jobs, tracking job status using the provided execution delegate. func execute(jobs: [Job], delegate: JobExecutionDelegate, @@ -45,6 +57,13 @@ public protocol DriverExecutor { recordedInputMetadata: [TypedVirtualPath: FileMetadata] ) throws + func execute(jobs: [Job], + delegate: JobExecutionDelegate, + numParallelJobs: Int, + forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath: TimePoint] + ) throws + /// Launch a process with the given command line and report the result. @discardableResult func checkNonZeroExit(args: String..., environment: [String: String]) throws -> String @@ -53,6 +72,34 @@ public protocol DriverExecutor { func description(of job: Job, forceResponseFiles: Bool) throws -> String } +extension DriverExecutor { + public func execute(job: Job, + forceResponseFiles: Bool, + recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> ProcessResult + { + return try execute(job: job, forceResponseFiles: forceResponseFiles, recordedInputModificationDates: recordedInputMetadata.mapValues { $0.mTime }) + } + + + public func execute(workload: DriverExecutorWorkload, + delegate: JobExecutionDelegate, + numParallelJobs: Int, + forceResponseFiles: Bool, + recordedInputMetadata: [TypedVirtualPath: FileMetadata] + ) throws { + try execute(workload: workload, delegate: delegate, numParallelJobs: numParallelJobs, forceResponseFiles: forceResponseFiles, recordedInputModificationDates: recordedInputMetadata.mapValues { $0.mTime }) + } + + public func execute(jobs: [Job], + delegate: JobExecutionDelegate, + numParallelJobs: Int, + forceResponseFiles: Bool, + recordedInputMetadata: [TypedVirtualPath: FileMetadata] + ) throws { + try execute(jobs: jobs, delegate: delegate, numParallelJobs: numParallelJobs, forceResponseFiles: forceResponseFiles, recordedInputModificationDates: recordedInputMetadata.mapValues { $0.mTime }) + } +} + public struct DriverExecutorWorkload { public let continueBuildingAfterErrors: Bool public enum Kind { @@ -121,14 +168,14 @@ extension DriverExecutor { delegate: JobExecutionDelegate, numParallelJobs: Int, forceResponseFiles: Bool, - recordedInputMetadata: [TypedVirtualPath: FileMetadata] + recordedInputModificationDates: [TypedVirtualPath: TimePoint] ) throws { try execute( workload: .all(jobs), delegate: delegate, numParallelJobs: numParallelJobs, forceResponseFiles: forceResponseFiles, - recordedInputMetadata: recordedInputMetadata) + recordedInputModificationDates: recordedInputModificationDates) } static func computeReturnCode(exitStatus: ProcessResult.ExitStatus) -> Int { diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift index 1c94b8129..0629dc78c 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift @@ -195,6 +195,8 @@ internal extension InterModuleDependencyGraph { reporter?.report("Unable to 'stat' \(inputPath.description)") return false } + // SHA256 hashes for these files from the previous build are not + // currently stored, so we can only check timestamps if inputModTime > outputModTime { reporter?.reportExplicitDependencyOutOfDate(moduleName, inputPath: inputPath.description) diff --git a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift index 45d136259..a8a1147d4 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift @@ -322,6 +322,8 @@ extension IncrementalCompilationState.FirstWaveComputer { let previousModTime = inputInfo?.previousModTime let previousHash = inputInfo?.hash + assert(metadata.hash != nil || !useHashes) + switch previousCompilationStatus { case .upToDate where metadata.mTime == previousModTime: reporter?.report("May skip current input:", input) diff --git a/Sources/SwiftDriver/IncrementalCompilation/InputInfo.swift b/Sources/SwiftDriver/IncrementalCompilation/InputInfo.swift index dcce647fb..a3c351460 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/InputInfo.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/InputInfo.swift @@ -26,9 +26,9 @@ import struct TSCBasic.SHA256 /// The last known modification time of this input. /*@_spi(Testing)*/ public let previousModTime: TimePoint - /*@_spi(Testing)*/ public let hash: String + /*@_spi(Testing)*/ public let hash: String? - /*@_spi(Testing)*/ public init(status: Status, previousModTime: TimePoint, hash: String) { + /*@_spi(Testing)*/ public init(status: Status, previousModTime: TimePoint, hash: String?) { self.status = status self.previousModTime = previousModTime self.hash = hash diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift index 0abeaaa03..0b3bb386b 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift @@ -14,6 +14,7 @@ import SwiftOptions import protocol TSCBasic.FileSystem import struct TSCBasic.ByteString +import struct TSCBasic.SHA256 import class Dispatch.DispatchQueue import struct Foundation.TimeInterval @@ -63,11 +64,12 @@ import struct Foundation.TimeInterval _ phase: Phase, _ internedStringTable: InternedStringTable, _ nodeFinder: NodeFinder, - _ fingerprintedExternalDependencies: Set + _ fingerprintedExternalDependencies: Set, + _ externalDependencyFileHashes: Dictionary ) { self.buildRecord = buildRecord self.currencyCache = ExternalDependencyCurrencyCache( - info.fileSystem, buildStartTime: buildRecord.buildStartTime) + info.fileSystem, buildStartTime: buildRecord.buildStartTime, externalDependencyFileHashes: externalDependencyFileHashes, useFileHashes: info.useFileHashesInModuleDependencyGraph) self.info = info self.dotFileWriter = info.emitDependencyDotFileAfterEveryImport ? DependencyGraphDotFileWriter(info) @@ -88,7 +90,7 @@ import struct Foundation.TimeInterval "If updating from prior, should be supplying more ingredients") self.init(buildRecord, info, phase, InternedStringTable(info.incrementalCompilationQueue), NodeFinder(), - Set()) + Set(), Dictionary()) } public static func createFromPrior( @@ -96,14 +98,16 @@ import struct Foundation.TimeInterval _ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup, _ internedStringTable: InternedStringTable, _ nodeFinder: NodeFinder, - _ fingerprintedExternalDependencies: Set + _ fingerprintedExternalDependencies: Set, + _ externalDependencyFileHashes: Dictionary ) -> Self { self.init(buildRecord, info, .updatingFromAPrior, internedStringTable, nodeFinder, - fingerprintedExternalDependencies) + fingerprintedExternalDependencies, + externalDependencyFileHashes) } public static func createForBuildingFromSwiftDeps( @@ -567,10 +571,24 @@ extension ModuleDependencyGraph { private let fileSystem: FileSystem private let buildStartTime: TimePoint private var currencyCache = [ExternalDependency: Bool]() + public internal(set) var externalDependencyFileHashes: [ExternalDependency: String] + private let useFileHashes: Bool - init(_ fileSystem: FileSystem, buildStartTime: TimePoint) { + init(_ fileSystem: FileSystem, buildStartTime: TimePoint, externalDependencyFileHashes: Dictionary, useFileHashes: Bool) { self.fileSystem = fileSystem self.buildStartTime = buildStartTime + self.externalDependencyFileHashes = externalDependencyFileHashes + self.useFileHashes = useFileHashes + } + + mutating func updateHash(_ externalDependency: ExternalDependency) { + guard useFileHashes else { + return + } + if let depFile = externalDependency.path, + let data = try? fileSystem.readFileContents(depFile) { + externalDependencyFileHashes[externalDependency] = SHA256().hash(data).hexadecimalRepresentation + } } mutating func beCurrent(_ externalDependency: ExternalDependency) { @@ -581,11 +599,31 @@ extension ModuleDependencyGraph { if let cachedResult = self.currencyCache[externalDependency] { return cachedResult } - let uncachedResult = isCurrentWRTFileSystem(externalDependency) + var uncachedResult = isCurrentWRTFileSystem(externalDependency) + if useFileHashes && !uncachedResult { + uncachedResult = isCurrentWRTFileHash(externalDependency) + } self.currencyCache[externalDependency] = uncachedResult + if !uncachedResult { + self.updateHash(externalDependency) + } return uncachedResult } + mutating func ensureHash(_ externalDependency: ExternalDependency) { + if (self.externalDependencyFileHashes[externalDependency] ?? "").isEmpty { + self.updateHash(externalDependency) + } + } + + private func isCurrentWRTFileHash(_ externalDependency: ExternalDependency) -> Bool { + if let depFile = externalDependency.path, + let data = try? fileSystem.readFileContents(depFile) { + return self.externalDependencyFileHashes[externalDependency] == SHA256().hash(data).hexadecimalRepresentation + } + return false + } + private func isCurrentWRTFileSystem(_ externalDependency: ExternalDependency) -> Bool { if let depFile = externalDependency.path, let fileModTime = try? self.fileSystem.lastModificationTime(for: depFile), @@ -753,6 +791,7 @@ extension ModuleDependencyGraph { private var currentDefKey: DependencyKey? = nil private var nodeUses: [(DependencyKey, Int)] = [] private var fingerprintedExternalDependencies = Set() + private var externalDependencyFileHashes = Dictionary() /// Deserialized nodes, in order appearing in the priors file. If `nil`, the node is for a removed source file. /// @@ -789,7 +828,8 @@ extension ModuleDependencyGraph { info, internedStringTable, nodeFinder, - fingerprintedExternalDependencies) + fingerprintedExternalDependencies, + externalDependencyFileHashes) for (dependencyKey, useID) in self.nodeUses { guard let use = self.potentiallyUsedNodes[useID] else { // Don't record uses of defs of removed files. @@ -982,16 +1022,19 @@ extension ModuleDependencyGraph { } self.nodeUses.append( (key, Int(record.fields[0])) ) case .externalDepNode: - guard record.fields.count == 2 + guard record.fields.count == 2, + case .blob(let hashBlob) = record.payload else { throw malformedError } let path = try internedString(field: 0) let fingerprint = try nonemptyInternedString(field: 1) + let hash = String(decoding: hashBlob, as: UTF8.self) + let externalDependency = ExternalDependency(fileName: path, internedStringTable) + fingerprintedExternalDependencies.insert( - FingerprintedExternalDependency( - ExternalDependency(fileName: path, internedStringTable), - fingerprint)) + FingerprintedExternalDependency(externalDependency, fingerprint)) + externalDependencyFileHashes[externalDependency] = hash case .identifierNode: guard record.fields.count == 0, case .blob(let identifierBlob) = record.payload @@ -1170,7 +1213,7 @@ extension ModuleDependencyGraph { $0.append(inputInfo.previousModTime) $0.append(inputInfo.status.code) $0.append(pathID) - }, blob: inputInfo.hash) + }, blob: inputInfo.hash ?? "") } } @@ -1282,6 +1325,8 @@ extension ModuleDependencyGraph { .vbr(chunkBitWidth: 13), // fingerprint ID .vbr(chunkBitWidth: 13), + // file hash + .blob, ]) self.abbreviate(.identifierNode, [ .literal(RecordID.identifierNode.rawValue), @@ -1356,13 +1401,15 @@ extension ModuleDependencyGraph { } } for fingerprintedExternalDependency in graph.fingerprintedExternalDependencies { - serializer.stream.writeRecord(serializer.abbreviations[.externalDepNode]!) { + graph.currencyCache.ensureHash(fingerprintedExternalDependency.externalDependency) + serializer.stream.writeRecord(serializer.abbreviations[.externalDepNode]!, { $0.append(RecordID.externalDepNode) $0.append(serializer.lookupIdentifierCode( for: fingerprintedExternalDependency.externalDependency.fileName)) $0.append( serializer.lookupIdentifierCode( for: fingerprintedExternalDependency.fingerprint)) - } + }, + blob: graph.currencyCache.externalDependencyFileHashes[fingerprintedExternalDependency.externalDependency] ?? "") } } return ByteString(serializer.stream.data) diff --git a/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift b/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift index 4d6330a9c..b9c9a8c77 100644 --- a/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift +++ b/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift @@ -45,9 +45,45 @@ import class TSCBasic.Process public func execute(workload: DriverExecutorWorkload, delegate: JobExecutionDelegate, numParallelJobs: Int, forceResponseFiles: Bool, recordedInputMetadata: [TypedVirtualPath : FileMetadata]) throws { - fatalError("Unsupported operation on current executor") + fatalError("Unsupported operation on current executor") } + public func execute(job: Job, + forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws -> ProcessResult { + fatalError("Unsupported legacy operation on current executor") + } + + public func execute(workload: DriverExecutorWorkload, delegate: JobExecutionDelegate, + numParallelJobs: Int, forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws { + fatalError("Unsupported operation on current executor") + } + + public func execute(jobs: [Job], + delegate: JobExecutionDelegate, + numParallelJobs: Int, + forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws { + fatalError("Unsupported legacy operation on current executor") + } + + public func execute( + jobs: [Job], + delegate: JobExecutionDelegate, + numParallelJobs: Int, + forceResponseFiles: Bool, + recordedInputMetadata: [TypedVirtualPath: FileMetadata] + ) throws { + try execute( + workload: .all(jobs), + delegate: delegate, + numParallelJobs: numParallelJobs, + forceResponseFiles: forceResponseFiles, + recordedInputMetadata: recordedInputMetadata) + } + + public func checkNonZeroExit(args: String..., environment: [String : String]) throws -> String { try Process.checkNonZeroExit(arguments: args, environment: environment) } diff --git a/Sources/SwiftDriver/Utilities/FileMetadata.swift b/Sources/SwiftDriver/Utilities/FileMetadata.swift new file mode 100644 index 000000000..2b9bfbe5b --- /dev/null +++ b/Sources/SwiftDriver/Utilities/FileMetadata.swift @@ -0,0 +1,25 @@ +//===--------------- Driver.swift - Swift Driver --------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +public struct FileMetadata { + public let mTime: TimePoint + public let hash: String? + init(mTime: TimePoint, hash: String? = nil) { + self.mTime = mTime + if let hash = hash, !hash.isEmpty { + self.hash = hash + } else { + self.hash = nil + } + } +} + diff --git a/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift b/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift index cf2ea16c7..d23647219 100644 --- a/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift +++ b/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift @@ -70,6 +70,12 @@ public final class SwiftDriverExecutor: DriverExecutor { } } + public func execute(job: Job, + forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws -> ProcessResult { + fatalError("Unsuppored legacy operation on current executor") + } + public func execute(workload: DriverExecutorWorkload, delegate: JobExecutionDelegate, numParallelJobs: Int = 1, @@ -88,6 +94,14 @@ public final class SwiftDriverExecutor: DriverExecutor { try llbuildExecutor.execute(env: env, fileSystem: fileSystem) } + public func execute(workload: DriverExecutorWorkload, + delegate: JobExecutionDelegate, + numParallelJobs: Int = 1, + forceResponseFiles: Bool = false, + recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws { + fatalError("Unsuppored legacy operation on current executor") + } + @discardableResult public func checkNonZeroExit(args: String..., environment: [String: String] = ProcessEnv.vars) throws -> String { return try Process.checkNonZeroExit(arguments: args, environment: environment) diff --git a/Tests/SwiftDriverTests/IncrementalCompilationTests.swift b/Tests/SwiftDriverTests/IncrementalCompilationTests.swift index 4c73b4a0b..1cfe420c9 100644 --- a/Tests/SwiftDriverTests/IncrementalCompilationTests.swift +++ b/Tests/SwiftDriverTests/IncrementalCompilationTests.swift @@ -345,6 +345,44 @@ extension IncrementalCompilationTests { try checkReactionToTouchingAll(checkDiagnostics: true, explicitModuleBuild: true) } + // Source file and external deps timestamps updated but contents are the same, and file-hashing is enabled + func testExplicitIncrementalBuildWithHashing() throws { + replace(contentsOf: "other", with: "import E;let bar = foo") + try buildInitialState(extraArguments: ["-enable-incremental-file-hashing"], explicitModuleBuild: true) + touch("main") + touch("other") + touch(try AbsolutePath(validating: explicitSwiftDependenciesPath.appending(component: "E.swiftinterface").pathString)) + let driver = try checkNullBuild(extraArguments: ["-enable-incremental-file-hashing"], explicitModuleBuild: true) + let mandatoryJobs = try XCTUnwrap(driver.incrementalCompilationState?.mandatoryJobsInOrder) + let mandatoryJobInputs = mandatoryJobs.flatMap { $0.inputs } .map { $0.file.basename } + XCTAssertFalse(mandatoryJobInputs.contains("main.swift")) + XCTAssertFalse(mandatoryJobInputs.contains("other.swift")) + } + + // External deps timestamp updated but contents are the same, and file-hashing is explicitly disabled + func testExplicitIncrementalBuildExternalDepsWithoutHashing() throws { + replace(contentsOf: "other", with: "import E;let bar = foo") + try buildInitialState(extraArguments: ["-disable-incremental-file-hashing"], explicitModuleBuild: true) + touch(try AbsolutePath(validating: explicitSwiftDependenciesPath.appending(component: "E.swiftinterface").pathString)) + let driver = try checkNullBuild(extraArguments: ["-disable-incremental-file-hashing"], explicitModuleBuild: true) + let mandatoryJobs = try XCTUnwrap(driver.incrementalCompilationState?.mandatoryJobsInOrder) + let mandatoryJobInputs = mandatoryJobs.flatMap { $0.inputs } .map { $0.file.basename } + XCTAssertTrue(mandatoryJobInputs.contains("other.swift")) + XCTAssertTrue(mandatoryJobInputs.contains("main.swift")) + } + + // Source file timestamps updated but contents are the same, and file-hashing is explicitly disabled + func testExplicitIncrementalBuildSourceFilesWithoutHashing() throws { + try buildInitialState(extraArguments: ["-disable-incremental-file-hashing"], explicitModuleBuild: true) + touch("main") + touch("other") + let driver = try checkNullBuild(extraArguments: ["-disable-incremental-file-hashing"], explicitModuleBuild: true) + let mandatoryJobs = try XCTUnwrap(driver.incrementalCompilationState?.mandatoryJobsInOrder) + let mandatoryJobInputs = mandatoryJobs.flatMap { $0.inputs } .map { $0.file.basename } + XCTAssertTrue(mandatoryJobInputs.contains("other.swift")) + XCTAssertTrue(mandatoryJobInputs.contains("main.swift")) + } + // Adding an import invalidates prior inter-module dependency graph. func testExplicitIncrementalBuildNewImport() throws { try buildInitialState(checkDiagnostics: false, explicitModuleBuild: true) diff --git a/Tests/SwiftDriverTests/SwiftDriverTests.swift b/Tests/SwiftDriverTests/SwiftDriverTests.swift index 25a6099b3..1d5baf3af 100644 --- a/Tests/SwiftDriverTests/SwiftDriverTests.swift +++ b/Tests/SwiftDriverTests/SwiftDriverTests.swift @@ -5740,7 +5740,29 @@ final class SwiftDriverTests: XCTestCase { func description(of job: Job, forceResponseFiles: Bool) throws -> String { fatalError() } - } + + public func execute(job: Job, + forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws -> ProcessResult { + fatalError("This DriverExecutor protocol method is only for backwards compatibility and should not be called directly") + } + + public func execute(workload: DriverExecutorWorkload, + delegate: JobExecutionDelegate, + numParallelJobs: Int, + forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws { + fatalError("This DriverExecutor protocol method is only for backwards compatibility and should not be called directly") + } + + public func execute(jobs: [Job], + delegate: JobExecutionDelegate, + numParallelJobs: Int, + forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws { + fatalError("This DriverExecutor protocol method is only for backwards compatibility and should not be called directly") + } + } // Override path to libSwiftScan to force the fallback of using the executor var hideSwiftScanEnv = ProcessEnv.vars From d1c7f969eeca4bedd49f452890727f0b3a13ebf0 Mon Sep 17 00:00:00 2001 From: Ben Blackburne Date: Thu, 29 May 2025 11:42:48 +0100 Subject: [PATCH 5/5] Bump serialized version --- .../IncrementalCompilation/ModuleDependencyGraph.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift index 0b3bb386b..e539e47fb 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift @@ -666,7 +666,8 @@ extension ModuleDependencyGraph { /// - Minor number 2: Use `.swift` files instead of `.swiftdeps` in ``DependencySource`` /// - Minor number 3: Use interned strings, including for fingerprints and use empty dependency source file for no DependencySource /// - Minor number 4: Absorb the data in the ``BuildRecord`` into the module dependency graph. - @_spi(Testing) public static let serializedGraphVersion = Version(1, 4, 0) + /// - Minor number 5: SHA256 hashes for files in externalDepNode and inputInfo blobs. + @_spi(Testing) public static let serializedGraphVersion = Version(1, 5, 0) /// The IDs of the records used by the module dependency graph. fileprivate enum RecordID: UInt64 {