From ccc4024de3ad0b8f185ae3ebc8efec62a6258493 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Thu, 29 Apr 2021 17:05:37 -0700 Subject: [PATCH] Refactor `IncrementalCompilationState` to compute initial state before job-generation. In the near future, planning of `PrecompileModuleDependenciesJobs`, for explicit module dependencies, will require access to the incremental state in order to incrementalize expensive actions such as dependency scanning. This change refactors the creation of incremental compilation state in order to move computation of the initial state (dependency graph) to occur before job-generation. The first wave of jobs is then computed after job-generation, using the initial state, and the set of jobs in the plan, as input. - Separate `InitialIncrementalStateComputer` to run early, by the `Driver`, to compute the dependency graph and the set of changed inputs. Rename `InitialIncrementalStateComputer` into `IncrementalDependencyAndInputSetup`. - Introduce `FirstWaveComputer` which the constructor of the `IncrementalCompilationContext` uses to compute the `mandatoryJobsInOrder` set of jobs the executors *must* run. --- Sources/SwiftDriver/CMakeLists.txt | 3 +- .../DependencyGraphDotFileWriter.swift | 4 +- ...Computer.swift => FirstWaveComputer.swift} | 246 ++++------------ .../IncrementalCompilationState.swift | 98 +++---- .../IncrementalDependencyAndInputSetup.swift | 272 ++++++++++++++++++ .../ModuleDependencyGraph.swift | 10 +- Sources/SwiftDriver/Jobs/Planning.swift | 88 +++--- .../MockingIncrementalCompilation.swift | 15 +- .../IncrementalCompilationTests.swift | 2 +- .../ModuleDependencyGraphTests.swift | 2 +- 10 files changed, 423 insertions(+), 317 deletions(-) rename Sources/SwiftDriver/IncrementalCompilation/{InitialStateComputer.swift => FirstWaveComputer.swift} (52%) create mode 100644 Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift diff --git a/Sources/SwiftDriver/CMakeLists.txt b/Sources/SwiftDriver/CMakeLists.txt index 4300c981b..b39d96d1d 100644 --- a/Sources/SwiftDriver/CMakeLists.txt +++ b/Sources/SwiftDriver/CMakeLists.txt @@ -47,7 +47,8 @@ add_library(SwiftDriver "IncrementalCompilation/DirectAndTransitiveCollections.swift" "IncrementalCompilation/ExternalDependencyAndFingerprintEnforcer.swift" "IncrementalCompilation/IncrementalCompilationState.swift" - "IncrementalCompilation/InitialStateComputer.swift" + "IncrementalCompilation/IncrementalDependencyAndInputSetup.swift" + "IncrementalCompilation/FirstWaveComputer.swift" "IncrementalCompilation/InputInfo.swift" "IncrementalCompilation/KeyAndFingerprintHolder.swift" "IncrementalCompilation/ModuleDependencyGraph.swift" diff --git a/Sources/SwiftDriver/IncrementalCompilation/DependencyGraphDotFileWriter.swift b/Sources/SwiftDriver/IncrementalCompilation/DependencyGraphDotFileWriter.swift index 8ff43355a..97fb5273a 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/DependencyGraphDotFileWriter.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/DependencyGraphDotFileWriter.swift @@ -14,11 +14,11 @@ import TSCBasic // MARK: - Asking to write dot files / interface public class DependencyGraphDotFileWriter { /// Holds file-system and options - private let info: IncrementalCompilationState.InitialStateComputer + private let info: IncrementalCompilationState.IncrementalDependencyAndInputSetup private var versionNumber = 0 - init(_ info: IncrementalCompilationState.InitialStateComputer) { + init(_ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup) { self.info = info } diff --git a/Sources/SwiftDriver/IncrementalCompilation/InitialStateComputer.swift b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift similarity index 52% rename from Sources/SwiftDriver/IncrementalCompilation/InitialStateComputer.swift rename to Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift index fff3ebbcd..362d803c8 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/InitialStateComputer.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift @@ -1,8 +1,9 @@ -//===--------------- IncrementalCompilationStateComputer.swift - Incremental -----------===// +import Foundation +//===--------------- FirstWaveComputer.swift - Incremental --------------===// // // This source file is part of the Swift.org open source project // -// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -10,200 +11,65 @@ // //===----------------------------------------------------------------------===// import TSCBasic -import Foundation - -/// Builds the IncrementalCompilationState -/// Also bundles up an bunch of configuration info extension IncrementalCompilationState { - public struct InitialStateComputer { - @_spi(Testing) public let jobsInPhases: JobsInPhases - @_spi(Testing) public let outputFileMap: OutputFileMap - @_spi(Testing) public let buildRecordInfo: BuildRecordInfo - @_spi(Testing) public let maybeBuildRecord: BuildRecord? - @_spi(Testing) public let reporter: IncrementalCompilationState.Reporter? - @_spi(Testing) public let inputFiles: [TypedVirtualPath] - @_spi(Testing) public let fileSystem: FileSystem - @_spi(Testing) public let showJobLifecycle: Bool - @_spi(Testing) public let sourceFiles: SourceFiles - @_spi(Testing) public let diagnosticEngine: DiagnosticsEngine - @_spi(Testing) public let readPriorsFromModuleDependencyGraph: Bool - @_spi(Testing) public let alwaysRebuildDependents: Bool - @_spi(Testing) public let isCrossModuleIncrementalBuildEnabled: Bool - @_spi(Testing) public let verifyDependencyGraphAfterEveryImport: Bool - @_spi(Testing) public let emitDependencyDotFileAfterEveryImport: Bool - - /// Options, someday - @_spi(Testing) public let dependencyDotFilesIncludeExternals: Bool = true - @_spi(Testing) public let dependencyDotFilesIncludeAPINotes: Bool = false - - @_spi(Testing) public let buildStartTime: Date - @_spi(Testing) public let buildEndTime: Date + struct FirstWaveComputer { + let moduleDependencyGraph: ModuleDependencyGraph + let jobsInPhases: JobsInPhases + let inputsInvalidatedByExternals: TransitivelyInvalidatedInputSet + let inputFiles: [TypedVirtualPath] + let sourceFiles: SourceFiles + let buildRecordInfo: BuildRecordInfo + let maybeBuildRecord: BuildRecord? + let fileSystem: FileSystem + let showJobLifecycle: Bool + let alwaysRebuildDependents: Bool + /// If non-null outputs information for `-driver-show-incremental` for input path + private let reporter: Reporter? @_spi(Testing) public init( - _ options: IncrementalCompilationState.Options, - _ jobsInPhases: JobsInPhases, - _ outputFileMap: OutputFileMap, - _ buildRecordInfo: BuildRecordInfo, - _ buildRecord: BuildRecord?, - _ reporter: IncrementalCompilationState.Reporter?, - _ inputFiles: [TypedVirtualPath], - _ fileSystem: FileSystem, - showJobLifecycle: Bool, - _ diagnosticEngine: DiagnosticsEngine + initialState: IncrementalCompilationState.InitialStateForPlanning, + jobsInPhases: JobsInPhases, + driver: Driver, + reporter: Reporter? ) { + self.moduleDependencyGraph = initialState.graph self.jobsInPhases = jobsInPhases - self.outputFileMap = outputFileMap - self.buildRecordInfo = buildRecordInfo - self.maybeBuildRecord = buildRecord + self.inputsInvalidatedByExternals = initialState.inputsInvalidatedByExternals + self.inputFiles = driver.inputFiles + self.sourceFiles = SourceFiles( + inputFiles: inputFiles, + buildRecord: initialState.maybeBuildRecord) + self.buildRecordInfo = initialState.buildRecordInfo + self.maybeBuildRecord = initialState.maybeBuildRecord + self.fileSystem = driver.fileSystem + self.showJobLifecycle = driver.showJobLifecycle + self.alwaysRebuildDependents = initialState.incrementalOptions.contains( + .alwaysRebuildDependents) self.reporter = reporter - self.inputFiles = inputFiles - self.fileSystem = fileSystem - self.showJobLifecycle = showJobLifecycle - assert(outputFileMap.onlySourceFilesHaveSwiftDeps()) - self.sourceFiles = SourceFiles(inputFiles: inputFiles, - buildRecord: buildRecord) - self.diagnosticEngine = diagnosticEngine - // Do not try to reuse a graph from a different compilation, so check - // the build record. - self.readPriorsFromModuleDependencyGraph = maybeBuildRecord != nil && - options.contains(.readPriorsFromModuleDependencyGraph) - self.alwaysRebuildDependents = options.contains(.alwaysRebuildDependents) - self.isCrossModuleIncrementalBuildEnabled = options.contains(.enableCrossModuleIncrementalBuild) - self.verifyDependencyGraphAfterEveryImport = options.contains(.verifyDependencyGraphAfterEveryImport) - self.emitDependencyDotFileAfterEveryImport = options.contains(.emitDependencyDotFileAfterEveryImport) - self.buildStartTime = maybeBuildRecord?.buildStartTime ?? .distantPast - self.buildEndTime = maybeBuildRecord?.buildEndTime ?? .distantFuture } - func compute(batchJobFormer: inout Driver) - throws -> InitialState? { - guard sourceFiles.disappeared.isEmpty else { - // Would have to cleanse nodes of disappeared inputs from graph - // and would have to schedule files dependening on defs from disappeared nodes - if let reporter = reporter { - reporter.report( - "Incremental compilation has been disabled, " + - " because the following inputs were used in the previous compilation but not in this one: " - + sourceFiles.disappeared.map {$0.basename} .joined(separator: ", ")) - } - return nil - } - - guard let (graph, inputsInvalidatedByExternals) = - computeGraphAndInputsInvalidatedByExternals() - else { - return nil - } - let (skippedCompileGroups: skippedCompileGroups, mandatoryJobsInOrder) = - try computeInputsAndGroups( - graph, - inputsInvalidatedByExternals, - batchJobFormer: &batchJobFormer) - - return InitialState(graph: graph, - skippedCompileGroups: skippedCompileGroups, - mandatoryJobsInOrder: mandatoryJobsInOrder, - buildStartTime: buildStartTime, - buildEndTime: buildEndTime) + public func compute(batchJobFormer: inout Driver) throws -> FirstWave { + let (skippedCompileGroups, mandatoryJobsInOrder) = + try computeInputsAndGroups(batchJobFormer: &batchJobFormer) + return FirstWave( + skippedCompileGroups: skippedCompileGroups, + mandatoryJobsInOrder: mandatoryJobsInOrder) } } } -// MARK: - building/reading the ModuleDependencyGraph & scheduling externals for 1st wave -extension IncrementalCompilationState.InitialStateComputer { - - /// Builds or reads the graph - /// Returns nil if some input (i.e. .swift file) has no corresponding swiftdeps file. - /// Does not cope with disappeared inputs -- would be left in graph - /// For inputs with swiftDeps in OFM, but no readable file, puts input in graph map, but no nodes in graph: - /// caller must ensure scheduling of those - private func computeGraphAndInputsInvalidatedByExternals() - -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)? { - precondition(sourceFiles.disappeared.isEmpty, - "Would have to remove nodes from the graph if reading prior") - if readPriorsFromModuleDependencyGraph { - return readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals() - } - // Every external is added, but don't want to compile an unchanged input that has an import - // so just changed, not changedOrAdded - return buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals() - } - - private func readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals( - ) -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)? - { - let dependencyGraphPath = buildRecordInfo.dependencyGraphPath - let graphIfPresent: ModuleDependencyGraph? - do { - graphIfPresent = try ModuleDependencyGraph.read( from: dependencyGraphPath, info: self) - } - catch { - diagnosticEngine.emit( - warning: "Could not read \(dependencyGraphPath), will not do cross-module incremental builds") - graphIfPresent = nil - } - guard let graph = graphIfPresent - else { - return buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals() - } - guard graph.populateInputDependencySourceMap() else { - return nil - } - graph.dotFileWriter?.write(graph) - - // Any externals not already in graph must be additions which should trigger - // recompilation. Thus, `ChangedOrAdded`. - let nodesDirectlyInvalidatedByExternals = graph.collectNodesInvalidatedByChangedOrAddedExternals() - // Wait till the last minute to do the transitive closure as an optimization. - let inputsInvalidatedByExternals = graph.collectInputsUsingInvalidated( - nodes: nodesDirectlyInvalidatedByExternals) - return (graph, inputsInvalidatedByExternals) - } - - /// Builds a graph - /// Returns nil if some input (i.e. .swift file) has no corresponding swiftdeps file. - /// Does not cope with disappeared inputs - /// For inputs with swiftDeps in OFM, but no readable file, puts input in graph map, but no nodes in graph: - /// caller must ensure scheduling of those - /// For externalDependencies, puts then in graph.fingerprintedExternalDependencies, but otherwise - /// does nothing special. - private func buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals( - ) -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)? - { - let graph = ModuleDependencyGraph(self, .buildingWithoutAPrior) - assert(outputFileMap.onlySourceFilesHaveSwiftDeps()) - guard graph.populateInputDependencySourceMap() else { - return nil - } - - var inputsInvalidatedByChangedExternals = TransitivelyInvalidatedInputSet() - for input in sourceFiles.currentInOrder { - guard let invalidatedInputs = graph.collectInputsRequiringCompilationFromExternalsFoundByCompiling(input: input) - else { - return nil - } - inputsInvalidatedByChangedExternals.formUnion(invalidatedInputs) - } - reporter?.report("Created dependency graph from swiftdeps files") - return (graph, inputsInvalidatedByChangedExternals) - } -} - // MARK: - Preparing the first wave -extension IncrementalCompilationState.InitialStateComputer { - +extension IncrementalCompilationState.FirstWaveComputer { /// At this stage the graph will have all external dependencies found in the swiftDeps or in the priors /// listed in fingerprintExternalDependencies. - private func computeInputsAndGroups( - _ moduleDependencyGraph: ModuleDependencyGraph, - _ inputsInvalidatedByExternals: TransitivelyInvalidatedInputSet, - batchJobFormer: inout Driver - ) throws -> (skippedCompileGroups: [TypedVirtualPath: CompileJobGroup], - mandatoryJobsInOrder: [Job]) + private func computeInputsAndGroups(batchJobFormer: inout Driver) + throws -> (skippedCompileGroups: [TypedVirtualPath: CompileJobGroup], + mandatoryJobsInOrder: [Job]) { precondition(sourceFiles.disappeared.isEmpty, "unimplemented") + let compileGroups = Dictionary(uniqueKeysWithValues: jobsInPhases.compileGroups.map { ($0.primaryInput, $0) }) @@ -225,13 +91,12 @@ extension IncrementalCompilationState.InitialStateComputer { } moduleDependencyGraph.phase = .updatingAfterCompilation - let skippedInputs = computeSkippedCompilationInputs( inputsInvalidatedByExternals: inputsInvalidatedByExternals, moduleDependencyGraph, buildRecord) - let skippedCompileGroups = compileGroups.filter {skippedInputs.contains($0.key)} + let skippedCompileGroups = compileGroups.filter { skippedInputs.contains($0.key) } let mandatoryCompileGroupsInOrder = inputFiles.compactMap { input -> CompileJobGroup? in @@ -250,7 +115,7 @@ extension IncrementalCompilationState.InitialStateComputer { mandatoryJobsInOrder: mandatoryJobsInOrder) } - /// Figure out which compilation inputs are *not* mandatory + // Figure out which compilation inputs are *not* mandatory private func computeSkippedCompilationInputs( inputsInvalidatedByExternals: TransitivelyInvalidatedInputSet, _ moduleDependencyGraph: ModuleDependencyGraph, @@ -258,7 +123,7 @@ extension IncrementalCompilationState.InitialStateComputer { ) -> Set { let allGroups = jobsInPhases.compileGroups // Input == source file - let changedInputs = computeChangedInputs( moduleDependencyGraph, buildRecord) + let changedInputs = computeChangedInputs(moduleDependencyGraph, buildRecord) if let reporter = reporter { for input in inputsInvalidatedByExternals { @@ -268,7 +133,8 @@ extension IncrementalCompilationState.InitialStateComputer { let inputsHavingMalformedDependencySources = sourceFiles.currentInOrder.filter { sourceFile in - !moduleDependencyGraph.containsNodes(forSourceFile: sourceFile)} + !moduleDependencyGraph.containsNodes(forSourceFile: sourceFile) + } if let reporter = reporter { for input in inputsHavingMalformedDependencySources { @@ -276,7 +142,7 @@ extension IncrementalCompilationState.InitialStateComputer { } } let inputsMissingOutputs = allGroups.compactMap { - $0.outputs.contains {(try? !fileSystem.exists($0.file)) ?? true} + $0.outputs.contains { (try? !fileSystem.exists($0.file)) ?? true } ? $0.primaryInput : nil } @@ -319,7 +185,7 @@ extension IncrementalCompilationState.InitialStateComputer { let skippedInputs = Set(buildRecordInfo.compilationInputModificationDates.keys) .subtracting(immediatelyCompiledInputs) if let reporter = reporter { - for skippedInput in sortByCommandLineOrder(skippedInputs) { + for skippedInput in sortByCommandLineOrder(skippedInputs) { reporter.report("Skipping input:", skippedInput) } } @@ -344,7 +210,7 @@ extension IncrementalCompilationState.InitialStateComputer { let datesMatch: Bool } - /// Find the inputs that have changed since last compilation, or were marked as needed a build + // Find the inputs that have changed since last compilation, or were marked as needed a build private func computeChangedInputs( _ moduleDependencyGraph: ModuleDependencyGraph, _ outOfDateBuildRecord: BuildRecord @@ -384,10 +250,10 @@ extension IncrementalCompilationState.InitialStateComputer { } } - /// Returns the cascaded files to compile in the first wave, even though it may not be need. - /// The needs[Non}CascadingBuild stuff was cargo-culted from the legacy driver. - /// TODO: something better, e.g. return nothing here, but process changed dependencySource - /// before the whole frontend job finished. + // Returns the cascaded files to compile in the first wave, even though it may not be need. + // The needs[Non}CascadingBuild stuff was cargo-culted from the legacy driver. + // TODO: something better, e.g. return nothing here, but process changed dependencySource + // before the whole frontend job finished. private func collectInputsToBeSpeculativelyRecompiled( changedInputs: [ChangedInput], externalDependents: TransitivelyInvalidatedInputSet, @@ -414,7 +280,7 @@ extension IncrementalCompilationState.InitialStateComputer { } } - // Collect the files that will be compiled whose dependents should be schedule + //Collect the files that will be compiled whose dependents should be schedule private func computeCascadingChangedInputs( from changedInputs: [ChangedInput], inputsMissingOutputs: Set diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift index 42c723aaa..74e56d3da 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift @@ -69,81 +69,48 @@ public final class IncrementalCompilationState { /// be protected by the confinement queue. private var skippedCompileGroups = [TypedVirtualPath: CompileJobGroup]() -// MARK: - Creating IncrementalCompilationState if possible + // MARK: - Creating IncrementalCompilationState /// Return nil if not compiling incrementally - init?( + internal init( driver: inout Driver, - options: Options, - jobsInPhases: JobsInPhases + jobsInPhases: JobsInPhases, + initialState: InitialStateForPlanning ) throws { - guard driver.shouldAttemptIncrementalCompilation else { return nil } - - if options.contains(.showIncremental) { + if initialState.incrementalOptions.contains(.showIncremental) { self.reporter = Reporter(diagnosticEngine: driver.diagnosticEngine, outputFileMap: driver.outputFileMap) } else { self.reporter = nil } - let enablingOrDisabling = options.contains(.enableCrossModuleIncrementalBuild) + let enablingOrDisabling = + initialState.incrementalOptions.contains(.enableCrossModuleIncrementalBuild) ? "Enabling" : "Disabling" reporter?.report( "\(enablingOrDisabling) incremental cross-module building") + let firstWave = + try FirstWaveComputer(initialState: initialState, jobsInPhases: jobsInPhases, + driver: driver, reporter: reporter).compute(batchJobFormer: &driver) - guard let outputFileMap = driver.outputFileMap else { - driver.diagnosticEngine.emit(.warning_incremental_requires_output_file_map) - return nil - } - - guard let buildRecordInfo = driver.buildRecordInfo else { - reporter?.reportDisablingIncrementalBuild("no build record path") - return nil - } - - // FIXME: This should work without an output file map. We should have - // another way to specify a build record and where to put intermediates. - let maybeBuildRecord = buildRecordInfo.populateOutOfDateBuildRecord( - inputFiles: driver.inputFiles, reporter: reporter) - - // Forming batch jobs requires passing in the driver "inout". But that's the - // only "inout" use needed, among many other values needed from the driver. - // So, pass the other values individually, and pass the driver "inout" as - // the "batchJobFormer". Maybe someday there will be a better way. - guard - let initial = try InitialStateComputer( - options, - jobsInPhases, - outputFileMap, - buildRecordInfo, - maybeBuildRecord, - self.reporter, - driver.inputFiles, - driver.fileSystem, - showJobLifecycle: driver.showJobLifecycle, - driver.diagnosticEngine) - .compute(batchJobFormer: &driver) - else { - return nil - } - - self.skippedCompileGroups = initial.skippedCompileGroups - self.mandatoryJobsInOrder = initial.mandatoryJobsInOrder + self.skippedCompileGroups = firstWave.skippedCompileGroups + self.mandatoryJobsInOrder = firstWave.mandatoryJobsInOrder self.jobsAfterCompiles = jobsInPhases.afterCompiles - self.moduleDependencyGraph = initial.graph - self.buildStartTime = initial.buildStartTime - self.buildEndTime = initial.buildEndTime + self.moduleDependencyGraph = initialState.graph + self.buildStartTime = initialState.buildStartTime + self.buildEndTime = initialState.buildEndTime self.fileSystem = driver.fileSystem self.driver = driver } } // MARK: - Initial State - extension IncrementalCompilationState { - /// The initial state of an incremental compilation plan. - @_spi(Testing) public struct InitialState { + /// The initial state of an incremental compilation plan that consists of the module dependency graph + /// and computes which inputs were invalidated by external changes. + /// This set of incremental information is used during planning - job-generation, and is computed early. + @_spi(Testing) public struct InitialStateForPlanning { /// The dependency graph. /// /// In a status quo build, the dependency graph is derived from the state @@ -154,6 +121,25 @@ extension IncrementalCompilationState { /// In a cross-module build, the dependency graph is derived from prior /// state that is serialized alongside the build record. let graph: ModuleDependencyGraph + /// Information about the last known compilation, incl. the location of build artifacts such as the dependency graph. + let buildRecordInfo: BuildRecordInfo + /// Record about existence and time of the last compile. + let maybeBuildRecord: BuildRecord? + /// A set of inputs invalidated by external chagnes. + let inputsInvalidatedByExternals: TransitivelyInvalidatedInputSet + /// Compiler options related to incremental builds. + let incrementalOptions: IncrementalCompilationState.Options + /// The last time this compilation was started. Used to compare against e.g. input file mod dates. + let buildStartTime: Date + /// The last time this compilation finished. Used to compare against output file mod dates + let buildEndTime: Date + } +} + +// MARK: - First Wave +extension IncrementalCompilationState { + /// The first set of mandatory jobs for inputs which *must* be built + struct FirstWave { /// The set of compile jobs we can definitely skip given the state of the /// incremental dependency graph and the status of the input files for this /// incremental build. @@ -161,10 +147,6 @@ extension IncrementalCompilationState { /// All of the pre-compile or compilation job (groups) known to be required /// for the first wave to execute. let mandatoryJobsInOrder: [Job] - /// The last time this compilation was started. Used to compare against e.g. input file mod dates. - let buildStartTime: Date - /// The last time this compilation finished. Used to compare against output file mod dates - let buildEndTime: Date } } @@ -204,7 +186,7 @@ fileprivate extension CompilerMode { } extension Diagnostic.Message { - fileprivate static var warning_incremental_requires_output_file_map: Diagnostic.Message { + static var warning_incremental_requires_output_file_map: Diagnostic.Message { .warning("ignoring -incremental (currently requires an output file map)") } static var warning_incremental_requires_build_record_entry: Diagnostic.Message { @@ -220,7 +202,7 @@ extension Diagnostic.Message { return .remark("Incremental compilation has been disabled: \(why)") } - fileprivate static func remark_incremental_compilation(because why: String) -> Diagnostic.Message { + static func remark_incremental_compilation(because why: String) -> Diagnostic.Message { .remark("Incremental compilation: \(why)") } } diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift new file mode 100644 index 000000000..05d7a3b06 --- /dev/null +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift @@ -0,0 +1,272 @@ +import Foundation +import SwiftOptions +//===----- IncrementalDependencyAndInputSetup.swift - Incremental --------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import TSCBasic + +// Initial incremental state computation +extension IncrementalCompilationState { + static func computeIncrementalStateForPlanning(driver: inout Driver) + throws -> IncrementalCompilationState.InitialStateForPlanning? + { + guard driver.shouldAttemptIncrementalCompilation else { return nil } + + let options = computeIncrementalOptions(driver: &driver) + + guard let outputFileMap = driver.outputFileMap else { + driver.diagnosticEngine.emit(.warning_incremental_requires_output_file_map) + return nil + } + + let reporter: IncrementalCompilationState.Reporter? + if options.contains(.showIncremental) { + reporter = IncrementalCompilationState.Reporter( + diagnosticEngine: driver.diagnosticEngine, + outputFileMap: outputFileMap) + } else { + reporter = nil + } + + guard let buildRecordInfo = driver.buildRecordInfo else { + reporter?.reportDisablingIncrementalBuild("no build record path") + return nil + } + + // FIXME: This should work without an output file map. We should have + // another way to specify a build record and where to put intermediates. + let maybeBuildRecord = + buildRecordInfo.populateOutOfDateBuildRecord( + inputFiles: driver.inputFiles, + reporter: reporter) + + guard + let initialState = + try IncrementalCompilationState + .IncrementalDependencyAndInputSetup( + options, outputFileMap, + buildRecordInfo, maybeBuildRecord, + reporter, driver.inputFiles, + driver.fileSystem, + driver.diagnosticEngine + ).compute() + else { + return nil + } + + return initialState + } + + // Extract options relevant to incremental builds + static func computeIncrementalOptions(driver: inout Driver) -> IncrementalCompilationState.Options { + var options: IncrementalCompilationState.Options = [] + if driver.parsedOptions.contains(.driverAlwaysRebuildDependents) { + options.formUnion(.alwaysRebuildDependents) + } + if driver.parsedOptions.contains(.driverShowIncremental) || driver.showJobLifecycle { + options.formUnion(.showIncremental) + } + let emitOpt = Option.driverEmitFineGrainedDependencyDotFileAfterEveryImport + if driver.parsedOptions.contains(emitOpt) { + options.formUnion(.emitDependencyDotFileAfterEveryImport) + } + let veriOpt = Option.driverVerifyFineGrainedDependencyGraphAfterEveryImport + if driver.parsedOptions.contains(veriOpt) { + options.formUnion(.verifyDependencyGraphAfterEveryImport) + } + if driver.parsedOptions.hasFlag(positive: .enableIncrementalImports, + negative: .disableIncrementalImports, + default: true) { + options.formUnion(.enableCrossModuleIncrementalBuild) + options.formUnion(.readPriorsFromModuleDependencyGraph) + } + return options + } +} + +/// Builds the `InitialState` +/// Also bundles up an bunch of configuration info +extension IncrementalCompilationState { + + public struct IncrementalDependencyAndInputSetup { + @_spi(Testing) public let outputFileMap: OutputFileMap + @_spi(Testing) public let buildRecordInfo: BuildRecordInfo + @_spi(Testing) public let maybeBuildRecord: BuildRecord? + @_spi(Testing) public let reporter: IncrementalCompilationState.Reporter? + @_spi(Testing) public let options: IncrementalCompilationState.Options + @_spi(Testing) public let inputFiles: [TypedVirtualPath] + @_spi(Testing) public let fileSystem: FileSystem + @_spi(Testing) public let sourceFiles: SourceFiles + @_spi(Testing) public let diagnosticEngine: DiagnosticsEngine + + /// Options, someday + @_spi(Testing) public let dependencyDotFilesIncludeExternals: Bool = true + @_spi(Testing) public let dependencyDotFilesIncludeAPINotes: Bool = false + + @_spi(Testing) public let buildStartTime: Date + @_spi(Testing) public let buildEndTime: Date + + // Do not try to reuse a graph from a different compilation, so check + // the build record. + @_spi(Testing) public var readPriorsFromModuleDependencyGraph: Bool { + maybeBuildRecord != nil && options.contains(.readPriorsFromModuleDependencyGraph) + } + @_spi(Testing) public var alwaysRebuildDependents: Bool { + options.contains(.alwaysRebuildDependents) + } + @_spi(Testing) public var isCrossModuleIncrementalBuildEnabled: Bool { + options.contains(.enableCrossModuleIncrementalBuild) + } + @_spi(Testing) public var verifyDependencyGraphAfterEveryImport: Bool { + options.contains(.verifyDependencyGraphAfterEveryImport) + } + @_spi(Testing) public var emitDependencyDotFileAfterEveryImport: Bool { + options.contains(.emitDependencyDotFileAfterEveryImport) + } + + @_spi(Testing) public init( + _ options: Options, + _ outputFileMap: OutputFileMap, + _ buildRecordInfo: BuildRecordInfo, + _ buildRecord: BuildRecord?, + _ reporter: IncrementalCompilationState.Reporter?, + _ inputFiles: [TypedVirtualPath], + _ fileSystem: FileSystem, + _ diagnosticEngine: DiagnosticsEngine + ) { + self.outputFileMap = outputFileMap + self.buildRecordInfo = buildRecordInfo + self.maybeBuildRecord = buildRecord + self.reporter = reporter + self.options = options + self.inputFiles = inputFiles + self.fileSystem = fileSystem + assert(outputFileMap.onlySourceFilesHaveSwiftDeps()) + self.sourceFiles = SourceFiles( + inputFiles: inputFiles, + buildRecord: buildRecord) + self.diagnosticEngine = diagnosticEngine + self.buildStartTime = maybeBuildRecord?.buildStartTime ?? .distantPast + self.buildEndTime = maybeBuildRecord?.buildEndTime ?? .distantFuture + } + + func compute() throws -> InitialStateForPlanning? { + guard sourceFiles.disappeared.isEmpty else { + // Would have to cleanse nodes of disappeared inputs from graph + // and would have to schedule files dependening on defs from disappeared nodes + if let reporter = reporter { + reporter.report( + "Incremental compilation has been disabled, " + + " because the following inputs were used in the previous compilation but not in this one: " + + sourceFiles.disappeared.map { $0.basename }.joined(separator: ", ")) + } + return nil + } + + guard + let (graph, inputsInvalidatedByExternals) = + computeGraphAndInputsInvalidatedByExternals() + else { + return nil + } + + return InitialStateForPlanning( + graph: graph, buildRecordInfo: buildRecordInfo, + maybeBuildRecord: maybeBuildRecord, + inputsInvalidatedByExternals: inputsInvalidatedByExternals, + incrementalOptions: options, buildStartTime: buildStartTime, + buildEndTime: buildEndTime) + } + } +} + +// MARK: - building/reading the ModuleDependencyGraph & scheduling externals for 1st wave +extension IncrementalCompilationState.IncrementalDependencyAndInputSetup { + /// Builds or reads the graph + /// Returns nil if some input (i.e. .swift file) has no corresponding swiftdeps file. + /// Does not cope with disappeared inputs -- would be left in graph + /// For inputs with swiftDeps in OFM, but no readable file, puts input in graph map, but no nodes in graph: + /// caller must ensure scheduling of those + private func computeGraphAndInputsInvalidatedByExternals() + -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)? + { + precondition( + sourceFiles.disappeared.isEmpty, + "Would have to remove nodes from the graph if reading prior") + if readPriorsFromModuleDependencyGraph { + return readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals() + } + // Every external is added, but don't want to compile an unchanged input that has an import + // so just changed, not changedOrAdded + return buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals() + } + + private func readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals( + ) -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)? + { + let dependencyGraphPath = buildRecordInfo.dependencyGraphPath + let graphIfPresent: ModuleDependencyGraph? + do { + graphIfPresent = try ModuleDependencyGraph.read( from: dependencyGraphPath, info: self) + } + catch { + diagnosticEngine.emit( + warning: "Could not read \(dependencyGraphPath), will not do cross-module incremental builds") + graphIfPresent = nil + } + guard let graph = graphIfPresent + else { + return buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals() + } + guard graph.populateInputDependencySourceMap() else { + return nil + } + graph.dotFileWriter?.write(graph) + + // Any externals not already in graph must be additions which should trigger + // recompilation. Thus, `ChangedOrAdded`. + let nodesDirectlyInvalidatedByExternals = + graph.collectNodesInvalidatedByChangedOrAddedExternals() + // Wait till the last minute to do the transitive closure as an optimization. + let inputsInvalidatedByExternals = graph.collectInputsUsingInvalidated( + nodes: nodesDirectlyInvalidatedByExternals) + return (graph, inputsInvalidatedByExternals) + } + + /// Builds a graph + /// Returns nil if some input (i.e. .swift file) has no corresponding swiftdeps file. + /// Does not cope with disappeared inputs + /// For inputs with swiftDeps in OFM, but no readable file, puts input in graph map, but no nodes in graph: + /// caller must ensure scheduling of those + /// For externalDependencies, puts then in graph.fingerprintedExternalDependencies, but otherwise + /// does nothing special. + private func buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals() + -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)? + { + let graph = ModuleDependencyGraph(self, .buildingWithoutAPrior) + assert(outputFileMap.onlySourceFilesHaveSwiftDeps()) + guard graph.populateInputDependencySourceMap() else { + return nil + } + + var inputsInvalidatedByChangedExternals = TransitivelyInvalidatedInputSet() + for input in sourceFiles.currentInOrder { + guard let invalidatedInputs = + graph.collectInputsRequiringCompilationFromExternalsFoundByCompiling(input: input) + else { + return nil + } + inputsInvalidatedByChangedExternals.formUnion(invalidatedInputs) + } + reporter?.report("Created dependency graph from swiftdeps files") + return (graph, inputsInvalidatedByChangedExternals) + } +} diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift index 91b7c4de6..d828ff035 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift @@ -30,7 +30,7 @@ import SwiftOptions public internal(set) var fingerprintedExternalDependencies = Set() /// A lot of initial state that it's handy to have around. - @_spi(Testing) public let info: IncrementalCompilationState.InitialStateComputer + @_spi(Testing) public let info: IncrementalCompilationState.IncrementalDependencyAndInputSetup /// For debugging, something to write out files for visualizing graphs let dotFileWriter: DependencyGraphDotFileWriter? @@ -40,7 +40,7 @@ import SwiftOptions /// Minimize the number of file system modification-time queries. private var externalDependencyModTimeCache = [ExternalDependency: Bool]() - public init(_ info: IncrementalCompilationState.InitialStateComputer, + public init(_ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup, _ phase: Phase ) { self.info = info @@ -339,7 +339,7 @@ extension ModuleDependencyGraph { /// Return nil if it's not incremental, or if an error occurs. private func collectNodesInvalidatedByAttemptingToProcess( _ fed: FingerprintedExternalDependency, - _ info: IncrementalCompilationState.InitialStateComputer) -> DirectlyInvalidatedNodeSet? { + _ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup) -> DirectlyInvalidatedNodeSet? { fed.incrementalDependencySource? .read(in: info.fileSystem, reporter: info.reporter) .map { unserializedDepGraph in @@ -462,7 +462,7 @@ extension ModuleDependencyGraph { /// - Returns: A fully deserialized ModuleDependencyGraph, or nil if nothing is there @_spi(Testing) public static func read( from path: VirtualPath, - info: IncrementalCompilationState.InitialStateComputer + info: IncrementalCompilationState.IncrementalDependencyAndInputSetup ) throws -> ModuleDependencyGraph? { guard try info.fileSystem.exists(path) else { return nil @@ -483,7 +483,7 @@ extension ModuleDependencyGraph { private var inputDependencySourceMap: [(TypedVirtualPath, DependencySource)] = [] public private(set) var allNodes: [Node] = [] - init(_ info: IncrementalCompilationState.InitialStateComputer) { + init(_ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup) { self.fileSystem = info.fileSystem self.graph = ModuleDependencyGraph(info, .updatingFromAPrior) } diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index 4ea060f9a..83997559f 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -82,6 +82,40 @@ extension Driver { precondition(compilerMode.isStandardCompilationForPlanning, "compiler mode \(compilerMode) is handled elsewhere") + // Determine the initial state for incremental compilation that is required during + // the planning process. This state contains the module dependency graph and + // cross-module dependency information. + let initialIncrementalState = + try IncrementalCompilationState.computeIncrementalStateForPlanning(driver: &self) + + // Compute the set of all jobs required to build this module + let jobsInPhases = try computeJobsForPhasedStandardBuild() + + // Determine the state for incremental compilation + let incrementalCompilationState: IncrementalCompilationState? + // If no initial state was computed, we will not be performing an incremental build + if let initialState = initialIncrementalState { + incrementalCompilationState = + try IncrementalCompilationState(driver: &self, jobsInPhases: jobsInPhases, + initialState: initialState) + } else { + incrementalCompilationState = nil + } + + return try ( + // For compatibility with swiftpm, the driver produces batched jobs + // for every job, even when run in incremental mode, so that all jobs + // can be returned from `planBuild`. + // But in that case, don't emit lifecycle messages. + formBatchedJobs(jobsInPhases.allJobs, + showJobLifecycle: showJobLifecycle && incrementalCompilationState == nil), + incrementalCompilationState + ) + } + + /// Construct a build plan consisting of *all* jobs required for building the current module (non-incrementally). + /// At build time, incremental state will be used to distinguish which of these jobs must run. + mutating private func computeJobsForPhasedStandardBuild() throws -> JobsInPhases { // Centralize job accumulation here. // For incremental compilation, must separate jobs happening before, // during, and after compilation. @@ -104,7 +138,8 @@ extension Driver { try addPrecompileModuleDependenciesJobs(addJob: addJobBeforeCompiles) try addPrecompileBridgingHeaderJob(addJob: addJobBeforeCompiles) - try addEmitModuleJob(addJobBeforeCompiles: addJobBeforeCompiles, addJobAfterCompiles: addJobAfterCompiles) + try addEmitModuleJob(addJobBeforeCompiles: addJobBeforeCompiles, + addJobAfterCompiles: addJobAfterCompiles) let linkerInputs = try addJobsFeedingLinker( addJobBeforeCompiles: addJobBeforeCompiles, addCompileJobGroup: addCompileJobGroup, @@ -113,52 +148,9 @@ extension Driver { debugInfo: debugInfo, addJob: addJobAfterCompiles) - let jobsInPhases = JobsInPhases( - beforeCompiles: jobsBeforeCompiles, - compileGroups: compileJobGroups, - afterCompiles: jobsAfterCompiles - ) - - // Determine the state for incremental compilation - let incrementalCompilationState = try IncrementalCompilationState( - driver: &self, - options: self.computeIncrementalOptions(), - jobsInPhases: jobsInPhases) - - return try ( - // For compatibility with swiftpm, the driver produces batched jobs - // for every job, even when run in incremental mode, so that all jobs - // can be returned from `planBuild`. - // But in that case, don't emit lifecycle messages. - formBatchedJobs(jobsInPhases.allJobs, - showJobLifecycle: showJobLifecycle && incrementalCompilationState == nil), - incrementalCompilationState - ) - } - - mutating func computeIncrementalOptions() -> IncrementalCompilationState.Options { - var options: IncrementalCompilationState.Options = [] - if self.parsedOptions.contains(.driverAlwaysRebuildDependents) { - options.formUnion(.alwaysRebuildDependents) - } - if self.parsedOptions.contains(.driverShowIncremental) || self.showJobLifecycle { - options.formUnion(.showIncremental) - } - let emitOpt = Option.driverEmitFineGrainedDependencyDotFileAfterEveryImport - if self.parsedOptions.contains(emitOpt) { - options.formUnion(.emitDependencyDotFileAfterEveryImport) - } - let veriOpt = Option.driverVerifyFineGrainedDependencyGraphAfterEveryImport - if self.parsedOptions.contains(veriOpt) { - options.formUnion(.verifyDependencyGraphAfterEveryImport) - } - if self.parsedOptions.hasFlag(positive: .enableIncrementalImports, - negative: .disableIncrementalImports, - default: true) { - options.formUnion(.enableCrossModuleIncrementalBuild) - options.formUnion(.readPriorsFromModuleDependencyGraph) - } - return options + return JobsInPhases(beforeCompiles: jobsBeforeCompiles, + compileGroups: compileJobGroups, + afterCompiles: jobsAfterCompiles) } private mutating func addPrecompileModuleDependenciesJobs(addJob: (Job) -> Void) throws { @@ -324,7 +316,7 @@ extension Driver { // Generate a compile job for primary inputs here. guard compilerMode.usesPrimaryFileInputs else { break } - assert(input.type.isPartOfSwiftCompilation) + assert(input.type.isPartOfSwiftCompilation) // We can skip the compile jobs if all we want is a module when it's // built separately. let canSkipIfOnlyModule = compilerOutputType == .swiftModule && shouldCreateEmitModuleJob diff --git a/Tests/SwiftDriverTests/Helpers/MockingIncrementalCompilation.swift b/Tests/SwiftDriverTests/Helpers/MockingIncrementalCompilation.swift index 8438bac5d..cdb1dbf24 100644 --- a/Tests/SwiftDriverTests/Helpers/MockingIncrementalCompilation.swift +++ b/Tests/SwiftDriverTests/Helpers/MockingIncrementalCompilation.swift @@ -116,23 +116,16 @@ extension BuildRecordInfo { } } -extension IncrementalCompilationState.InitialStateComputer { +extension IncrementalCompilationState.IncrementalDependencyAndInputSetup { static func mock( options: IncrementalCompilationState.Options = [.verifyDependencyGraphAfterEveryImport], diagnosticEngine: DiagnosticsEngine = DiagnosticsEngine(), fileSystem: FileSystem = localFileSystem) -> Self { let diagnosticsEngine = DiagnosticsEngine() let outputFileMap = OutputFileMap() - return Self(options, - JobsInPhases.none, - outputFileMap, - BuildRecordInfo.mock(diagnosticsEngine, outputFileMap), - nil, - nil, - [], - fileSystem, - showJobLifecycle: false, - diagnosticsEngine) + return Self(options, outputFileMap, + BuildRecordInfo.mock(diagnosticsEngine, outputFileMap), + nil, nil, [], fileSystem, diagnosticsEngine) } } diff --git a/Tests/SwiftDriverTests/IncrementalCompilationTests.swift b/Tests/SwiftDriverTests/IncrementalCompilationTests.swift index d2f152ccd..953ec9dcf 100644 --- a/Tests/SwiftDriverTests/IncrementalCompilationTests.swift +++ b/Tests/SwiftDriverTests/IncrementalCompilationTests.swift @@ -472,7 +472,7 @@ extension IncrementalCompilationTests { func testOptionsParsing() throws { let optionPairs: [( - Option, (IncrementalCompilationState.InitialStateComputer) -> Bool + Option, (IncrementalCompilationState.IncrementalDependencyAndInputSetup) -> Bool )] = [ (.driverAlwaysRebuildDependents, {$0.alwaysRebuildDependents}), (.driverShowIncremental, {$0.reporter != nil}), diff --git a/Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift b/Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift index 7bb1bb1ab..a5a057388 100644 --- a/Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift +++ b/Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift @@ -941,7 +941,7 @@ extension ModuleDependencyGraph { mock diagnosticEngine: DiagnosticsEngine, options: IncrementalCompilationState.Options = [ .verifyDependencyGraphAfterEveryImport ] ) { - self.init(IncrementalCompilationState.InitialStateComputer.mock(), .buildingWithoutAPrior) + self.init(IncrementalCompilationState.IncrementalDependencyAndInputSetup.mock(), .buildingWithoutAPrior) }