Revert "Deprecate Legacy driver by removing incremental builds and adding warnings"

include/swift/AST/DiagnosticsDriver.def

@@ -188,9 +188,6 @@ WARNING(warn_drv_darwin_sdk_invalid_settings, none,
 WARNING(warning_unsupported_driver_option,none,
         "option '%0' is only supported in swift-driver", (StringRef))
 
-WARNING(new_driver_not_found,none,
-        "using (deprecated) legacy driver, Swift installation does not contain swift-driver at: '%0'", (StringRef))
-
 WARNING(old_driver_deprecated,none,
         "legacy driver is now deprecated; consider avoiding specifying '%0'", (StringRef))
 #define UNDEFINE_DIAGNOSTIC_MACROS

include/swift/Driver/Action.h

@@ -128,16 +128,78 @@ class JobAction : public Action {
   }
 };
 
+class IncrementalJobAction : public JobAction {
+public:
+  struct InputInfo {
+    /// The status of an input known to the driver. These are used to affect
+    /// the scheduling decisions made during an incremental build.
+    ///
+    /// \Note The order of cases matters. They are ordered from least to
+    /// greatest impact on the incremental build schedule.
+    enum class Status {
+      /// The input to this job is up to date.
+      UpToDate,
+      /// The input to this job has changed in a way that requires this job to
+      /// be rerun, but not in such a way that it requires a cascading rebuild.
+      NeedsNonCascadingBuild,
+      /// The input to this job has changed in a way that requires this job to
+      /// be rerun, and in such a way that all jobs dependent upon this one
+      /// must be scheduled as well.
+      NeedsCascadingBuild,
+      /// The input to this job was not known to the driver when it was last
+      /// run.
+      NewlyAdded
+    };
+
+  public:
+    Status status = Status::UpToDate;
+    llvm::sys::TimePoint<> previousModTime;
+
+    InputInfo() = default;
+    InputInfo(Status stat, llvm::sys::TimePoint<> time)
+        : status(stat), previousModTime(time) {}
+
+    static InputInfo makeNewlyAdded() {
+      return {Status::NewlyAdded, llvm::sys::TimePoint<>::max()};
+    }
+
+    static InputInfo makeNeedsCascadingRebuild() {
+      return {Status::NeedsCascadingBuild, llvm::sys::TimePoint<>::min()};
+    }
+  };
+
+private:
+  virtual void anchor() override;
+  InputInfo inputInfo;
+
+public:
+  IncrementalJobAction(Kind Kind, ArrayRef<const Action *> Inputs,
+                       file_types::ID Type, InputInfo info)
+      : JobAction(Kind, Inputs, Type), inputInfo(info) {}
+
+public:
+  InputInfo getInputInfo() const {
+    return inputInfo;
+  }
+
+public:
+  static bool classof(const Action *A) {
+    return A->getKind() == Action::Kind::CompileJob ||
+           A->getKind() == Action::Kind::MergeModuleJob;
+  }
+};
 
-class CompileJobAction : public JobAction {
+class CompileJobAction : public IncrementalJobAction {
 private:
   virtual void anchor() override;
 
 public:
   CompileJobAction(file_types::ID OutputType)
-      : JobAction(Action::Kind::CompileJob, llvm::None, OutputType) {}
-  CompileJobAction(Action *Input, file_types::ID OutputType)
-      : JobAction(Action::Kind::CompileJob, Input, OutputType) {}
+      : IncrementalJobAction(Action::Kind::CompileJob, llvm::None, OutputType,
+                             {}) {}
+  CompileJobAction(Action *Input, file_types::ID OutputType, InputInfo info)
+      : IncrementalJobAction(Action::Kind::CompileJob, Input, OutputType,
+                             info) {}
 
   static bool classof(const Action *A) {
     return A->getKind() == Action::Kind::CompileJob;
@@ -221,12 +283,12 @@ class REPLJobAction : public JobAction {
   }
 };
 
-class MergeModuleJobAction : public JobAction {
+class MergeModuleJobAction : public IncrementalJobAction {
   virtual void anchor() override;
 public:
-  MergeModuleJobAction(ArrayRef<const Action *> Inputs)
-      : JobAction(Action::Kind::MergeModuleJob, Inputs,
-                  file_types::TY_SwiftModuleFile) {}
+  MergeModuleJobAction(ArrayRef<const Action *> Inputs, InputInfo input)
+      : IncrementalJobAction(Action::Kind::MergeModuleJob, Inputs,
+                             file_types::TY_SwiftModuleFile, input) {}
 
   static bool classof(const Action *A) {
     return A->getKind() == Action::Kind::MergeModuleJob;

include/swift/Driver/Compilation.h

@@ -24,6 +24,7 @@
 #include "swift/Basic/OutputFileMap.h"
 #include "swift/Basic/Statistic.h"
 #include "swift/Driver/Driver.h"
+#include "swift/Driver/FineGrainedDependencyDriverGraph.h"
 #include "swift/Driver/Job.h"
 #include "swift/Driver/Util.h"
 #include "llvm/ADT/StringRef.h"
@@ -83,9 +84,15 @@ class Compilation {
     bool hadAbnormalExit;
     /// The exit code of this driver process.
     int exitCode;
+    /// The dependency graph built up during the compilation of this module.
+    ///
+    /// This data is used for cross-module module dependencies.
+    fine_grained_dependencies::ModuleDepGraph depGraph;
 
-    Result(bool hadAbnormalExit, int exitCode)
-        : hadAbnormalExit(hadAbnormalExit), exitCode(exitCode) {}
+    Result(bool hadAbnormalExit, int exitCode,
+           fine_grained_dependencies::ModuleDepGraph depGraph)
+        : hadAbnormalExit(hadAbnormalExit), exitCode(exitCode),
+          depGraph(std::move(depGraph)) {}
 
     Result(const Result &) = delete;
     Result &operator=(const Result &) = delete;
@@ -95,7 +102,8 @@ class Compilation {
 
     /// Construct a \c Compilation::Result from just an exit code.
     static Result code(int code) {
-      return Compilation::Result{false, code};
+      return Compilation::Result{false, code,
+                                 fine_grained_dependencies::ModuleDepGraph()};
     }
   };
 
@@ -163,10 +171,33 @@ class Compilation {
   /// These apply whether the compilation succeeds or fails. If the
   llvm::StringMap<PreserveOnSignal> TempFilePaths;
 
+  /// Write information about this compilation to this file.
+  ///
+  /// This is used for incremental builds.
+  std::string CompilationRecordPath;
+
+  /// A hash representing all the arguments that could trigger a full rebuild.
+  std::string ArgsHash;
+
+  /// When the build was started.
+  ///
+  /// This should be as close as possible to when the driver was invoked, since
+  /// it's used as a lower bound.
+  llvm::sys::TimePoint<> BuildStartTime;
+
+  /// The time of the last build.
+  ///
+  /// If unknown, this will be some time in the past.
+  llvm::sys::TimePoint<> LastBuildTime = llvm::sys::TimePoint<>::min();
+
   /// Indicates whether this Compilation should continue execution of subtasks
   /// even if they returned an error status.
   bool ContinueBuildingAfterErrors = false;
 
+  /// Indicates whether tasks should only be executed if their output is out
+  /// of date.
+  bool EnableIncrementalBuild;
+
   /// Indicates whether groups of parallel frontend jobs should be merged
   /// together and run in composite "batch jobs" when possible, to reduce
   /// redundant work.
@@ -225,6 +256,17 @@ class Compilation {
   /// -disable-only-one-dependency-file on the command line.
   const bool OnlyOneDependencyFile;
 
+private:
+  /// Helpful for debugging, but slows down the driver. So, only turn on when
+  /// needed.
+  const bool VerifyFineGrainedDependencyGraphAfterEveryImport;
+  /// Helpful for debugging, but slows down the driver. So, only turn on when
+  /// needed.
+  const bool EmitFineGrainedDependencyDotFileAfterEveryImport;
+
+  /// (experimental) Enable cross-module incremental build scheduling.
+  const bool EnableCrossModuleIncrementalBuild;
+
 private:
   template <typename T>
   static T *unwrap(const std::unique_ptr<T> &p) {
@@ -243,15 +285,22 @@ class Compilation {
               std::unique_ptr<llvm::opt::InputArgList> InputArgs,
               std::unique_ptr<llvm::opt::DerivedArgList> TranslatedArgs,
               InputFileList InputsWithTypes,
+              std::string CompilationRecordPath,
+              StringRef ArgsHash, llvm::sys::TimePoint<> StartTime,
+              llvm::sys::TimePoint<> LastBuildTime,
               size_t FilelistThreshold,
+              bool EnableIncrementalBuild = false,
               bool EnableBatchMode = false,
               unsigned BatchSeed = 0,
               llvm::Optional<unsigned> BatchCount = llvm::None,
               llvm::Optional<unsigned> BatchSizeLimit = llvm::None,
               bool SaveTemps = false,
               bool ShowDriverTimeCompilation = false,
               std::unique_ptr<UnifiedStatsReporter> Stats = nullptr,
-              bool OnlyOneDependencyFile = false);
+              bool OnlyOneDependencyFile = false,
+              bool VerifyFineGrainedDependencyGraphAfterEveryImport = false,
+              bool EmitFineGrainedDependencyDotFileAfterEveryImport = false,
+              bool EnableCrossModuleIncrementalBuild = false);
   // clang-format on
   ~Compilation();
 
@@ -304,6 +353,19 @@ class Compilation {
     return DerivedOutputFileMap;
   }
 
+  bool getIncrementalBuildEnabled() const {
+    return EnableIncrementalBuild;
+  }
+  void disableIncrementalBuild(Twine why);
+
+  bool getVerifyFineGrainedDependencyGraphAfterEveryImport() const {
+    return VerifyFineGrainedDependencyGraphAfterEveryImport;
+  }
+
+  bool getEmitFineGrainedDependencyDotFileAfterEveryImport() const {
+    return EmitFineGrainedDependencyDotFileAfterEveryImport;
+  }
+
   bool getBatchModeEnabled() const {
     return EnableBatchMode;
   }
@@ -315,6 +377,13 @@ class Compilation {
     ContinueBuildingAfterErrors = Value;
   }
 
+  bool getShowIncrementalBuildDecisions() const {
+    return ShowIncrementalBuildDecisions;
+  }
+  void setShowIncrementalBuildDecisions(bool value = true) {
+    ShowIncrementalBuildDecisions = value;
+  }
+
   bool getShowJobLifecycle() const {
     return ShowJobLifecycle;
   }
@@ -326,6 +395,10 @@ class Compilation {
     return ShowDriverTimeCompilation;
   }
 
+  bool getEnableCrossModuleIncrementalBuild() const {
+    return EnableCrossModuleIncrementalBuild;
+  }
+
   size_t getFilelistThreshold() const {
     return FilelistThreshold;
   }
@@ -347,7 +420,7 @@ class Compilation {
   /// True if extra work has to be done when tracing through the dependency
   /// graph, either in order to print dependencies or to collect statistics.
   bool getTraceDependencies() const {
-    return getStatsReporter();
+    return getShowIncrementalBuildDecisions() || getStatsReporter();
   }
 
   OutputLevel getOutputLevel() const {
@@ -358,10 +431,12 @@ class Compilation {
     return BatchSeed;
   }
 
-  llvm::Optional<unsigned> getBatchCount() const {
-    return BatchCount;
+  llvm::sys::TimePoint<> getLastBuildTime() const {
+    return LastBuildTime;
   }
 
+  llvm::Optional<unsigned> getBatchCount() const { return BatchCount; }
+
   llvm::Optional<unsigned> getBatchSizeLimit() const { return BatchSizeLimit; }
 
   /// Requests the path to a file containing all input source files. This can
@@ -373,6 +448,17 @@ class Compilation {
   /// \sa types::isPartOfSwiftCompilation
   const char *getAllSourcesPath() const;
 
+  /// Retrieve the path to the external swift deps file.
+  ///
+  /// For cross-module incremental builds, this file contains the dependencies
+  /// from all the modules integrated over the prior build.
+  ///
+  /// Currently this patch is relative to the build record, but we may want
+  /// to allow the output file map to customize this at some point.
+  std::string getExternalSwiftDepsFilePath() const {
+    return CompilationRecordPath + ".external";
+  }
+
   /// Asks the Compilation to perform the Jobs which it knows about.
   ///
   /// \param TQ The TaskQueue used to schedule jobs for execution.

include/swift/Driver/Driver.h

@@ -329,9 +329,12 @@ class Driver {
   /// \param[out] TopLevelActions The main Actions to build Jobs for.
   /// \param TC the default host tool chain.
   /// \param OI The OutputInfo for which Actions should be generated.
+  /// \param OutOfDateMap If present, information used to decide which files
+  /// need to be rebuilt.
   /// \param C The Compilation to which Actions should be added.
   void buildActions(SmallVectorImpl<const Action *> &TopLevelActions,
                     const ToolChain &TC, const OutputInfo &OI,
+                    const InputInfoMap *OutOfDateMap,
                     Compilation &C) const;
 
   /// Construct the OutputFileMap for the driver from the given arguments.