[clang][deps] Compute command-lines for dependencies immediately

Instead of delaying the generation of command-lines to after all
dependencies are reported, compute them immediately. This is partly in
preparation for splitting the TU driver command into its constituent cc1
and other jobs, but it also just simplifies working with the compiler
invocation for modules if they are not "without paths".

Also change the computation of the default output path in
clang-scan-deps to scrape the implicit module cache from the
command-line rather than get it from the dependency, since that is now
unavailable at the time we make the callback.

Differential Revision: https://reviews.llvm.org/D131934

Author: benlangmuir

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

@@ -23,6 +23,10 @@ namespace clang {
 namespace tooling {
 namespace dependencies {
 
+/// A callback to lookup module outputs for "-fmodule-file=", "-o" etc.
+using LookupModuleOutputCallback =
+    llvm::function_ref<std::string(const ModuleID &, ModuleOutputKind)>;
+
 /// The full dependencies and module graph for a specific input.
 struct FullDependencies {
   /// The identifier of the C++20 module this translation unit exports.
@@ -45,17 +49,8 @@ struct FullDependencies {
   /// determined that the differences are benign for this compilation.
   std::vector<ModuleID> ClangModuleDeps;
 
-  /// The original command line of the TU (excluding the compiler executable).
-  std::vector<std::string> OriginalCommandLine;
-
-  /// Get the full command line.
-  ///
-  /// \param LookupModuleOutput This function is called to fill in
-  ///                           "-fmodule-file=", "-o" and other output
-  ///                           arguments for dependencies.
-  std::vector<std::string> getCommandLine(
-      llvm::function_ref<std::string(const ModuleID &, ModuleOutputKind)>
-          LookupModuleOutput) const;
+  /// The command line of the TU (excluding the compiler executable).
+  std::vector<std::string> CommandLine;
 };
 
 struct FullDependenciesResult {
@@ -92,12 +87,16 @@ class DependencyScanningTool {
   ///                    function for a single \c DependencyScanningTool in a
   ///                    single build. Use a different one for different tools,
   ///                    and clear it between builds.
+  /// \param LookupModuleOutput This function is called to fill in
+  ///                           "-fmodule-file=", "-o" and other output
+  ///                           arguments for dependencies.
   ///
   /// \returns a \c StringError with the diagnostic output if clang errors
   /// occurred, \c FullDependencies otherwise.
   llvm::Expected<FullDependenciesResult>
   getFullDependencies(const std::vector<std::string> &CommandLine,
                       StringRef CWD, const llvm::StringSet<> &AlreadySeen,
+                      LookupModuleOutputCallback LookupModuleOutput,
                       llvm::Optional<StringRef> ModuleName = None);
 
 private:
@@ -106,8 +105,9 @@ class DependencyScanningTool {
 
 class FullDependencyConsumer : public DependencyConsumer {
 public:
-  FullDependencyConsumer(const llvm::StringSet<> &AlreadySeen)
-      : AlreadySeen(AlreadySeen) {}
+  FullDependencyConsumer(const llvm::StringSet<> &AlreadySeen,
+                         LookupModuleOutputCallback LookupModuleOutput)
+      : AlreadySeen(AlreadySeen), LookupModuleOutput(LookupModuleOutput) {}
 
   void handleDependencyOutputOpts(const DependencyOutputOptions &) override {}
 
@@ -127,6 +127,11 @@ class FullDependencyConsumer : public DependencyConsumer {
     ContextHash = std::move(Hash);
   }
 
+  std::string lookupModuleOutput(const ModuleID &ID,
+                                 ModuleOutputKind Kind) override {
+    return LookupModuleOutput(ID, Kind);
+  }
+
   FullDependenciesResult getFullDependencies(
       const std::vector<std::string> &OriginalCommandLine) const;
 
@@ -138,6 +143,7 @@ class FullDependencyConsumer : public DependencyConsumer {
   std::string ContextHash;
   std::vector<std::string> OutputPaths;
   const llvm::StringSet<> &AlreadySeen;
+  LookupModuleOutputCallback LookupModuleOutput;
 };
 
 } // end namespace dependencies

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

@@ -42,6 +42,9 @@ class DependencyConsumer {
   virtual void handleModuleDependency(ModuleDeps MD) = 0;
 
   virtual void handleContextHash(std::string Hash) = 0;
+
+  virtual std::string lookupModuleOutput(const ModuleID &ID,
+                                         ModuleOutputKind Kind) = 0;
 };
 
 /// An individual dependency scanning worker that is able to run on its own

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -119,25 +119,11 @@ struct ModuleDeps {
   // the primary TU.
   bool ImportedByMainFile = false;
 
-  /// Whether the TU had a dependency file. The path in \c BuildInvocation is
-  /// cleared to avoid leaking the specific path from the TU into the module.
-  bool HadDependencyFile = false;
-
-  /// Whether the TU had serialized diagnostics. The path in \c BuildInvocation
-  /// is cleared to avoid leaking the specific path from the TU into the module.
-  bool HadSerializedDiagnostics = false;
-
   /// Compiler invocation that can be used to build this module (without paths).
   CompilerInvocation BuildInvocation;
 
   /// Gets the canonical command line suitable for passing to clang.
-  ///
-  /// \param LookupModuleOutput This function is called to fill in
-  ///                           "-fmodule-file=", "-o" and other output
-  ///                           arguments.
-  std::vector<std::string> getCanonicalCommandLine(
-      llvm::function_ref<std::string(const ModuleID &, ModuleOutputKind)>
-          LookupModuleOutput) const;
+  std::vector<std::string> getCanonicalCommandLine() const;
 };
 
 class ModuleDepCollector;
@@ -237,9 +223,12 @@ class ModuleDepCollector final : public DependencyCollector {
   /// Constructs a CompilerInvocation that can be used to build the given
   /// module, excluding paths to discovered modular dependencies that are yet to
   /// be built.
-  CompilerInvocation makeInvocationForModuleBuildWithoutPaths(
+  CompilerInvocation makeInvocationForModuleBuildWithoutOutputs(
       const ModuleDeps &Deps,
       llvm::function_ref<void(CompilerInvocation &)> Optimize) const;
+
+  /// Add paths that require looking up outputs to the given dependencies.
+  void addOutputPaths(ModuleDeps &Deps);
 };
 
 } // end namespace dependencies

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

@@ -13,18 +13,12 @@ using namespace clang;
 using namespace tooling;
 using namespace dependencies;
 
-std::vector<std::string> FullDependencies::getCommandLine(
-    llvm::function_ref<std::string(const ModuleID &, ModuleOutputKind)>
-        LookupModuleOutput) const {
+static std::vector<std::string>
+makeTUCommandLineWithoutPaths(ArrayRef<std::string> OriginalCommandLine) {
   std::vector<std::string> Args = OriginalCommandLine;
 
   Args.push_back("-fno-implicit-modules");
   Args.push_back("-fno-implicit-module-maps");
-  for (const PrebuiltModuleDep &PMD : PrebuiltModuleDeps)
-    Args.push_back("-fmodule-file=" + PMD.PCMFile);
-  for (ModuleID MID : ClangModuleDeps)
-    Args.push_back("-fmodule-file=" +
-                   LookupModuleOutput(MID, ModuleOutputKind::ModuleFile));
 
   // These arguments are unused in explicit compiles.
   llvm::erase_if(Args, [](StringRef Arg) {
@@ -72,6 +66,11 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
 
     void handleContextHash(std::string Hash) override {}
 
+    std::string lookupModuleOutput(const ModuleID &ID,
+                                   ModuleOutputKind Kind) override {
+      llvm::report_fatal_error("unexpected call to lookupModuleOutput");
+    }
+
     void printDependencies(std::string &S) {
       assert(Opts && "Handled dependency output options.");
 
@@ -113,8 +112,9 @@ llvm::Expected<FullDependenciesResult>
 DependencyScanningTool::getFullDependencies(
     const std::vector<std::string> &CommandLine, StringRef CWD,
     const llvm::StringSet<> &AlreadySeen,
+    LookupModuleOutputCallback LookupModuleOutput,
     llvm::Optional<StringRef> ModuleName) {
-  FullDependencyConsumer Consumer(AlreadySeen);
+  FullDependencyConsumer Consumer(AlreadySeen, LookupModuleOutput);
   llvm::Error Result =
       Worker.computeDependencies(CWD, CommandLine, Consumer, ModuleName);
   if (Result)
@@ -126,16 +126,24 @@ FullDependenciesResult FullDependencyConsumer::getFullDependencies(
     const std::vector<std::string> &OriginalCommandLine) const {
   FullDependencies FD;
 
-  FD.OriginalCommandLine = ArrayRef<std::string>(OriginalCommandLine).slice(1);
+  FD.CommandLine = makeTUCommandLineWithoutPaths(
+      ArrayRef<std::string>(OriginalCommandLine).slice(1));
 
   FD.ID.ContextHash = std::move(ContextHash);
 
   FD.FileDeps.assign(Dependencies.begin(), Dependencies.end());
 
+  for (const PrebuiltModuleDep &PMD : PrebuiltModuleDeps)
+    FD.CommandLine.push_back("-fmodule-file=" + PMD.PCMFile);
+
   for (auto &&M : ClangModuleDeps) {
     auto &MD = M.second;
-    if (MD.ImportedByMainFile)
+    if (MD.ImportedByMainFile) {
       FD.ClangModuleDeps.push_back(MD.ID);
+      FD.CommandLine.push_back(
+          "-fmodule-file=" +
+          LookupModuleOutput(MD.ID, ModuleOutputKind::ModuleFile));
+    }
   }
 
   FD.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps);