[clang][deps] Generate command lines lazily (llvm#65691)

This patch makes the generation of command lines for modular
dependencies lazy/on-demand. That operation is somewhat expensive and
prior to this patch used to be performed multiple times for the
identical `ModuleDeps` (i.e. when they were imported from multiple
different TUs).

Author: jansvoboda11

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -24,6 +24,7 @@
 #include <optional>
 #include <string>
 #include <unordered_map>
+#include <variant>
 
 namespace clang {
 namespace tooling {
@@ -148,9 +149,15 @@ struct ModuleDeps {
   /// The \c ActionCache key for this module, if any.
   std::optional<std::string> ModuleCacheKey;
 
-  /// Compiler invocation that can be used to build this module. Does not
-  /// include argv[0].
-  std::vector<std::string> BuildArguments;
+  /// Get (or compute) the compiler invocation that can be used to build this
+  /// module. Does not include argv[0].
+  const std::vector<std::string> &getBuildArguments();
+
+private:
+  friend class ModuleDepCollectorPP;
+
+  std::variant<std::monostate, CowCompilerInvocation, std::vector<std::string>>
+      BuildInfo;
 };
 
 class ModuleDepCollector;

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

@@ -26,6 +26,14 @@ using namespace clang;
 using namespace tooling;
 using namespace dependencies;
 
+const std::vector<std::string> &ModuleDeps::getBuildArguments() {
+  assert(!std::holds_alternative<std::monostate>(BuildInfo) &&
+         "Using uninitialized ModuleDeps");
+  if (const auto *CI = std::get_if<CowCompilerInvocation>(&BuildInfo))
+    BuildInfo = CI->getCC1CommandLine();
+  return std::get<std::vector<std::string>>(BuildInfo);
+}
+
 static void optimizeHeaderSearchOpts(HeaderSearchOptions &Opts,
                                      ASTReader &Reader,
                                      const serialization::ModuleFile &MF) {
@@ -638,7 +646,7 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   }
 #endif
 
-  MD.BuildArguments = CI.getCC1CommandLine();
+  MD.BuildInfo = std::move(CI);
 
   return MD.ID;
 }

clang/tools/clang-scan-deps/ClangScanDeps.cpp

@@ -613,14 +613,23 @@ class FullDeps {
   }
 
   void mergeDeps(ModuleDepsGraph Graph, size_t InputIndex) {
-    std::unique_lock<std::mutex> ul(Lock);
-    for (const ModuleDeps &MD : Graph) {
-      auto I = Modules.find({MD.ID, 0});
-      if (I != Modules.end()) {
-        I->first.InputIndex = std::min(I->first.InputIndex, InputIndex);
-        continue;
+    std::vector<ModuleDeps *> NewMDs;
+    {
+      std::unique_lock<std::mutex> ul(Lock);
+      for (const ModuleDeps &MD : Graph) {
+        auto I = Modules.find({MD.ID, 0});
+        if (I != Modules.end()) {
+          I->first.InputIndex = std::min(I->first.InputIndex, InputIndex);
+          continue;
+        }
+        auto Res = Modules.insert(I, {{MD.ID, InputIndex}, std::move(MD)});
+        NewMDs.push_back(&Res->second);
       }
-      Modules.insert(I, {{MD.ID, InputIndex}, std::move(MD)});
+      // First call to \c getBuildArguments is somewhat expensive. Let's call it
+      // on the current thread (instead of the main one), and outside the
+      // critical section.
+      for (ModuleDeps *MD : NewMDs)
+        (void)MD->getBuildArguments();
     }
   }
 
@@ -644,7 +653,7 @@ class FullDeps {
                                             /*ShouldOwnClient=*/false);
 
     for (auto &&M : Modules)
-      if (roundTripCommand(M.second.BuildArguments, *Diags))
+      if (roundTripCommand(M.second.getBuildArguments(), *Diags))
         return true;
 
     for (auto &&I : Inputs)
@@ -673,7 +682,7 @@ class FullDeps {
           {"file-deps", toJSONSorted(MD.FileDeps)},
           {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)},
           {"clang-modulemap-file", MD.ClangModuleMapFile},
-          {"command-line", MD.BuildArguments},
+          {"command-line", MD.getBuildArguments()},
       };
       if (MD.ModuleCacheKey)
         O.try_emplace("cache-key", MD.ModuleCacheKey);

clang/tools/libclang/CDependencies.cpp

@@ -266,16 +266,16 @@ static CXErrorCode getFullDependencies(DependencyScanningWorker *Worker,
     MDS->Modules = new CXModuleDependency[MDS->Count];
     for (int I = 0; I < MDS->Count; ++I) {
       CXModuleDependency &M = MDS->Modules[I];
-      const ModuleDeps &MD = TU.ModuleGraph[I];
+      ModuleDeps &MD = TU.ModuleGraph[I];
       M.Name = cxstring::createDup(MD.ID.ModuleName);
       M.ContextHash = cxstring::createDup(MD.ID.ContextHash);
       M.ModuleMapPath = cxstring::createDup(MD.ClangModuleMapFile);
       M.FileDeps = cxstring::createSet(MD.FileDeps);
       std::vector<std::string> Modules;
-      for (const ModuleID &MID : MD.ClangModuleDeps)
+      for (ModuleID &MID : MD.ClangModuleDeps)
         Modules.push_back(MID.ModuleName + ":" + MID.ContextHash);
       M.ModuleDeps = cxstring::createSet(Modules);
-      M.BuildArguments = cxstring::createSet(MD.BuildArguments);
+      M.BuildArguments = cxstring::createSet(MD.getBuildArguments());
     }
     MDC(Context, MDS);
   }
@@ -594,7 +594,8 @@ clang_experimental_DepGraphModule_getModuleDeps(CXDepGraphModule CXDepMod) {
 CXCStringArray
 clang_experimental_DepGraphModule_getBuildArguments(CXDepGraphModule CXDepMod) {
   ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps;
-  return unwrap(CXDepMod)->StrMgr.createCStringsRef(ModDeps.BuildArguments);
+  return unwrap(CXDepMod)->StrMgr.createCStringsRef(
+      ModDeps.getBuildArguments());
 }
 
 const char *