[clang][deps][cas] Ensure AST file inputs are tracked in cas-fs

The scanner may not need to load all inputs of an AST file, but the
compilation could trigger lazy loading. Ensure we access all input files
when caching with cas-fs so that they are tracked correctly.

Fixes a nasty bug where including a module from a PCH could produce
fatal errors about missing files in the importing compilation.

(cherry picked from commit 277e0d6)

Author: benlangmuir

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

@@ -609,6 +609,23 @@ Error FullDependencyConsumer::initialize(CompilerInstance &ScanInstance,
   return Error::success();
 }
 
+/// Ensure that all files reachable from imported modules/pch are tracked.
+/// These could be loaded lazily during compilation.
+static void trackASTFileInputs(CompilerInstance &CI,
+                               llvm::cas::CachingOnDiskFileSystem &CacheFS) {
+  auto Reader = CI.getASTReader();
+  if (!Reader)
+    return;
+
+  for (serialization::ModuleFile &MF : Reader->getModuleManager()) {
+    Reader->visitInputFiles(
+        MF, /*IncludeSystem=*/true, /*Complain=*/false,
+        [](const serialization::InputFile &IF, bool isSystem) {
+          // Visiting input files triggers the file system lookup.
+        });
+  }
+}
+
 Error FullDependencyConsumer::finalize(CompilerInstance &ScanInstance,
                                        CompilerInvocation &NewInvocation) {
   if (CacheFS) {
@@ -624,6 +641,8 @@ Error FullDependencyConsumer::finalize(CompilerInstance &ScanInstance,
     (void)CacheFS->status(NewInvocation.getCodeGenOpts().SampleProfileFile);
     (void)CacheFS->status(NewInvocation.getCodeGenOpts().ProfileRemappingFile);
 
+    trackASTFileInputs(ScanInstance, *CacheFS);
+
     auto E = CacheFS
                  ->createTreeFromNewAccesses(
                      [&](const llvm::vfs::CachedDirectoryEntry &Entry,
@@ -670,6 +689,8 @@ Error FullDependencyConsumer::initializeModuleBuild(
 Error FullDependencyConsumer::finalizeModuleBuild(
     CompilerInstance &ModuleScanInstance) {
   if (CacheFS) {
+    trackASTFileInputs(ModuleScanInstance, *CacheFS);
+
     std::optional<cas::CASID> RootID;
     auto E = CacheFS
                  ->createTreeFromNewAccesses(

clang/test/ClangScanDeps/modules-cas-trees-input-files.c

@@ -0,0 +1,92 @@
+// Ensure all files references by a cached module can be found.
+
+// REQUIRES: ondisk_cas
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json
+// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
+
+// == Scan PCH
+// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \
+// RUN:   -cas-path %t/cas -action-cache-path %t/cache -module-files-dir %t/outputs \
+// RUN:   -format experimental-full -mode preprocess-dependency-directives \
+// RUN:   > %t/deps_pch.json
+
+// == Build PCH
+// RUN: %deps-to-rsp %t/deps_pch.json --module-name A > %t/A.rsp
+// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp
+
+// RUN: %clang @%t/A.rsp
+// RUN: %clang @%t/pch.rsp
+
+// == Scan TU, including PCH
+// RUN: clang-scan-deps -compilation-database %t/cdb.json \
+// RUN:   -cas-path %t/cas -action-cache-path %t/cache -module-files-dir %t/outputs \
+// RUN:   -format experimental-full -mode preprocess-dependency-directives \
+// RUN:   > %t/deps.json
+
+// == Build TU, including PCH
+// RUN: %deps-to-rsp %t/deps.json --module-name C > %t/C.rsp
+// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp
+
+// RUN: %clang @%t/C.rsp
+// RUN: %clang @%t/tu.rsp
+
+//--- cdb_pch.json.template
+[
+  {
+    "directory" : "DIR",
+    "command" : "clang_tool -I DIR -x c-header DIR/prefix.h -o DIR/prefix.h.pch -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache",
+    "file" : "DIR/prefix.h"
+  },
+]
+
+//--- cdb.json.template
+[
+  {
+    "directory" : "DIR",
+    "command" : "clang_tool -I DIR -fsyntax-only DIR/tu.c -include DIR/prefix.h -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache",
+    "file" : "DIR/tu.c"
+  },
+]
+
+// The test below is a specific instance that was failing in the past. The test
+// complexity is required to trigger looking up an input file of a module that
+// would not be automatically visited when importing that module during
+// dependency scanning, because it is only triggered incidentally. The anatomy
+// of the specific test case is:
+// * module A has a reference to B's modulemap but does not import B
+//   because it was for an excluded header
+// * module C imports A and performs macro expansion of module_macro
+// * looking up the location for the macro definition does binary search and
+//   incidentally deserializes the SLocEntry for B's modulemap. With small
+//   modules such as in this test case, the binary search is *likely* to hit
+//   that specific SLocEntry, and it did at the time this test was written.
+// * module A must be "prebuilt"; otherwise the scanner will visit all inputs
+//   during implicit module validation; so we load it via PCH.
+
+//--- A/module.modulemap
+module A { header "A.h" }
+
+//--- A/A.h
+#include "B/B.h"
+#define module_macro int
+
+//--- B/module.modulemap
+module B { exclude header "B.h" }
+
+//--- B/B.h
+
+//--- module.modulemap
+module C { header "C.h" }
+
+//--- C.h
+#include "A/A.h"
+module_macro x;
+
+//--- prefix.h
+#include "A/A.h"
+
+//--- tu.c
+#include "C.h"