Merge pull request #37723 from artemcm/SaveReuseDependencyScannerCache

[Dependency Scanning] Add ability for `-scan-dependencies` action to serialize and deserialize dependency scanner cache from a `.moddepcache` file.

Author: artemcm

include/swift-c/DependencyScan/DependencyScan.h

@@ -337,6 +337,30 @@ swiftscan_batch_scan_result_create(swiftscan_scanner_t scanner,
 SWIFTSCAN_PUBLIC swiftscan_import_set_t swiftscan_import_set_create(
     swiftscan_scanner_t scanner, swiftscan_scan_invocation_t invocation);
 
+//=== Scanner Cache Operations --------------------------------------------===//
+// The following operations expose an implementation detail of the dependency
+// scanner: its module dependencies cache. This is done in order
+// to allow clients to perform incremental dependency scans by having the
+// scanner's state be serializable and re-usable.
+
+/// For the specified \c scanner instance, serialize its state to the specified file-system \c path .
+SWIFTSCAN_PUBLIC void
+swiftscan_scanner_cache_serialize(swiftscan_scanner_t scanner,
+                                  const char * path);
+
+/// For the specified \c scanner instance, load in scanner state from a file at
+/// the specified file-system \c path .
+SWIFTSCAN_PUBLIC bool
+swiftscan_scanner_cache_load(swiftscan_scanner_t scanner,
+                             const char * path);
+
+/// For the specified \c scanner instance, reset its internal state, ensuring subsequent
+/// scanning queries are done "from-scratch".
+SWIFTSCAN_PUBLIC void
+swiftscan_scanner_cache_reset(swiftscan_scanner_t scanner);
+
+//===----------------------------------------------------------------------===//
+
 SWIFTSCAN_END_DECLS
 
 #endif // SWIFT_C_DEPENDENCY_SCAN_H

include/swift/AST/DiagnosticsCommon.def

@@ -192,6 +192,15 @@ ERROR(scanner_find_cycle, none,
 ERROR(scanner_arguments_invalid, none,
       "dependencies scanner cannot be configured with arguments: '%0'", (StringRef))
 
+WARNING(warn_scaner_deserialize_failed, none,
+        "Failed to load module scanning dependency cache from: '%0', re-building scanner cache from scratch.", (StringRef))
+
+REMARK(remark_reuse_cache, none,
+       "Re-using serialized module scanning dependency cache from: '%0'.", (StringRef))
+
+REMARK(remark_save_cache, none,
+       "Serializing module scanning dependency cache to: '%0'.", (StringRef))
+
 //------------------------------------------------------------------------------
 // MARK: custom attribute diagnostics
 //------------------------------------------------------------------------------

include/swift/DependencyScan/DependencyScanningTool.h

@@ -57,6 +57,14 @@ class DependencyScanningTool {
                   const std::vector<BatchScanInput> &BatchInput,
                   const llvm::StringSet<> &PlaceholderModules);
 
+  /// Writes the current `SharedCache` instance to a specified FileSystem path.
+  void serializeCache(llvm::StringRef path);
+  /// Loads an instance of a `ModuleDependenciesCache` to serve as the `SharedCache`
+  /// from a specified FileSystem path.
+  bool loadCache(llvm::StringRef path);
+  /// Discard the tool's current `SharedCache` and start anew.
+  void resetCache();
+
 private:
   /// Using the specified invocation command, instantiate a CompilerInstance
   /// that will be used for this scan.

include/swift/Frontend/FrontendOptions.h

@@ -313,6 +313,9 @@ class FrontendOptions {
   /// The path at which to either serialize or deserialize the dependency scanner cache.
   std::string SerializedDependencyScannerCachePath;
 
+  /// Emit remarks indicating use of the serialized module dependency scanning cache
+  bool EmitDependencyScannerCacheRemarks = false;
+
   /// After performing a dependency scanning action, serialize and deserialize the
   /// dependency cache before producing the result.
   bool TestDependencyScannerCacheSerialization = false;

include/swift/Option/FrontendOptions.td

@@ -194,6 +194,9 @@ def reuse_dependency_scan_cache : Flag<["-"], "load-dependency-scan-cache">,
 def dependency_scan_cache_path : Separate<["-"], "dependency-scan-cache-path">,
   HelpText<"The path to output the dependency scanner's internal state.">;
 
+def dependency_scan_cache_remarks : Flag<["-"], "Rdependency-scan-cache">,
+  HelpText<"Emit remarks indicating use of the serialized module dependency scanning cache.">;
+
 def enable_copy_propagation : Flag<["-"], "enable-copy-propagation">,
   HelpText<"Run SIL copy propagation to shorten object lifetime.">;
 def disable_copy_propagation : Flag<["-"], "disable-copy-propagation">,

lib/DependencyScan/DependencyScanningTool.cpp

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "swift/DependencyScan/DependencyScanningTool.h"
+#include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h"
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/DiagnosticsFrontend.h"
 #include "swift/Basic/LLVMInitialize.h"
@@ -77,11 +78,37 @@ DependencyScanningTool::getDependencies(
         BatchInput.size(), std::make_error_code(std::errc::invalid_argument));
   auto Instance = std::move(*InstanceOrErr);
 
-  auto batchScanResults = performBatchModuleScan(
+  auto BatchScanResults = performBatchModuleScan(
       *Instance.get(), *SharedCache, VersionedPCMInstanceCacheCache.get(),
       Saver, BatchInput);
 
-  return batchScanResults;
+  return BatchScanResults;
+}
+
+void DependencyScanningTool::serializeCache(llvm::StringRef path) {
+  SourceManager SM;
+  DiagnosticEngine Diags(SM);
+  Diags.addConsumer(PDC);
+  module_dependency_cache_serialization::writeInterModuleDependenciesCache(
+      Diags, path, *SharedCache);
+}
+
+bool DependencyScanningTool::loadCache(llvm::StringRef path) {
+  SourceManager SM;
+  DiagnosticEngine Diags(SM);
+  Diags.addConsumer(PDC);
+  SharedCache = std::make_unique<ModuleDependenciesCache>();
+  bool Success =
+      module_dependency_cache_serialization::readInterModuleDependenciesCache(
+          path, *SharedCache);
+  if (!Success) {
+    Diags.diagnose(SourceLoc(), diag::warn_scaner_deserialize_failed, path);
+  }
+  return Success;
+}
+
+void DependencyScanningTool::resetCache() {
+  SharedCache.reset(new ModuleDependenciesCache());
 }
 
 llvm::ErrorOr<std::unique_ptr<CompilerInstance>>

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -472,7 +472,7 @@ bool swift::dependencies::module_dependency_cache_serialization::
       "loading inter-module dependency graph", path);
   auto buffer = llvm::MemoryBuffer::getFile(path);
   if (!buffer)
-    return false;
+    return true;
 
   return readInterModuleDependenciesCache(*buffer.get(), cache);
 }

lib/DependencyScan/ScanDependencies.cpp

@@ -282,7 +282,16 @@ static void discoverCrosssImportOverlayDependencies(
 
   // Record the dummy main module's direct dependencies. The dummy main module
   // only directly depend on these newly discovered overlay modules.
-  cache.recordDependencies(dummyMainName, dummyMainDependencies);
+  if (cache.findDependencies(dummyMainName,
+                             ModuleDependenciesKind::SwiftTextual)) {
+    cache.updateDependencies(
+        std::make_pair(dummyMainName.str(),
+                       ModuleDependenciesKind::SwiftTextual),
+        dummyMainDependencies);
+  } else {
+    cache.recordDependencies(dummyMainName, dummyMainDependencies);
+  }
+
   llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
                   std::set<ModuleDependencyID>>
       allModules;
@@ -1100,17 +1109,30 @@ identifyMainModuleDependencies(CompilerInstance &instance) {
 
 } // namespace
 
-static void serializeDependencyCache(DiagnosticEngine &diags,
-                                     const std::string &path,
+static void serializeDependencyCache(CompilerInstance &instance,
                                      const ModuleDependenciesCache &cache) {
+  const FrontendOptions &opts = instance.getInvocation().getFrontendOptions();
+  ASTContext &Context = instance.getASTContext();
+  auto savePath = opts.SerializedDependencyScannerCachePath;
   module_dependency_cache_serialization::writeInterModuleDependenciesCache(
-      diags, path, cache);
+      Context.Diags, savePath, cache);
+  if (opts.EmitDependencyScannerCacheRemarks) {
+    Context.Diags.diagnose(SourceLoc(), diag::remark_save_cache, savePath);
+  }
 }
 
-static void deserializeDependencyCache(const std::string &path,
+static void deserializeDependencyCache(CompilerInstance &instance,
                                        ModuleDependenciesCache &cache) {
-  module_dependency_cache_serialization::readInterModuleDependenciesCache(
-      path, cache);
+  const FrontendOptions &opts = instance.getInvocation().getFrontendOptions();
+  ASTContext &Context = instance.getASTContext();
+  auto loadPath = opts.SerializedDependencyScannerCachePath;
+  if (module_dependency_cache_serialization::readInterModuleDependenciesCache(
+          loadPath, cache)) {
+    Context.Diags.diagnose(SourceLoc(), diag::warn_scaner_deserialize_failed,
+                           loadPath);
+  } else if (opts.EmitDependencyScannerCacheRemarks) {
+    Context.Diags.diagnose(SourceLoc(), diag::remark_reuse_cache, loadPath);
+  }
 }
 
 bool swift::dependencies::scanDependencies(CompilerInstance &instance) {
@@ -1119,22 +1141,32 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) {
   std::string path = opts.InputsAndOutputs.getSingleOutputFilename();
   std::error_code EC;
   llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::F_None);
-
-  // `-scan-dependencies` invocations use a single new instance
-  // of a module cache
-  ModuleDependenciesCache cache;
   if (out.has_error() || EC) {
     Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path,
                            EC.message());
     out.clear_error();
     return true;
   }
 
-  // Execute scan, and write JSON output to the output stream
+  // `-scan-dependencies` invocations use a single new instance
+  // of a module cache
+  ModuleDependenciesCache cache;
+
+  if (opts.ReuseDependencyScannerCache)
+    deserializeDependencyCache(instance, cache);
+
+  // Execute scan
   auto dependenciesOrErr = performModuleScan(instance, cache);
+
+  // Serialize the dependency cache if -serialize-dependency-scan-cache
+  // is specified
+  if (opts.SerializeDependencyScannerCache)
+    serializeDependencyCache(instance, cache);
+  
   if (dependenciesOrErr.getError())
     return true;
   auto dependencies = std::move(*dependenciesOrErr);
+
   // Write out the JSON description.
   writeJSON(out, dependencies);
   // This process succeeds regardless of whether any errors occurred.
@@ -1253,7 +1285,19 @@ swift::dependencies::performModuleScan(CompilerInstance &instance,
                   std::set<ModuleDependencyID>> allModules;
 
   allModules.insert({mainModuleName.str(), mainDependencies.getKind()});
-  cache.recordDependencies(mainModuleName, std::move(mainDependencies));
+
+  // We may be re-using an instance of the cache which already contains
+  // an entry for this module.
+  if (cache.findDependencies(mainModuleName,
+                             ModuleDependenciesKind::SwiftTextual)) {
+    cache.updateDependencies(
+        std::make_pair(mainModuleName.str(),
+                       ModuleDependenciesKind::SwiftTextual),
+        std::move(mainDependencies));
+  } else {
+    cache.recordDependencies(mainModuleName, std::move(mainDependencies));
+  }
+
   auto &ctx = instance.getASTContext();
   auto ModuleCachePath = getModuleCachePathFromClang(
       ctx.getClangModuleLoader()->getClangInstance());
@@ -1295,15 +1339,17 @@ swift::dependencies::performModuleScan(CompilerInstance &instance,
   ModuleDependenciesCache loadedCache;
   if (FEOpts.TestDependencyScannerCacheSerialization) {
     llvm::SmallString<128> buffer;
-    auto EC = llvm::sys::fs::createTemporaryFile("depscan_cache", "moddepcache", buffer);
+    auto EC = llvm::sys::fs::createTemporaryFile("depscan_cache", "moddepcache",
+                                                 buffer);
     if (EC) {
-      instance.getDiags().diagnose(SourceLoc(),
-                                   diag::error_unable_to_make_temporary_file,
-                                   EC.message());
+      instance.getDiags().diagnose(
+          SourceLoc(), diag::error_unable_to_make_temporary_file, EC.message());
     } else {
       std::string path = buffer.str().str();
-      serializeDependencyCache(instance.getDiags(), path, cache);
-      deserializeDependencyCache(path, loadedCache);
+      module_dependency_cache_serialization::writeInterModuleDependenciesCache(
+          instance.getDiags(), path, cache);
+      module_dependency_cache_serialization::readInterModuleDependenciesCache(
+          path, loadedCache);
       resultCache = &loadedCache;
     }
   }

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -119,6 +119,7 @@ bool ArgsToFrontendOptionsConverter::convert(
 
   Opts.SerializeDependencyScannerCache |= Args.hasArg(OPT_serialize_dependency_scan_cache);
   Opts.ReuseDependencyScannerCache |= Args.hasArg(OPT_reuse_dependency_scan_cache);
+  Opts.EmitDependencyScannerCacheRemarks |= Args.hasArg(OPT_dependency_scan_cache_remarks);
   Opts.TestDependencyScannerCacheSerialization |= Args.hasArg(OPT_debug_test_dependency_scan_cache_serialization);
   if (const Arg *A = Args.getLastArg(OPT_dependency_scan_cache_path)) {
     Opts.SerializedDependencyScannerCachePath = A->getValue();