Merge pull request #38704 from artemcm/DependencyScanningCherryFarming

[5.5][Dependency Scanning] Cherry-pick dependency scanner changes required for Explicit Module Builds

Author: artemcm

include/swift-c/DependencyScan/DependencyScan.h

@@ -337,6 +337,30 @@ swiftscan_batch_scan_result_create(swiftscan_scanner_t scanner,
 SWIFTSCAN_PUBLIC swiftscan_import_set_t swiftscan_import_set_create(
     swiftscan_scanner_t scanner, swiftscan_scan_invocation_t invocation);
 
+//=== Scanner Cache Operations --------------------------------------------===//
+// The following operations expose an implementation detail of the dependency
+// scanner: its module dependencies cache. This is done in order
+// to allow clients to perform incremental dependency scans by having the
+// scanner's state be serializable and re-usable.
+
+/// For the specified \c scanner instance, serialize its state to the specified file-system \c path .
+SWIFTSCAN_PUBLIC void
+swiftscan_scanner_cache_serialize(swiftscan_scanner_t scanner,
+                                  const char * path);
+
+/// For the specified \c scanner instance, load in scanner state from a file at
+/// the specified file-system \c path .
+SWIFTSCAN_PUBLIC bool
+swiftscan_scanner_cache_load(swiftscan_scanner_t scanner,
+                             const char * path);
+
+/// For the specified \c scanner instance, reset its internal state, ensuring subsequent
+/// scanning queries are done "from-scratch".
+SWIFTSCAN_PUBLIC void
+swiftscan_scanner_cache_reset(swiftscan_scanner_t scanner);
+
+//===----------------------------------------------------------------------===//
+
 SWIFTSCAN_END_DECLS
 
 #endif // SWIFT_C_DEPENDENCY_SCAN_H

include/swift/AST/ASTContext.h

@@ -37,6 +37,7 @@
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/Support/Allocator.h"
@@ -825,6 +826,13 @@ class ASTContext final {
       ModuleDependenciesCache &cache,
       InterfaceSubContextDelegate &delegate);
 
+  /// Compute the extra implicit framework search paths on Apple platforms:
+  /// $SDKROOT/System/Library/Frameworks/ and $SDKROOT/Library/Frameworks/.
+  std::vector<std::string> getDarwinImplicitFrameworkSearchPaths() const;
+
+  /// Return a set of all possible filesystem locations where modules can be found.
+  llvm::StringSet<> getAllModuleSearchPathsSet() const;
+
   /// Load extensions to the given nominal type from the external
   /// module loaders.
   ///

include/swift/AST/DiagnosticsCommon.def

@@ -192,6 +192,15 @@ ERROR(scanner_find_cycle, none,
 ERROR(scanner_arguments_invalid, none,
       "dependencies scanner cannot be configured with arguments: '%0'", (StringRef))
 
+WARNING(warn_scaner_deserialize_failed, none,
+        "Failed to load module scanning dependency cache from: '%0', re-building scanner cache from scratch.", (StringRef))
+
+REMARK(remark_reuse_cache, none,
+       "Re-using serialized module scanning dependency cache from: '%0'.", (StringRef))
+
+REMARK(remark_save_cache, none,
+       "Serializing module scanning dependency cache to: '%0'.", (StringRef))
+
 //------------------------------------------------------------------------------
 // MARK: custom attribute diagnostics
 //------------------------------------------------------------------------------

include/swift/AST/ModuleDependencies.h

@@ -25,6 +25,7 @@
 #include "llvm/ADT/StringSet.h"
 #include <string>
 #include <vector>
+#include <unordered_map>
 
 namespace swift {
 
@@ -35,7 +36,8 @@ class Identifier;
 
 /// Which kind of module dependencies we are looking for.
 enum class ModuleDependenciesKind : int8_t {
-  SwiftTextual,
+  FirstKind,
+  SwiftTextual = FirstKind,
   SwiftBinary,
   // Placeholder dependencies are a kind of dependencies used only by the
   // dependency scanner. They are swift modules that the scanner will not be
@@ -61,6 +63,20 @@ enum class ModuleDependenciesKind : int8_t {
   // of all targets, individually, have been computed.
   SwiftPlaceholder,
   Clang,
+  LastKind = Clang + 1
+};
+
+struct ModuleDependenciesKindHash {
+  std::size_t operator()(ModuleDependenciesKind k) const {
+    using UnderlyingType = std::underlying_type<ModuleDependenciesKind>::type;
+    return std::hash<UnderlyingType>{}(static_cast<UnderlyingType>(k));
+  }
+};
+
+/// Details of a given module used for dependency scanner cache queries.
+struct ModuleLookupSpecifics {
+  Optional<ModuleDependenciesKind> kind;
+  llvm::StringSet<> currentSearchPaths;
 };
 
 /// Base class for the variant storage of ModuleDependencies.
@@ -275,8 +291,8 @@ class ModuleDependencies {
 
   /// Describe the module dependencies for a Swift module that can be
   /// built from a Swift interface file (\c .swiftinterface).
-  static ModuleDependencies forSwiftInterface(
-      const std::string &swiftInterfaceFile,
+  static ModuleDependencies forSwiftTextualModule(
+      const Optional<std::string> &swiftInterfaceFile,
       ArrayRef<std::string> compiledCandidates,
       ArrayRef<StringRef> buildCommands,
       ArrayRef<StringRef> extraPCMArgs,
@@ -386,6 +402,9 @@ class ModuleDependencies {
   /// Add a bridging header to a Swift module's dependencies.
   void addBridgingHeader(StringRef bridgingHeader);
 
+  /// Add source files
+  void addSourceFile(StringRef sourceFile);
+
   /// Add source files that the bridging header depends on.
   void addBridgingSourceFile(StringRef bridgingSourceFile);
 
@@ -400,25 +419,30 @@ class ModuleDependencies {
 };
 
 using ModuleDependencyID = std::pair<std::string, ModuleDependenciesKind>;
+using ModuleDependenciesVector = llvm::SmallVector<ModuleDependencies, 1>;
 
 /// A cache describing the set of module dependencies that has been queried
-/// thus far.
-class ModuleDependenciesCache {
+/// thus far. This cache records/stores the actual Dependency values and can be
+/// preserved across different scanning actions (e.g. in
+/// `DependencyScanningTool`). It is not to be queried directly, but is rather
+/// meant to be wrapped in an instance of `ModuleDependenciesCache`, responsible
+/// for recording new dependencies and answering cache queries in a given scan.
+/// Queries to this cache must be disambiguated with a set of search paths to
+/// ensure that the returned cached dependency was one that can be found in the
+/// current scanning action's filesystem view.
+class GlobalModuleDependenciesCache {
   /// All cached module dependencies, in the order in which they were
   /// encountered.
   std::vector<ModuleDependencyID> AllModules;
 
-  /// Dependencies for Textual Swift modules that have already been computed.
-  llvm::StringMap<ModuleDependencies> SwiftTextualModuleDependencies;
-
-  /// Dependencies for Binary Swift modules that have already been computed.
-  llvm::StringMap<ModuleDependencies> SwiftBinaryModuleDependencies;
-
-  /// Dependencies for Swift placeholder dependency modules that have already been computed.
-  llvm::StringMap<ModuleDependencies> SwiftPlaceholderModuleDependencies;
-
-  /// Dependencies for Clang modules that have already been computed.
-  llvm::StringMap<ModuleDependencies> ClangModuleDependencies;
+  /// Dependencies for modules that have already been computed.
+  /// This maps a dependency kind to a map of a module's name to a vector of Dependency objects,
+  /// which correspond to instances of the same module that may have been found
+  /// in different sets of search paths.
+  std::unordered_map<ModuleDependenciesKind,
+                     llvm::StringMap<ModuleDependenciesVector>,
+                     ModuleDependenciesKindHash>
+      ModuleDependenciesKindMap;
 
   /// Additional information needed for Clang dependency scanning.
   ClangModuleDependenciesCacheImpl *clangImpl = nullptr;
@@ -434,46 +458,148 @@ class ModuleDependenciesCache {
 
   /// Retrieve the dependencies map that corresponds to the given dependency
   /// kind.
-  llvm::StringMap<ModuleDependencies> &getDependenciesMap(
-      ModuleDependenciesKind kind);
-  const llvm::StringMap<ModuleDependencies> &getDependenciesMap(
-      ModuleDependenciesKind kind) const;
+  llvm::StringMap<ModuleDependenciesVector> &
+  getDependenciesMap(ModuleDependenciesKind kind);
+  const llvm::StringMap<ModuleDependenciesVector> &
+  getDependenciesMap(ModuleDependenciesKind kind) const;
 
 public:
-  ModuleDependenciesCache() { }
+  GlobalModuleDependenciesCache();
+  GlobalModuleDependenciesCache(const GlobalModuleDependenciesCache &) = delete;
+  GlobalModuleDependenciesCache &
+  operator=(const GlobalModuleDependenciesCache &) = delete;
 
-  ModuleDependenciesCache(const ModuleDependenciesCache &) = delete;
-  ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete;
+  virtual ~GlobalModuleDependenciesCache() { destroyClangImpl(); }
 
-  ~ModuleDependenciesCache() {
-    destroyClangImpl();
-  }
+private:
+  /// Enforce clients not being allowed to query this cache directly, it must be
+  /// wrapped in an instance of `ModuleDependenciesCache`.
+  friend class ModuleDependenciesCache;
 
   /// Set the Clang-specific implementation data.
-  void setClangImpl(
-      ClangModuleDependenciesCacheImpl *clangImpl,
-      void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *)) {
+  virtual void
+  setClangImpl(ClangModuleDependenciesCacheImpl *clangImpl,
+               void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *)) {
     destroyClangImpl();
 
     this->clangImpl = clangImpl;
     this->clangImplDeleter = clangImplDeleter;
   }
 
+  /// Retrieve the Clang-specific implementation data;
+  ClangModuleDependenciesCacheImpl *getClangImpl() const { return clangImpl; }
+
+  /// Whether we have cached dependency information for the given module.
+  bool hasDependencies(StringRef moduleName,
+                       ModuleLookupSpecifics details) const;
+
+  /// Look for module dependencies for a module with the given name given
+  /// current search paths.
+  ///
+  /// \returns the cached result, or \c None if there is no cached entry.
+  Optional<ModuleDependencies>
+  findDependencies(StringRef moduleName, ModuleLookupSpecifics details) const;
+
+public:
+  /// Look for module dependencies for a module with the given name.
+  /// This method has a deliberately-obtuse name to indicate that it is not to
+  /// be used for general queries.
+  ///
+  /// \returns the cached result, or \c None if there is no cached entry.
+  Optional<ModuleDependenciesVector>
+  findAllDependenciesIrrespectiveOfSearchPaths(
+      StringRef moduleName, Optional<ModuleDependenciesKind> kind) const;
+
+  /// Record dependencies for the given module.
+  const ModuleDependencies *recordDependencies(StringRef moduleName,
+                                               ModuleDependencies dependencies);
+
+  /// Update stored dependencies for the given module.
+  const ModuleDependencies *updateDependencies(ModuleDependencyID moduleID,
+                                               ModuleDependencies dependencies);
+
+  /// Reference the list of all module dependencies.
+  const std::vector<ModuleDependencyID> &getAllModules() const {
+    return AllModules;
+  }
+};
+
+/// This "local" dependencies cache persists only for the duration of a given
+/// scanning action, and wraps an instance of a `GlobalModuleDependenciesCache`
+/// which may carry cached scanning information from prior scanning actions.
+/// This cache maintains a store of references to all dependencies found within
+/// the current scanning action (with their values stored in the global Cache),
+/// since these do not require clients to disambiguate them with search paths.
+class ModuleDependenciesCache {
+private:
+  GlobalModuleDependenciesCache &globalCache;
+
+  /// References to data in `globalCache` for dependencies accimulated during
+  /// the current scanning action.
+  std::unordered_map<ModuleDependenciesKind,
+                     llvm::StringMap<const ModuleDependencies *>,
+                     ModuleDependenciesKindHash>
+      ModuleDependenciesKindMap;
+
+  /// Retrieve the dependencies map that corresponds to the given dependency
+  /// kind.
+  llvm::StringMap<const ModuleDependencies *> &
+  getDependencyReferencesMap(ModuleDependenciesKind kind);
+  const llvm::StringMap<const ModuleDependencies *> &
+  getDependencyReferencesMap(ModuleDependenciesKind kind) const;
+
+  /// Local cache results lookup, only for modules which were discovered during
+  /// the current scanner invocation.
+  bool hasDependencies(StringRef moduleName,
+                       Optional<ModuleDependenciesKind> kind) const;
+
+  /// Local cache results lookup, only for modules which were discovered during
+  /// the current scanner invocation.
+  Optional<const ModuleDependencies *>
+  findDependencies(StringRef moduleName,
+                   Optional<ModuleDependenciesKind> kind) const;
+
+public:
+  ModuleDependenciesCache(GlobalModuleDependenciesCache &globalCache);
+  ModuleDependenciesCache(const ModuleDependenciesCache &) = delete;
+  ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete;
+  virtual ~ModuleDependenciesCache() {}
+
+public:
+  /// Set the Clang-specific implementation data.
+  void
+  setClangImpl(ClangModuleDependenciesCacheImpl *clangImpl,
+               void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *)) {
+    globalCache.setClangImpl(clangImpl, clangImplDeleter);
+  }
+
   /// Retrieve the Clang-specific implementation data;
   ClangModuleDependenciesCacheImpl *getClangImpl() const {
-    return clangImpl;
+    return globalCache.getClangImpl();
   }
 
   /// Whether we have cached dependency information for the given module.
   bool hasDependencies(StringRef moduleName,
-                       Optional<ModuleDependenciesKind> kind) const;
+                       ModuleLookupSpecifics details) const;
+
+  /// Look for module dependencies for a module with the given name given
+  /// current search paths.
+  ///
+  /// \returns the cached result, or \c None if there is no cached entry.
+  Optional<ModuleDependencies>
+  findDependencies(StringRef moduleName, ModuleLookupSpecifics details) const;
 
   /// Look for module dependencies for a module with the given name.
+  /// This method has a deliberately-obtuse name to indicate that it is not to
+  /// be used for general queries.
   ///
   /// \returns the cached result, or \c None if there is no cached entry.
-  Optional<ModuleDependencies> findDependencies(
-      StringRef moduleName,
-      Optional<ModuleDependenciesKind> kind) const;
+  Optional<ModuleDependenciesVector>
+  findAllDependenciesIrrespectiveOfSearchPaths(
+      StringRef moduleName, Optional<ModuleDependenciesKind> kind) const {
+    return globalCache.findAllDependenciesIrrespectiveOfSearchPaths(moduleName,
+                                                                    kind);
+  }
 
   /// Record dependencies for the given module.
   void recordDependencies(StringRef moduleName,
@@ -485,10 +611,10 @@ class ModuleDependenciesCache {
 
   /// Reference the list of all module dependencies.
   const std::vector<ModuleDependencyID> &getAllModules() const {
-    return AllModules;
+    return globalCache.getAllModules();
   }
 };
 
-}
+} // namespace swift
 
 #endif /* SWIFT_AST_MODULE_DEPENDENCIES_H */

include/swift/DependencyScan/DependencyScanningTool.h

@@ -57,6 +57,14 @@ class DependencyScanningTool {
                   const std::vector<BatchScanInput> &BatchInput,
                   const llvm::StringSet<> &PlaceholderModules);
 
+  /// Writes the current `SharedCache` instance to a specified FileSystem path.
+  void serializeCache(llvm::StringRef path);
+  /// Loads an instance of a `GlobalModuleDependenciesCache` to serve as the `SharedCache`
+  /// from a specified FileSystem path.
+  bool loadCache(llvm::StringRef path);
+  /// Discard the tool's current `SharedCache` and start anew.
+  void resetCache();
+
 private:
   /// Using the specified invocation command, instantiate a CompilerInstance
   /// that will be used for this scan.
@@ -65,7 +73,7 @@ class DependencyScanningTool {
 
   /// Shared cache of module dependencies, re-used by individual full-scan queries
   /// during the lifetime of this Tool.
-  std::unique_ptr<ModuleDependenciesCache> SharedCache;
+  std::unique_ptr<GlobalModuleDependenciesCache> SharedCache;
 
   /// Shared cache of compiler instances created during batch scanning, corresponding to
   /// command-line options specified in the batch scan input entry.

include/swift/DependencyScan/ScanDependencies.h

@@ -27,12 +27,18 @@ namespace swift {
 class CompilerInvocation;
 class CompilerInstance;
 class ModuleDependenciesCache;
+class GlobalModuleDependenciesCache;
 
 namespace dependencies {
 
+//using CompilerArgInstanceCacheMap =
+//    llvm::StringMap<std::pair<std::unique_ptr<CompilerInstance>,
+//                              std::unique_ptr<ModuleDependenciesCache>>>;
+
 using CompilerArgInstanceCacheMap =
-    llvm::StringMap<std::pair<std::unique_ptr<CompilerInstance>,
-                              std::unique_ptr<ModuleDependenciesCache>>>;
+    llvm::StringMap<std::tuple<std::unique_ptr<CompilerInstance>,
+                               std::unique_ptr<GlobalModuleDependenciesCache>,
+                               std::unique_ptr<ModuleDependenciesCache>>>;
 
 struct BatchScanInput {
   llvm::StringRef moduleName;