diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index 18d09245839f5..c4ccbf8e861dc 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -25,6 +25,7 @@ #include "llvm/ADT/StringSet.h" #include #include +#include namespace swift { @@ -35,7 +36,8 @@ class Identifier; /// Which kind of module dependencies we are looking for. enum class ModuleDependenciesKind : int8_t { - SwiftTextual, + FirstKind, + SwiftTextual = FirstKind, SwiftBinary, // Placeholder dependencies are a kind of dependencies used only by the // dependency scanner. They are swift modules that the scanner will not be @@ -61,6 +63,14 @@ enum class ModuleDependenciesKind : int8_t { // of all targets, individually, have been computed. SwiftPlaceholder, Clang, + LastKind = Clang + 1 +}; + +struct ModuleDependenciesKindHash { + std::size_t operator()(ModuleDependenciesKind k) const { + using UnderlyingType = std::underlying_type::type; + return std::hash{}(static_cast(k)); + } }; /// Details of a given module used for dependency scanner cache queries. @@ -409,29 +419,30 @@ class ModuleDependencies { }; using ModuleDependencyID = std::pair; +using ModuleDependenciesVector = llvm::SmallVector; /// A cache describing the set of module dependencies that has been queried -/// thus far. -class ModuleDependenciesCache { - using ModuleDependenciesVector = llvm::SmallVector; - +/// thus far. This cache records/stores the actual Dependency values and can be +/// preserved across different scanning actions (e.g. in +/// `DependencyScanningTool`). It is not to be queried directly, but is rather +/// meant to be wrapped in an instance of `ModuleDependenciesCache`, responsible +/// for recording new dependencies and answering cache queries in a given scan. +/// Queries to this cache must be disambiguated with a set of search paths to +/// ensure that the returned cached dependency was one that can be found in the +/// current scanning action's filesystem view. +class GlobalModuleDependenciesCache { /// All cached module dependencies, in the order in which they were /// encountered. std::vector AllModules; - /// Dependencies for Textual Swift modules that have already been computed. - /// This maps a module's id (name, kind) to a vector of Dependency objects, which correspond - /// to instances of the same module that may have been found in different sets of search paths. - llvm::StringMap SwiftTextualModuleDependencies; - - /// Dependencies for Binary Swift modules that have already been computed. - llvm::StringMap SwiftBinaryModuleDependencies; - - /// Dependencies for Swift placeholder dependency modules that have already been computed. - llvm::StringMap SwiftPlaceholderModuleDependencies; - - /// Dependencies for Clang modules that have already been computed. - llvm::StringMap ClangModuleDependencies; + /// Dependencies for modules that have already been computed. + /// This maps a dependency kind to a map of a module's name to a vector of Dependency objects, + /// which correspond to instances of the same module that may have been found + /// in different sets of search paths. + std::unordered_map, + ModuleDependenciesKindHash> + ModuleDependenciesKindMap; /// Additional information needed for Clang dependency scanning. ClangModuleDependenciesCacheImpl *clangImpl = nullptr; @@ -447,55 +458,148 @@ class ModuleDependenciesCache { /// Retrieve the dependencies map that corresponds to the given dependency /// kind. - llvm::StringMap &getDependenciesMap( - ModuleDependenciesKind kind); - const llvm::StringMap &getDependenciesMap( - ModuleDependenciesKind kind) const; + llvm::StringMap & + getDependenciesMap(ModuleDependenciesKind kind); + const llvm::StringMap & + getDependenciesMap(ModuleDependenciesKind kind) const; public: - ModuleDependenciesCache() { } + GlobalModuleDependenciesCache(); + GlobalModuleDependenciesCache(const GlobalModuleDependenciesCache &) = delete; + GlobalModuleDependenciesCache & + operator=(const GlobalModuleDependenciesCache &) = delete; - ModuleDependenciesCache(const ModuleDependenciesCache &) = delete; - ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete; + virtual ~GlobalModuleDependenciesCache() { destroyClangImpl(); } - ~ModuleDependenciesCache() { - destroyClangImpl(); - } +private: + /// Enforce clients not being allowed to query this cache directly, it must be + /// wrapped in an instance of `ModuleDependenciesCache`. + friend class ModuleDependenciesCache; /// Set the Clang-specific implementation data. - void setClangImpl( - ClangModuleDependenciesCacheImpl *clangImpl, - void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *)) { + virtual void + setClangImpl(ClangModuleDependenciesCacheImpl *clangImpl, + void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *)) { destroyClangImpl(); this->clangImpl = clangImpl; this->clangImplDeleter = clangImplDeleter; } + /// Retrieve the Clang-specific implementation data; + ClangModuleDependenciesCacheImpl *getClangImpl() const { return clangImpl; } + + /// Whether we have cached dependency information for the given module. + bool hasDependencies(StringRef moduleName, + ModuleLookupSpecifics details) const; + + /// Look for module dependencies for a module with the given name given + /// current search paths. + /// + /// \returns the cached result, or \c None if there is no cached entry. + Optional + findDependencies(StringRef moduleName, ModuleLookupSpecifics details) const; + +public: + /// Look for module dependencies for a module with the given name. + /// This method has a deliberately-obtuse name to indicate that it is not to + /// be used for general queries. + /// + /// \returns the cached result, or \c None if there is no cached entry. + Optional + findAllDependenciesIrrespectiveOfSearchPaths( + StringRef moduleName, Optional kind) const; + + /// Record dependencies for the given module. + const ModuleDependencies *recordDependencies(StringRef moduleName, + ModuleDependencies dependencies); + + /// Update stored dependencies for the given module. + const ModuleDependencies *updateDependencies(ModuleDependencyID moduleID, + ModuleDependencies dependencies); + + /// Reference the list of all module dependencies. + const std::vector &getAllModules() const { + return AllModules; + } +}; + +/// This "local" dependencies cache persists only for the duration of a given +/// scanning action, and wraps an instance of a `GlobalModuleDependenciesCache` +/// which may carry cached scanning information from prior scanning actions. +/// This cache maintains a store of references to all dependencies found within +/// the current scanning action (with their values stored in the global Cache), +/// since these do not require clients to disambiguate them with search paths. +class ModuleDependenciesCache { +private: + GlobalModuleDependenciesCache &globalCache; + + /// References to data in `globalCache` for dependencies accimulated during + /// the current scanning action. + std::unordered_map, + ModuleDependenciesKindHash> + ModuleDependenciesKindMap; + + /// Retrieve the dependencies map that corresponds to the given dependency + /// kind. + llvm::StringMap & + getDependencyReferencesMap(ModuleDependenciesKind kind); + const llvm::StringMap & + getDependencyReferencesMap(ModuleDependenciesKind kind) const; + + /// Local cache results lookup, only for modules which were discovered during + /// the current scanner invocation. + bool hasDependencies(StringRef moduleName, + Optional kind) const; + + /// Local cache results lookup, only for modules which were discovered during + /// the current scanner invocation. + Optional + findDependencies(StringRef moduleName, + Optional kind) const; + +public: + ModuleDependenciesCache(GlobalModuleDependenciesCache &globalCache); + ModuleDependenciesCache(const ModuleDependenciesCache &) = delete; + ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete; + virtual ~ModuleDependenciesCache() {} + +public: + /// Set the Clang-specific implementation data. + void + setClangImpl(ClangModuleDependenciesCacheImpl *clangImpl, + void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *)) { + globalCache.setClangImpl(clangImpl, clangImplDeleter); + } + /// Retrieve the Clang-specific implementation data; ClangModuleDependenciesCacheImpl *getClangImpl() const { - return clangImpl; + return globalCache.getClangImpl(); } /// Whether we have cached dependency information for the given module. bool hasDependencies(StringRef moduleName, ModuleLookupSpecifics details) const; - /// Look for module dependencies for a module with the given name given current search paths. + /// Look for module dependencies for a module with the given name given + /// current search paths. /// /// \returns the cached result, or \c None if there is no cached entry. - Optional findDependencies( - StringRef moduleName, - ModuleLookupSpecifics details) const; + Optional + findDependencies(StringRef moduleName, ModuleLookupSpecifics details) const; /// Look for module dependencies for a module with the given name. - /// This method has a deliberately-obtuse name to indicate that it is not to be used for general - /// queries. + /// This method has a deliberately-obtuse name to indicate that it is not to + /// be used for general queries. /// /// \returns the cached result, or \c None if there is no cached entry. - Optional findAllDependenciesIrrespectiveOfSearchPaths( - StringRef moduleName, - Optional kind) const; + Optional + findAllDependenciesIrrespectiveOfSearchPaths( + StringRef moduleName, Optional kind) const { + return globalCache.findAllDependenciesIrrespectiveOfSearchPaths(moduleName, + kind); + } /// Record dependencies for the given module. void recordDependencies(StringRef moduleName, @@ -507,10 +611,10 @@ class ModuleDependenciesCache { /// Reference the list of all module dependencies. const std::vector &getAllModules() const { - return AllModules; + return globalCache.getAllModules(); } }; -} +} // namespace swift #endif /* SWIFT_AST_MODULE_DEPENDENCIES_H */ diff --git a/include/swift/DependencyScan/DependencyScanningTool.h b/include/swift/DependencyScan/DependencyScanningTool.h index e0913ac312e69..54783b6735631 100644 --- a/include/swift/DependencyScan/DependencyScanningTool.h +++ b/include/swift/DependencyScan/DependencyScanningTool.h @@ -59,7 +59,7 @@ class DependencyScanningTool { /// Writes the current `SharedCache` instance to a specified FileSystem path. void serializeCache(llvm::StringRef path); - /// Loads an instance of a `ModuleDependenciesCache` to serve as the `SharedCache` + /// Loads an instance of a `GlobalModuleDependenciesCache` to serve as the `SharedCache` /// from a specified FileSystem path. bool loadCache(llvm::StringRef path); /// Discard the tool's current `SharedCache` and start anew. @@ -73,7 +73,7 @@ class DependencyScanningTool { /// Shared cache of module dependencies, re-used by individual full-scan queries /// during the lifetime of this Tool. - std::unique_ptr SharedCache; + std::unique_ptr SharedCache; /// Shared cache of compiler instances created during batch scanning, corresponding to /// command-line options specified in the batch scan input entry. diff --git a/include/swift/DependencyScan/ScanDependencies.h b/include/swift/DependencyScan/ScanDependencies.h index e1d5edb7970e5..4a9fd7ee0b676 100644 --- a/include/swift/DependencyScan/ScanDependencies.h +++ b/include/swift/DependencyScan/ScanDependencies.h @@ -27,12 +27,18 @@ namespace swift { class CompilerInvocation; class CompilerInstance; class ModuleDependenciesCache; +class GlobalModuleDependenciesCache; namespace dependencies { +//using CompilerArgInstanceCacheMap = +// llvm::StringMap, +// std::unique_ptr>>; + using CompilerArgInstanceCacheMap = - llvm::StringMap, - std::unique_ptr>>; + llvm::StringMap, + std::unique_ptr, + std::unique_ptr>>; struct BatchScanInput { llvm::StringRef moduleName; diff --git a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h index 0ff3a1562d5d7..1b77a7510992c 100644 --- a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h +++ b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h @@ -23,7 +23,7 @@ class MemoryBuffer; namespace swift { class DiagnosticEngine; -class ModuleDependenciesCache; +class GlobalModuleDependenciesCache; namespace dependencies { namespace module_dependency_cache_serialization { @@ -160,23 +160,23 @@ using ClangModuleDetailsLayout = /// Tries to read the dependency graph from the given buffer. /// Returns \c true if there was an error. bool readInterModuleDependenciesCache(llvm::MemoryBuffer &buffer, - ModuleDependenciesCache &cache); + GlobalModuleDependenciesCache &cache); /// Tries to read the dependency graph from the given path name. /// Returns true if there was an error. bool readInterModuleDependenciesCache(llvm::StringRef path, - ModuleDependenciesCache &cache); + GlobalModuleDependenciesCache &cache); /// Tries to write the dependency graph to the given path name. /// Returns true if there was an error. bool writeInterModuleDependenciesCache(DiagnosticEngine &diags, llvm::StringRef path, - const ModuleDependenciesCache &cache); + const GlobalModuleDependenciesCache &cache); /// Tries to write out the given dependency cache with the given /// bitstream writer. void writeInterModuleDependenciesCache(llvm::BitstreamWriter &Out, - const ModuleDependenciesCache &cache); + const GlobalModuleDependenciesCache &cache); } // end namespace module_dependency_cache_serialization } // end namespace dependencies diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h index 07b74b8fe1a9f..b34c17b7b6526 100644 --- a/include/swift/Frontend/FrontendOptions.h +++ b/include/swift/Frontend/FrontendOptions.h @@ -310,10 +310,6 @@ class FrontendOptions { /// Emit remarks indicating use of the serialized module dependency scanning cache bool EmitDependencyScannerCacheRemarks = false; - /// After performing a dependency scanning action, serialize and deserialize the - /// dependency cache before producing the result. - bool TestDependencyScannerCacheSerialization = false; - /// When performing an incremental build, ensure that cross-module incremental /// build metadata is available in any swift modules emitted by this frontend /// job. diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index c97e6fdd3c43e..885104e28e70f 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -18,6 +18,14 @@ #include "swift/AST/SourceFile.h" using namespace swift; +ModuleDependenciesKind& operator++(ModuleDependenciesKind& e) { + if (e == ModuleDependenciesKind::LastKind) { + llvm_unreachable("Attempting to incrementa last enum value on ModuleDependenciesKind"); + } + e = ModuleDependenciesKind(static_cast::type>(e) + 1); + return e; +} + ModuleDependenciesStorageBase::~ModuleDependenciesStorageBase() { } bool ModuleDependencies::isSwiftModule() const { @@ -131,34 +139,18 @@ void ModuleDependencies::addBridgingModuleDependency( swiftStorage->bridgingModuleDependencies.push_back(module.str()); } -llvm::StringMap & -ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) { - switch (kind) { - case ModuleDependenciesKind::SwiftTextual: - return SwiftTextualModuleDependencies; - case ModuleDependenciesKind::SwiftBinary: - return SwiftBinaryModuleDependencies; - case ModuleDependenciesKind::SwiftPlaceholder: - return SwiftPlaceholderModuleDependencies; - case ModuleDependenciesKind::Clang: - return ClangModuleDependencies; - } - llvm_unreachable("invalid dependency kind"); -} - -const llvm::StringMap & -ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) const { - switch (kind) { - case ModuleDependenciesKind::SwiftTextual: - return SwiftTextualModuleDependencies; - case ModuleDependenciesKind::SwiftBinary: - return SwiftBinaryModuleDependencies; - case ModuleDependenciesKind::SwiftPlaceholder: - return SwiftPlaceholderModuleDependencies; - case ModuleDependenciesKind::Clang: - return ClangModuleDependencies; - } - llvm_unreachable("invalid dependency kind"); +llvm::StringMap & +GlobalModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) { + auto it = ModuleDependenciesKindMap.find(kind); + assert(it != ModuleDependenciesKindMap.end() && "invalid dependency kind"); + return it->second; +} + +const llvm::StringMap & +GlobalModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) const { + auto it = ModuleDependenciesKindMap.find(kind); + assert(it != ModuleDependenciesKindMap.end() && "invalid dependency kind"); + return it->second; } static std::string moduleBasePath(const StringRef modulePath) { @@ -190,23 +182,36 @@ static bool moduleContainedInImportPathSet(const ModuleDependencies &module, const llvm::StringSet<> &importPaths) { std::string modulePath = ""; - if (auto *swiftDep = module.getAsSwiftTextualModule()) { - if (swiftDep->swiftInterfaceFile) - modulePath = *(swiftDep->swiftInterfaceFile); - else { - // If we encountered a Swift textual dependency without an interface - // file, we are seeing the main scan module itself. This means that - // our search-path disambiguation is not necessary here. + switch (module.getKind()) { + case swift::ModuleDependenciesKind::SwiftTextual: { + auto *swiftDep = module.getAsSwiftTextualModule(); + if (swiftDep->swiftInterfaceFile) + modulePath = *(swiftDep->swiftInterfaceFile); + else { + // If we encountered a Swift textual dependency without an interface + // file, we are seeing the main scan module itself. This means that + // our search-path disambiguation is not necessary here. + return true; + } + break; + } + case swift::ModuleDependenciesKind::SwiftBinary: { + auto *swiftBinaryDep = module.getAsSwiftBinaryModule(); + modulePath = swiftBinaryDep->compiledModulePath; + break; + } + case swift::ModuleDependenciesKind::Clang: { + auto *clangDep = module.getAsClangModule(); + modulePath = clangDep->moduleMapFile; + break; + } + case swift::ModuleDependenciesKind::SwiftPlaceholder: { + // Placeholders are resolved as `true` because they are not associated with + // any specific search path. return true; } - } else if (auto *swiftBinaryDep = module.getAsSwiftBinaryModule()) { - modulePath = swiftBinaryDep->compiledModulePath; - } else if (auto *placehodlerDep = module.getAsPlaceholderDependencyModule()) { - // Placeholders are resolved as `true` because they are not associated with - // any specific search path. - return true; - } else if (auto *clangDep = module.getAsClangModule()) { - modulePath = clangDep->moduleMapFile; + default: + llvm_unreachable("Unhandled dependency kind."); } if (moduleContainedInImportPathSet(modulePath, importPaths)) { @@ -215,56 +220,35 @@ static bool moduleContainedInImportPathSet(const ModuleDependencies &module, return false; } -bool ModuleDependenciesCache::hasDependencies( - StringRef moduleName, - ModuleLookupSpecifics details) const { - if (!details.kind) { - return hasDependencies(moduleName, - {ModuleDependenciesKind::SwiftTextual, - details.currentSearchPaths}) || - hasDependencies(moduleName, - {ModuleDependenciesKind::SwiftBinary, - details.currentSearchPaths}) || - hasDependencies(moduleName, - {ModuleDependenciesKind::SwiftPlaceholder, - details.currentSearchPaths}) || - hasDependencies(moduleName, - {ModuleDependenciesKind::Clang, - details.currentSearchPaths}); +GlobalModuleDependenciesCache::GlobalModuleDependenciesCache() { + for (auto kind = ModuleDependenciesKind::FirstKind; + kind != ModuleDependenciesKind::LastKind; ++kind) { + ModuleDependenciesKindMap.insert( + {kind, llvm::StringMap()}); } - const auto &map = getDependenciesMap(*details.kind); - auto known = map.find(moduleName); - if (known != map.end()) { - assert(!known->second.empty()); - for (auto &dep : known->second) { - if (moduleContainedInImportPathSet(dep, details.currentSearchPaths)) - return true; - } - return false; - } - return false; + ModuleDependenciesKindMap.insert( + {ModuleDependenciesKind::SwiftBinary, + llvm::StringMap()}); + ModuleDependenciesKindMap.insert( + {ModuleDependenciesKind::SwiftPlaceholder, + llvm::StringMap()}); + ModuleDependenciesKindMap.insert( + {ModuleDependenciesKind::Clang, + llvm::StringMap()}); } -Optional ModuleDependenciesCache::findDependencies( +Optional GlobalModuleDependenciesCache::findDependencies( StringRef moduleName, ModuleLookupSpecifics details) const { if (!details.kind) { - if (auto swiftTextualDep = findDependencies( - moduleName, - {ModuleDependenciesKind::SwiftTextual, details.currentSearchPaths})) - return swiftTextualDep; - else if (auto swiftBinaryDep = findDependencies( - moduleName, - {ModuleDependenciesKind::SwiftBinary, details.currentSearchPaths})) - return swiftBinaryDep; - else if (auto swiftPlaceholderDep = findDependencies( - moduleName, - {ModuleDependenciesKind::SwiftPlaceholder, details.currentSearchPaths})) - return swiftPlaceholderDep; - else - return findDependencies(moduleName, - {ModuleDependenciesKind::Clang, details.currentSearchPaths}); + for (auto kind = ModuleDependenciesKind::FirstKind; + kind != ModuleDependenciesKind::LastKind; ++kind) { + auto dep = findDependencies(moduleName, {kind, details.currentSearchPaths}); + if (dep.hasValue()) + return dep.getValue(); + } + return None; } const auto &map = getDependenciesMap(*details.kind); @@ -280,24 +264,23 @@ Optional ModuleDependenciesCache::findDependencies( return None; } -Optional -ModuleDependenciesCache::findAllDependenciesIrrespectiveOfSearchPaths( +bool GlobalModuleDependenciesCache::hasDependencies( + StringRef moduleName, + ModuleLookupSpecifics details) const { + return findDependencies(moduleName, details).hasValue(); +} + +Optional +GlobalModuleDependenciesCache::findAllDependenciesIrrespectiveOfSearchPaths( StringRef moduleName, Optional kind) const { if (!kind) { - if (auto swiftTextualDeps = findAllDependenciesIrrespectiveOfSearchPaths( - moduleName, ModuleDependenciesKind::SwiftTextual)) - return swiftTextualDeps; - else if (auto swiftBinaryDeps = - findAllDependenciesIrrespectiveOfSearchPaths( - moduleName, ModuleDependenciesKind::SwiftBinary)) - return swiftBinaryDeps; - else if (auto swiftPlaceholderDeps = - findAllDependenciesIrrespectiveOfSearchPaths( - moduleName, ModuleDependenciesKind::SwiftPlaceholder)) - return swiftPlaceholderDeps; - else - return findAllDependenciesIrrespectiveOfSearchPaths( - moduleName, ModuleDependenciesKind::Clang); + for (auto kind = ModuleDependenciesKind::FirstKind; + kind != ModuleDependenciesKind::LastKind; ++kind) { + auto deps = findAllDependenciesIrrespectiveOfSearchPaths(moduleName, kind); + if (deps.hasValue()) + return deps.getValue(); + } + return None; } const auto &map = getDependenciesMap(*kind); @@ -311,18 +294,31 @@ ModuleDependenciesCache::findAllDependenciesIrrespectiveOfSearchPaths( static std::string modulePathForVerification(const ModuleDependencies &module) { std::string existingModulePath = ""; - if (auto *swiftDep = module.getAsSwiftTextualModule()) { - if (swiftDep->swiftInterfaceFile) - existingModulePath = *(swiftDep->swiftInterfaceFile); - } else if (auto *swiftBinaryDep = module.getAsSwiftBinaryModule()) { - existingModulePath = swiftBinaryDep->compiledModulePath; - } else if (auto *clangDep = module.getAsClangModule()) { - existingModulePath = clangDep->moduleMapFile; + switch (module.getKind()) { + case swift::ModuleDependenciesKind::SwiftTextual: { + auto *swiftDep = module.getAsSwiftTextualModule(); + if (swiftDep->swiftInterfaceFile) + existingModulePath = *(swiftDep->swiftInterfaceFile); + break; + } + case swift::ModuleDependenciesKind::SwiftBinary: { + auto *swiftBinaryDep = module.getAsSwiftBinaryModule(); + existingModulePath = swiftBinaryDep->compiledModulePath; + break; + } + case swift::ModuleDependenciesKind::Clang: { + auto *clangDep = module.getAsClangModule(); + existingModulePath = clangDep->moduleMapFile; + break; + } + case swift::ModuleDependenciesKind::SwiftPlaceholder: + default: + llvm_unreachable("Unhandled dependency kind."); } return existingModulePath; } -void ModuleDependenciesCache::recordDependencies( +const ModuleDependencies* GlobalModuleDependenciesCache::recordDependencies( StringRef moduleName, ModuleDependencies dependencies) { auto kind = dependencies.getKind(); @@ -332,24 +328,107 @@ void ModuleDependenciesCache::recordDependencies( // Ensure that the existing dependencies objects are at a different path. // i.e. do not record duplicate dependencies. auto newModulePath = modulePathForVerification(dependencies); - bool newEntry = true; for (auto &existingDeps : map[moduleName]) { if (modulePathForVerification(existingDeps) == newModulePath) - newEntry = false; + return &existingDeps; } - if (newEntry) - map[moduleName].emplace_back(std::move(dependencies)); + + map[moduleName].emplace_back(std::move(dependencies)); + return map[moduleName].end()-1; } else { map.insert({moduleName, ModuleDependenciesVector{std::move(dependencies)}}); AllModules.push_back({moduleName.str(), kind}); + return &(map[moduleName].front()); } } -void ModuleDependenciesCache::updateDependencies( +const ModuleDependencies* GlobalModuleDependenciesCache::updateDependencies( ModuleDependencyID moduleID, ModuleDependencies dependencies) { auto &map = getDependenciesMap(moduleID.second); auto known = map.find(moduleID.first); assert(known != map.end() && "Not yet added to map"); - assert(known->second.size() == 1 && "Cannot update dependency with multiple candidates."); + assert(known->second.size() == 1 && + "Cannot update dependency with multiple candidates."); known->second[0] = std::move(dependencies); + return &(known->second[0]); +} + +llvm::StringMap & +ModuleDependenciesCache::getDependencyReferencesMap(ModuleDependenciesKind kind) { + auto it = ModuleDependenciesKindMap.find(kind); + assert(it != ModuleDependenciesKindMap.end() && "invalid dependency kind"); + return it->second; +} + +const llvm::StringMap & +ModuleDependenciesCache::getDependencyReferencesMap(ModuleDependenciesKind kind) const { + auto it = ModuleDependenciesKindMap.find(kind); + assert(it != ModuleDependenciesKindMap.end() && "invalid dependency kind"); + return it->second; +} + +ModuleDependenciesCache::ModuleDependenciesCache(GlobalModuleDependenciesCache &globalCache) +: globalCache(globalCache) { + for (auto kind = ModuleDependenciesKind::FirstKind; + kind != ModuleDependenciesKind::LastKind; ++kind) { + ModuleDependenciesKindMap.insert( + {kind, llvm::StringMap()}); + } +} + +Optional ModuleDependenciesCache::findDependencies( + StringRef moduleName, Optional kind) const { + if (!kind) { + for (auto kind = ModuleDependenciesKind::FirstKind; + kind != ModuleDependenciesKind::LastKind; ++kind) { + auto dep = findDependencies(moduleName, kind); + if (dep.hasValue()) + return dep.getValue(); + } + return None; + } + + const auto &map = getDependencyReferencesMap(*kind); + auto known = map.find(moduleName); + if (known != map.end()) + return known->second; + + return None; +} + +Optional +ModuleDependenciesCache::findDependencies(StringRef moduleName, + ModuleLookupSpecifics details) const { + // 1. Query the local scan results + // 2. If no module found, query the global cache using the module details + // lookup + auto localResult = findDependencies(moduleName, details.kind); + if (localResult.hasValue()) + return *(localResult.getValue()); + else + return globalCache.findDependencies(moduleName, details); +} + +bool ModuleDependenciesCache::hasDependencies( + StringRef moduleName, ModuleLookupSpecifics details) const { + return findDependencies(moduleName, details).hasValue(); +} + +void ModuleDependenciesCache::recordDependencies( + StringRef moduleName, ModuleDependencies dependencies) { + auto globalDepPtr = globalCache.recordDependencies(moduleName, dependencies); + auto kind = globalDepPtr->getKind(); + auto &map = getDependencyReferencesMap(kind); + assert(map.count(moduleName) == 0 && "Already added to map"); + map.insert({moduleName, globalDepPtr}); +} + +void ModuleDependenciesCache::updateDependencies( + ModuleDependencyID moduleID, ModuleDependencies dependencies) { + auto globalDepRef = globalCache.updateDependencies(moduleID, dependencies); + auto &map = getDependencyReferencesMap(moduleID.second); + auto known = map.find(moduleID.first); + if (known != map.end()) + map.erase(known); + map.insert({moduleID.first, globalDepRef}); } diff --git a/lib/DependencyScan/DependencyScanningTool.cpp b/lib/DependencyScan/DependencyScanningTool.cpp index 2d3aa357c19c0..8c4f70afc7e87 100644 --- a/lib/DependencyScan/DependencyScanningTool.cpp +++ b/lib/DependencyScan/DependencyScanningTool.cpp @@ -25,7 +25,7 @@ namespace swift { namespace dependencies { DependencyScanningTool::DependencyScanningTool() - : SharedCache(std::make_unique()), + : SharedCache(std::make_unique()), VersionedPCMInstanceCacheCache( std::make_unique()), PDC(), Alloc(), Saver(Alloc) {} @@ -40,8 +40,10 @@ DependencyScanningTool::getDependencies( return EC; auto Instance = std::move(*InstanceOrErr); + // Local scan cache instance, wrapping the shared global cache. + ModuleDependenciesCache cache(*SharedCache); // Execute the scanning action, retreiving the in-memory result - auto DependenciesOrErr = performModuleScan(*Instance.get(), *SharedCache); + auto DependenciesOrErr = performModuleScan(*Instance.get(), cache); if (DependenciesOrErr.getError()) return std::make_error_code(std::errc::not_supported); auto Dependencies = std::move(*DependenciesOrErr); @@ -78,8 +80,10 @@ DependencyScanningTool::getDependencies( BatchInput.size(), std::make_error_code(std::errc::invalid_argument)); auto Instance = std::move(*InstanceOrErr); + // Local scan cache instance, wrapping the shared global cache. + ModuleDependenciesCache cache(*SharedCache); auto BatchScanResults = performBatchModuleScan( - *Instance.get(), *SharedCache, VersionedPCMInstanceCacheCache.get(), + *Instance.get(), cache, VersionedPCMInstanceCacheCache.get(), Saver, BatchInput); return BatchScanResults; @@ -97,7 +101,7 @@ bool DependencyScanningTool::loadCache(llvm::StringRef path) { SourceManager SM; DiagnosticEngine Diags(SM); Diags.addConsumer(PDC); - SharedCache = std::make_unique(); + SharedCache = std::make_unique(); bool readFailed = module_dependency_cache_serialization::readInterModuleDependenciesCache( path, *SharedCache); @@ -108,7 +112,7 @@ bool DependencyScanningTool::loadCache(llvm::StringRef path) { } void DependencyScanningTool::resetCache() { - SharedCache.reset(new ModuleDependenciesCache()); + SharedCache.reset(new GlobalModuleDependenciesCache()); } llvm::ErrorOr> diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index 97128e1ecc6c5..6ef405496e390 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -36,14 +36,14 @@ class Deserializer { bool readSignature(); bool enterGraphBlock(); bool readMetadata(); - bool readGraph(ModuleDependenciesCache &cache); + bool readGraph(GlobalModuleDependenciesCache &cache); llvm::Optional getIdentifier(unsigned n); llvm::Optional> getArray(unsigned n); public: Deserializer(llvm::MemoryBufferRef Data) : Cursor(Data) {} - bool readInterModuleDependenciesCache(ModuleDependenciesCache &cache); + bool readInterModuleDependenciesCache(GlobalModuleDependenciesCache &cache); }; } // end namespace @@ -146,7 +146,7 @@ bool Deserializer::readMetadata() { /// all of the file's identifiers and arrays of identifiers, followed by /// consuming individual module info records and registering them into the /// cache. -bool Deserializer::readGraph(ModuleDependenciesCache &cache) { +bool Deserializer::readGraph(GlobalModuleDependenciesCache &cache) { using namespace graph_block; bool hasCurrentModule = false; @@ -406,7 +406,7 @@ bool Deserializer::readGraph(ModuleDependenciesCache &cache) { } bool Deserializer::readInterModuleDependenciesCache( - ModuleDependenciesCache &cache) { + GlobalModuleDependenciesCache &cache) { using namespace graph_block; if (readSignature()) @@ -460,14 +460,14 @@ llvm::Optional> Deserializer::getArray(unsigned n) { bool swift::dependencies::module_dependency_cache_serialization:: readInterModuleDependenciesCache(llvm::MemoryBuffer &buffer, - ModuleDependenciesCache &cache) { + GlobalModuleDependenciesCache &cache) { Deserializer deserializer(buffer.getMemBufferRef()); return deserializer.readInterModuleDependenciesCache(cache); } bool swift::dependencies::module_dependency_cache_serialization:: readInterModuleDependenciesCache(StringRef path, - ModuleDependenciesCache &cache) { + GlobalModuleDependenciesCache &cache) { PrettyStackTraceStringAction stackTrace( "loading inter-module dependency graph", path); auto buffer = llvm::MemoryBuffer::getFile(path); @@ -571,7 +571,7 @@ class Serializer { AbbrCodes[Layout::Code] = Layout::emitAbbrev(Out); } - void collectStringsAndArrays(const ModuleDependenciesCache &cache); + void collectStringsAndArrays(const GlobalModuleDependenciesCache &cache); void emitBlockID(unsigned ID, StringRef name, SmallVectorImpl &nameBuffer); @@ -593,7 +593,7 @@ class Serializer { Serializer(llvm::BitstreamWriter &ExistingOut) : Out(ExistingOut) {} public: - void writeInterModuleDependenciesCache(const ModuleDependenciesCache &cache); + void writeInterModuleDependenciesCache(const GlobalModuleDependenciesCache &cache); }; } // end namespace @@ -742,6 +742,8 @@ void Serializer::writeModuleInfo(ModuleDependencyID moduleID, break; } + default: + llvm_unreachable("Unhandled dependency kind."); } } @@ -812,7 +814,7 @@ unsigned Serializer::getArray(ModuleDependencyID moduleID, return arrayIter->second; } -void Serializer::collectStringsAndArrays(const ModuleDependenciesCache &cache) { +void Serializer::collectStringsAndArrays(const GlobalModuleDependenciesCache &cache) { for (auto &moduleID : cache.getAllModules()) { auto dependencyInfos = cache.findAllDependenciesIrrespectiveOfSearchPaths( moduleID.first, moduleID.second); @@ -876,13 +878,15 @@ void Serializer::collectStringsAndArrays(const ModuleDependenciesCache &cache) { clangDeps->fileDependencies); break; } + default: + llvm_unreachable("Unhandled dependency kind."); } } } } void Serializer::writeInterModuleDependenciesCache( - const ModuleDependenciesCache &cache) { + const GlobalModuleDependenciesCache &cache) { // Write the header writeSignature(); writeBlockInfoBlock(); @@ -930,14 +934,14 @@ void Serializer::writeInterModuleDependenciesCache( void swift::dependencies::module_dependency_cache_serialization:: writeInterModuleDependenciesCache(llvm::BitstreamWriter &Out, - const ModuleDependenciesCache &cache) { + const GlobalModuleDependenciesCache &cache) { Serializer serializer{Out}; serializer.writeInterModuleDependenciesCache(cache); } bool swift::dependencies::module_dependency_cache_serialization:: writeInterModuleDependenciesCache(DiagnosticEngine &diags, StringRef path, - const ModuleDependenciesCache &cache) { + const GlobalModuleDependenciesCache &cache) { PrettyStackTraceStringAction stackTrace( "saving inter-module dependency graph", path); return withOutputFile(diags, path, [&](llvm::raw_ostream &out) { diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 48e2603225a35..a8365779b2050 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -734,6 +734,8 @@ static std::string createEncodedModuleKindAndName(ModuleDependencyID id) { return "swiftPlaceholder:" + id.first; case ModuleDependenciesKind::Clang: return "clang:" + id.first; + default: + llvm_unreachable("Unhandled dependency kind."); } } @@ -871,6 +873,8 @@ generateFullDependencyGraph(CompilerInstance &instance, case ModuleDependenciesKind::Clang: dependencyKindAndName = "clang"; break; + default: + llvm_unreachable("Unhandled dependency kind."); } dependencyKindAndName += ":"; dependencyKindAndName += dep.first; @@ -944,10 +948,6 @@ static bool diagnoseCycle(CompilerInstance &instance, return false; } -using CompilerArgInstanceCacheMap = - llvm::StringMap, - std::unique_ptr>>; - static void updateCachedInstanceOpts(CompilerInstance &cachedInstance, const CompilerInstance &invocationInstance, llvm::StringRef entryArguments) { @@ -1006,8 +1006,8 @@ forEachBatchEntry(CompilerInstance &invocationInstance, } else if (subInstanceMap->count(entry.arguments)) { // Use the previously created instance if we've seen the arguments // before. - pInstance = (*subInstanceMap)[entry.arguments].first.get(); - pCache = (*subInstanceMap)[entry.arguments].second.get(); + pInstance = std::get<0>((*subInstanceMap)[entry.arguments]).get(); + pCache = std::get<2>((*subInstanceMap)[entry.arguments]).get(); // We must update the search paths of this instance to instead reflect // those of the current scanner invocation. updateCachedInstanceOpts(*pInstance, invocationInstance, entry.arguments); @@ -1018,13 +1018,15 @@ forEachBatchEntry(CompilerInstance &invocationInstance, llvm::cl::ResetAllOptionOccurrences(); // Create a new instance by the arguments and save it in the map. + auto newGlobalCache = std::make_unique(); subInstanceMap->insert( {entry.arguments, - std::make_pair(std::make_unique(), - std::make_unique())}); + std::make_tuple(std::make_unique(), + std::move(newGlobalCache), + std::make_unique(*newGlobalCache))}); - pInstance = (*subInstanceMap)[entry.arguments].first.get(); - pCache = (*subInstanceMap)[entry.arguments].second.get(); + pInstance = std::get<0>((*subInstanceMap)[entry.arguments]).get(); + pCache = std::get<2>((*subInstanceMap)[entry.arguments]).get(); SmallVector args; llvm::cl::TokenizeGNUCommandLine(entry.arguments, saver, args); CompilerInvocation subInvok = invok; @@ -1129,7 +1131,7 @@ identifyMainModuleDependencies(CompilerInstance &instance) { } // namespace static void serializeDependencyCache(CompilerInstance &instance, - const ModuleDependenciesCache &cache) { + const GlobalModuleDependenciesCache &cache) { const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); ASTContext &Context = instance.getASTContext(); auto savePath = opts.SerializedDependencyScannerCachePath; @@ -1141,7 +1143,7 @@ static void serializeDependencyCache(CompilerInstance &instance, } static void deserializeDependencyCache(CompilerInstance &instance, - ModuleDependenciesCache &cache) { + GlobalModuleDependenciesCache &cache) { const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); ASTContext &Context = instance.getASTContext(); auto loadPath = opts.SerializedDependencyScannerCachePath; @@ -1169,10 +1171,11 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { // `-scan-dependencies` invocations use a single new instance // of a module cache - ModuleDependenciesCache cache; - + GlobalModuleDependenciesCache globalCache; if (opts.ReuseDependencyScannerCache) - deserializeDependencyCache(instance, cache); + deserializeDependencyCache(instance, globalCache); + + ModuleDependenciesCache cache(globalCache); // Execute scan auto dependenciesOrErr = performModuleScan(instance, cache); @@ -1180,7 +1183,7 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { // Serialize the dependency cache if -serialize-dependency-scan-cache // is specified if (opts.SerializeDependencyScannerCache) - serializeDependencyCache(instance, cache); + serializeDependencyCache(instance, globalCache); if (dependenciesOrErr.getError()) return true; @@ -1204,7 +1207,8 @@ bool swift::dependencies::prescanDependencies(CompilerInstance &instance) { llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::F_None); // `-scan-dependencies` invocations use a single new instance // of a module cache - ModuleDependenciesCache cache; + GlobalModuleDependenciesCache singleUseGlobalCache; + ModuleDependenciesCache cache(singleUseGlobalCache); if (out.has_error() || EC) { Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, EC.message()); @@ -1232,7 +1236,8 @@ bool swift::dependencies::batchScanDependencies( CompilerInstance &instance, llvm::StringRef batchInputFile) { // The primary cache used for scans carried out with the compiler instance // we have created - ModuleDependenciesCache cache; + GlobalModuleDependenciesCache singleUseGlobalCache; + ModuleDependenciesCache cache(singleUseGlobalCache); (void)instance.getMainModule(); llvm::BumpPtrAllocator alloc; llvm::StringSaver saver(alloc); @@ -1264,7 +1269,8 @@ bool swift::dependencies::batchPrescanDependencies( CompilerInstance &instance, llvm::StringRef batchInputFile) { // The primary cache used for scans carried out with the compiler instance // we have created - ModuleDependenciesCache cache; + GlobalModuleDependenciesCache singleUseGlobalCache; + ModuleDependenciesCache cache(singleUseGlobalCache); (void)instance.getMainModule(); llvm::BumpPtrAllocator alloc; llvm::StringSaver saver(alloc); @@ -1356,34 +1362,13 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, ASTDelegate)) return std::make_error_code(std::errc::not_supported); - // Test dependency cache serialization/deserialization if - // `-test-dependency-scan-cache-serialization` is specified. - auto *resultCache = &cache; - ModuleDependenciesCache loadedCache; - if (FEOpts.TestDependencyScannerCacheSerialization) { - llvm::SmallString<128> buffer; - auto EC = llvm::sys::fs::createTemporaryFile("depscan_cache", "moddepcache", - buffer); - if (EC) { - instance.getDiags().diagnose( - SourceLoc(), diag::error_unable_to_make_temporary_file, EC.message()); - } else { - std::string path = buffer.str().str(); - module_dependency_cache_serialization::writeInterModuleDependenciesCache( - instance.getDiags(), path, cache); - module_dependency_cache_serialization::readInterModuleDependenciesCache( - path, loadedCache); - resultCache = &loadedCache; - } - } - auto dependencyGraph = generateFullDependencyGraph( - instance, *resultCache, ASTDelegate, + instance, cache, ASTDelegate, allModules.getArrayRef(), currentImportPathSet); // Update the dependency tracker. if (auto depTracker = instance.getDependencyTracker()) { for (auto module : allModules) { - auto deps = resultCache->findDependencies(module.first, + auto deps = cache.findDependencies(module.first, {module.second, currentImportPathSet}); if (!deps) continue; diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp index dece951514d03..e3051fac170e7 100644 --- a/lib/Driver/Driver.cpp +++ b/lib/Driver/Driver.cpp @@ -178,8 +178,6 @@ static void validateDependencyScanningArgs(DiagnosticEngine &diags, args.getLastArg(options::OPT_reuse_dependency_scan_cache); const Arg *CacheSerializationPath = args.getLastArg(options::OPT_dependency_scan_cache_path); - const Arg *TestSerialization = args.getLastArg( - options::OPT_debug_test_dependency_scan_cache_serialization); if (ExternalDependencyMap && !ScanDependencies) { diags.diagnose(SourceLoc(), diag::error_requirement_not_met, @@ -202,11 +200,6 @@ static void validateDependencyScanningArgs(DiagnosticEngine &diags, diags.diagnose(SourceLoc(), diag::error_requirement_not_met, "-load-dependency-scan-cache", "-scan-dependencies"); } - if (TestSerialization && !ScanDependencies) { - diags.diagnose(SourceLoc(), diag::error_requirement_not_met, - "-test-dependency-scan-cache-serialization", - "-scan-dependencies"); - } if (SerializeCache && !CacheSerializationPath) { diags.diagnose(SourceLoc(), diag::error_requirement_not_met, "-serialize-dependency-scan-cache", diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index 9d41f36c4934c..13180f5123998 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -120,7 +120,6 @@ bool ArgsToFrontendOptionsConverter::convert( Opts.SerializeDependencyScannerCache |= Args.hasArg(OPT_serialize_dependency_scan_cache); Opts.ReuseDependencyScannerCache |= Args.hasArg(OPT_reuse_dependency_scan_cache); Opts.EmitDependencyScannerCacheRemarks |= Args.hasArg(OPT_dependency_scan_cache_remarks); - Opts.TestDependencyScannerCacheSerialization |= Args.hasArg(OPT_debug_test_dependency_scan_cache_serialization); if (const Arg *A = Args.getLastArg(OPT_dependency_scan_cache_path)) { Opts.SerializedDependencyScannerCachePath = A->getValue(); } diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index 3c8e65bdc65b3..46d5307bbd4a0 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -1069,7 +1069,6 @@ withSemanticAnalysis(CompilerInstance &Instance, FrontendObserver *observer, static bool performScanDependencies(CompilerInstance &Instance) { auto batchScanInput = Instance.getASTContext().SearchPathOpts.BatchScanInputFilePath; - ModuleDependenciesCache SingleUseCache; if (batchScanInput.empty()) { if (Instance.getInvocation().getFrontendOptions().ImportPrescan) return dependencies::prescanDependencies(Instance); diff --git a/test/ScanDependencies/local_cache_consistency.swift b/test/ScanDependencies/local_cache_consistency.swift new file mode 100644 index 0000000000000..c969a648b89cb --- /dev/null +++ b/test/ScanDependencies/local_cache_consistency.swift @@ -0,0 +1,26 @@ +// REQUIRES: executable_test +// REQUIRES: objc_interop +// REQUIRES: OS=macosx + +// RUN: %empty-directory(%t) +// RUN: mkdir -p %t/clang-module-cache + +// Run the scanner once, ensuring CoreFoundation dependencies are as expected +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -swift-version 4 +// RUN: %FileCheck %s < %t/deps.json + +import CoreFoundation + +// CHECK: "clang": "CoreFoundation" + +// CHECK: "directDependencies": [ +// CHECK: { +// CHECK: "clang": "Darwin" +// CHECK: }, +// CHECK: { +// CHECK: "clang": "Dispatch" +// CHECK: } +// CHECK: ], + +// Make sure the transitive dependency on os_object is present +// CHECK: "clang": "os_object"