From bf771379d14e44b7b978dffcec42ca1b10c25934 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Fri, 21 May 2021 10:46:19 -0700 Subject: [PATCH 1/5] [Dependency Scanning] Add a binary serialization format for the Inter-Module Dependencies Cache - Adds serialization format based on the LLVM Bitcode File Format (https://llvm.org/docs/BitCodeFormat.html). - Adds Serialization and Deserialization code. --- include/swift/AST/ModuleDependencies.h | 7 +- .../SerializedModuleDependencyCacheFormat.h | 185 ++++ include/swift/Frontend/FrontendOptions.h | 13 + include/swift/Option/FrontendOptions.td | 12 + lib/AST/ModuleDependencies.cpp | 5 + lib/DependencyScan/CMakeLists.txt | 1 + .../ModuleDependencyCacheSerialization.cpp | 920 ++++++++++++++++++ lib/DependencyScan/ScanDependencies.cpp | 50 +- lib/Driver/Driver.cpp | 43 +- .../ArgsToFrontendOptionsConverter.cpp | 7 + lib/Serialization/ModuleDependencyScanner.cpp | 2 +- test/ScanDependencies/bin_mod_import.swift | 5 +- .../ScanDependencies/binary_module_only.swift | 4 + .../can_import_placeholder.swift | 5 +- test/ScanDependencies/module_deps.swift | 4 + .../module_deps_cross_import_overlay.swift | 5 +- .../module_deps_external.swift | 4 + test/ScanDependencies/module_framework.swift | 8 +- 18 files changed, 1264 insertions(+), 16 deletions(-) create mode 100644 include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h create mode 100644 lib/DependencyScan/ModuleDependencyCacheSerialization.cpp diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index 992df4e4a3764..20edf419a0bcc 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -275,8 +275,8 @@ class ModuleDependencies { /// Describe the module dependencies for a Swift module that can be /// built from a Swift interface file (\c .swiftinterface). - static ModuleDependencies forSwiftInterface( - const std::string &swiftInterfaceFile, + static ModuleDependencies forSwiftTextualModule( + const Optional &swiftInterfaceFile, ArrayRef compiledCandidates, ArrayRef buildCommands, ArrayRef extraPCMArgs, @@ -386,6 +386,9 @@ class ModuleDependencies { /// Add a bridging header to a Swift module's dependencies. void addBridgingHeader(StringRef bridgingHeader); + /// Add source files + void addSourceFile(StringRef sourceFile); + /// Add source files that the bridging header depends on. void addBridgingSourceFile(StringRef bridgingSourceFile); diff --git a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h new file mode 100644 index 0000000000000..0ff3a1562d5d7 --- /dev/null +++ b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h @@ -0,0 +1,185 @@ +//=== SerializedModuleDependencyCacheFormat.h - serialized format -*- C++-*-=// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_DEPENDENCY_SERIALIZEDCACHEFORMAT_H +#define SWIFT_DEPENDENCY_SERIALIZEDCACHEFORMAT_H + +#include "llvm/Bitcode/BitcodeConvenience.h" +#include "llvm/Bitstream/BitCodes.h" + +namespace llvm { +class MemoryBuffer; +} + +namespace swift { + +class DiagnosticEngine; +class ModuleDependenciesCache; + +namespace dependencies { +namespace module_dependency_cache_serialization { + +using llvm::BCArray; +using llvm::BCBlob; +using llvm::BCFixed; +using llvm::BCRecordLayout; +using llvm::BCVBR; + +/// Every .moddepcache file begins with these 4 bytes, for easy identification. +const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D', + 'C'}; +const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 1; +/// Increment this on every change. +const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 0; + +/// Various identifiers in this format will rely on having their strings mapped +/// using this ID. +using IdentifierIDField = BCVBR<13>; +using FileIDField = IdentifierIDField; +using ModuleIDField = IdentifierIDField; +using CompilerFlagField = IdentifierIDField; +using ContextHashField = IdentifierIDField; + +/// A bit that indicates whether or not a module is a framework +using IsFrameworkField = BCFixed<1>; + +/// Arrays of various identifiers, distinguised for readability +using IdentifierIDArryField = llvm::BCArray; + +/// Identifiers used to refer to the above arrays +using FileIDArrayIDField = IdentifierIDField; +using DependencyIDArrayIDField = IdentifierIDField; +using FlagIDArrayIDField = IdentifierIDField; + +/// The ID of the top-level block containing the dependency graph +const unsigned GRAPH_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID; + +/// The .moddepcache file format consists of a METADATA record, followed by +/// zero or more IDENTIFIER records that contain various strings seen in the graph +/// (e.g. file names or compiler flags), followed by zero or more IDENTIFIER_ARRAY records +/// which are arrays of identifiers seen in the graph (e.g. list of source files or list of compile flags), +/// followed by zero or more MODULE_NODE, *_DETAILS_NODE pairs of records. +namespace graph_block { +enum { + METADATA = 1, + MODULE_NODE, + SWIFT_TEXTUAL_MODULE_DETAILS_NODE, + SWIFT_PLACEHOLDER_MODULE_DETAILS_NODE, + SWIFT_BINARY_MODULE_DETAILS_NODE, + CLANG_MODULE_DETAILS_NODE, + IDENTIFIER_NODE, + IDENTIFIER_ARRAY_NODE +}; + +// Always the first record in the file. +using MetadataLayout = BCRecordLayout< + METADATA, // ID + BCFixed<16>, // Inter-Module Dependency graph format major version + BCFixed<16>, // Inter-Module Dependency graph format minor version + BCBlob // Compiler version string + >; + +// After the metadata record, we have zero or more identifier records, +// for each unique string that is referenced in the graph. +// +// Identifiers are referenced by their sequence number, starting from 1. +// The identifier value 0 is special; it always represents the empty string. +// There is no IDENTIFIER_NODE serialized that corresponds to it, instead +// the first IDENTIFIER_NODE always has a sequence number of 1. +using IdentifierNodeLayout = BCRecordLayout; + +// After the identifier records we have zero or more identifier array records. +// +// These arrays are also referenced by their sequence number, +// starting from 1, similar to identifiers above. Value 0 indicates an +// empty array. This record is used because individiual array fields must +// appear as the last field of whatever record they belong to, and several of +// the below record layouts contain multiple arrays. +using IdentifierArrayLayout = + BCRecordLayout; + +// After the array records, we have a sequence of Module info +// records, each of which is followed by one of: +// - SwiftTextualModuleDetails +// - SwiftBinaryModuleDetails +// - SwiftPlaceholderModuleDetails +// - ClangModuleDetails +using ModuleInfoLayout = + BCRecordLayout; + +using SwiftTextualModuleDetailsLayout = + BCRecordLayout; + +using SwiftBinaryModuleDetailsLayout = + BCRecordLayout; + +using SwiftPlaceholderModuleDetailsLayout = + BCRecordLayout; + +using ClangModuleDetailsLayout = + BCRecordLayout; +} // namespace graph_block + +/// Tries to read the dependency graph from the given buffer. +/// Returns \c true if there was an error. +bool readInterModuleDependenciesCache(llvm::MemoryBuffer &buffer, + ModuleDependenciesCache &cache); + +/// Tries to read the dependency graph from the given path name. +/// Returns true if there was an error. +bool readInterModuleDependenciesCache(llvm::StringRef path, + ModuleDependenciesCache &cache); + +/// Tries to write the dependency graph to the given path name. +/// Returns true if there was an error. +bool writeInterModuleDependenciesCache(DiagnosticEngine &diags, + llvm::StringRef path, + const ModuleDependenciesCache &cache); + +/// Tries to write out the given dependency cache with the given +/// bitstream writer. +void writeInterModuleDependenciesCache(llvm::BitstreamWriter &Out, + const ModuleDependenciesCache &cache); + +} // end namespace module_dependency_cache_serialization +} // end namespace dependencies +} // end namespace swift + +#endif diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h index 7fc04742ffdab..84295a9a09b21 100644 --- a/include/swift/Frontend/FrontendOptions.h +++ b/include/swift/Frontend/FrontendOptions.h @@ -297,6 +297,19 @@ class FrontendOptions { /// of the main Swift module's source files. bool ImportPrescan = false; + /// After performing a dependency scanning action, serialize the scanner's internal state. + bool SerializeDependencyScannerCache = false; + + /// Load and re-use a prior serialized dependency scanner cache. + bool ReuseDependencyScannerCache = false; + + /// The path at which to either serialize or deserialize the dependency scanner cache. + std::string SerializedDependencyScannerCachePath; + + /// After performing a dependency scanning action, serialize and deserialize the + /// dependency cache before producing the result. + bool TestDependencyScannerCacheSerialization = false; + /// When performing an incremental build, ensure that cross-module incremental /// build metadata is available in any swift modules emitted by this frontend /// job. diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index ef31eba97e762..d2bb716c350cf 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -188,6 +188,15 @@ def batch_scan_input_file def import_prescan : Flag<["-"], "import-prescan">, HelpText<"When performing a dependency scan, only dentify all imports of the main Swift module sources">; +def serialize_dependency_scan_cache : Flag<["-"], "serialize-dependency-scan-cache">, + HelpText<"After performing a dependency scan, serialize the scanner's internal state.">; + +def reuse_dependency_scan_cache : Flag<["-"], "load-dependency-scan-cache">, + HelpText<"After performing a dependency scan, serialize the scanner's internal state.">; + +def dependency_scan_cache_path : Separate<["-"], "dependency-scan-cache-path">, + HelpText<"The path to output the dependency scanner's internal state.">; + def enable_copy_propagation : Flag<["-"], "enable-copy-propagation">, HelpText<"Run SIL copy propagation to shorten object lifetime.">; def disable_copy_propagation : Flag<["-"], "disable-copy-propagation">, @@ -286,6 +295,9 @@ def debug_crash_immediately : Flag<["-"], "debug-crash-immediately">, def debug_crash_after_parse : Flag<["-"], "debug-crash-after-parse">, DebugCrashOpt, HelpText<"Force a crash after parsing">; +def debug_test_dependency_scan_cache_serialization: Flag<["-"], "test-dependency-scan-cache-serialization">, + HelpText<"After performing a dependency scan, serialize and then deserialize the scanner's internal state.">; + def debugger_support : Flag<["-"], "debugger-support">, HelpText<"Process swift code as if running in the debugger">; diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index d33dddf5c90fe..d63dc739f884c 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -118,6 +118,11 @@ void ModuleDependencies::addBridgingSourceFile(StringRef bridgingSourceFile) { swiftStorage->bridgingSourceFiles.push_back(bridgingSourceFile.str()); } +void ModuleDependencies::addSourceFile(StringRef sourceFile) { + auto swiftStorage = cast(storage.get()); + swiftStorage->sourceFiles.push_back(sourceFile.str()); +} + /// Add (Clang) module on which the bridging header depends. void ModuleDependencies::addBridgingModuleDependency( StringRef module, llvm::StringSet<> &alreadyAddedModules) { diff --git a/lib/DependencyScan/CMakeLists.txt b/lib/DependencyScan/CMakeLists.txt index e5190d22e7a59..b582874368646 100644 --- a/lib/DependencyScan/CMakeLists.txt +++ b/lib/DependencyScan/CMakeLists.txt @@ -2,6 +2,7 @@ add_swift_host_library(swiftDependencyScan STATIC DependencyScanningTool.cpp + ModuleDependencyCacheSerialization.cpp ScanDependencies.cpp StringUtils.cpp) diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp new file mode 100644 index 0000000000000..da99ac609f7fa --- /dev/null +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -0,0 +1,920 @@ +//=== ModuleDependencyCacheSerialization.cpp - serialized format -*- C++ -*-==// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "swift/AST/FileSystem.h" +#include "swift/AST/ModuleDependencies.h" +#include "swift/Basic/PrettyStackTrace.h" +#include "swift/Basic/Version.h" +#include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h" +#include "llvm/ADT/DenseMap.h" +#include + +using namespace swift; +using namespace dependencies; +using namespace module_dependency_cache_serialization; + +// MARK: Deserialization +namespace { + +class Deserializer { + std::vector Identifiers; + std::vector> ArraysOfIdentifierIDs; + llvm::BitstreamCursor Cursor; + SmallVector Scratch; + StringRef BlobData; + + // These return true if there was an error. + bool readSignature(); + bool enterGraphBlock(); + bool readMetadata(); + bool readGraph(ModuleDependenciesCache &cache); + + llvm::Optional getIdentifier(unsigned n); + llvm::Optional> getArray(unsigned n); + +public: + Deserializer(llvm::MemoryBufferRef Data) : Cursor(Data) {} + bool readInterModuleDependenciesCache(ModuleDependenciesCache &cache); +}; + +} // end namespace + +/// Read in the expected signature: IMDC +bool Deserializer::readSignature() { + for (unsigned char byte : MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE) { + if (Cursor.AtEndOfStream()) + return true; + if (auto maybeRead = Cursor.Read(8)) { + if (maybeRead.get() != byte) + return true; + } else { + return true; + } + } + return false; +} + +/// Read in the info block and enter the top-level block which represents the +/// graph +bool Deserializer::enterGraphBlock() { + // Read the BLOCKINFO_BLOCK, which contains metadata used when dumping + // the binary data with llvm-bcanalyzer. + { + auto next = Cursor.advance(); + if (!next) { + consumeError(next.takeError()); + return true; + } + + if (next->Kind != llvm::BitstreamEntry::SubBlock) + return true; + + if (next->ID != llvm::bitc::BLOCKINFO_BLOCK_ID) + return true; + + if (!Cursor.ReadBlockInfoBlock()) + return true; + } + + // Enters our top-level subblock, + // which contains the actual module dependency information. + { + auto next = Cursor.advance(); + if (!next) { + consumeError(next.takeError()); + return true; + } + + if (next->Kind != llvm::BitstreamEntry::SubBlock) + return true; + + if (next->ID != GRAPH_BLOCK_ID) + return true; + + if (auto err = Cursor.EnterSubBlock(GRAPH_BLOCK_ID)) { + consumeError(std::move(err)); + return true; + } + } + return false; +} + +/// Read in the serialized file's format version, error/exit if not matching +/// current version. +bool Deserializer::readMetadata() { + using namespace graph_block; + + auto entry = Cursor.advance(); + if (!entry) { + consumeError(entry.takeError()); + return true; + } + + if (entry->Kind != llvm::BitstreamEntry::Record) + return true; + + auto recordID = Cursor.readRecord(entry->ID, Scratch, &BlobData); + if (!recordID) { + consumeError(recordID.takeError()); + return true; + } + + if (*recordID != METADATA) + return true; + + unsigned majorVersion, minorVersion; + + MetadataLayout::readRecord(Scratch, majorVersion, minorVersion); + if (majorVersion != MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR || + minorVersion != MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR) { + return true; + } + + return false; +} + +/// Read in the top-level block's graph structure by first reading in +/// all of the file's identifiers and arrays of identifiers, followed by +/// consuming individual module info records and registering them into the +/// cache. +bool Deserializer::readGraph(ModuleDependenciesCache &cache) { + using namespace graph_block; + + bool hasCurrentModule = false; + std::string currentModuleName; + llvm::Optional> currentModuleDependencies; + + while (!Cursor.AtEndOfStream()) { + auto entry = cantFail(Cursor.advance(), "Advance bitstream cursor"); + + if (entry.Kind == llvm::BitstreamEntry::EndBlock) { + Cursor.ReadBlockEnd(); + assert(Cursor.GetCurrentBitNo() % CHAR_BIT == 0); + break; + } + + if (entry.Kind != llvm::BitstreamEntry::Record) + llvm::report_fatal_error("Bad bitstream entry kind"); + + Scratch.clear(); + unsigned recordID = + cantFail(Cursor.readRecord(entry.ID, Scratch, &BlobData), + "Read bitstream record"); + + switch (recordID) { + case METADATA: { + // METADATA must appear at the beginning and is read by readMetadata(). + llvm::report_fatal_error("Unexpected METADATA record"); + break; + } + + case IDENTIFIER_NODE: { + // IDENTIFIER_NODE must come before MODULE_NODEs. + if (hasCurrentModule) + llvm::report_fatal_error("Unexpected IDENTIFIER_NODE record"); + IdentifierNodeLayout::readRecord(Scratch); + Identifiers.push_back(BlobData.str()); + break; + } + + case IDENTIFIER_ARRAY_NODE: { + // IDENTIFIER_ARRAY_NODE must come before MODULE_NODEs. + if (hasCurrentModule) + llvm::report_fatal_error("Unexpected IDENTIFIER_NODE record"); + ArrayRef identifierIDs; + IdentifierArrayLayout::readRecord(Scratch, identifierIDs); + ArraysOfIdentifierIDs.push_back(identifierIDs.vec()); + break; + } + + case MODULE_NODE: { + hasCurrentModule = true; + unsigned moduleNameID, moduleDependenciesArrayID; + ModuleInfoLayout::readRecord(Scratch, moduleNameID, + moduleDependenciesArrayID); + auto moduleName = getIdentifier(moduleNameID); + if (!moduleName) + llvm::report_fatal_error("Bad module name"); + currentModuleName = *moduleName; + + currentModuleDependencies = getArray(moduleDependenciesArrayID); + if (!currentModuleDependencies) + llvm::report_fatal_error("Bad direct dependencies"); + break; + } + + case SWIFT_TEXTUAL_MODULE_DETAILS_NODE: { + if (!hasCurrentModule) + llvm::report_fatal_error( + "Unexpected SWIFT_TEXTUAL_MODULE_DETAILS_NODE record"); + unsigned interfaceFileID, compiledModuleCandidatesArrayID, + buildCommandLineArrayID, extraPCMArgsArrayID, contextHashID, + isFramework, bridgingHeaderFileID, sourceFilesArrayID, + bridgingSourceFilesArrayID, bridgingModuleDependenciesArrayID; + SwiftTextualModuleDetailsLayout::readRecord( + Scratch, interfaceFileID, compiledModuleCandidatesArrayID, + buildCommandLineArrayID, extraPCMArgsArrayID, contextHashID, + isFramework, bridgingHeaderFileID, sourceFilesArrayID, + bridgingSourceFilesArrayID, bridgingModuleDependenciesArrayID); + + Optional optionalSwiftInterfaceFile; + if (interfaceFileID != 0) { + auto swiftInterfaceFile = getIdentifier(interfaceFileID); + if (!swiftInterfaceFile) + llvm::report_fatal_error("Bad swift interface file path"); + optionalSwiftInterfaceFile = *swiftInterfaceFile; + } + auto compiledModuleCandidates = getArray(compiledModuleCandidatesArrayID); + if (!compiledModuleCandidates) + llvm::report_fatal_error("Bad compiled module candidates"); + auto commandLine = getArray(buildCommandLineArrayID); + if (!commandLine) + llvm::report_fatal_error("Bad command line"); + auto extraPCMArgs = getArray(extraPCMArgsArrayID); + if (!extraPCMArgs) + llvm::report_fatal_error("Bad PCM Args set"); + auto contextHash = getIdentifier(contextHashID); + if (!contextHash) + llvm::report_fatal_error("Bad context hash"); + + // forSwiftInterface API demands references here. + std::vector buildCommandRefs; + for (auto &arg : *commandLine) + buildCommandRefs.push_back(arg); + std::vector extraPCMRefs; + for (auto &arg : *extraPCMArgs) + extraPCMRefs.push_back(arg); + + // Form the dependencies storage object + auto moduleDep = ModuleDependencies::forSwiftTextualModule( + optionalSwiftInterfaceFile, *compiledModuleCandidates, + buildCommandRefs, extraPCMRefs, *contextHash, isFramework); + + // Add dependencies of this module + for (const auto &moduleName : *currentModuleDependencies) + moduleDep.addModuleDependency(moduleName); + + // Add bridging header file path + if (bridgingHeaderFileID != 0) { + auto bridgingHeaderFile = getIdentifier(bridgingHeaderFileID); + if (!bridgingHeaderFile) + llvm::report_fatal_error("Bad bridging header path"); + + moduleDep.addBridgingHeader(*bridgingHeaderFile); + } + + // Add bridging source files + auto bridgingSourceFiles = getArray(bridgingSourceFilesArrayID); + if (!bridgingSourceFiles) + llvm::report_fatal_error("Bad bridging source files"); + for (const auto &file : *bridgingSourceFiles) + moduleDep.addBridgingSourceFile(file); + + // Add source files + auto sourceFiles = getArray(sourceFilesArrayID); + if (!sourceFiles) + llvm::report_fatal_error("Bad bridging source files"); + for (const auto &file : *sourceFiles) + moduleDep.addSourceFile(file); + + // Add bridging module dependencies + auto bridgingModuleDeps = getArray(bridgingModuleDependenciesArrayID); + if (!bridgingModuleDeps) + llvm::report_fatal_error("Bad bridging module dependencies"); + llvm::StringSet<> alreadyAdded; + for (const auto &mod : *bridgingModuleDeps) + moduleDep.addBridgingModuleDependency(mod, alreadyAdded); + + cache.recordDependencies(currentModuleName, std::move(moduleDep)); + hasCurrentModule = false; + break; + } + + case SWIFT_BINARY_MODULE_DETAILS_NODE: { + if (!hasCurrentModule) + llvm::report_fatal_error( + "Unexpected SWIFT_BINARY_MODULE_DETAILS_NODE record"); + unsigned compiledModulePathID, moduleDocPathID, moduleSourceInfoPathID, + isFramework; + SwiftBinaryModuleDetailsLayout::readRecord( + Scratch, compiledModulePathID, moduleDocPathID, + moduleSourceInfoPathID, isFramework); + + auto compiledModulePath = getIdentifier(compiledModulePathID); + if (!compiledModulePath) + llvm::report_fatal_error("Bad compiled module path"); + auto moduleDocPath = getIdentifier(moduleDocPathID); + if (!moduleDocPath) + llvm::report_fatal_error("Bad module doc path"); + auto moduleSourceInfoPath = getIdentifier(moduleSourceInfoPathID); + if (!moduleSourceInfoPath) + llvm::report_fatal_error("Bad module source info path"); + + // Form the dependencies storage object + auto moduleDep = ModuleDependencies::forSwiftBinaryModule( + *compiledModulePath, *moduleDocPath, *moduleSourceInfoPath, + isFramework); + // Add dependencies of this module + for (const auto &moduleName : *currentModuleDependencies) + moduleDep.addModuleDependency(moduleName); + + cache.recordDependencies(currentModuleName, std::move(moduleDep)); + hasCurrentModule = false; + break; + } + + case SWIFT_PLACEHOLDER_MODULE_DETAILS_NODE: { + if (!hasCurrentModule) + llvm::report_fatal_error( + "Unexpected SWIFT_PLACEHOLDER_MODULE_DETAILS_NODE record"); + unsigned compiledModulePathID, moduleDocPathID, moduleSourceInfoPathID; + SwiftPlaceholderModuleDetailsLayout::readRecord( + Scratch, compiledModulePathID, moduleDocPathID, + moduleSourceInfoPathID); + + auto compiledModulePath = getIdentifier(compiledModulePathID); + if (!compiledModulePath) + llvm::report_fatal_error("Bad compiled module path"); + auto moduleDocPath = getIdentifier(moduleDocPathID); + if (!moduleDocPath) + llvm::report_fatal_error("Bad module doc path"); + auto moduleSourceInfoPath = getIdentifier(moduleSourceInfoPathID); + if (!moduleSourceInfoPath) + llvm::report_fatal_error("Bad module source info path"); + + // Form the dependencies storage object + auto moduleDep = ModuleDependencies::forPlaceholderSwiftModuleStub( + *compiledModulePath, *moduleDocPath, *moduleSourceInfoPath); + // Add dependencies of this module + for (const auto &moduleName : *currentModuleDependencies) + moduleDep.addModuleDependency(moduleName); + + cache.recordDependencies(currentModuleName, std::move(moduleDep)); + hasCurrentModule = false; + break; + } + + case CLANG_MODULE_DETAILS_NODE: { + if (!hasCurrentModule) + llvm::report_fatal_error("Unexpected CLANG_MODULE_DETAILS_NODE record"); + unsigned moduleMapPathID, contextHashID, commandLineArrayID, + fileDependenciesArrayID; + ClangModuleDetailsLayout::readRecord(Scratch, moduleMapPathID, + contextHashID, commandLineArrayID, + fileDependenciesArrayID); + auto moduleMapPath = getIdentifier(moduleMapPathID); + if (!moduleMapPath) + llvm::report_fatal_error("Bad module map path"); + auto contextHash = getIdentifier(contextHashID); + if (!contextHash) + llvm::report_fatal_error("Bad context hash"); + auto commandLineArgs = getArray(commandLineArrayID); + if (!commandLineArgs) + llvm::report_fatal_error("Bad command line"); + auto fileDependencies = getArray(fileDependenciesArrayID); + if (!fileDependencies) + llvm::report_fatal_error("Bad file dependencies"); + + // Form the dependencies storage object + auto moduleDep = ModuleDependencies::forClangModule( + *moduleMapPath, *contextHash, *commandLineArgs, *fileDependencies); + // Add dependencies of this module + for (const auto &moduleName : *currentModuleDependencies) + moduleDep.addModuleDependency(moduleName); + + cache.recordDependencies(currentModuleName, std::move(moduleDep)); + hasCurrentModule = false; + break; + } + + default: { + llvm::report_fatal_error("Unknown record ID"); + } + } + } + + return false; +} + +bool Deserializer::readInterModuleDependenciesCache( + ModuleDependenciesCache &cache) { + using namespace graph_block; + + if (readSignature()) + return true; + + if (enterGraphBlock()) + return true; + + if (readMetadata()) + return true; + + if (readGraph(cache)) + return true; + + return false; +} + +llvm::Optional Deserializer::getIdentifier(unsigned n) { + if (n == 0) + return std::string(); + + --n; + if (n >= Identifiers.size()) + return None; + + return Identifiers[n]; +} + +llvm::Optional> Deserializer::getArray(unsigned n) { + if (n == 0) + return std::vector(); + + --n; + if (n >= ArraysOfIdentifierIDs.size()) + return None; + + auto &identifierIDs = ArraysOfIdentifierIDs[n]; + + auto IDtoStringMap = [this](unsigned id) { + auto identifier = getIdentifier(id); + if (!identifier) + llvm::report_fatal_error("Bad identifier array element"); + return *identifier; + }; + std::vector result; + result.reserve(identifierIDs.size()); + std::transform(identifierIDs.begin(), identifierIDs.end(), + std::back_inserter(result), IDtoStringMap); + return result; +} + +bool swift::dependencies::module_dependency_cache_serialization:: + readInterModuleDependenciesCache(llvm::MemoryBuffer &buffer, + ModuleDependenciesCache &cache) { + Deserializer deserializer(buffer.getMemBufferRef()); + return deserializer.readInterModuleDependenciesCache(cache); +} + +bool swift::dependencies::module_dependency_cache_serialization:: + readInterModuleDependenciesCache(StringRef path, + ModuleDependenciesCache &cache) { + PrettyStackTraceStringAction stackTrace( + "loading inter-module dependency graph", path); + auto buffer = llvm::MemoryBuffer::getFile(path); + if (!buffer) + return false; + + return readInterModuleDependenciesCache(*buffer.get(), cache); +} + +// MARK: Serialization + +/// Kinds of arrays that we track being serialized. Used to query serialized +/// array ID for a given module. +enum ModuleIdentifierArrayKind : uint8_t { + Empty = 0, + DirectDependencies, + CompiledModuleCandidates, + BuildCommandLine, + ExtraPCMArgs, + SourceFiles, + BridgingSourceFiles, + BridgingModuleDependencies, + NonPathCommandLine, + FileDependencies, + LastArrayKind +}; + +using ModuleIdentifierArrayKey = + std::pair; + +// DenseMap Infos for hashing of ModuleIdentifierArrayKind +template <> +struct llvm::DenseMapInfo { + using UnderlyingType = std::underlying_type::type; + using UnerlyingInfo = DenseMapInfo; + + static inline ModuleIdentifierArrayKind getEmptyKey() { + return ModuleIdentifierArrayKind::Empty; + } + static inline ModuleIdentifierArrayKind getTombstoneKey() { + return ModuleIdentifierArrayKind::LastArrayKind; + } + static unsigned getHashValue(const ModuleIdentifierArrayKind &arrKind) { + auto underlyingValue = static_cast(arrKind); + return UnerlyingInfo::getHashValue(underlyingValue); + } + static bool isEqual(const ModuleIdentifierArrayKind &LHS, + const ModuleIdentifierArrayKind &RHS) { + return LHS == RHS; + } +}; + +namespace std { +template <> +struct hash { + using UnderlyingKindType = std::underlying_type::type; + std::size_t operator()(const ModuleDependencyID &id) const { + auto underlyingKindValue = static_cast(id.second); + + return (hash()(id.first) ^ + (hash()(underlyingKindValue))); + } +}; +} // namespace std + +namespace { + +class Serializer { + llvm::StringMap IdentifierIDs; + std::unordered_map> + ArrayIDs; + unsigned LastIdentifierID = 0; + unsigned LastArrayID = 0; + std::vector Identifiers; + std::vector> ArraysOfIdentifiers; + + llvm::BitstreamWriter &Out; + + /// A reusable buffer for emitting records. + SmallVector ScratchRecord; + std::array AbbrCodes; + + // Returns the identifier ID of the added identifier, either + // new or previously-hashed + unsigned addIdentifier(const std::string &str); + unsigned getIdentifier(const std::string &str) const; + + // Returns the array ID of the added array + void addArray(ModuleDependencyID moduleID, + ModuleIdentifierArrayKind arrayKind, + const std::vector &vec); + unsigned getArray(ModuleDependencyID moduleID, + ModuleIdentifierArrayKind arrayKind) const; + + template + void registerRecordAbbr() { + using AbbrArrayTy = decltype(AbbrCodes); + static_assert(Layout::Code <= std::tuple_size::value, + "layout has invalid record code"); + AbbrCodes[Layout::Code] = Layout::emitAbbrev(Out); + } + + void collectStringsAndArrays(const ModuleDependenciesCache &cache); + + void emitBlockID(unsigned ID, StringRef name, + SmallVectorImpl &nameBuffer); + + void emitRecordID(unsigned ID, StringRef name, + SmallVectorImpl &nameBuffer); + + void writeSignature(); + void writeBlockInfoBlock(); + + void writeMetadata(); + void writeIdentifiers(); + void writeArraysOfIdentifiers(); + + void writeModuleInfo(ModuleDependencyID moduleID, + const ModuleDependencies &dependencyInfo); + +public: + Serializer(llvm::BitstreamWriter &ExistingOut) : Out(ExistingOut) {} + +public: + void writeInterModuleDependenciesCache(const ModuleDependenciesCache &cache); +}; + +} // end namespace + +/// Record the name of a block. +void Serializer::emitBlockID(unsigned ID, StringRef name, + SmallVectorImpl &nameBuffer) { + SmallVector idBuffer; + idBuffer.push_back(ID); + Out.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, idBuffer); + + // Emit the block name if present. + if (name.empty()) + return; + nameBuffer.resize(name.size()); + memcpy(nameBuffer.data(), name.data(), name.size()); + Out.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, nameBuffer); +} + +/// Record the name of a record. +void Serializer::emitRecordID(unsigned ID, StringRef name, + SmallVectorImpl &nameBuffer) { + assert(ID < 256 && "can't fit record ID in next to name"); + nameBuffer.resize(name.size() + 1); + nameBuffer[0] = ID; + memcpy(nameBuffer.data() + 1, name.data(), name.size()); + Out.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, nameBuffer); +} + +void Serializer::writeBlockInfoBlock() { + llvm::BCBlockRAII restoreBlock(Out, llvm::bitc::BLOCKINFO_BLOCK_ID, 2); + + SmallVector nameBuffer; +#define BLOCK(X) emitBlockID(X##_ID, #X, nameBuffer) +#define BLOCK_RECORD(K, X) emitRecordID(K::X, #X, nameBuffer) + + BLOCK(GRAPH_BLOCK); + BLOCK_RECORD(graph_block, METADATA); + BLOCK_RECORD(graph_block, IDENTIFIER_NODE); + BLOCK_RECORD(graph_block, IDENTIFIER_ARRAY_NODE); + + BLOCK_RECORD(graph_block, MODULE_NODE); + BLOCK_RECORD(graph_block, SWIFT_TEXTUAL_MODULE_DETAILS_NODE); + BLOCK_RECORD(graph_block, SWIFT_BINARY_MODULE_DETAILS_NODE); + BLOCK_RECORD(graph_block, SWIFT_PLACEHOLDER_MODULE_DETAILS_NODE); + BLOCK_RECORD(graph_block, CLANG_MODULE_DETAILS_NODE); +} + +void Serializer::writeSignature() { + for (auto c : MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE) + Out.Emit((unsigned)c, 8); +} + +void Serializer::writeMetadata() { + using namespace graph_block; + + MetadataLayout::emitRecord(Out, ScratchRecord, + AbbrCodes[MetadataLayout::Code], + MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR, + MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR, + version::getSwiftFullVersion()); +} + +void Serializer::writeIdentifiers() { + using namespace graph_block; + for (auto str : Identifiers) { + IdentifierNodeLayout::emitRecord( + Out, ScratchRecord, AbbrCodes[IdentifierNodeLayout::Code], str); + } +} + +void Serializer::writeArraysOfIdentifiers() { + using namespace graph_block; + for (auto vec : ArraysOfIdentifiers) { + IdentifierArrayLayout::emitRecord( + Out, ScratchRecord, AbbrCodes[IdentifierArrayLayout::Code], vec); + } +} + +void Serializer::writeModuleInfo(ModuleDependencyID moduleID, + const ModuleDependencies &dependencyInfo) { + using namespace graph_block; + + ModuleInfoLayout::emitRecord( + Out, ScratchRecord, AbbrCodes[ModuleInfoLayout::Code], + getIdentifier(moduleID.first), + getArray(moduleID, ModuleIdentifierArrayKind::DirectDependencies)); + + if (auto swiftTextDeps = dependencyInfo.getAsSwiftTextualModule()) { + unsigned swiftInterfaceFileId = + swiftTextDeps->swiftInterfaceFile + ? getIdentifier(swiftTextDeps->swiftInterfaceFile.getValue()) + : 0; + unsigned bridgingHeaderFileId = + swiftTextDeps->bridgingHeaderFile + ? getIdentifier(swiftTextDeps->bridgingHeaderFile.getValue()) + : 0; + SwiftTextualModuleDetailsLayout::emitRecord( + Out, ScratchRecord, AbbrCodes[SwiftTextualModuleDetailsLayout::Code], + swiftInterfaceFileId, + getArray(moduleID, ModuleIdentifierArrayKind::CompiledModuleCandidates), + getArray(moduleID, ModuleIdentifierArrayKind::BuildCommandLine), + getArray(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs), + getIdentifier(swiftTextDeps->contextHash), swiftTextDeps->isFramework, + bridgingHeaderFileId, + getArray(moduleID, ModuleIdentifierArrayKind::SourceFiles), + getArray(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles), + getArray(moduleID, + ModuleIdentifierArrayKind::BridgingModuleDependencies)); + + } else if (auto swiftBinDeps = dependencyInfo.getAsSwiftBinaryModule()) { + SwiftBinaryModuleDetailsLayout::emitRecord( + Out, ScratchRecord, AbbrCodes[SwiftBinaryModuleDetailsLayout::Code], + getIdentifier(swiftBinDeps->compiledModulePath), + getIdentifier(swiftBinDeps->moduleDocPath), + getIdentifier(swiftBinDeps->sourceInfoPath), swiftBinDeps->isFramework); + + } else if (auto swiftPHDeps = + dependencyInfo.getAsPlaceholderDependencyModule()) { + SwiftPlaceholderModuleDetailsLayout::emitRecord( + Out, ScratchRecord, + AbbrCodes[SwiftPlaceholderModuleDetailsLayout::Code], + getIdentifier(swiftPHDeps->compiledModulePath), + getIdentifier(swiftPHDeps->moduleDocPath), + getIdentifier(swiftPHDeps->sourceInfoPath)); + + } else if (auto clangDeps = dependencyInfo.getAsClangModule()) { + ClangModuleDetailsLayout::emitRecord( + Out, ScratchRecord, AbbrCodes[ClangModuleDetailsLayout::Code], + getIdentifier(clangDeps->moduleMapFile), + getIdentifier(clangDeps->contextHash), + getArray(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine), + getArray(moduleID, ModuleIdentifierArrayKind::FileDependencies)); + + } else { + llvm_unreachable("Unexpected module dependency kind."); + } +} + +unsigned Serializer::addIdentifier(const std::string &str) { + if (str.empty()) + return 0; + + decltype(IdentifierIDs)::iterator iter; + bool isNew; + std::tie(iter, isNew) = IdentifierIDs.insert({str, LastIdentifierID + 1}); + + if (!isNew) + return iter->getValue(); + + ++LastIdentifierID; + // Note that we use the string data stored in the StringMap. + Identifiers.push_back(iter->getKey()); + return iter->getValue(); +} + +unsigned Serializer::getIdentifier(const std::string &str) const { + if (str.empty()) + return 0; + + auto iter = IdentifierIDs.find(str); + assert(iter != IdentifierIDs.end()); + assert(iter->second != 0); + return iter->second; +} + +void Serializer::addArray(ModuleDependencyID moduleID, + ModuleIdentifierArrayKind arrayKind, + const std::vector &vec) { + if (ArrayIDs.find(moduleID) != ArrayIDs.end()) { + // Already have arrays for this module + llvm::DenseMap::iterator iter; + bool isNew; + std::tie(iter, isNew) = + ArrayIDs[moduleID].insert({arrayKind, LastArrayID + 1}); + if (!isNew) + return; + } else { + // Do not yet have any arrays for this module + ArrayIDs[moduleID] = llvm::DenseMap(); + ArrayIDs[moduleID].insert({arrayKind, LastArrayID + 1}); + } + + ++LastArrayID; + + // Add in the individual identifiers in the array + std::vector identifierIDs; + identifierIDs.reserve(vec.size()); + for (const auto &id : vec) { + identifierIDs.push_back(addIdentifier(id)); + } + + ArraysOfIdentifiers.push_back(identifierIDs); + return; +} + +unsigned Serializer::getArray(ModuleDependencyID moduleID, + ModuleIdentifierArrayKind arrayKind) const { + auto iter = ArrayIDs.find(moduleID); + assert(iter != ArrayIDs.end()); + auto &innerMap = iter->second; + auto arrayIter = innerMap.find(arrayKind); + assert(arrayIter != innerMap.end()); + return arrayIter->second; +} + +void Serializer::collectStringsAndArrays(const ModuleDependenciesCache &cache) { + for (auto &moduleID : cache.getAllModules()) { + auto dependencyInfo = + cache.findDependencies(moduleID.first, moduleID.second); + assert(dependencyInfo.hasValue() && "Expected dependency info."); + // Add the module's name + addIdentifier(moduleID.first); + // Add the module's dependencies + addArray(moduleID, ModuleIdentifierArrayKind::DirectDependencies, + dependencyInfo->getModuleDependencies()); + + // Add the dependency-kind-specific data + if (auto swiftTextDeps = dependencyInfo->getAsSwiftTextualModule()) { + if (swiftTextDeps->swiftInterfaceFile) + addIdentifier(swiftTextDeps->swiftInterfaceFile.getValue()); + addArray(moduleID, ModuleIdentifierArrayKind::CompiledModuleCandidates, + swiftTextDeps->compiledModuleCandidates); + addArray(moduleID, ModuleIdentifierArrayKind::BuildCommandLine, + swiftTextDeps->buildCommandLine); + addArray(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs, + swiftTextDeps->extraPCMArgs); + addIdentifier(swiftTextDeps->contextHash); + if (swiftTextDeps->bridgingHeaderFile) + addIdentifier(swiftTextDeps->bridgingHeaderFile.getValue()); + addArray(moduleID, ModuleIdentifierArrayKind::SourceFiles, + swiftTextDeps->sourceFiles); + addArray(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles, + swiftTextDeps->bridgingSourceFiles); + addArray(moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies, + swiftTextDeps->bridgingModuleDependencies); + } else if (auto swiftBinDeps = dependencyInfo->getAsSwiftBinaryModule()) { + addIdentifier(swiftBinDeps->compiledModulePath); + addIdentifier(swiftBinDeps->moduleDocPath); + addIdentifier(swiftBinDeps->sourceInfoPath); + } else if (auto swiftPHDeps = + dependencyInfo->getAsPlaceholderDependencyModule()) { + addIdentifier(swiftPHDeps->compiledModulePath); + addIdentifier(swiftPHDeps->moduleDocPath); + addIdentifier(swiftPHDeps->sourceInfoPath); + } else if (auto clangDeps = dependencyInfo->getAsClangModule()) { + addIdentifier(clangDeps->moduleMapFile); + addIdentifier(clangDeps->contextHash); + addArray(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine, + clangDeps->nonPathCommandLine); + addArray(moduleID, ModuleIdentifierArrayKind::FileDependencies, + clangDeps->fileDependencies); + } else { + llvm_unreachable("Unexpected module dependency kind."); + } + } +} + +void Serializer::writeInterModuleDependenciesCache( + const ModuleDependenciesCache &cache) { + // Write the header + writeSignature(); + writeBlockInfoBlock(); + + // Enter the main graph block + unsigned blockID = GRAPH_BLOCK_ID; + llvm::BCBlockRAII restoreBlock(Out, blockID, 8); + + using namespace graph_block; + + registerRecordAbbr(); + registerRecordAbbr(); + registerRecordAbbr(); + registerRecordAbbr(); + registerRecordAbbr(); + registerRecordAbbr(); + registerRecordAbbr(); + registerRecordAbbr(); + + // Make a pass to collect all unique strings and arrays + // of strings + collectStringsAndArrays(cache); + + // Write the version information + writeMetadata(); + + // Write the strings + writeIdentifiers(); + + // Write the arrays + writeArraysOfIdentifiers(); + + // Write the core graph + for (auto &moduleID : cache.getAllModules()) { + auto dependencyInfo = + cache.findDependencies(moduleID.first, moduleID.second); + assert(dependencyInfo.hasValue() && "Expected dependency info."); + writeModuleInfo(moduleID, dependencyInfo.getValue()); + } + + return; +} + +void swift::dependencies::module_dependency_cache_serialization:: + writeInterModuleDependenciesCache(llvm::BitstreamWriter &Out, + const ModuleDependenciesCache &cache) { + Serializer serializer{Out}; + serializer.writeInterModuleDependenciesCache(cache); +} + +bool swift::dependencies::module_dependency_cache_serialization:: + writeInterModuleDependenciesCache(DiagnosticEngine &diags, StringRef path, + const ModuleDependenciesCache &cache) { + PrettyStackTraceStringAction stackTrace( + "saving inter-module dependency graph", path); + return withOutputFile(diags, path, [&](llvm::raw_ostream &out) { + SmallVector Buffer; + llvm::BitstreamWriter Writer{Buffer}; + writeInterModuleDependenciesCache(Writer, cache); + out.write(Buffer.data(), Buffer.size()); + out.flush(); + return false; + }); +} diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 32c2aa1cef5f2..7ab6508e3f17b 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -11,10 +11,12 @@ //===----------------------------------------------------------------------===// #include "swift/DependencyScan/ScanDependencies.h" +#include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h" #include "swift/AST/ASTContext.h" #include "swift/AST/Decl.h" #include "swift/AST/DiagnosticEngine.h" #include "swift/AST/DiagnosticsFrontend.h" +#include "swift/AST/DiagnosticsDriver.h" #include "swift/AST/Module.h" #include "swift/AST/ModuleDependencies.h" #include "swift/AST/ModuleLoader.h" @@ -733,8 +735,13 @@ generateFullDependencyGraph(CompilerInstance &instance, for (size_t i = 0; i < allModules.size(); ++i) { const auto &module = allModules[i]; // Grab the completed module dependencies. - auto moduleDeps = *cache.findDependencies(module.first, module.second); + auto moduleDepsQuery = cache.findDependencies(module.first, module.second); + if (!moduleDepsQuery) { + std::string err = "Module Dependency Cache missing module" + module.first; + llvm::report_fatal_error(err); + } + auto moduleDeps = *moduleDepsQuery; // Collect all the required pieces to build a ModuleInfo auto swiftPlaceholderDeps = moduleDeps.getAsPlaceholderDependencyModule(); auto swiftTextualDeps = moduleDeps.getAsSwiftTextualModule(); @@ -1098,12 +1105,26 @@ identifyMainModuleDependencies(CompilerInstance &instance) { } // namespace +static void serializeDependencyCache(DiagnosticEngine &diags, + const std::string &path, + const ModuleDependenciesCache &cache) { + module_dependency_cache_serialization::writeInterModuleDependenciesCache( + diags, path, cache); +} + +static void deserializeDependencyCache(const std::string &path, + ModuleDependenciesCache &cache) { + module_dependency_cache_serialization::readInterModuleDependenciesCache( + path, cache); +} + bool swift::dependencies::scanDependencies(CompilerInstance &instance) { ASTContext &Context = instance.getASTContext(); const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); std::error_code EC; llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::F_None); + // `-scan-dependencies` invocations use a single new instance // of a module cache ModuleDependenciesCache cache; @@ -1234,8 +1255,7 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, // Add the main module. StringRef mainModuleName = mainModule->getNameStr(); llvm::SetVector, - std::set> - allModules; + std::set> allModules; allModules.insert({mainModuleName.str(), mainDependencies.getKind()}); cache.recordDependencies(mainModuleName, std::move(mainDependencies)); @@ -1272,12 +1292,32 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, if (diagnoseCycle(instance, cache, /*MainModule*/ allModules.front(), ASTDelegate)) return std::make_error_code(std::errc::not_supported); + + // Test dependency cache serialization/deserialization if + // `-test-dependency-scan-cache-serialization` is specified. + auto *resultCache = &cache; + ModuleDependenciesCache loadedCache; + if (FEOpts.TestDependencyScannerCacheSerialization) { + llvm::SmallString<128> buffer; + auto EC = llvm::sys::fs::createTemporaryFile("depscan_cache", "moddepcache", buffer); + if (EC) { + instance.getDiags().diagnose(SourceLoc(), + diag::error_unable_to_make_temporary_file, + EC.message()); + } else { + std::string path = buffer.str().str(); + serializeDependencyCache(instance.getDiags(), path, cache); + deserializeDependencyCache(path, loadedCache); + resultCache = &loadedCache; + } + } + auto dependencyGraph = generateFullDependencyGraph( - instance, cache, ASTDelegate, allModules.getArrayRef()); + instance, *resultCache, ASTDelegate, allModules.getArrayRef()); // Update the dependency tracker. if (auto depTracker = instance.getDependencyTracker()) { for (auto module : allModules) { - auto deps = cache.findDependencies(module.first, module.second); + auto deps = resultCache->findDependencies(module.first, module.second); if (!deps) continue; diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp index 74f7ce32b92dc..94a221cf25bbc 100644 --- a/lib/Driver/Driver.cpp +++ b/lib/Driver/Driver.cpp @@ -167,17 +167,56 @@ static void validateProfilingArgs(DiagnosticEngine &diags, static void validateDependencyScanningArgs(DiagnosticEngine &diags, const ArgList &args) { - const Arg *ExternalDependencyMap = args.getLastArg(options::OPT_placeholder_dependency_module_map); + const Arg *ExternalDependencyMap = + args.getLastArg(options::OPT_placeholder_dependency_module_map); const Arg *ScanDependencies = args.getLastArg(options::OPT_scan_dependencies); const Arg *Prescan = args.getLastArg(options::OPT_import_prescan); + + const Arg *SerializeCache = + args.getLastArg(options::OPT_serialize_dependency_scan_cache); + const Arg *ReuseCache = + args.getLastArg(options::OPT_reuse_dependency_scan_cache); + const Arg *CacheSerializationPath = + args.getLastArg(options::OPT_dependency_scan_cache_path); + const Arg *TestSerialization = args.getLastArg( + options::OPT_debug_test_dependency_scan_cache_serialization); + if (ExternalDependencyMap && !ScanDependencies) { diags.diagnose(SourceLoc(), diag::error_requirement_not_met, - "-placeholder-dependency-module-map-file", "-scan-dependencies"); + "-placeholder-dependency-module-map-file", + "-scan-dependencies"); + } + if (Prescan && !ScanDependencies) { + diags.diagnose(SourceLoc(), diag::error_requirement_not_met, + "-import-prescan", "-scan-dependencies"); } if (Prescan && !ScanDependencies) { diags.diagnose(SourceLoc(), diag::error_requirement_not_met, "-import-prescan", "-scan-dependencies"); } + if (SerializeCache && !ScanDependencies) { + diags.diagnose(SourceLoc(), diag::error_requirement_not_met, + "-serialize-dependency-scan-cache", "-scan-dependencies"); + } + if (ReuseCache && !ScanDependencies) { + diags.diagnose(SourceLoc(), diag::error_requirement_not_met, + "-load-dependency-scan-cache", "-scan-dependencies"); + } + if (TestSerialization && !ScanDependencies) { + diags.diagnose(SourceLoc(), diag::error_requirement_not_met, + "-test-dependency-scan-cache-serialization", + "-scan-dependencies"); + } + if (SerializeCache && !CacheSerializationPath) { + diags.diagnose(SourceLoc(), diag::error_requirement_not_met, + "-serialize-dependency-scan-cache", + "-dependency-scan-cache-path"); + } + if (ReuseCache && !CacheSerializationPath) { + diags.diagnose(SourceLoc(), diag::error_requirement_not_met, + "-serialize-dependency-scan-cache", + "-dependency-scan-cache-path"); + } } static void validateDebugInfoArgs(DiagnosticEngine &diags, diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index 114b9173377fb..9dd86bfb25af7 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -103,6 +103,13 @@ bool ArgsToFrontendOptionsConverter::convert( Opts.ImportPrescan |= Args.hasArg(OPT_import_prescan); + Opts.SerializeDependencyScannerCache |= Args.hasArg(OPT_serialize_dependency_scan_cache); + Opts.ReuseDependencyScannerCache |= Args.hasArg(OPT_reuse_dependency_scan_cache); + Opts.TestDependencyScannerCacheSerialization |= Args.hasArg(OPT_debug_test_dependency_scan_cache_serialization); + if (const Arg *A = Args.getLastArg(OPT_dependency_scan_cache_path)) { + Opts.SerializedDependencyScannerCachePath = A->getValue(); + } + Opts.DisableCrossModuleIncrementalBuild |= Args.hasArg(OPT_disable_incremental_imports); diff --git a/lib/Serialization/ModuleDependencyScanner.cpp b/lib/Serialization/ModuleDependencyScanner.cpp index a24cb856a3d63..0b338db50fc44 100644 --- a/lib/Serialization/ModuleDependencyScanner.cpp +++ b/lib/Serialization/ModuleDependencyScanner.cpp @@ -123,7 +123,7 @@ ErrorOr ModuleDependencyScanner::scanInterfaceFile( std::string InPath = moduleInterfacePath.str(); auto compiledCandidates = getCompiledCandidates(Ctx, moduleName.str(), InPath); - Result = ModuleDependencies::forSwiftInterface(InPath, + Result = ModuleDependencies::forSwiftTextualModule(InPath, compiledCandidates, Args, PCMArgs, diff --git a/test/ScanDependencies/bin_mod_import.swift b/test/ScanDependencies/bin_mod_import.swift index 360a8c2309c1c..ef03f2f9562db 100644 --- a/test/ScanDependencies/bin_mod_import.swift +++ b/test/ScanDependencies/bin_mod_import.swift @@ -8,7 +8,10 @@ import EWrapper // RUN: %target-swift-frontend -compile-module-from-interface %S/Inputs/Swift/EWrapper.swiftinterface -o %t/EWrapper.swiftmodule -I %t // Step 3: scan dependency should give us the binary module and a textual swift dependency from it // RUN: %target-swift-frontend -scan-dependencies %s -o %t/deps.json -I %t -// Step 4: Verify +// RUN: %FileCheck %s < %t/deps.json + +// Step 4: Ensure that round-trip serialization does not affect result +// RUN: %target-swift-frontend -scan-dependencies -test-dependency-scan-cache-serialization %s -o %t/deps.json -I %t // RUN: %FileCheck %s < %t/deps.json // CHECK: "modulePath": "{{.*}}EWrapper.swiftmodule" diff --git a/test/ScanDependencies/binary_module_only.swift b/test/ScanDependencies/binary_module_only.swift index 488c95cc526ed..a2bb7178f5e7f 100644 --- a/test/ScanDependencies/binary_module_only.swift +++ b/test/ScanDependencies/binary_module_only.swift @@ -23,3 +23,7 @@ import Foo // Step 3: scan dependencies, pointed only at the binary module file should detect it as a swiftBinaryModule kind of dependency // RUN: %target-swift-frontend -scan-dependencies %s -o %t/deps.json -I %t/binaryModuleOnly -emit-dependencies -emit-dependencies-path %t/deps.d -sdk %t -prebuilt-module-cache-path %t/ResourceDir/%target-sdk-name/prebuilt-modules // RUN: %FileCheck %s -check-prefix=BINARY_MODULE_ONLY < %t/deps.json + +// Step 4: Ensure that round-trip serialization does not affect result +// RUN: %target-swift-frontend -scan-dependencies -test-dependency-scan-cache-serialization %s -o %t/deps.json -I %t/binaryModuleOnly -emit-dependencies -emit-dependencies-path %t/deps.d -sdk %t -prebuilt-module-cache-path %t/ResourceDir/%target-sdk-name/prebuilt-modules +// RUN: %FileCheck %s -check-prefix=BINARY_MODULE_ONLY < %t/deps.json diff --git a/test/ScanDependencies/can_import_placeholder.swift b/test/ScanDependencies/can_import_placeholder.swift index a04e103bfe76b..8fb9900d5a123 100644 --- a/test/ScanDependencies/can_import_placeholder.swift +++ b/test/ScanDependencies/can_import_placeholder.swift @@ -11,8 +11,10 @@ // RUN: echo "}]" >> %/t/inputs/map.json // RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -placeholder-dependency-module-map-file %t/inputs/map.json -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 +// RUN: %FileCheck %s < %t/deps.json -// Check the contents of the JSON output +// Ensure that round-trip serialization does not affect result +// RUN: %target-swift-frontend -scan-dependencies -test-dependency-scan-cache-serialization -module-cache-path %t/clang-module-cache %s -placeholder-dependency-module-map-file %t/inputs/map.json -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 // RUN: %FileCheck %s < %t/deps.json // REQUIRES: executable_test @@ -45,4 +47,3 @@ import SomeExternalModule // CHECK-NEXT: "swift": "_Concurrency" // CHECK-NEXT: } // CHECK-NEXT: ], - diff --git a/test/ScanDependencies/module_deps.swift b/test/ScanDependencies/module_deps.swift index 0556e8fc3f948..c951041071a47 100644 --- a/test/ScanDependencies/module_deps.swift +++ b/test/ScanDependencies/module_deps.swift @@ -25,6 +25,10 @@ // Check the contents of the JSON output // RUN: %FileCheck -check-prefix CHECK_CLANG_TARGET %s < %t/deps_clang_target.json +// Ensure that round-trip serialization does not affect result +// RUN: %target-swift-frontend -scan-dependencies -test-dependency-scan-cache-serialization -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 +// RUN: %FileCheck %s < %t/deps.json + // REQUIRES: executable_test // REQUIRES: objc_interop diff --git a/test/ScanDependencies/module_deps_cross_import_overlay.swift b/test/ScanDependencies/module_deps_cross_import_overlay.swift index 8998196c0156a..951517430976e 100644 --- a/test/ScanDependencies/module_deps_cross_import_overlay.swift +++ b/test/ScanDependencies/module_deps_cross_import_overlay.swift @@ -1,10 +1,13 @@ // RUN: %empty-directory(%t) // RUN: mkdir -p %t/clang-module-cache // RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 - // Check the contents of the JSON output // RUN: %FileCheck %s < %t/deps.json +// Ensure that round-trip serialization does not affect result +// RUN: %target-swift-frontend -scan-dependencies -test-dependency-scan-cache-serialization -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 +// RUN: %FileCheck %s < %t/deps.json + // REQUIRES: executable_test // REQUIRES: objc_interop diff --git a/test/ScanDependencies/module_deps_external.swift b/test/ScanDependencies/module_deps_external.swift index cc0144a0fed64..f896911d0d9a9 100644 --- a/test/ScanDependencies/module_deps_external.swift +++ b/test/ScanDependencies/module_deps_external.swift @@ -27,6 +27,10 @@ // RUN: %target-codesign %t/main // RUN: %target-run %t/main %t/deps.json +// Ensure that round-trip serialization does not affect result +// RUN: %target-swift-frontend -scan-dependencies -test-dependency-scan-cache-serialization -module-cache-path %t/clang-module-cache %s -placeholder-dependency-module-map-file %t/inputs/map.json -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 +// RUN: %FileCheck %s < %t/deps.json + // REQUIRES: executable_test // REQUIRES: objc_interop import SomeExternalModule diff --git a/test/ScanDependencies/module_framework.swift b/test/ScanDependencies/module_framework.swift index 21fa3d95efb4f..bbd0933efe143 100644 --- a/test/ScanDependencies/module_framework.swift +++ b/test/ScanDependencies/module_framework.swift @@ -1,10 +1,14 @@ // RUN: %empty-directory(%t) // RUN: %target-swift-frontend -scan-dependencies %s -o %t/deps.json -emit-dependencies -emit-dependencies-path %t/deps.d -swift-version 4 -Xcc -Xclang - // Check the contents of the JSON output // RUN: %FileCheck %s < %t/deps.json + +// Ensure that round-trip serialization does not affect result +// RUN: %target-swift-frontend -scan-dependencies -test-dependency-scan-cache-serialization %s -o %t/deps.json -emit-dependencies -emit-dependencies-path %t/deps.d -swift-version 4 -Xcc -Xclang +// RUN: %FileCheck %s < %t/deps.json + // REQUIRES: OS=macosx - + import CryptoKit // CHECK: "mainModuleName": "deps" From 5edb6e9b4b511238f4c4780c25325f7b53da49c3 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Tue, 25 May 2021 15:00:04 -0700 Subject: [PATCH 2/5] [Dependency Scanning] Add ability for `-scan-dependencies` action to serialize and deserialize dependency scanner cache from a `.moddepcache` file. Using the serialization format added in https://github.com/apple/swift/pull/37585. - Add load/save code for the `-scan-dependencies` code-path. - Add `libSwiftDriver` entry-points to load/store the cache of a given scanner instance. --- .../swift-c/DependencyScan/DependencyScan.h | 24 +++ include/swift/AST/DiagnosticsCommon.def | 9 + .../DependencyScan/DependencyScanningTool.h | 8 + include/swift/Frontend/FrontendOptions.h | 3 + include/swift/Option/FrontendOptions.td | 3 + lib/DependencyScan/DependencyScanningTool.cpp | 31 ++- .../ModuleDependencyCacheSerialization.cpp | 2 +- lib/DependencyScan/ScanDependencies.cpp | 84 ++++++-- .../ArgsToFrontendOptionsConverter.cpp | 1 + .../module_deps_cache_reuse.swift | 195 ++++++++++++++++++ tools/libSwiftScan/libSwiftScan.cpp | 19 ++ tools/libSwiftScan/libSwiftScan.exports | 3 + 12 files changed, 360 insertions(+), 22 deletions(-) create mode 100644 test/ScanDependencies/module_deps_cache_reuse.swift diff --git a/include/swift-c/DependencyScan/DependencyScan.h b/include/swift-c/DependencyScan/DependencyScan.h index be2c9257c494c..8a3589a227737 100644 --- a/include/swift-c/DependencyScan/DependencyScan.h +++ b/include/swift-c/DependencyScan/DependencyScan.h @@ -337,6 +337,30 @@ swiftscan_batch_scan_result_create(swiftscan_scanner_t scanner, SWIFTSCAN_PUBLIC swiftscan_import_set_t swiftscan_import_set_create( swiftscan_scanner_t scanner, swiftscan_scan_invocation_t invocation); +//=== Scanner Cache Operations --------------------------------------------===// +// The following operations expose an implementation detail of the dependency +// scanner: its module dependencies cache. This is done in order +// to allow clients to perform incremental dependency scans by having the +// scanner's state be serializable and re-usable. + +/// For the specified \c scanner instance, serialize its state to the specified file-system \c path . +SWIFTSCAN_PUBLIC void +swiftscan_scanner_cache_serialize(swiftscan_scanner_t scanner, + const char * path); + +/// For the specified \c scanner instance, load in scanner state from a file at +/// the specified file-system \c path . +SWIFTSCAN_PUBLIC bool +swiftscan_scanner_cache_load(swiftscan_scanner_t scanner, + const char * path); + +/// For the specified \c scanner instance, reset its internal state, ensuring subsequent +/// scanning queries are done "from-scratch". +SWIFTSCAN_PUBLIC void +swiftscan_scanner_cache_reset(swiftscan_scanner_t scanner); + +//===----------------------------------------------------------------------===// + SWIFTSCAN_END_DECLS #endif // SWIFT_C_DEPENDENCY_SCAN_H diff --git a/include/swift/AST/DiagnosticsCommon.def b/include/swift/AST/DiagnosticsCommon.def index d79dfbc137669..da90df5d3d5d9 100644 --- a/include/swift/AST/DiagnosticsCommon.def +++ b/include/swift/AST/DiagnosticsCommon.def @@ -192,6 +192,15 @@ ERROR(scanner_find_cycle, none, ERROR(scanner_arguments_invalid, none, "dependencies scanner cannot be configured with arguments: '%0'", (StringRef)) +WARNING(warn_scaner_deserialize_failed, none, + "Failed to load module scanning dependency cache from: '%0', re-building scanner cache from scratch.", (StringRef)) + +REMARK(remark_reuse_cache, none, + "Re-using serialized module scanning dependency cache from: '%0'.", (StringRef)) + +REMARK(remark_save_cache, none, + "Serializing module scanning dependency cache to: '%0'.", (StringRef)) + //------------------------------------------------------------------------------ // MARK: custom attribute diagnostics //------------------------------------------------------------------------------ diff --git a/include/swift/DependencyScan/DependencyScanningTool.h b/include/swift/DependencyScan/DependencyScanningTool.h index d18d5bf09167d..75430ebb791c4 100644 --- a/include/swift/DependencyScan/DependencyScanningTool.h +++ b/include/swift/DependencyScan/DependencyScanningTool.h @@ -57,6 +57,14 @@ class DependencyScanningTool { const std::vector &BatchInput, const llvm::StringSet<> &PlaceholderModules); + /// Writes the current `SharedCache` instance to a specified FileSystem path. + void serializeCache(llvm::StringRef path); + /// Loads an instance of a `ModuleDependenciesCache` to serve as the `SharedCache` + /// from a specified FileSystem path. + bool loadCache(llvm::StringRef path); + /// Discard the tool's current `SharedCache` and start anew. + void resetCache(); + private: /// Using the specified invocation command, instantiate a CompilerInstance /// that will be used for this scan. diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h index 84295a9a09b21..d7fa6667b549c 100644 --- a/include/swift/Frontend/FrontendOptions.h +++ b/include/swift/Frontend/FrontendOptions.h @@ -306,6 +306,9 @@ class FrontendOptions { /// The path at which to either serialize or deserialize the dependency scanner cache. std::string SerializedDependencyScannerCachePath; + /// Emit remarks indicating use of the serialized module dependency scanning cache + bool EmitDependencyScannerCacheRemarks = false; + /// After performing a dependency scanning action, serialize and deserialize the /// dependency cache before producing the result. bool TestDependencyScannerCacheSerialization = false; diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index d2bb716c350cf..4452ba55f3d4b 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -197,6 +197,9 @@ def reuse_dependency_scan_cache : Flag<["-"], "load-dependency-scan-cache">, def dependency_scan_cache_path : Separate<["-"], "dependency-scan-cache-path">, HelpText<"The path to output the dependency scanner's internal state.">; +def dependency_scan_cache_remarks : Flag<["-"], "Rdependency-scan-cache">, + HelpText<"Emit remarks indicating use of the serialized module dependency scanning cache.">; + def enable_copy_propagation : Flag<["-"], "enable-copy-propagation">, HelpText<"Run SIL copy propagation to shorten object lifetime.">; def disable_copy_propagation : Flag<["-"], "disable-copy-propagation">, diff --git a/lib/DependencyScan/DependencyScanningTool.cpp b/lib/DependencyScan/DependencyScanningTool.cpp index 26c12e5c03205..c69c597f40c11 100644 --- a/lib/DependencyScan/DependencyScanningTool.cpp +++ b/lib/DependencyScan/DependencyScanningTool.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "swift/DependencyScan/DependencyScanningTool.h" +#include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h" #include "swift/AST/DiagnosticEngine.h" #include "swift/AST/DiagnosticsFrontend.h" #include "swift/Basic/LLVMInitialize.h" @@ -77,11 +78,37 @@ DependencyScanningTool::getDependencies( BatchInput.size(), std::make_error_code(std::errc::invalid_argument)); auto Instance = std::move(*InstanceOrErr); - auto batchScanResults = performBatchModuleScan( + auto BatchScanResults = performBatchModuleScan( *Instance.get(), *SharedCache, VersionedPCMInstanceCacheCache.get(), Saver, BatchInput); - return batchScanResults; + return BatchScanResults; +} + +void DependencyScanningTool::serializeCache(llvm::StringRef path) { + SourceManager SM; + DiagnosticEngine Diags(SM); + Diags.addConsumer(PDC); + module_dependency_cache_serialization::writeInterModuleDependenciesCache( + Diags, path, *SharedCache); +} + +bool DependencyScanningTool::loadCache(llvm::StringRef path) { + SourceManager SM; + DiagnosticEngine Diags(SM); + Diags.addConsumer(PDC); + SharedCache = std::make_unique(); + bool Success = + module_dependency_cache_serialization::readInterModuleDependenciesCache( + path, *SharedCache); + if (!Success) { + Diags.diagnose(SourceLoc(), diag::warn_scaner_deserialize_failed, path); + } + return Success; +} + +void DependencyScanningTool::resetCache() { + SharedCache.reset(new ModuleDependenciesCache()); } llvm::ErrorOr> diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index da99ac609f7fa..142de49e16985 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -472,7 +472,7 @@ bool swift::dependencies::module_dependency_cache_serialization:: "loading inter-module dependency graph", path); auto buffer = llvm::MemoryBuffer::getFile(path); if (!buffer) - return false; + return true; return readInterModuleDependenciesCache(*buffer.get(), cache); } diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 7ab6508e3f17b..e3cef5d01daa2 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -282,7 +282,16 @@ static void discoverCrosssImportOverlayDependencies( // Record the dummy main module's direct dependencies. The dummy main module // only directly depend on these newly discovered overlay modules. - cache.recordDependencies(dummyMainName, dummyMainDependencies); + if (cache.findDependencies(dummyMainName, + ModuleDependenciesKind::SwiftTextual)) { + cache.updateDependencies( + std::make_pair(dummyMainName.str(), + ModuleDependenciesKind::SwiftTextual), + dummyMainDependencies); + } else { + cache.recordDependencies(dummyMainName, dummyMainDependencies); + } + llvm::SetVector, std::set> allModules; @@ -1105,17 +1114,30 @@ identifyMainModuleDependencies(CompilerInstance &instance) { } // namespace -static void serializeDependencyCache(DiagnosticEngine &diags, - const std::string &path, +static void serializeDependencyCache(CompilerInstance &instance, const ModuleDependenciesCache &cache) { + const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); + ASTContext &Context = instance.getASTContext(); + auto savePath = opts.SerializedDependencyScannerCachePath; module_dependency_cache_serialization::writeInterModuleDependenciesCache( - diags, path, cache); + Context.Diags, savePath, cache); + if (opts.EmitDependencyScannerCacheRemarks) { + Context.Diags.diagnose(SourceLoc(), diag::remark_save_cache, savePath); + } } -static void deserializeDependencyCache(const std::string &path, +static void deserializeDependencyCache(CompilerInstance &instance, ModuleDependenciesCache &cache) { - module_dependency_cache_serialization::readInterModuleDependenciesCache( - path, cache); + const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); + ASTContext &Context = instance.getASTContext(); + auto loadPath = opts.SerializedDependencyScannerCachePath; + if (module_dependency_cache_serialization::readInterModuleDependenciesCache( + loadPath, cache)) { + Context.Diags.diagnose(SourceLoc(), diag::warn_scaner_deserialize_failed, + loadPath); + } else if (opts.EmitDependencyScannerCacheRemarks) { + Context.Diags.diagnose(SourceLoc(), diag::remark_reuse_cache, loadPath); + } } bool swift::dependencies::scanDependencies(CompilerInstance &instance) { @@ -1124,10 +1146,6 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); std::error_code EC; llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::F_None); - - // `-scan-dependencies` invocations use a single new instance - // of a module cache - ModuleDependenciesCache cache; if (out.has_error() || EC) { Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, EC.message()); @@ -1135,11 +1153,25 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { return true; } - // Execute scan, and write JSON output to the output stream + // `-scan-dependencies` invocations use a single new instance + // of a module cache + ModuleDependenciesCache cache; + + if (opts.ReuseDependencyScannerCache) + deserializeDependencyCache(instance, cache); + + // Execute scan auto dependenciesOrErr = performModuleScan(instance, cache); + + // Serialize the dependency cache if -serialize-dependency-scan-cache + // is specified + if (opts.SerializeDependencyScannerCache) + serializeDependencyCache(instance, cache); + if (dependenciesOrErr.getError()) return true; auto dependencies = std::move(*dependenciesOrErr); + // Write out the JSON description. writeJSON(out, dependencies); // This process succeeds regardless of whether any errors occurred. @@ -1258,7 +1290,19 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, std::set> allModules; allModules.insert({mainModuleName.str(), mainDependencies.getKind()}); - cache.recordDependencies(mainModuleName, std::move(mainDependencies)); + + // We may be re-using an instance of the cache which already contains + // an entry for this module. + if (cache.findDependencies(mainModuleName, + ModuleDependenciesKind::SwiftTextual)) { + cache.updateDependencies( + std::make_pair(mainModuleName.str(), + ModuleDependenciesKind::SwiftTextual), + std::move(mainDependencies)); + } else { + cache.recordDependencies(mainModuleName, std::move(mainDependencies)); + } + auto &ctx = instance.getASTContext(); auto ModuleCachePath = getModuleCachePathFromClang( ctx.getClangModuleLoader()->getClangInstance()); @@ -1299,15 +1343,17 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, ModuleDependenciesCache loadedCache; if (FEOpts.TestDependencyScannerCacheSerialization) { llvm::SmallString<128> buffer; - auto EC = llvm::sys::fs::createTemporaryFile("depscan_cache", "moddepcache", buffer); + auto EC = llvm::sys::fs::createTemporaryFile("depscan_cache", "moddepcache", + buffer); if (EC) { - instance.getDiags().diagnose(SourceLoc(), - diag::error_unable_to_make_temporary_file, - EC.message()); + instance.getDiags().diagnose( + SourceLoc(), diag::error_unable_to_make_temporary_file, EC.message()); } else { std::string path = buffer.str().str(); - serializeDependencyCache(instance.getDiags(), path, cache); - deserializeDependencyCache(path, loadedCache); + module_dependency_cache_serialization::writeInterModuleDependenciesCache( + instance.getDiags(), path, cache); + module_dependency_cache_serialization::readInterModuleDependenciesCache( + path, loadedCache); resultCache = &loadedCache; } } diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index 9dd86bfb25af7..c705289570532 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -105,6 +105,7 @@ bool ArgsToFrontendOptionsConverter::convert( Opts.SerializeDependencyScannerCache |= Args.hasArg(OPT_serialize_dependency_scan_cache); Opts.ReuseDependencyScannerCache |= Args.hasArg(OPT_reuse_dependency_scan_cache); + Opts.EmitDependencyScannerCacheRemarks |= Args.hasArg(OPT_dependency_scan_cache_remarks); Opts.TestDependencyScannerCacheSerialization |= Args.hasArg(OPT_debug_test_dependency_scan_cache_serialization); if (const Arg *A = Args.getLastArg(OPT_dependency_scan_cache_path)) { Opts.SerializedDependencyScannerCachePath = A->getValue(); diff --git a/test/ScanDependencies/module_deps_cache_reuse.swift b/test/ScanDependencies/module_deps_cache_reuse.swift new file mode 100644 index 0000000000000..ce74581081b76 --- /dev/null +++ b/test/ScanDependencies/module_deps_cache_reuse.swift @@ -0,0 +1,195 @@ +// RUN: %empty-directory(%t) +// RUN: mkdir -p %t/clang-module-cache + +// Run the scanner once, emitting the serialized scanner cache +// RUN: %target-swift-frontend -scan-dependencies -Rdependency-scan-cache -serialize-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/clang-module-cache %s -o %t/deps_initial.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 2>&1 | %FileCheck %s -check-prefix CHECK-REMARK-SAVE + +// Run the scanner again, but now re-using previously-serialized cache +// RUN: %target-swift-frontend -scan-dependencies -Rdependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 2>&1 | %FileCheck %s -check-prefix CHECK-REMARK-LOAD + +// Check the contents of the JSON output +// RUN: %FileCheck %s < %t/deps.json + +// REQUIRES: executable_test +// REQUIRES: objc_interop + +import C +import E +import G +import SubE + +// CHECK-REMARK-SAVE: remark: Serializing module scanning dependency cache to: +// CHECK-REMARK-LOAD: remark: Re-using serialized module scanning dependency cache from: + +// CHECK: "mainModuleName": "deps" + +/// --------Main module +// CHECK-LABEL: "modulePath": "deps.swiftmodule", +// CHECK-NEXT: sourceFiles +// CHECK-NEXT: module_deps_cache_reuse.swift +// CHECK-NEXT: ], +// CHECK-NEXT: "directDependencies": [ +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "A" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "C" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "E" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "F" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "G" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "SubE" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "SwiftOnoneSupport" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "_Concurrency" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "_cross_import_E" +// CHECK-NEXT: } +// CHECK-NEXT: ], + +// CHECK: "extraPcmArgs": [ +// CHECK-NEXT: "-Xcc", +// CHECK-NEXT: "-target", +// CHECK-NEXT: "-Xcc", +// CHECK: "-fapinotes-swift-version=4" +// CHECK-NOT: "error: cannot open Swift placeholder dependency module map from" +// CHECK: "bridgingHeader": +// CHECK-NEXT: "path": +// CHECK-SAME: Bridging.h + +// CHECK-NEXT: "sourceFiles": +// CHECK-NEXT: Bridging.h +// CHECK-NEXT: BridgingOther.h + +// CHECK: "moduleDependencies": [ +// CHECK-NEXT: "F" +// CHECK-NEXT: ] + +/// --------Swift module A +// CHECK-LABEL: "modulePath": "A.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "A" +// CHECK-NEXT: } +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" +// CHECK-NEXT: }, + +/// --------Clang module C +// CHECK-LABEL: "modulePath": "C.pcm", + +// CHECK: "sourceFiles": [ +// CHECK-DAG: module.modulemap +// CHECK-DAG: C.h + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "B" + +// CHECK: "moduleMapPath" +// CHECK-SAME: module.modulemap + +// CHECK: "contextHash" +// CHECK-SAME: "{{.*}}" + +// CHECK: "commandLine": [ +// CHECK-NEXT: "-frontend" +// CHECK-NEXT: "-only-use-extra-clang-opts" +// CHECK-NEXT: "-Xcc" +// CHECK-NEXT: "clang" +// CHECK: "-fsystem-module", +// CHECK-NEXT: "-emit-pcm", +// CHECK-NEXT: "-module-name", +// CHECK-NEXT: "C" + +/// --------Swift module E +// CHECK: "swift": "E" +// CHECK-LABEL: modulePath": "E.swiftmodule" +// CHECK: "directDependencies" +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" + +// CHECK: "moduleInterfacePath" +// CHECK-SAME: E.swiftinterface + +/// --------Swift module F +// CHECK: "modulePath": "F.swiftmodule", +// CHECK-NEXT: "sourceFiles": [ +// CHECK-NEXT: ], +// CHECK-NEXT: "directDependencies": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "F" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "SwiftOnoneSupport" +// CHECK-NEXT: } +// CHECK-NEXT: ], + +/// --------Swift module G +// CHECK-LABEL: "modulePath": "G.swiftmodule" +// CHECK: "directDependencies" +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "G" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "SwiftOnoneSupport" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "details": { + +// CHECK: "contextHash": "{{.*}}", +// CHECK: "commandLine": [ +// CHECK: "-compile-module-from-interface" +// CHECK: "-target" +// CHECK: "-module-name" +// CHECK: "G" +// CHECK: "-swift-version" +// CHECK: "5" +// CHECK: ], +// CHECK" "extraPcmArgs": [ +// CHECK" "-target", +// CHECK" "-fapinotes-swift-version=5" +// CHECK" ] + +/// --------Swift module Swift +// CHECK-LABEL: "modulePath": "Swift.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "SwiftShims" + +/// --------Clang module B +// CHECK-LABEL: "modulePath": "B.pcm" + +// CHECK-NEXT: sourceFiles +// CHECK-DAG: module.modulemap +// CHECK-DAG: B.h + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "A" +// CHECK-NEXT: } + +/// --------Clang module SwiftShims +// CHECK-LABEL: "modulePath": "SwiftShims.pcm", diff --git a/tools/libSwiftScan/libSwiftScan.cpp b/tools/libSwiftScan/libSwiftScan.cpp index e8c423515b5fd..842f05839aa6d 100644 --- a/tools/libSwiftScan/libSwiftScan.cpp +++ b/tools/libSwiftScan/libSwiftScan.cpp @@ -95,6 +95,25 @@ void swiftscan_dependency_set_dispose(swiftscan_dependency_set_t *set) { delete set; } +//=== Scanner Cache Operations --------------------------------------------===// + +void swiftscan_scanner_cache_serialize(swiftscan_scanner_t scanner, + const char * path) { + DependencyScanningTool *ScanningTool = unwrap(scanner); + ScanningTool->serializeCache(path); +} + +bool swiftscan_scanner_cache_load(swiftscan_scanner_t scanner, + const char * path) { + DependencyScanningTool *ScanningTool = unwrap(scanner); + return ScanningTool->loadCache(path); +} + +void swiftscan_scanner_cache_reset(swiftscan_scanner_t scanner) { + DependencyScanningTool *ScanningTool = unwrap(scanner); + ScanningTool->resetCache(); +} + //=== Scanner Functions ---------------------------------------------------===// swiftscan_scanner_t swiftscan_scanner_create(void) { diff --git a/tools/libSwiftScan/libSwiftScan.exports b/tools/libSwiftScan/libSwiftScan.exports index e2c62ef1faa25..b16110ae98e6d 100644 --- a/tools/libSwiftScan/libSwiftScan.exports +++ b/tools/libSwiftScan/libSwiftScan.exports @@ -54,3 +54,6 @@ swiftscan_import_set_dispose swiftscan_scanner_dispose swiftscan_compiler_supported_arguments_query swiftscan_compiler_supported_features_query +swiftscan_scanner_cache_serialize +swiftscan_scanner_cache_load +swiftscan_scanner_cache_reset From 3e8fea93a66661db925b2699974ff92e1706456d Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Thu, 29 Jul 2021 10:46:48 -0700 Subject: [PATCH 3/5] [Dependency Scanning] Have the scanner cache answer queries relevant to current search paths only. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dependency scanner's cache persists across different queries and answering a subsequent query's module lookup with a module not in the query's search path is not correct. For example, suppose we are looking for a Swift module `Foo` with a set of search paths `SP`. And dependency scanner cache already contains a module `Foo`, for which we found an interface file at location `L`. If `L`∉`SP`, then we cannot re-use the cached entry because we’d be resolving the scanning query to a filesystem location that the current scanning context is not aware of. Resolves rdar://81175942 --- include/swift/AST/ASTContext.h | 8 + include/swift/AST/ModuleDependencies.h | 35 +++- lib/AST/ASTContext.cpp | 83 +++++++- lib/AST/ModuleDependencies.cpp | 181 +++++++++++++++--- .../ClangModuleDependencyScanner.cpp | 24 ++- .../ModuleDependencyCacheSerialization.cpp | 149 ++++++++------ lib/DependencyScan/ScanDependencies.cpp | 58 ++++-- lib/Serialization/ModuleDependencyScanner.cpp | 10 +- lib/Serialization/SerializedModuleLoader.cpp | 16 +- .../Inputs/SwiftDifferent/A.swiftinterface | 5 + ...module_deps_different_paths_no_reuse.swift | 50 +++++ 11 files changed, 479 insertions(+), 140 deletions(-) create mode 100644 test/ScanDependencies/Inputs/SwiftDifferent/A.swiftinterface create mode 100644 test/ScanDependencies/module_deps_different_paths_no_reuse.swift diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h index d6ee9029947a5..fe28d98ac94df 100644 --- a/include/swift/AST/ASTContext.h +++ b/include/swift/AST/ASTContext.h @@ -37,6 +37,7 @@ #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/StringSet.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/TinyPtrVector.h" #include "llvm/Support/Allocator.h" @@ -825,6 +826,13 @@ class ASTContext final { ModuleDependenciesCache &cache, InterfaceSubContextDelegate &delegate); + /// Compute the extra implicit framework search paths on Apple platforms: + /// $SDKROOT/System/Library/Frameworks/ and $SDKROOT/Library/Frameworks/. + std::vector getDarwinImplicitFrameworkSearchPaths() const; + + /// Return a set of all possible filesystem locations where modules can be found. + llvm::StringSet<> getAllModuleSearchPathsSet() const; + /// Load extensions to the given nominal type from the external /// module loaders. /// diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index 20edf419a0bcc..18d09245839f5 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -63,6 +63,12 @@ enum class ModuleDependenciesKind : int8_t { Clang, }; +/// Details of a given module used for dependency scanner cache queries. +struct ModuleLookupSpecifics { + Optional kind; + llvm::StringSet<> currentSearchPaths; +}; + /// Base class for the variant storage of ModuleDependencies. /// /// This class is mostly an implementation detail for \c ModuleDependencies. @@ -407,21 +413,25 @@ using ModuleDependencyID = std::pair; /// A cache describing the set of module dependencies that has been queried /// thus far. class ModuleDependenciesCache { + using ModuleDependenciesVector = llvm::SmallVector; + /// All cached module dependencies, in the order in which they were /// encountered. std::vector AllModules; /// Dependencies for Textual Swift modules that have already been computed. - llvm::StringMap SwiftTextualModuleDependencies; + /// This maps a module's id (name, kind) to a vector of Dependency objects, which correspond + /// to instances of the same module that may have been found in different sets of search paths. + llvm::StringMap SwiftTextualModuleDependencies; /// Dependencies for Binary Swift modules that have already been computed. - llvm::StringMap SwiftBinaryModuleDependencies; + llvm::StringMap SwiftBinaryModuleDependencies; /// Dependencies for Swift placeholder dependency modules that have already been computed. - llvm::StringMap SwiftPlaceholderModuleDependencies; + llvm::StringMap SwiftPlaceholderModuleDependencies; /// Dependencies for Clang modules that have already been computed. - llvm::StringMap ClangModuleDependencies; + llvm::StringMap ClangModuleDependencies; /// Additional information needed for Clang dependency scanning. ClangModuleDependenciesCacheImpl *clangImpl = nullptr; @@ -437,9 +447,9 @@ class ModuleDependenciesCache { /// Retrieve the dependencies map that corresponds to the given dependency /// kind. - llvm::StringMap &getDependenciesMap( + llvm::StringMap &getDependenciesMap( ModuleDependenciesKind kind); - const llvm::StringMap &getDependenciesMap( + const llvm::StringMap &getDependenciesMap( ModuleDependenciesKind kind) const; public: @@ -469,12 +479,21 @@ class ModuleDependenciesCache { /// Whether we have cached dependency information for the given module. bool hasDependencies(StringRef moduleName, - Optional kind) const; + ModuleLookupSpecifics details) const; - /// Look for module dependencies for a module with the given name. + /// Look for module dependencies for a module with the given name given current search paths. /// /// \returns the cached result, or \c None if there is no cached entry. Optional findDependencies( + StringRef moduleName, + ModuleLookupSpecifics details) const; + + /// Look for module dependencies for a module with the given name. + /// This method has a deliberately-obtuse name to indicate that it is not to be used for general + /// queries. + /// + /// \returns the cached result, or \c None if there is no cached entry. + Optional findAllDependenciesIrrespectiveOfSearchPaths( StringRef moduleName, Optional kind) const; diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp index 4e7b6343f1e73..a7df4e76e3c81 100644 --- a/lib/AST/ASTContext.cpp +++ b/lib/AST/ASTContext.cpp @@ -41,6 +41,7 @@ #include "swift/AST/PropertyWrappers.h" #include "swift/AST/ProtocolConformance.h" #include "swift/AST/RawComment.h" +#include "swift/AST/SearchPathOptions.h" #include "swift/AST/SILLayout.h" #include "swift/AST/SemanticAttrs.h" #include "swift/AST/SourceFile.h" @@ -1573,20 +1574,23 @@ Optional ASTContext::getModuleDependencies( bool cacheOnly) { // Retrieve the dependencies for this module. if (cacheOnly) { + auto searchPathSet = getAllModuleSearchPathsSet(); // Check whether we've cached this result. if (!isUnderlyingClangModule) { - if (auto found = cache.findDependencies(moduleName, - ModuleDependenciesKind::SwiftTextual)) + if (auto found = cache.findDependencies( + moduleName, + {ModuleDependenciesKind::SwiftTextual, searchPathSet})) return found; - if (auto found = cache.findDependencies(moduleName, - ModuleDependenciesKind::SwiftTextual)) + if (auto found = cache.findDependencies( + moduleName, {ModuleDependenciesKind::SwiftBinary, searchPathSet})) return found; - if (auto found = cache.findDependencies(moduleName, - ModuleDependenciesKind::SwiftPlaceholder)) + if (auto found = cache.findDependencies( + moduleName, + {ModuleDependenciesKind::SwiftPlaceholder, searchPathSet})) return found; } - if (auto found = cache.findDependencies(moduleName, - ModuleDependenciesKind::Clang)) + if (auto found = cache.findDependencies( + moduleName, {ModuleDependenciesKind::Clang, searchPathSet})) return found; } else { for (auto &loader : getImpl().ModuleLoaders) { @@ -1594,8 +1598,8 @@ Optional ASTContext::getModuleDependencies( loader.get() != getImpl().TheClangModuleLoader) continue; - if (auto dependencies = loader->getModuleDependencies(moduleName, cache, - delegate)) + if (auto dependencies = + loader->getModuleDependencies(moduleName, cache, delegate)) return dependencies; } } @@ -1618,6 +1622,65 @@ ASTContext::getSwiftModuleDependencies(StringRef moduleName, return None; } +namespace { + static StringRef + pathStringFromFrameworkSearchPath(const SearchPathOptions::FrameworkSearchPath &next) { + return next.Path; + }; +} + +std::vector ASTContext::getDarwinImplicitFrameworkSearchPaths() +const { + assert(LangOpts.Target.isOSDarwin()); + SmallString<128> systemFrameworksScratch; + systemFrameworksScratch = SearchPathOpts.SDKPath; + llvm::sys::path::append(systemFrameworksScratch, "System", "Library", "Frameworks"); + + SmallString<128> frameworksScratch; + frameworksScratch = SearchPathOpts.SDKPath; + llvm::sys::path::append(frameworksScratch, "Library", "Frameworks"); + return {systemFrameworksScratch.str().str(), frameworksScratch.str().str()}; +} + +llvm::StringSet<> ASTContext::getAllModuleSearchPathsSet() +const { + llvm::StringSet<> result; + result.insert(SearchPathOpts.ImportSearchPaths.begin(), + SearchPathOpts.ImportSearchPaths.end()); + + // Framework paths are "special", they contain more than path strings, + // but path strings are all we care about here. + using FrameworkPathView = ArrayRefView; + FrameworkPathView frameworkPathsOnly{SearchPathOpts.FrameworkSearchPaths}; + result.insert(frameworkPathsOnly.begin(), frameworkPathsOnly.end()); + + if (LangOpts.Target.isOSDarwin()) { + auto implicitFrameworkSearchPaths = getDarwinImplicitFrameworkSearchPaths(); + result.insert(implicitFrameworkSearchPaths.begin(), + implicitFrameworkSearchPaths.end()); + } + result.insert(SearchPathOpts.RuntimeLibraryImportPaths.begin(), + SearchPathOpts.RuntimeLibraryImportPaths.end()); + + // ClangImporter special-cases the path for SwiftShims, so do the same here + // If there are no shims in the resource dir, add a search path in the SDK. + SmallString<128> shimsPath(SearchPathOpts.RuntimeResourcePath); + llvm::sys::path::append(shimsPath, "shims"); + if (!llvm::sys::fs::exists(shimsPath)) { + shimsPath = SearchPathOpts.SDKPath; + llvm::sys::path::append(shimsPath, "usr", "lib", "swift", "shims"); + } + result.insert(shimsPath.str()); + + // Clang system modules are found in the SDK root + SmallString<128> clangSysRootPath(SearchPathOpts.SDKPath); + llvm::sys::path::append(clangSysRootPath, "usr", "include"); + result.insert(clangSysRootPath.str()); + return result; +} + void ASTContext::loadExtensions(NominalTypeDecl *nominal, unsigned previousGeneration) { PrettyStackTraceDecl stackTrace("loading extensions for", nominal); diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index d63dc739f884c..c97e6fdd3c43e 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -131,7 +131,7 @@ void ModuleDependencies::addBridgingModuleDependency( swiftStorage->bridgingModuleDependencies.push_back(module.str()); } -llvm::StringMap & +llvm::StringMap & ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) { switch (kind) { case ModuleDependenciesKind::SwiftTextual: @@ -146,7 +146,7 @@ ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) { llvm_unreachable("invalid dependency kind"); } -const llvm::StringMap & +const llvm::StringMap & ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) const { switch (kind) { case ModuleDependenciesKind::SwiftTextual: @@ -161,54 +161,188 @@ ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) const { llvm_unreachable("invalid dependency kind"); } +static std::string moduleBasePath(const StringRef modulePath) { + auto parent = llvm::sys::path::parent_path(modulePath); + // If the modulePath is that of a .swiftinterface contained in a .swiftmodule, + // disambiguate further to parent. + if (llvm::sys::path::extension(parent) == ".swiftmodule") { + parent = llvm::sys::path::parent_path(parent); + } + + // If the module is a part of a framework, disambiguate to the framework's parent + if (llvm::sys::path::filename(parent) == "Modules") { + auto grandParent = llvm::sys::path::parent_path(parent); + if (llvm::sys::path::extension(grandParent) == ".framework") { + parent = llvm::sys::path::parent_path(grandParent); + } + } + + return parent.str(); +} + +static bool moduleContainedInImportPathSet(const StringRef modulePath, + const llvm::StringSet<> &importPaths) +{ + return importPaths.contains(moduleBasePath(modulePath)); +} + +static bool moduleContainedInImportPathSet(const ModuleDependencies &module, + const llvm::StringSet<> &importPaths) +{ + std::string modulePath = ""; + if (auto *swiftDep = module.getAsSwiftTextualModule()) { + if (swiftDep->swiftInterfaceFile) + modulePath = *(swiftDep->swiftInterfaceFile); + else { + // If we encountered a Swift textual dependency without an interface + // file, we are seeing the main scan module itself. This means that + // our search-path disambiguation is not necessary here. + return true; + } + } else if (auto *swiftBinaryDep = module.getAsSwiftBinaryModule()) { + modulePath = swiftBinaryDep->compiledModulePath; + } else if (auto *placehodlerDep = module.getAsPlaceholderDependencyModule()) { + // Placeholders are resolved as `true` because they are not associated with + // any specific search path. + return true; + } else if (auto *clangDep = module.getAsClangModule()) { + modulePath = clangDep->moduleMapFile; + } + + if (moduleContainedInImportPathSet(modulePath, importPaths)) { + return true; + } + return false; +} + bool ModuleDependenciesCache::hasDependencies( StringRef moduleName, - Optional kind) const { - if (!kind) { - return hasDependencies(moduleName, ModuleDependenciesKind::SwiftTextual) || - hasDependencies(moduleName, ModuleDependenciesKind::SwiftBinary) || - hasDependencies(moduleName, ModuleDependenciesKind::SwiftPlaceholder) || - hasDependencies(moduleName, ModuleDependenciesKind::Clang); + ModuleLookupSpecifics details) const { + if (!details.kind) { + return hasDependencies(moduleName, + {ModuleDependenciesKind::SwiftTextual, + details.currentSearchPaths}) || + hasDependencies(moduleName, + {ModuleDependenciesKind::SwiftBinary, + details.currentSearchPaths}) || + hasDependencies(moduleName, + {ModuleDependenciesKind::SwiftPlaceholder, + details.currentSearchPaths}) || + hasDependencies(moduleName, + {ModuleDependenciesKind::Clang, + details.currentSearchPaths}); } - const auto &map = getDependenciesMap(*kind); - return map.count(moduleName) > 0; + const auto &map = getDependenciesMap(*details.kind); + auto known = map.find(moduleName); + if (known != map.end()) { + assert(!known->second.empty()); + for (auto &dep : known->second) { + if (moduleContainedInImportPathSet(dep, details.currentSearchPaths)) + return true; + } + return false; + } + return false; } Optional ModuleDependenciesCache::findDependencies( StringRef moduleName, - Optional kind) const { - if (!kind) { + ModuleLookupSpecifics details) const { + if (!details.kind) { if (auto swiftTextualDep = findDependencies( - moduleName, ModuleDependenciesKind::SwiftTextual)) + moduleName, + {ModuleDependenciesKind::SwiftTextual, details.currentSearchPaths})) return swiftTextualDep; else if (auto swiftBinaryDep = findDependencies( - moduleName, ModuleDependenciesKind::SwiftBinary)) + moduleName, + {ModuleDependenciesKind::SwiftBinary, details.currentSearchPaths})) return swiftBinaryDep; else if (auto swiftPlaceholderDep = findDependencies( - moduleName, ModuleDependenciesKind::SwiftPlaceholder)) + moduleName, + {ModuleDependenciesKind::SwiftPlaceholder, details.currentSearchPaths})) return swiftPlaceholderDep; else - return findDependencies(moduleName, ModuleDependenciesKind::Clang); + return findDependencies(moduleName, + {ModuleDependenciesKind::Clang, details.currentSearchPaths}); + } + + const auto &map = getDependenciesMap(*details.kind); + auto known = map.find(moduleName); + if (known != map.end()) { + assert(!known->second.empty()); + for (auto &dep : known->second) { + if (moduleContainedInImportPathSet(dep, details.currentSearchPaths)) + return dep; + } + return None; + } + return None; +} + +Optional +ModuleDependenciesCache::findAllDependenciesIrrespectiveOfSearchPaths( + StringRef moduleName, Optional kind) const { + if (!kind) { + if (auto swiftTextualDeps = findAllDependenciesIrrespectiveOfSearchPaths( + moduleName, ModuleDependenciesKind::SwiftTextual)) + return swiftTextualDeps; + else if (auto swiftBinaryDeps = + findAllDependenciesIrrespectiveOfSearchPaths( + moduleName, ModuleDependenciesKind::SwiftBinary)) + return swiftBinaryDeps; + else if (auto swiftPlaceholderDeps = + findAllDependenciesIrrespectiveOfSearchPaths( + moduleName, ModuleDependenciesKind::SwiftPlaceholder)) + return swiftPlaceholderDeps; + else + return findAllDependenciesIrrespectiveOfSearchPaths( + moduleName, ModuleDependenciesKind::Clang); } const auto &map = getDependenciesMap(*kind); auto known = map.find(moduleName); - if (known != map.end()) + if (known != map.end()) { + assert(!known->second.empty()); return known->second; - + } return None; } +static std::string modulePathForVerification(const ModuleDependencies &module) { + std::string existingModulePath = ""; + if (auto *swiftDep = module.getAsSwiftTextualModule()) { + if (swiftDep->swiftInterfaceFile) + existingModulePath = *(swiftDep->swiftInterfaceFile); + } else if (auto *swiftBinaryDep = module.getAsSwiftBinaryModule()) { + existingModulePath = swiftBinaryDep->compiledModulePath; + } else if (auto *clangDep = module.getAsClangModule()) { + existingModulePath = clangDep->moduleMapFile; + } + return existingModulePath; +} + void ModuleDependenciesCache::recordDependencies( StringRef moduleName, ModuleDependencies dependencies) { auto kind = dependencies.getKind(); auto &map = getDependenciesMap(kind); - assert(map.count(moduleName) == 0 && "Already added to map"); - map.insert({moduleName, std::move(dependencies)}); - - AllModules.push_back({moduleName.str(), kind}); + // Cache may already have a dependency for this module + if (map.count(moduleName) != 0) { + // Ensure that the existing dependencies objects are at a different path. + // i.e. do not record duplicate dependencies. + auto newModulePath = modulePathForVerification(dependencies); + bool newEntry = true; + for (auto &existingDeps : map[moduleName]) { + if (modulePathForVerification(existingDeps) == newModulePath) + newEntry = false; + } + if (newEntry) + map[moduleName].emplace_back(std::move(dependencies)); + } else { + map.insert({moduleName, ModuleDependenciesVector{std::move(dependencies)}}); + AllModules.push_back({moduleName.str(), kind}); + } } void ModuleDependenciesCache::updateDependencies( @@ -216,5 +350,6 @@ void ModuleDependenciesCache::updateDependencies( auto &map = getDependenciesMap(moduleID.second); auto known = map.find(moduleID.first); assert(known != map.end() && "Not yet added to map"); - known->second = std::move(dependencies); + assert(known->second.size() == 1 && "Cannot update dependency with multiple candidates."); + known->second[0] = std::move(dependencies); } diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp index dc57a590cf446..ee8458fba819e 100644 --- a/lib/ClangImporter/ClangModuleDependencyScanner.cpp +++ b/lib/ClangImporter/ClangModuleDependencyScanner.cpp @@ -197,11 +197,14 @@ void ClangImporter::recordModuleDependencies( std::string PCMPath; std::string ModuleMapPath; }; + auto &ctx = Impl.SwiftContext; + auto currentSwiftSearchPathSet = ctx.getAllModuleSearchPathsSet(); for (const auto &clangModuleDep : clangDependencies.DiscoveredModules) { // If we've already cached this information, we're done. - if (cache.hasDependencies(clangModuleDep.ModuleName, - ModuleDependenciesKind::Clang)) + if (cache.hasDependencies( + clangModuleDep.ModuleName, + {ModuleDependenciesKind::Clang, currentSwiftSearchPathSet})) continue; // File dependencies for this module. @@ -303,9 +306,12 @@ void ClangImporter::recordModuleDependencies( Optional ClangImporter::getModuleDependencies( StringRef moduleName, ModuleDependenciesCache &cache, InterfaceSubContextDelegate &delegate) { + auto &ctx = Impl.SwiftContext; + auto currentSwiftSearchPathSet = ctx.getAllModuleSearchPathsSet(); // Check whether there is already a cached result. if (auto found = cache.findDependencies( - moduleName, ModuleDependenciesKind::Clang)) + moduleName, + {ModuleDependenciesKind::Clang, currentSwiftSearchPathSet})) return found; // Retrieve or create the shared state. @@ -319,7 +325,7 @@ Optional ClangImporter::getModuleDependencies( } // Determine the command-line arguments for dependency scanning. - auto &ctx = Impl.SwiftContext; + std::vector commandLineArgs = getClangDepScanningInvocationArguments(ctx, *importHackFile); @@ -338,14 +344,19 @@ Optional ClangImporter::getModuleDependencies( // Record module dependencies for each module we found. recordModuleDependencies(cache, *clangDependencies); - return cache.findDependencies(moduleName, ModuleDependenciesKind::Clang); + return cache.findDependencies( + moduleName, + {ModuleDependenciesKind::Clang, currentSwiftSearchPathSet}); } bool ClangImporter::addBridgingHeaderDependencies( StringRef moduleName, ModuleDependenciesCache &cache) { + auto &ctx = Impl.SwiftContext; + auto currentSwiftSearchPathSet = ctx.getAllModuleSearchPathsSet(); auto targetModule = *cache.findDependencies( - moduleName, ModuleDependenciesKind::SwiftTextual); + moduleName, + {ModuleDependenciesKind::SwiftTextual,currentSwiftSearchPathSet}); // If we've already recorded bridging header dependencies, we're done. auto swiftDeps = targetModule.getAsSwiftTextualModule(); @@ -360,7 +371,6 @@ bool ClangImporter::addBridgingHeaderDependencies( std::string bridgingHeader = *targetModule.getBridgingHeader(); // Determine the command-line arguments for dependency scanning. - auto &ctx = Impl.SwiftContext; std::vector commandLineArgs = getClangDepScanningInvocationArguments(ctx, bridgingHeader); diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index 142de49e16985..97128e1ecc6c5 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -682,7 +682,10 @@ void Serializer::writeModuleInfo(ModuleDependencyID moduleID, getIdentifier(moduleID.first), getArray(moduleID, ModuleIdentifierArrayKind::DirectDependencies)); - if (auto swiftTextDeps = dependencyInfo.getAsSwiftTextualModule()) { + switch (dependencyInfo.getKind()) { + case swift::ModuleDependenciesKind::SwiftTextual: { + auto swiftTextDeps = dependencyInfo.getAsSwiftTextualModule(); + assert(swiftTextDeps); unsigned swiftInterfaceFileId = swiftTextDeps->swiftInterfaceFile ? getIdentifier(swiftTextDeps->swiftInterfaceFile.getValue()) @@ -703,24 +706,33 @@ void Serializer::writeModuleInfo(ModuleDependencyID moduleID, getArray(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles), getArray(moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies)); - - } else if (auto swiftBinDeps = dependencyInfo.getAsSwiftBinaryModule()) { + break; + } + case swift::ModuleDependenciesKind::SwiftBinary: { + auto swiftBinDeps = dependencyInfo.getAsSwiftBinaryModule(); + assert(swiftBinDeps); SwiftBinaryModuleDetailsLayout::emitRecord( Out, ScratchRecord, AbbrCodes[SwiftBinaryModuleDetailsLayout::Code], getIdentifier(swiftBinDeps->compiledModulePath), getIdentifier(swiftBinDeps->moduleDocPath), getIdentifier(swiftBinDeps->sourceInfoPath), swiftBinDeps->isFramework); - } else if (auto swiftPHDeps = - dependencyInfo.getAsPlaceholderDependencyModule()) { + break; + } + case swift::ModuleDependenciesKind::SwiftPlaceholder: { + auto swiftPHDeps = dependencyInfo.getAsPlaceholderDependencyModule(); + assert(swiftPHDeps); SwiftPlaceholderModuleDetailsLayout::emitRecord( Out, ScratchRecord, AbbrCodes[SwiftPlaceholderModuleDetailsLayout::Code], getIdentifier(swiftPHDeps->compiledModulePath), getIdentifier(swiftPHDeps->moduleDocPath), getIdentifier(swiftPHDeps->sourceInfoPath)); - - } else if (auto clangDeps = dependencyInfo.getAsClangModule()) { + break; + } + case swift::ModuleDependenciesKind::Clang: { + auto clangDeps = dependencyInfo.getAsClangModule(); + assert(clangDeps); ClangModuleDetailsLayout::emitRecord( Out, ScratchRecord, AbbrCodes[ClangModuleDetailsLayout::Code], getIdentifier(clangDeps->moduleMapFile), @@ -728,8 +740,8 @@ void Serializer::writeModuleInfo(ModuleDependencyID moduleID, getArray(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine), getArray(moduleID, ModuleIdentifierArrayKind::FileDependencies)); - } else { - llvm_unreachable("Unexpected module dependency kind."); + break; + } } } @@ -802,52 +814,69 @@ unsigned Serializer::getArray(ModuleDependencyID moduleID, void Serializer::collectStringsAndArrays(const ModuleDependenciesCache &cache) { for (auto &moduleID : cache.getAllModules()) { - auto dependencyInfo = - cache.findDependencies(moduleID.first, moduleID.second); - assert(dependencyInfo.hasValue() && "Expected dependency info."); - // Add the module's name - addIdentifier(moduleID.first); - // Add the module's dependencies - addArray(moduleID, ModuleIdentifierArrayKind::DirectDependencies, - dependencyInfo->getModuleDependencies()); - - // Add the dependency-kind-specific data - if (auto swiftTextDeps = dependencyInfo->getAsSwiftTextualModule()) { - if (swiftTextDeps->swiftInterfaceFile) - addIdentifier(swiftTextDeps->swiftInterfaceFile.getValue()); - addArray(moduleID, ModuleIdentifierArrayKind::CompiledModuleCandidates, - swiftTextDeps->compiledModuleCandidates); - addArray(moduleID, ModuleIdentifierArrayKind::BuildCommandLine, - swiftTextDeps->buildCommandLine); - addArray(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs, - swiftTextDeps->extraPCMArgs); - addIdentifier(swiftTextDeps->contextHash); - if (swiftTextDeps->bridgingHeaderFile) - addIdentifier(swiftTextDeps->bridgingHeaderFile.getValue()); - addArray(moduleID, ModuleIdentifierArrayKind::SourceFiles, - swiftTextDeps->sourceFiles); - addArray(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles, - swiftTextDeps->bridgingSourceFiles); - addArray(moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies, - swiftTextDeps->bridgingModuleDependencies); - } else if (auto swiftBinDeps = dependencyInfo->getAsSwiftBinaryModule()) { - addIdentifier(swiftBinDeps->compiledModulePath); - addIdentifier(swiftBinDeps->moduleDocPath); - addIdentifier(swiftBinDeps->sourceInfoPath); - } else if (auto swiftPHDeps = - dependencyInfo->getAsPlaceholderDependencyModule()) { - addIdentifier(swiftPHDeps->compiledModulePath); - addIdentifier(swiftPHDeps->moduleDocPath); - addIdentifier(swiftPHDeps->sourceInfoPath); - } else if (auto clangDeps = dependencyInfo->getAsClangModule()) { - addIdentifier(clangDeps->moduleMapFile); - addIdentifier(clangDeps->contextHash); - addArray(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine, - clangDeps->nonPathCommandLine); - addArray(moduleID, ModuleIdentifierArrayKind::FileDependencies, - clangDeps->fileDependencies); - } else { - llvm_unreachable("Unexpected module dependency kind."); + auto dependencyInfos = cache.findAllDependenciesIrrespectiveOfSearchPaths( + moduleID.first, moduleID.second); + assert(dependencyInfos.hasValue() && "Expected dependency info."); + for (auto &dependencyInfo : *dependencyInfos) { + // Add the module's name + addIdentifier(moduleID.first); + // Add the module's dependencies + addArray(moduleID, ModuleIdentifierArrayKind::DirectDependencies, + dependencyInfo.getModuleDependencies()); + + // Add the dependency-kind-specific data + switch (dependencyInfo.getKind()) { + case swift::ModuleDependenciesKind::SwiftTextual: { + auto swiftTextDeps = dependencyInfo.getAsSwiftTextualModule(); + assert(swiftTextDeps); + if (swiftTextDeps->swiftInterfaceFile) + addIdentifier(swiftTextDeps->swiftInterfaceFile.getValue()); + addArray(moduleID, ModuleIdentifierArrayKind::CompiledModuleCandidates, + swiftTextDeps->compiledModuleCandidates); + addArray(moduleID, ModuleIdentifierArrayKind::BuildCommandLine, + swiftTextDeps->buildCommandLine); + addArray(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs, + swiftTextDeps->extraPCMArgs); + addIdentifier(swiftTextDeps->contextHash); + if (swiftTextDeps->bridgingHeaderFile) + addIdentifier(swiftTextDeps->bridgingHeaderFile.getValue()); + addArray(moduleID, ModuleIdentifierArrayKind::SourceFiles, + swiftTextDeps->sourceFiles); + addArray(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles, + swiftTextDeps->bridgingSourceFiles); + addArray(moduleID, + ModuleIdentifierArrayKind::BridgingModuleDependencies, + swiftTextDeps->bridgingModuleDependencies); + break; + } + case swift::ModuleDependenciesKind::SwiftBinary: { + auto swiftBinDeps = dependencyInfo.getAsSwiftBinaryModule(); + assert(swiftBinDeps); + addIdentifier(swiftBinDeps->compiledModulePath); + addIdentifier(swiftBinDeps->moduleDocPath); + addIdentifier(swiftBinDeps->sourceInfoPath); + break; + } + case swift::ModuleDependenciesKind::SwiftPlaceholder: { + auto swiftPHDeps = dependencyInfo.getAsPlaceholderDependencyModule(); + assert(swiftPHDeps); + addIdentifier(swiftPHDeps->compiledModulePath); + addIdentifier(swiftPHDeps->moduleDocPath); + addIdentifier(swiftPHDeps->sourceInfoPath); + break; + } + case swift::ModuleDependenciesKind::Clang: { + auto clangDeps = dependencyInfo.getAsClangModule(); + assert(clangDeps); + addIdentifier(clangDeps->moduleMapFile); + addIdentifier(clangDeps->contextHash); + addArray(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine, + clangDeps->nonPathCommandLine); + addArray(moduleID, ModuleIdentifierArrayKind::FileDependencies, + clangDeps->fileDependencies); + break; + } + } } } } @@ -888,10 +917,12 @@ void Serializer::writeInterModuleDependenciesCache( // Write the core graph for (auto &moduleID : cache.getAllModules()) { - auto dependencyInfo = - cache.findDependencies(moduleID.first, moduleID.second); - assert(dependencyInfo.hasValue() && "Expected dependency info."); - writeModuleInfo(moduleID, dependencyInfo.getValue()); + auto dependencyInfos = cache.findAllDependenciesIrrespectiveOfSearchPaths( + moduleID.first, moduleID.second); + assert(dependencyInfos.hasValue() && "Expected dependency info."); + for (auto &dependencyInfo : *dependencyInfos) { + writeModuleInfo(moduleID, dependencyInfo); + } } return; diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index e3cef5d01daa2..f18565f341f0e 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -143,8 +143,10 @@ static void findAllImportedClangModules(ASTContext &ctx, StringRef moduleName, return; allModules.push_back(moduleName.str()); + auto currentImportPathSet = ctx.getAllModuleSearchPathsSet(); auto dependencies = - cache.findDependencies(moduleName, ModuleDependenciesKind::Clang); + cache.findDependencies(moduleName, + {ModuleDependenciesKind::Clang, currentImportPathSet}); if (!dependencies) return; @@ -160,7 +162,10 @@ resolveDirectDependencies(CompilerInstance &instance, ModuleDependencyID module, InterfaceSubContextDelegate &ASTDelegate, bool cacheOnly = false) { auto &ctx = instance.getASTContext(); - auto knownDependencies = *cache.findDependencies(module.first, module.second); + auto currentImportSet = ctx.getAllModuleSearchPathsSet(); + auto knownDependencies = *cache.findDependencies( + module.first, + {module.second,currentImportSet}); auto isSwiftInterface = knownDependencies.isSwiftTextualModule(); auto isSwift = isSwiftInterface || knownDependencies.isSwiftBinaryModule(); @@ -188,7 +193,8 @@ resolveDirectDependencies(CompilerInstance &instance, ModuleDependencyID module, // Grab the updated module dependencies. // FIXME: This is such a hack. knownDependencies = - *cache.findDependencies(module.first, module.second); + *cache.findDependencies(module.first, + {module.second, currentImportSet}); // Add the Clang modules referenced from the bridging header to the // set of Clang modules we know about. @@ -234,12 +240,15 @@ static void discoverCrosssImportOverlayDependencies( CompilerInstance &instance, StringRef mainModuleName, ArrayRef allDependencies, ModuleDependenciesCache &cache, InterfaceSubContextDelegate &ASTDelegate, - llvm::function_ref action) { + llvm::function_ref action, + llvm::StringSet<> ¤tImportPathSet) { // Modules explicitly imported. Only these can be secondary module. llvm::SetVector newOverlays; for (auto dep : allDependencies) { auto moduleName = dep.first; - auto dependencies = *cache.findDependencies(moduleName, dep.second); + auto dependencies = *cache.findDependencies( + moduleName, + {dep.second, currentImportPathSet}); // Collect a map from secondary module name to cross-import overlay names. auto overlayMap = dependencies.collectCrossImportOverlayNames( instance.getASTContext(), moduleName); @@ -270,8 +279,9 @@ static void discoverCrosssImportOverlayDependencies( auto dummyMainDependencies = ModuleDependencies::forMainSwiftModule({}); // Update main module's dependencies to include these new overlays. - auto mainDep = *cache.findDependencies(mainModuleName, - ModuleDependenciesKind::SwiftTextual); + auto mainDep = *cache.findDependencies( + mainModuleName, + {ModuleDependenciesKind::SwiftTextual, currentImportPathSet}); std::for_each(newOverlays.begin(), newOverlays.end(), [&](Identifier modName) { dummyMainDependencies.addModuleDependency(modName.str()); @@ -282,8 +292,9 @@ static void discoverCrosssImportOverlayDependencies( // Record the dummy main module's direct dependencies. The dummy main module // only directly depend on these newly discovered overlay modules. - if (cache.findDependencies(dummyMainName, - ModuleDependenciesKind::SwiftTextual)) { + if (cache.findDependencies( + dummyMainName, + {ModuleDependenciesKind::SwiftTextual,currentImportPathSet})) { cache.updateDependencies( std::make_pair(dummyMainName.str(), ModuleDependenciesKind::SwiftTextual), @@ -730,7 +741,8 @@ static swiftscan_dependency_graph_t generateFullDependencyGraph(CompilerInstance &instance, ModuleDependenciesCache &cache, InterfaceSubContextDelegate &ASTDelegate, - ArrayRef allModules) { + ArrayRef allModules, + llvm::StringSet<> ¤tImportPathSet) { if (allModules.empty()) { return nullptr; } @@ -744,7 +756,9 @@ generateFullDependencyGraph(CompilerInstance &instance, for (size_t i = 0; i < allModules.size(); ++i) { const auto &module = allModules[i]; // Grab the completed module dependencies. - auto moduleDepsQuery = cache.findDependencies(module.first, module.second); + auto moduleDepsQuery = cache.findDependencies( + module.first, + {module.second, currentImportPathSet}); if (!moduleDepsQuery) { std::string err = "Module Dependency Cache missing module" + module.first; llvm::report_fatal_error(err); @@ -1284,6 +1298,9 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, ModuleDecl *mainModule = instance.getMainModule(); // First, identify the dependencies of the main module auto mainDependencies = identifyMainModuleDependencies(instance); + auto &ctx = instance.getASTContext(); + auto currentImportPathSet = ctx.getAllModuleSearchPathsSet(); + // Add the main module. StringRef mainModuleName = mainModule->getNameStr(); llvm::SetVector, @@ -1293,8 +1310,9 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, // We may be re-using an instance of the cache which already contains // an entry for this module. - if (cache.findDependencies(mainModuleName, - ModuleDependenciesKind::SwiftTextual)) { + if (cache.findDependencies( + mainModuleName, + {ModuleDependenciesKind::SwiftTextual, currentImportPathSet})) { cache.updateDependencies( std::make_pair(mainModuleName.str(), ModuleDependenciesKind::SwiftTextual), @@ -1303,7 +1321,6 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, cache.recordDependencies(mainModuleName, std::move(mainDependencies)); } - auto &ctx = instance.getASTContext(); auto ModuleCachePath = getModuleCachePathFromClang( ctx.getClangModuleLoader()->getClangInstance()); auto &FEOpts = instance.getInvocation().getFrontendOptions(); @@ -1330,7 +1347,8 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, discoverCrosssImportOverlayDependencies( instance, mainModuleName, /*All transitive dependencies*/ allModules.getArrayRef().slice(1), cache, - ASTDelegate, [&](ModuleDependencyID id) { allModules.insert(id); }); + ASTDelegate, [&](ModuleDependencyID id) { allModules.insert(id); }, + currentImportPathSet); // Dignose cycle in dependency graph. if (diagnoseCycle(instance, cache, /*MainModule*/ allModules.front(), @@ -1359,11 +1377,13 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, } auto dependencyGraph = generateFullDependencyGraph( - instance, *resultCache, ASTDelegate, allModules.getArrayRef()); + instance, *resultCache, ASTDelegate, + allModules.getArrayRef(), currentImportPathSet); // Update the dependency tracker. if (auto depTracker = instance.getDependencyTracker()) { for (auto module : allModules) { - auto deps = resultCache->findDependencies(module.first, module.second); + auto deps = resultCache->findDependencies(module.first, + {module.second, currentImportPathSet}); if (!deps) continue; @@ -1414,6 +1434,7 @@ swift::dependencies::performBatchModuleScan( StringRef moduleName = entry.moduleName; bool isClang = !entry.isSwift; ASTContext &ctx = instance.getASTContext(); + auto currentImportPathSet = ctx.getAllModuleSearchPathsSet(); auto &FEOpts = instance.getInvocation().getFrontendOptions(); ModuleInterfaceLoaderOptions LoaderOpts(FEOpts); auto ModuleCachePath = getModuleCachePathFromClang( @@ -1463,7 +1484,8 @@ swift::dependencies::performBatchModuleScan( } batchScanResult.push_back(generateFullDependencyGraph( - instance, cache, ASTDelegate, allModules.getArrayRef())); + instance, cache, ASTDelegate, + allModules.getArrayRef(), currentImportPathSet)); }); return batchScanResult; diff --git a/lib/Serialization/ModuleDependencyScanner.cpp b/lib/Serialization/ModuleDependencyScanner.cpp index 0b338db50fc44..9774deba705fc 100644 --- a/lib/Serialization/ModuleDependencyScanner.cpp +++ b/lib/Serialization/ModuleDependencyScanner.cpp @@ -164,15 +164,19 @@ ErrorOr ModuleDependencyScanner::scanInterfaceFile( Optional SerializedModuleLoaderBase::getModuleDependencies( StringRef moduleName, ModuleDependenciesCache &cache, InterfaceSubContextDelegate &delegate) { + auto currentSearchPathSet = Ctx.getAllModuleSearchPathsSet(); // Check whether we've cached this result. if (auto found = cache.findDependencies( - moduleName, ModuleDependenciesKind::SwiftTextual)) + moduleName, + {ModuleDependenciesKind::SwiftTextual, currentSearchPathSet})) return found; if (auto found = cache.findDependencies( - moduleName, ModuleDependenciesKind::SwiftBinary)) + moduleName, + {ModuleDependenciesKind::SwiftBinary, currentSearchPathSet})) return found; if (auto found = cache.findDependencies( - moduleName, ModuleDependenciesKind::SwiftPlaceholder)) + moduleName, + {ModuleDependenciesKind::SwiftPlaceholder, currentSearchPathSet})) return found; auto moduleId = Ctx.getIdentifier(moduleName); diff --git a/lib/Serialization/SerializedModuleLoader.cpp b/lib/Serialization/SerializedModuleLoader.cpp index a725c1595209f..8d0758aed7fce 100644 --- a/lib/Serialization/SerializedModuleLoader.cpp +++ b/lib/Serialization/SerializedModuleLoader.cpp @@ -86,18 +86,10 @@ Optional forEachModuleSearchPath( // Apple platforms have extra implicit framework search paths: // $SDKROOT/System/Library/Frameworks/ and $SDKROOT/Library/Frameworks/. if (Ctx.LangOpts.Target.isOSDarwin()) { - SmallString<128> scratch; - scratch = Ctx.SearchPathOpts.SDKPath; - llvm::sys::path::append(scratch, "System", "Library", "Frameworks"); - if (auto result = - callback(scratch, SearchPathKind::Framework, /*isSystem=*/true)) - return result; - - scratch = Ctx.SearchPathOpts.SDKPath; - llvm::sys::path::append(scratch, "Library", "Frameworks"); - if (auto result = - callback(scratch, SearchPathKind::Framework, /*isSystem=*/true)) - return result; + for (const auto &path : Ctx.getDarwinImplicitFrameworkSearchPaths()) + if (auto result = + callback(path, SearchPathKind::Framework, /*isSystem=*/true)) + return result; } for (auto importPath : Ctx.SearchPathOpts.RuntimeLibraryImportPaths) { diff --git a/test/ScanDependencies/Inputs/SwiftDifferent/A.swiftinterface b/test/ScanDependencies/Inputs/SwiftDifferent/A.swiftinterface new file mode 100644 index 0000000000000..a5d04bec9041c --- /dev/null +++ b/test/ScanDependencies/Inputs/SwiftDifferent/A.swiftinterface @@ -0,0 +1,5 @@ +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name A +import Swift + +public func FuncAA() { } diff --git a/test/ScanDependencies/module_deps_different_paths_no_reuse.swift b/test/ScanDependencies/module_deps_different_paths_no_reuse.swift new file mode 100644 index 0000000000000..dd260173eccab --- /dev/null +++ b/test/ScanDependencies/module_deps_different_paths_no_reuse.swift @@ -0,0 +1,50 @@ +// RUN: %empty-directory(%t) +// RUN: mkdir -p %t/clang-module-cache + +// This test ensures that subsequent invocations of the dependency scanner that re-use previous cache state do not re-use cache entries that contain modules found outside of the current scanner invocation's search paths. + +// Run the scanner once, emitting the serialized scanner cache, with one set of search paths +// RUN: %target-swift-frontend -scan-dependencies -serialize-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/clang-module-cache %s -o %t/deps_initial.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 +// RUN: %FileCheck -input-file %t/deps_initial.json %s -check-prefix CHECK-INITIAL-SCAN + +// Run the scanner again, but now re-using previously-serialized cache and using a different search path for Swift modules +// RUN: %target-swift-frontend -scan-dependencies -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/SwiftDifferent -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 +// RUN: %FileCheck -input-file %t/deps.json %s -check-prefix CHECK-DIFFERENT + +// REQUIRES: executable_test +// REQUIRES: objc_interop + +import A + +// CHECK-INITIAL-SCAN: "modulePath": "A.swiftmodule", +// CHECK-INITIAL-SCAN-NEXT: "sourceFiles": [ +// CHECK-INITIAL-SCAN-NEXT: ], +// CHECK-INITIAL-SCAN-NEXT: "directDependencies": [ +// CHECK-INITIAL-SCAN-NEXT: { +// CHECK-INITIAL-SCAN-NEXT: "clang": "A" +// CHECK-INITIAL-SCAN-NEXT: }, +// CHECK-INITIAL-SCAN-NEXT: { +// CHECK-INITIAL-SCAN-NEXT: "swift": "Swift" +// CHECK-INITIAL-SCAN-NEXT: }, +// CHECK-INITIAL-SCAN-NEXT: { +// CHECK-INITIAL-SCAN-NEXT: "swift": "SwiftOnoneSupport" +// CHECK-INITIAL-SCAN-NEXT: } +// CHECK-INITIAL-SCAN-NEXT: ], +// CHECK-INITIAL-SCAN-NEXT: "details": { +// CHECK-INITIAL-SCAN-NEXT: "swift": { +// CHECK-INITIAL-SCAN-NEXT: "moduleInterfacePath": "{{.*}}/Swift/A.swiftinterface", + +// CHECK-DIFFERENT: "modulePath": "A.swiftmodule", +// CHECK-DIFFERENT-NEXT: "sourceFiles": [ +// CHECK-DIFFERENT-NEXT: ], +// CHECK-DIFFERENT-NEXT: "directDependencies": [ +// CHECK-DIFFERENT-NEXT: { +// CHECK-DIFFERENT-NEXT: "swift": "Swift" +// CHECK-DIFFERENT-NEXT: }, +// CHECK-DIFFERENT-NEXT: { +// CHECK-DIFFERENT-NEXT: "swift": "SwiftOnoneSupport" +// CHECK-DIFFERENT-NEXT: } +// CHECK-DIFFERENT-NEXT: ], +// CHECK-DIFFERENT-NEXT: "details": { +// CHECK-DIFFERENT-NEXT: "swift": { +// CHECK-DIFFERENT-NEXT: "moduleInterfacePath": "{{.*}}/SwiftDifferent/A.swiftinterface", From 2d89d9f5094e2921fee92821f284b154c60ae222 Mon Sep 17 00:00:00 2001 From: Xi Ge Date: Wed, 28 Jul 2021 17:32:21 -0700 Subject: [PATCH 4/5] DependenciesScanner: include -sdk in Swift module building command Swift module building commands need -sdk because dependencies sometimes use sdk-relative paths (prebuilt modules for example). Without -sdk, the command will not be able to local these dependencies, leading to unnecessary rebuilding from textual interfaces. rdar://81177968 --- lib/Frontend/ModuleInterfaceLoader.cpp | 7 +++++++ test/ScanDependencies/include-sdk-in-command.swift | 11 +++++++++++ test/ScanDependencies/module_deps.swift | 1 - 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 test/ScanDependencies/include-sdk-in-command.swift diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index 9a1a1d805a1fe..83d4517fe8850 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -1153,6 +1153,13 @@ void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface( genericSubInvocation.setImportSearchPaths(SearchPathOpts.ImportSearchPaths); genericSubInvocation.setFrameworkSearchPaths(SearchPathOpts.FrameworkSearchPaths); if (!SearchPathOpts.SDKPath.empty()) { + // Add -sdk arguments to the module building commands. + // Module building commands need this because dependencies sometimes use + // sdk-relative paths (prebuilt modules for example). Without -sdk, the command + // will not be able to local these dependencies, leading to unnecessary + // building from textual interfaces. + GenericArgs.push_back("-sdk"); + GenericArgs.push_back(ArgSaver.save(SearchPathOpts.SDKPath)); genericSubInvocation.setSDKPath(SearchPathOpts.SDKPath); } diff --git a/test/ScanDependencies/include-sdk-in-command.swift b/test/ScanDependencies/include-sdk-in-command.swift new file mode 100644 index 0000000000000..eefe5a51c21fb --- /dev/null +++ b/test/ScanDependencies/include-sdk-in-command.swift @@ -0,0 +1,11 @@ +// RUN: %empty-directory(%t) + +// RUN: %target-swift-frontend -scan-dependencies %s -o %t/deps.json -sdk %t/mysecretsdk.sdk + +// Check the contents of the JSON output +// RUN: %FileCheck %s < %t/deps.json + +func foo() { print(1) } + +// CHECK: "-sdk", +// CHECK-NEXT: mysecretsdk.sdk diff --git a/test/ScanDependencies/module_deps.swift b/test/ScanDependencies/module_deps.swift index c951041071a47..39efcfd5649ce 100644 --- a/test/ScanDependencies/module_deps.swift +++ b/test/ScanDependencies/module_deps.swift @@ -213,7 +213,6 @@ import SubE /// --------Clang module SwiftShims // CHECK-LABEL: "modulePath": "SwiftShims.pcm", -// CHECK-NO-SEARCH-PATHS-NOT: "-sdk" // CHECK-NO-SEARCH-PATHS-NOT: "-prebuilt-module-cache-path" // Check make-style dependencies From 09662770aa3fe0209a60f910a197f25d092af992 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Wed, 4 Aug 2021 18:19:36 -0700 Subject: [PATCH 5/5] [Dependency Scanning] Split the ModuleDependencyCache into two: current scan cache & global. This change causes the cache to be layered with a local "cache" that wraps the global cache, which will serve as the source of truth. The local cache persists only for the duration of a given scanning action, and has a store of references to dependencies resolved as a part of the current scanning action only, while the global cache is the one that persists across scanning actions (e.g. in `DependencyScanningTool`) and stores actual module dependency info values. Only the local cache can answer dependency lookup queries, checking current scanning action results first, before falling back to querying the global cache, with queries disambiguated by the current scannning action's search paths, ensuring we never resolve a dependency lookup query with a module info that could not be found in the current action's search paths. This change is required because search-path disambiguation can lead to false-negatives: for example, the Clang dependency scanner may find modules relative to the compiler's path that are not on the compiler's direct search paths. While such false-negative query responses should be functionally safe, we rely on the current scanning action's results being always-present-in-the-cache for the scanner's functionality. This layering ensures that the cache use-sites remain unchanged and that we get both: preserved global state which can be queried disambiguated with the search path details, and an always-consistent local (current action) cache state. --- include/swift/AST/ModuleDependencies.h | 190 ++++++++--- .../DependencyScan/DependencyScanningTool.h | 4 +- .../swift/DependencyScan/ScanDependencies.h | 10 +- .../SerializedModuleDependencyCacheFormat.h | 10 +- include/swift/Frontend/FrontendOptions.h | 4 - lib/AST/ModuleDependencies.cpp | 311 +++++++++++------- lib/DependencyScan/DependencyScanningTool.cpp | 20 +- .../ModuleDependencyCacheSerialization.cpp | 28 +- lib/DependencyScan/ScanDependencies.cpp | 69 ++-- lib/Driver/Driver.cpp | 7 - .../ArgsToFrontendOptionsConverter.cpp | 1 - lib/FrontendTool/FrontendTool.cpp | 1 - .../local_cache_consistency.swift | 26 ++ 13 files changed, 438 insertions(+), 243 deletions(-) create mode 100644 test/ScanDependencies/local_cache_consistency.swift diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index 18d09245839f5..c4ccbf8e861dc 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -25,6 +25,7 @@ #include "llvm/ADT/StringSet.h" #include #include +#include namespace swift { @@ -35,7 +36,8 @@ class Identifier; /// Which kind of module dependencies we are looking for. enum class ModuleDependenciesKind : int8_t { - SwiftTextual, + FirstKind, + SwiftTextual = FirstKind, SwiftBinary, // Placeholder dependencies are a kind of dependencies used only by the // dependency scanner. They are swift modules that the scanner will not be @@ -61,6 +63,14 @@ enum class ModuleDependenciesKind : int8_t { // of all targets, individually, have been computed. SwiftPlaceholder, Clang, + LastKind = Clang + 1 +}; + +struct ModuleDependenciesKindHash { + std::size_t operator()(ModuleDependenciesKind k) const { + using UnderlyingType = std::underlying_type::type; + return std::hash{}(static_cast(k)); + } }; /// Details of a given module used for dependency scanner cache queries. @@ -409,29 +419,30 @@ class ModuleDependencies { }; using ModuleDependencyID = std::pair; +using ModuleDependenciesVector = llvm::SmallVector; /// A cache describing the set of module dependencies that has been queried -/// thus far. -class ModuleDependenciesCache { - using ModuleDependenciesVector = llvm::SmallVector; - +/// thus far. This cache records/stores the actual Dependency values and can be +/// preserved across different scanning actions (e.g. in +/// `DependencyScanningTool`). It is not to be queried directly, but is rather +/// meant to be wrapped in an instance of `ModuleDependenciesCache`, responsible +/// for recording new dependencies and answering cache queries in a given scan. +/// Queries to this cache must be disambiguated with a set of search paths to +/// ensure that the returned cached dependency was one that can be found in the +/// current scanning action's filesystem view. +class GlobalModuleDependenciesCache { /// All cached module dependencies, in the order in which they were /// encountered. std::vector AllModules; - /// Dependencies for Textual Swift modules that have already been computed. - /// This maps a module's id (name, kind) to a vector of Dependency objects, which correspond - /// to instances of the same module that may have been found in different sets of search paths. - llvm::StringMap SwiftTextualModuleDependencies; - - /// Dependencies for Binary Swift modules that have already been computed. - llvm::StringMap SwiftBinaryModuleDependencies; - - /// Dependencies for Swift placeholder dependency modules that have already been computed. - llvm::StringMap SwiftPlaceholderModuleDependencies; - - /// Dependencies for Clang modules that have already been computed. - llvm::StringMap ClangModuleDependencies; + /// Dependencies for modules that have already been computed. + /// This maps a dependency kind to a map of a module's name to a vector of Dependency objects, + /// which correspond to instances of the same module that may have been found + /// in different sets of search paths. + std::unordered_map, + ModuleDependenciesKindHash> + ModuleDependenciesKindMap; /// Additional information needed for Clang dependency scanning. ClangModuleDependenciesCacheImpl *clangImpl = nullptr; @@ -447,55 +458,148 @@ class ModuleDependenciesCache { /// Retrieve the dependencies map that corresponds to the given dependency /// kind. - llvm::StringMap &getDependenciesMap( - ModuleDependenciesKind kind); - const llvm::StringMap &getDependenciesMap( - ModuleDependenciesKind kind) const; + llvm::StringMap & + getDependenciesMap(ModuleDependenciesKind kind); + const llvm::StringMap & + getDependenciesMap(ModuleDependenciesKind kind) const; public: - ModuleDependenciesCache() { } + GlobalModuleDependenciesCache(); + GlobalModuleDependenciesCache(const GlobalModuleDependenciesCache &) = delete; + GlobalModuleDependenciesCache & + operator=(const GlobalModuleDependenciesCache &) = delete; - ModuleDependenciesCache(const ModuleDependenciesCache &) = delete; - ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete; + virtual ~GlobalModuleDependenciesCache() { destroyClangImpl(); } - ~ModuleDependenciesCache() { - destroyClangImpl(); - } +private: + /// Enforce clients not being allowed to query this cache directly, it must be + /// wrapped in an instance of `ModuleDependenciesCache`. + friend class ModuleDependenciesCache; /// Set the Clang-specific implementation data. - void setClangImpl( - ClangModuleDependenciesCacheImpl *clangImpl, - void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *)) { + virtual void + setClangImpl(ClangModuleDependenciesCacheImpl *clangImpl, + void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *)) { destroyClangImpl(); this->clangImpl = clangImpl; this->clangImplDeleter = clangImplDeleter; } + /// Retrieve the Clang-specific implementation data; + ClangModuleDependenciesCacheImpl *getClangImpl() const { return clangImpl; } + + /// Whether we have cached dependency information for the given module. + bool hasDependencies(StringRef moduleName, + ModuleLookupSpecifics details) const; + + /// Look for module dependencies for a module with the given name given + /// current search paths. + /// + /// \returns the cached result, or \c None if there is no cached entry. + Optional + findDependencies(StringRef moduleName, ModuleLookupSpecifics details) const; + +public: + /// Look for module dependencies for a module with the given name. + /// This method has a deliberately-obtuse name to indicate that it is not to + /// be used for general queries. + /// + /// \returns the cached result, or \c None if there is no cached entry. + Optional + findAllDependenciesIrrespectiveOfSearchPaths( + StringRef moduleName, Optional kind) const; + + /// Record dependencies for the given module. + const ModuleDependencies *recordDependencies(StringRef moduleName, + ModuleDependencies dependencies); + + /// Update stored dependencies for the given module. + const ModuleDependencies *updateDependencies(ModuleDependencyID moduleID, + ModuleDependencies dependencies); + + /// Reference the list of all module dependencies. + const std::vector &getAllModules() const { + return AllModules; + } +}; + +/// This "local" dependencies cache persists only for the duration of a given +/// scanning action, and wraps an instance of a `GlobalModuleDependenciesCache` +/// which may carry cached scanning information from prior scanning actions. +/// This cache maintains a store of references to all dependencies found within +/// the current scanning action (with their values stored in the global Cache), +/// since these do not require clients to disambiguate them with search paths. +class ModuleDependenciesCache { +private: + GlobalModuleDependenciesCache &globalCache; + + /// References to data in `globalCache` for dependencies accimulated during + /// the current scanning action. + std::unordered_map, + ModuleDependenciesKindHash> + ModuleDependenciesKindMap; + + /// Retrieve the dependencies map that corresponds to the given dependency + /// kind. + llvm::StringMap & + getDependencyReferencesMap(ModuleDependenciesKind kind); + const llvm::StringMap & + getDependencyReferencesMap(ModuleDependenciesKind kind) const; + + /// Local cache results lookup, only for modules which were discovered during + /// the current scanner invocation. + bool hasDependencies(StringRef moduleName, + Optional kind) const; + + /// Local cache results lookup, only for modules which were discovered during + /// the current scanner invocation. + Optional + findDependencies(StringRef moduleName, + Optional kind) const; + +public: + ModuleDependenciesCache(GlobalModuleDependenciesCache &globalCache); + ModuleDependenciesCache(const ModuleDependenciesCache &) = delete; + ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete; + virtual ~ModuleDependenciesCache() {} + +public: + /// Set the Clang-specific implementation data. + void + setClangImpl(ClangModuleDependenciesCacheImpl *clangImpl, + void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *)) { + globalCache.setClangImpl(clangImpl, clangImplDeleter); + } + /// Retrieve the Clang-specific implementation data; ClangModuleDependenciesCacheImpl *getClangImpl() const { - return clangImpl; + return globalCache.getClangImpl(); } /// Whether we have cached dependency information for the given module. bool hasDependencies(StringRef moduleName, ModuleLookupSpecifics details) const; - /// Look for module dependencies for a module with the given name given current search paths. + /// Look for module dependencies for a module with the given name given + /// current search paths. /// /// \returns the cached result, or \c None if there is no cached entry. - Optional findDependencies( - StringRef moduleName, - ModuleLookupSpecifics details) const; + Optional + findDependencies(StringRef moduleName, ModuleLookupSpecifics details) const; /// Look for module dependencies for a module with the given name. - /// This method has a deliberately-obtuse name to indicate that it is not to be used for general - /// queries. + /// This method has a deliberately-obtuse name to indicate that it is not to + /// be used for general queries. /// /// \returns the cached result, or \c None if there is no cached entry. - Optional findAllDependenciesIrrespectiveOfSearchPaths( - StringRef moduleName, - Optional kind) const; + Optional + findAllDependenciesIrrespectiveOfSearchPaths( + StringRef moduleName, Optional kind) const { + return globalCache.findAllDependenciesIrrespectiveOfSearchPaths(moduleName, + kind); + } /// Record dependencies for the given module. void recordDependencies(StringRef moduleName, @@ -507,10 +611,10 @@ class ModuleDependenciesCache { /// Reference the list of all module dependencies. const std::vector &getAllModules() const { - return AllModules; + return globalCache.getAllModules(); } }; -} +} // namespace swift #endif /* SWIFT_AST_MODULE_DEPENDENCIES_H */ diff --git a/include/swift/DependencyScan/DependencyScanningTool.h b/include/swift/DependencyScan/DependencyScanningTool.h index 75430ebb791c4..2c207d692d717 100644 --- a/include/swift/DependencyScan/DependencyScanningTool.h +++ b/include/swift/DependencyScan/DependencyScanningTool.h @@ -59,7 +59,7 @@ class DependencyScanningTool { /// Writes the current `SharedCache` instance to a specified FileSystem path. void serializeCache(llvm::StringRef path); - /// Loads an instance of a `ModuleDependenciesCache` to serve as the `SharedCache` + /// Loads an instance of a `GlobalModuleDependenciesCache` to serve as the `SharedCache` /// from a specified FileSystem path. bool loadCache(llvm::StringRef path); /// Discard the tool's current `SharedCache` and start anew. @@ -73,7 +73,7 @@ class DependencyScanningTool { /// Shared cache of module dependencies, re-used by individual full-scan queries /// during the lifetime of this Tool. - std::unique_ptr SharedCache; + std::unique_ptr SharedCache; /// Shared cache of compiler instances created during batch scanning, corresponding to /// command-line options specified in the batch scan input entry. diff --git a/include/swift/DependencyScan/ScanDependencies.h b/include/swift/DependencyScan/ScanDependencies.h index e1d5edb7970e5..4a9fd7ee0b676 100644 --- a/include/swift/DependencyScan/ScanDependencies.h +++ b/include/swift/DependencyScan/ScanDependencies.h @@ -27,12 +27,18 @@ namespace swift { class CompilerInvocation; class CompilerInstance; class ModuleDependenciesCache; +class GlobalModuleDependenciesCache; namespace dependencies { +//using CompilerArgInstanceCacheMap = +// llvm::StringMap, +// std::unique_ptr>>; + using CompilerArgInstanceCacheMap = - llvm::StringMap, - std::unique_ptr>>; + llvm::StringMap, + std::unique_ptr, + std::unique_ptr>>; struct BatchScanInput { llvm::StringRef moduleName; diff --git a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h index 0ff3a1562d5d7..1b77a7510992c 100644 --- a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h +++ b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h @@ -23,7 +23,7 @@ class MemoryBuffer; namespace swift { class DiagnosticEngine; -class ModuleDependenciesCache; +class GlobalModuleDependenciesCache; namespace dependencies { namespace module_dependency_cache_serialization { @@ -160,23 +160,23 @@ using ClangModuleDetailsLayout = /// Tries to read the dependency graph from the given buffer. /// Returns \c true if there was an error. bool readInterModuleDependenciesCache(llvm::MemoryBuffer &buffer, - ModuleDependenciesCache &cache); + GlobalModuleDependenciesCache &cache); /// Tries to read the dependency graph from the given path name. /// Returns true if there was an error. bool readInterModuleDependenciesCache(llvm::StringRef path, - ModuleDependenciesCache &cache); + GlobalModuleDependenciesCache &cache); /// Tries to write the dependency graph to the given path name. /// Returns true if there was an error. bool writeInterModuleDependenciesCache(DiagnosticEngine &diags, llvm::StringRef path, - const ModuleDependenciesCache &cache); + const GlobalModuleDependenciesCache &cache); /// Tries to write out the given dependency cache with the given /// bitstream writer. void writeInterModuleDependenciesCache(llvm::BitstreamWriter &Out, - const ModuleDependenciesCache &cache); + const GlobalModuleDependenciesCache &cache); } // end namespace module_dependency_cache_serialization } // end namespace dependencies diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h index d7fa6667b549c..439d8dcbba538 100644 --- a/include/swift/Frontend/FrontendOptions.h +++ b/include/swift/Frontend/FrontendOptions.h @@ -309,10 +309,6 @@ class FrontendOptions { /// Emit remarks indicating use of the serialized module dependency scanning cache bool EmitDependencyScannerCacheRemarks = false; - /// After performing a dependency scanning action, serialize and deserialize the - /// dependency cache before producing the result. - bool TestDependencyScannerCacheSerialization = false; - /// When performing an incremental build, ensure that cross-module incremental /// build metadata is available in any swift modules emitted by this frontend /// job. diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index c97e6fdd3c43e..885104e28e70f 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -18,6 +18,14 @@ #include "swift/AST/SourceFile.h" using namespace swift; +ModuleDependenciesKind& operator++(ModuleDependenciesKind& e) { + if (e == ModuleDependenciesKind::LastKind) { + llvm_unreachable("Attempting to incrementa last enum value on ModuleDependenciesKind"); + } + e = ModuleDependenciesKind(static_cast::type>(e) + 1); + return e; +} + ModuleDependenciesStorageBase::~ModuleDependenciesStorageBase() { } bool ModuleDependencies::isSwiftModule() const { @@ -131,34 +139,18 @@ void ModuleDependencies::addBridgingModuleDependency( swiftStorage->bridgingModuleDependencies.push_back(module.str()); } -llvm::StringMap & -ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) { - switch (kind) { - case ModuleDependenciesKind::SwiftTextual: - return SwiftTextualModuleDependencies; - case ModuleDependenciesKind::SwiftBinary: - return SwiftBinaryModuleDependencies; - case ModuleDependenciesKind::SwiftPlaceholder: - return SwiftPlaceholderModuleDependencies; - case ModuleDependenciesKind::Clang: - return ClangModuleDependencies; - } - llvm_unreachable("invalid dependency kind"); -} - -const llvm::StringMap & -ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) const { - switch (kind) { - case ModuleDependenciesKind::SwiftTextual: - return SwiftTextualModuleDependencies; - case ModuleDependenciesKind::SwiftBinary: - return SwiftBinaryModuleDependencies; - case ModuleDependenciesKind::SwiftPlaceholder: - return SwiftPlaceholderModuleDependencies; - case ModuleDependenciesKind::Clang: - return ClangModuleDependencies; - } - llvm_unreachable("invalid dependency kind"); +llvm::StringMap & +GlobalModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) { + auto it = ModuleDependenciesKindMap.find(kind); + assert(it != ModuleDependenciesKindMap.end() && "invalid dependency kind"); + return it->second; +} + +const llvm::StringMap & +GlobalModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) const { + auto it = ModuleDependenciesKindMap.find(kind); + assert(it != ModuleDependenciesKindMap.end() && "invalid dependency kind"); + return it->second; } static std::string moduleBasePath(const StringRef modulePath) { @@ -190,23 +182,36 @@ static bool moduleContainedInImportPathSet(const ModuleDependencies &module, const llvm::StringSet<> &importPaths) { std::string modulePath = ""; - if (auto *swiftDep = module.getAsSwiftTextualModule()) { - if (swiftDep->swiftInterfaceFile) - modulePath = *(swiftDep->swiftInterfaceFile); - else { - // If we encountered a Swift textual dependency without an interface - // file, we are seeing the main scan module itself. This means that - // our search-path disambiguation is not necessary here. + switch (module.getKind()) { + case swift::ModuleDependenciesKind::SwiftTextual: { + auto *swiftDep = module.getAsSwiftTextualModule(); + if (swiftDep->swiftInterfaceFile) + modulePath = *(swiftDep->swiftInterfaceFile); + else { + // If we encountered a Swift textual dependency without an interface + // file, we are seeing the main scan module itself. This means that + // our search-path disambiguation is not necessary here. + return true; + } + break; + } + case swift::ModuleDependenciesKind::SwiftBinary: { + auto *swiftBinaryDep = module.getAsSwiftBinaryModule(); + modulePath = swiftBinaryDep->compiledModulePath; + break; + } + case swift::ModuleDependenciesKind::Clang: { + auto *clangDep = module.getAsClangModule(); + modulePath = clangDep->moduleMapFile; + break; + } + case swift::ModuleDependenciesKind::SwiftPlaceholder: { + // Placeholders are resolved as `true` because they are not associated with + // any specific search path. return true; } - } else if (auto *swiftBinaryDep = module.getAsSwiftBinaryModule()) { - modulePath = swiftBinaryDep->compiledModulePath; - } else if (auto *placehodlerDep = module.getAsPlaceholderDependencyModule()) { - // Placeholders are resolved as `true` because they are not associated with - // any specific search path. - return true; - } else if (auto *clangDep = module.getAsClangModule()) { - modulePath = clangDep->moduleMapFile; + default: + llvm_unreachable("Unhandled dependency kind."); } if (moduleContainedInImportPathSet(modulePath, importPaths)) { @@ -215,56 +220,35 @@ static bool moduleContainedInImportPathSet(const ModuleDependencies &module, return false; } -bool ModuleDependenciesCache::hasDependencies( - StringRef moduleName, - ModuleLookupSpecifics details) const { - if (!details.kind) { - return hasDependencies(moduleName, - {ModuleDependenciesKind::SwiftTextual, - details.currentSearchPaths}) || - hasDependencies(moduleName, - {ModuleDependenciesKind::SwiftBinary, - details.currentSearchPaths}) || - hasDependencies(moduleName, - {ModuleDependenciesKind::SwiftPlaceholder, - details.currentSearchPaths}) || - hasDependencies(moduleName, - {ModuleDependenciesKind::Clang, - details.currentSearchPaths}); +GlobalModuleDependenciesCache::GlobalModuleDependenciesCache() { + for (auto kind = ModuleDependenciesKind::FirstKind; + kind != ModuleDependenciesKind::LastKind; ++kind) { + ModuleDependenciesKindMap.insert( + {kind, llvm::StringMap()}); } - const auto &map = getDependenciesMap(*details.kind); - auto known = map.find(moduleName); - if (known != map.end()) { - assert(!known->second.empty()); - for (auto &dep : known->second) { - if (moduleContainedInImportPathSet(dep, details.currentSearchPaths)) - return true; - } - return false; - } - return false; + ModuleDependenciesKindMap.insert( + {ModuleDependenciesKind::SwiftBinary, + llvm::StringMap()}); + ModuleDependenciesKindMap.insert( + {ModuleDependenciesKind::SwiftPlaceholder, + llvm::StringMap()}); + ModuleDependenciesKindMap.insert( + {ModuleDependenciesKind::Clang, + llvm::StringMap()}); } -Optional ModuleDependenciesCache::findDependencies( +Optional GlobalModuleDependenciesCache::findDependencies( StringRef moduleName, ModuleLookupSpecifics details) const { if (!details.kind) { - if (auto swiftTextualDep = findDependencies( - moduleName, - {ModuleDependenciesKind::SwiftTextual, details.currentSearchPaths})) - return swiftTextualDep; - else if (auto swiftBinaryDep = findDependencies( - moduleName, - {ModuleDependenciesKind::SwiftBinary, details.currentSearchPaths})) - return swiftBinaryDep; - else if (auto swiftPlaceholderDep = findDependencies( - moduleName, - {ModuleDependenciesKind::SwiftPlaceholder, details.currentSearchPaths})) - return swiftPlaceholderDep; - else - return findDependencies(moduleName, - {ModuleDependenciesKind::Clang, details.currentSearchPaths}); + for (auto kind = ModuleDependenciesKind::FirstKind; + kind != ModuleDependenciesKind::LastKind; ++kind) { + auto dep = findDependencies(moduleName, {kind, details.currentSearchPaths}); + if (dep.hasValue()) + return dep.getValue(); + } + return None; } const auto &map = getDependenciesMap(*details.kind); @@ -280,24 +264,23 @@ Optional ModuleDependenciesCache::findDependencies( return None; } -Optional -ModuleDependenciesCache::findAllDependenciesIrrespectiveOfSearchPaths( +bool GlobalModuleDependenciesCache::hasDependencies( + StringRef moduleName, + ModuleLookupSpecifics details) const { + return findDependencies(moduleName, details).hasValue(); +} + +Optional +GlobalModuleDependenciesCache::findAllDependenciesIrrespectiveOfSearchPaths( StringRef moduleName, Optional kind) const { if (!kind) { - if (auto swiftTextualDeps = findAllDependenciesIrrespectiveOfSearchPaths( - moduleName, ModuleDependenciesKind::SwiftTextual)) - return swiftTextualDeps; - else if (auto swiftBinaryDeps = - findAllDependenciesIrrespectiveOfSearchPaths( - moduleName, ModuleDependenciesKind::SwiftBinary)) - return swiftBinaryDeps; - else if (auto swiftPlaceholderDeps = - findAllDependenciesIrrespectiveOfSearchPaths( - moduleName, ModuleDependenciesKind::SwiftPlaceholder)) - return swiftPlaceholderDeps; - else - return findAllDependenciesIrrespectiveOfSearchPaths( - moduleName, ModuleDependenciesKind::Clang); + for (auto kind = ModuleDependenciesKind::FirstKind; + kind != ModuleDependenciesKind::LastKind; ++kind) { + auto deps = findAllDependenciesIrrespectiveOfSearchPaths(moduleName, kind); + if (deps.hasValue()) + return deps.getValue(); + } + return None; } const auto &map = getDependenciesMap(*kind); @@ -311,18 +294,31 @@ ModuleDependenciesCache::findAllDependenciesIrrespectiveOfSearchPaths( static std::string modulePathForVerification(const ModuleDependencies &module) { std::string existingModulePath = ""; - if (auto *swiftDep = module.getAsSwiftTextualModule()) { - if (swiftDep->swiftInterfaceFile) - existingModulePath = *(swiftDep->swiftInterfaceFile); - } else if (auto *swiftBinaryDep = module.getAsSwiftBinaryModule()) { - existingModulePath = swiftBinaryDep->compiledModulePath; - } else if (auto *clangDep = module.getAsClangModule()) { - existingModulePath = clangDep->moduleMapFile; + switch (module.getKind()) { + case swift::ModuleDependenciesKind::SwiftTextual: { + auto *swiftDep = module.getAsSwiftTextualModule(); + if (swiftDep->swiftInterfaceFile) + existingModulePath = *(swiftDep->swiftInterfaceFile); + break; + } + case swift::ModuleDependenciesKind::SwiftBinary: { + auto *swiftBinaryDep = module.getAsSwiftBinaryModule(); + existingModulePath = swiftBinaryDep->compiledModulePath; + break; + } + case swift::ModuleDependenciesKind::Clang: { + auto *clangDep = module.getAsClangModule(); + existingModulePath = clangDep->moduleMapFile; + break; + } + case swift::ModuleDependenciesKind::SwiftPlaceholder: + default: + llvm_unreachable("Unhandled dependency kind."); } return existingModulePath; } -void ModuleDependenciesCache::recordDependencies( +const ModuleDependencies* GlobalModuleDependenciesCache::recordDependencies( StringRef moduleName, ModuleDependencies dependencies) { auto kind = dependencies.getKind(); @@ -332,24 +328,107 @@ void ModuleDependenciesCache::recordDependencies( // Ensure that the existing dependencies objects are at a different path. // i.e. do not record duplicate dependencies. auto newModulePath = modulePathForVerification(dependencies); - bool newEntry = true; for (auto &existingDeps : map[moduleName]) { if (modulePathForVerification(existingDeps) == newModulePath) - newEntry = false; + return &existingDeps; } - if (newEntry) - map[moduleName].emplace_back(std::move(dependencies)); + + map[moduleName].emplace_back(std::move(dependencies)); + return map[moduleName].end()-1; } else { map.insert({moduleName, ModuleDependenciesVector{std::move(dependencies)}}); AllModules.push_back({moduleName.str(), kind}); + return &(map[moduleName].front()); } } -void ModuleDependenciesCache::updateDependencies( +const ModuleDependencies* GlobalModuleDependenciesCache::updateDependencies( ModuleDependencyID moduleID, ModuleDependencies dependencies) { auto &map = getDependenciesMap(moduleID.second); auto known = map.find(moduleID.first); assert(known != map.end() && "Not yet added to map"); - assert(known->second.size() == 1 && "Cannot update dependency with multiple candidates."); + assert(known->second.size() == 1 && + "Cannot update dependency with multiple candidates."); known->second[0] = std::move(dependencies); + return &(known->second[0]); +} + +llvm::StringMap & +ModuleDependenciesCache::getDependencyReferencesMap(ModuleDependenciesKind kind) { + auto it = ModuleDependenciesKindMap.find(kind); + assert(it != ModuleDependenciesKindMap.end() && "invalid dependency kind"); + return it->second; +} + +const llvm::StringMap & +ModuleDependenciesCache::getDependencyReferencesMap(ModuleDependenciesKind kind) const { + auto it = ModuleDependenciesKindMap.find(kind); + assert(it != ModuleDependenciesKindMap.end() && "invalid dependency kind"); + return it->second; +} + +ModuleDependenciesCache::ModuleDependenciesCache(GlobalModuleDependenciesCache &globalCache) +: globalCache(globalCache) { + for (auto kind = ModuleDependenciesKind::FirstKind; + kind != ModuleDependenciesKind::LastKind; ++kind) { + ModuleDependenciesKindMap.insert( + {kind, llvm::StringMap()}); + } +} + +Optional ModuleDependenciesCache::findDependencies( + StringRef moduleName, Optional kind) const { + if (!kind) { + for (auto kind = ModuleDependenciesKind::FirstKind; + kind != ModuleDependenciesKind::LastKind; ++kind) { + auto dep = findDependencies(moduleName, kind); + if (dep.hasValue()) + return dep.getValue(); + } + return None; + } + + const auto &map = getDependencyReferencesMap(*kind); + auto known = map.find(moduleName); + if (known != map.end()) + return known->second; + + return None; +} + +Optional +ModuleDependenciesCache::findDependencies(StringRef moduleName, + ModuleLookupSpecifics details) const { + // 1. Query the local scan results + // 2. If no module found, query the global cache using the module details + // lookup + auto localResult = findDependencies(moduleName, details.kind); + if (localResult.hasValue()) + return *(localResult.getValue()); + else + return globalCache.findDependencies(moduleName, details); +} + +bool ModuleDependenciesCache::hasDependencies( + StringRef moduleName, ModuleLookupSpecifics details) const { + return findDependencies(moduleName, details).hasValue(); +} + +void ModuleDependenciesCache::recordDependencies( + StringRef moduleName, ModuleDependencies dependencies) { + auto globalDepPtr = globalCache.recordDependencies(moduleName, dependencies); + auto kind = globalDepPtr->getKind(); + auto &map = getDependencyReferencesMap(kind); + assert(map.count(moduleName) == 0 && "Already added to map"); + map.insert({moduleName, globalDepPtr}); +} + +void ModuleDependenciesCache::updateDependencies( + ModuleDependencyID moduleID, ModuleDependencies dependencies) { + auto globalDepRef = globalCache.updateDependencies(moduleID, dependencies); + auto &map = getDependencyReferencesMap(moduleID.second); + auto known = map.find(moduleID.first); + if (known != map.end()) + map.erase(known); + map.insert({moduleID.first, globalDepRef}); } diff --git a/lib/DependencyScan/DependencyScanningTool.cpp b/lib/DependencyScan/DependencyScanningTool.cpp index c69c597f40c11..8c4f70afc7e87 100644 --- a/lib/DependencyScan/DependencyScanningTool.cpp +++ b/lib/DependencyScan/DependencyScanningTool.cpp @@ -25,7 +25,7 @@ namespace swift { namespace dependencies { DependencyScanningTool::DependencyScanningTool() - : SharedCache(std::make_unique()), + : SharedCache(std::make_unique()), VersionedPCMInstanceCacheCache( std::make_unique()), PDC(), Alloc(), Saver(Alloc) {} @@ -40,8 +40,10 @@ DependencyScanningTool::getDependencies( return EC; auto Instance = std::move(*InstanceOrErr); + // Local scan cache instance, wrapping the shared global cache. + ModuleDependenciesCache cache(*SharedCache); // Execute the scanning action, retreiving the in-memory result - auto DependenciesOrErr = performModuleScan(*Instance.get(), *SharedCache); + auto DependenciesOrErr = performModuleScan(*Instance.get(), cache); if (DependenciesOrErr.getError()) return std::make_error_code(std::errc::not_supported); auto Dependencies = std::move(*DependenciesOrErr); @@ -78,8 +80,10 @@ DependencyScanningTool::getDependencies( BatchInput.size(), std::make_error_code(std::errc::invalid_argument)); auto Instance = std::move(*InstanceOrErr); + // Local scan cache instance, wrapping the shared global cache. + ModuleDependenciesCache cache(*SharedCache); auto BatchScanResults = performBatchModuleScan( - *Instance.get(), *SharedCache, VersionedPCMInstanceCacheCache.get(), + *Instance.get(), cache, VersionedPCMInstanceCacheCache.get(), Saver, BatchInput); return BatchScanResults; @@ -97,18 +101,18 @@ bool DependencyScanningTool::loadCache(llvm::StringRef path) { SourceManager SM; DiagnosticEngine Diags(SM); Diags.addConsumer(PDC); - SharedCache = std::make_unique(); - bool Success = + SharedCache = std::make_unique(); + bool readFailed = module_dependency_cache_serialization::readInterModuleDependenciesCache( path, *SharedCache); - if (!Success) { + if (readFailed) { Diags.diagnose(SourceLoc(), diag::warn_scaner_deserialize_failed, path); } - return Success; + return readFailed; } void DependencyScanningTool::resetCache() { - SharedCache.reset(new ModuleDependenciesCache()); + SharedCache.reset(new GlobalModuleDependenciesCache()); } llvm::ErrorOr> diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index 97128e1ecc6c5..6ef405496e390 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -36,14 +36,14 @@ class Deserializer { bool readSignature(); bool enterGraphBlock(); bool readMetadata(); - bool readGraph(ModuleDependenciesCache &cache); + bool readGraph(GlobalModuleDependenciesCache &cache); llvm::Optional getIdentifier(unsigned n); llvm::Optional> getArray(unsigned n); public: Deserializer(llvm::MemoryBufferRef Data) : Cursor(Data) {} - bool readInterModuleDependenciesCache(ModuleDependenciesCache &cache); + bool readInterModuleDependenciesCache(GlobalModuleDependenciesCache &cache); }; } // end namespace @@ -146,7 +146,7 @@ bool Deserializer::readMetadata() { /// all of the file's identifiers and arrays of identifiers, followed by /// consuming individual module info records and registering them into the /// cache. -bool Deserializer::readGraph(ModuleDependenciesCache &cache) { +bool Deserializer::readGraph(GlobalModuleDependenciesCache &cache) { using namespace graph_block; bool hasCurrentModule = false; @@ -406,7 +406,7 @@ bool Deserializer::readGraph(ModuleDependenciesCache &cache) { } bool Deserializer::readInterModuleDependenciesCache( - ModuleDependenciesCache &cache) { + GlobalModuleDependenciesCache &cache) { using namespace graph_block; if (readSignature()) @@ -460,14 +460,14 @@ llvm::Optional> Deserializer::getArray(unsigned n) { bool swift::dependencies::module_dependency_cache_serialization:: readInterModuleDependenciesCache(llvm::MemoryBuffer &buffer, - ModuleDependenciesCache &cache) { + GlobalModuleDependenciesCache &cache) { Deserializer deserializer(buffer.getMemBufferRef()); return deserializer.readInterModuleDependenciesCache(cache); } bool swift::dependencies::module_dependency_cache_serialization:: readInterModuleDependenciesCache(StringRef path, - ModuleDependenciesCache &cache) { + GlobalModuleDependenciesCache &cache) { PrettyStackTraceStringAction stackTrace( "loading inter-module dependency graph", path); auto buffer = llvm::MemoryBuffer::getFile(path); @@ -571,7 +571,7 @@ class Serializer { AbbrCodes[Layout::Code] = Layout::emitAbbrev(Out); } - void collectStringsAndArrays(const ModuleDependenciesCache &cache); + void collectStringsAndArrays(const GlobalModuleDependenciesCache &cache); void emitBlockID(unsigned ID, StringRef name, SmallVectorImpl &nameBuffer); @@ -593,7 +593,7 @@ class Serializer { Serializer(llvm::BitstreamWriter &ExistingOut) : Out(ExistingOut) {} public: - void writeInterModuleDependenciesCache(const ModuleDependenciesCache &cache); + void writeInterModuleDependenciesCache(const GlobalModuleDependenciesCache &cache); }; } // end namespace @@ -742,6 +742,8 @@ void Serializer::writeModuleInfo(ModuleDependencyID moduleID, break; } + default: + llvm_unreachable("Unhandled dependency kind."); } } @@ -812,7 +814,7 @@ unsigned Serializer::getArray(ModuleDependencyID moduleID, return arrayIter->second; } -void Serializer::collectStringsAndArrays(const ModuleDependenciesCache &cache) { +void Serializer::collectStringsAndArrays(const GlobalModuleDependenciesCache &cache) { for (auto &moduleID : cache.getAllModules()) { auto dependencyInfos = cache.findAllDependenciesIrrespectiveOfSearchPaths( moduleID.first, moduleID.second); @@ -876,13 +878,15 @@ void Serializer::collectStringsAndArrays(const ModuleDependenciesCache &cache) { clangDeps->fileDependencies); break; } + default: + llvm_unreachable("Unhandled dependency kind."); } } } } void Serializer::writeInterModuleDependenciesCache( - const ModuleDependenciesCache &cache) { + const GlobalModuleDependenciesCache &cache) { // Write the header writeSignature(); writeBlockInfoBlock(); @@ -930,14 +934,14 @@ void Serializer::writeInterModuleDependenciesCache( void swift::dependencies::module_dependency_cache_serialization:: writeInterModuleDependenciesCache(llvm::BitstreamWriter &Out, - const ModuleDependenciesCache &cache) { + const GlobalModuleDependenciesCache &cache) { Serializer serializer{Out}; serializer.writeInterModuleDependenciesCache(cache); } bool swift::dependencies::module_dependency_cache_serialization:: writeInterModuleDependenciesCache(DiagnosticEngine &diags, StringRef path, - const ModuleDependenciesCache &cache) { + const GlobalModuleDependenciesCache &cache) { PrettyStackTraceStringAction stackTrace( "saving inter-module dependency graph", path); return withOutputFile(diags, path, [&](llvm::raw_ostream &out) { diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index f18565f341f0e..3152168b87348 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -734,6 +734,8 @@ static std::string createEncodedModuleKindAndName(ModuleDependencyID id) { return "swiftPlaceholder:" + id.first; case ModuleDependenciesKind::Clang: return "clang:" + id.first; + default: + llvm_unreachable("Unhandled dependency kind."); } } @@ -871,6 +873,8 @@ generateFullDependencyGraph(CompilerInstance &instance, case ModuleDependenciesKind::Clang: dependencyKindAndName = "clang"; break; + default: + llvm_unreachable("Unhandled dependency kind."); } dependencyKindAndName += ":"; dependencyKindAndName += dep.first; @@ -944,10 +948,6 @@ static bool diagnoseCycle(CompilerInstance &instance, return false; } -using CompilerArgInstanceCacheMap = - llvm::StringMap, - std::unique_ptr>>; - static void updateCachedInstanceOpts(CompilerInstance &cachedInstance, const CompilerInstance &invocationInstance, llvm::StringRef entryArguments) { @@ -1006,8 +1006,8 @@ forEachBatchEntry(CompilerInstance &invocationInstance, } else if (subInstanceMap->count(entry.arguments)) { // Use the previously created instance if we've seen the arguments // before. - pInstance = (*subInstanceMap)[entry.arguments].first.get(); - pCache = (*subInstanceMap)[entry.arguments].second.get(); + pInstance = std::get<0>((*subInstanceMap)[entry.arguments]).get(); + pCache = std::get<2>((*subInstanceMap)[entry.arguments]).get(); // We must update the search paths of this instance to instead reflect // those of the current scanner invocation. updateCachedInstanceOpts(*pInstance, invocationInstance, entry.arguments); @@ -1018,13 +1018,15 @@ forEachBatchEntry(CompilerInstance &invocationInstance, llvm::cl::ResetAllOptionOccurrences(); // Create a new instance by the arguments and save it in the map. + auto newGlobalCache = std::make_unique(); subInstanceMap->insert( {entry.arguments, - std::make_pair(std::make_unique(), - std::make_unique())}); + std::make_tuple(std::make_unique(), + std::move(newGlobalCache), + std::make_unique(*newGlobalCache))}); - pInstance = (*subInstanceMap)[entry.arguments].first.get(); - pCache = (*subInstanceMap)[entry.arguments].second.get(); + pInstance = std::get<0>((*subInstanceMap)[entry.arguments]).get(); + pCache = std::get<2>((*subInstanceMap)[entry.arguments]).get(); SmallVector args; llvm::cl::TokenizeGNUCommandLine(entry.arguments, saver, args); CompilerInvocation subInvok = invok; @@ -1129,7 +1131,7 @@ identifyMainModuleDependencies(CompilerInstance &instance) { } // namespace static void serializeDependencyCache(CompilerInstance &instance, - const ModuleDependenciesCache &cache) { + const GlobalModuleDependenciesCache &cache) { const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); ASTContext &Context = instance.getASTContext(); auto savePath = opts.SerializedDependencyScannerCachePath; @@ -1141,7 +1143,7 @@ static void serializeDependencyCache(CompilerInstance &instance, } static void deserializeDependencyCache(CompilerInstance &instance, - ModuleDependenciesCache &cache) { + GlobalModuleDependenciesCache &cache) { const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); ASTContext &Context = instance.getASTContext(); auto loadPath = opts.SerializedDependencyScannerCachePath; @@ -1169,10 +1171,11 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { // `-scan-dependencies` invocations use a single new instance // of a module cache - ModuleDependenciesCache cache; - + GlobalModuleDependenciesCache globalCache; if (opts.ReuseDependencyScannerCache) - deserializeDependencyCache(instance, cache); + deserializeDependencyCache(instance, globalCache); + + ModuleDependenciesCache cache(globalCache); // Execute scan auto dependenciesOrErr = performModuleScan(instance, cache); @@ -1180,7 +1183,7 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { // Serialize the dependency cache if -serialize-dependency-scan-cache // is specified if (opts.SerializeDependencyScannerCache) - serializeDependencyCache(instance, cache); + serializeDependencyCache(instance, globalCache); if (dependenciesOrErr.getError()) return true; @@ -1204,7 +1207,8 @@ bool swift::dependencies::prescanDependencies(CompilerInstance &instance) { llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::F_None); // `-scan-dependencies` invocations use a single new instance // of a module cache - ModuleDependenciesCache cache; + GlobalModuleDependenciesCache singleUseGlobalCache; + ModuleDependenciesCache cache(singleUseGlobalCache); if (out.has_error() || EC) { Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, EC.message()); @@ -1232,7 +1236,8 @@ bool swift::dependencies::batchScanDependencies( CompilerInstance &instance, llvm::StringRef batchInputFile) { // The primary cache used for scans carried out with the compiler instance // we have created - ModuleDependenciesCache cache; + GlobalModuleDependenciesCache singleUseGlobalCache; + ModuleDependenciesCache cache(singleUseGlobalCache); (void)instance.getMainModule(); llvm::BumpPtrAllocator alloc; llvm::StringSaver saver(alloc); @@ -1264,7 +1269,8 @@ bool swift::dependencies::batchPrescanDependencies( CompilerInstance &instance, llvm::StringRef batchInputFile) { // The primary cache used for scans carried out with the compiler instance // we have created - ModuleDependenciesCache cache; + GlobalModuleDependenciesCache singleUseGlobalCache; + ModuleDependenciesCache cache(singleUseGlobalCache); (void)instance.getMainModule(); llvm::BumpPtrAllocator alloc; llvm::StringSaver saver(alloc); @@ -1355,34 +1361,13 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, ASTDelegate)) return std::make_error_code(std::errc::not_supported); - // Test dependency cache serialization/deserialization if - // `-test-dependency-scan-cache-serialization` is specified. - auto *resultCache = &cache; - ModuleDependenciesCache loadedCache; - if (FEOpts.TestDependencyScannerCacheSerialization) { - llvm::SmallString<128> buffer; - auto EC = llvm::sys::fs::createTemporaryFile("depscan_cache", "moddepcache", - buffer); - if (EC) { - instance.getDiags().diagnose( - SourceLoc(), diag::error_unable_to_make_temporary_file, EC.message()); - } else { - std::string path = buffer.str().str(); - module_dependency_cache_serialization::writeInterModuleDependenciesCache( - instance.getDiags(), path, cache); - module_dependency_cache_serialization::readInterModuleDependenciesCache( - path, loadedCache); - resultCache = &loadedCache; - } - } - auto dependencyGraph = generateFullDependencyGraph( - instance, *resultCache, ASTDelegate, + instance, cache, ASTDelegate, allModules.getArrayRef(), currentImportPathSet); // Update the dependency tracker. if (auto depTracker = instance.getDependencyTracker()) { for (auto module : allModules) { - auto deps = resultCache->findDependencies(module.first, + auto deps = cache.findDependencies(module.first, {module.second, currentImportPathSet}); if (!deps) continue; diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp index 94a221cf25bbc..9e5de90561a31 100644 --- a/lib/Driver/Driver.cpp +++ b/lib/Driver/Driver.cpp @@ -178,8 +178,6 @@ static void validateDependencyScanningArgs(DiagnosticEngine &diags, args.getLastArg(options::OPT_reuse_dependency_scan_cache); const Arg *CacheSerializationPath = args.getLastArg(options::OPT_dependency_scan_cache_path); - const Arg *TestSerialization = args.getLastArg( - options::OPT_debug_test_dependency_scan_cache_serialization); if (ExternalDependencyMap && !ScanDependencies) { diags.diagnose(SourceLoc(), diag::error_requirement_not_met, @@ -202,11 +200,6 @@ static void validateDependencyScanningArgs(DiagnosticEngine &diags, diags.diagnose(SourceLoc(), diag::error_requirement_not_met, "-load-dependency-scan-cache", "-scan-dependencies"); } - if (TestSerialization && !ScanDependencies) { - diags.diagnose(SourceLoc(), diag::error_requirement_not_met, - "-test-dependency-scan-cache-serialization", - "-scan-dependencies"); - } if (SerializeCache && !CacheSerializationPath) { diags.diagnose(SourceLoc(), diag::error_requirement_not_met, "-serialize-dependency-scan-cache", diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index c705289570532..adf4df00b0e6f 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -106,7 +106,6 @@ bool ArgsToFrontendOptionsConverter::convert( Opts.SerializeDependencyScannerCache |= Args.hasArg(OPT_serialize_dependency_scan_cache); Opts.ReuseDependencyScannerCache |= Args.hasArg(OPT_reuse_dependency_scan_cache); Opts.EmitDependencyScannerCacheRemarks |= Args.hasArg(OPT_dependency_scan_cache_remarks); - Opts.TestDependencyScannerCacheSerialization |= Args.hasArg(OPT_debug_test_dependency_scan_cache_serialization); if (const Arg *A = Args.getLastArg(OPT_dependency_scan_cache_path)) { Opts.SerializedDependencyScannerCachePath = A->getValue(); } diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index 540e50c1369eb..eb84b2ac55c7a 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -1125,7 +1125,6 @@ withSemanticAnalysis(CompilerInstance &Instance, FrontendObserver *observer, static bool performScanDependencies(CompilerInstance &Instance) { auto batchScanInput = Instance.getASTContext().SearchPathOpts.BatchScanInputFilePath; - ModuleDependenciesCache SingleUseCache; if (batchScanInput.empty()) { if (Instance.getInvocation().getFrontendOptions().ImportPrescan) return dependencies::prescanDependencies(Instance); diff --git a/test/ScanDependencies/local_cache_consistency.swift b/test/ScanDependencies/local_cache_consistency.swift new file mode 100644 index 0000000000000..c969a648b89cb --- /dev/null +++ b/test/ScanDependencies/local_cache_consistency.swift @@ -0,0 +1,26 @@ +// REQUIRES: executable_test +// REQUIRES: objc_interop +// REQUIRES: OS=macosx + +// RUN: %empty-directory(%t) +// RUN: mkdir -p %t/clang-module-cache + +// Run the scanner once, ensuring CoreFoundation dependencies are as expected +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -swift-version 4 +// RUN: %FileCheck %s < %t/deps.json + +import CoreFoundation + +// CHECK: "clang": "CoreFoundation" + +// CHECK: "directDependencies": [ +// CHECK: { +// CHECK: "clang": "Darwin" +// CHECK: }, +// CHECK: { +// CHECK: "clang": "Dispatch" +// CHECK: } +// CHECK: ], + +// Make sure the transitive dependency on os_object is present +// CHECK: "clang": "os_object"