diff --git a/include/swift-c/DependencyScan/DependencyScan.h b/include/swift-c/DependencyScan/DependencyScan.h index be2c9257c494c..8a3589a227737 100644 --- a/include/swift-c/DependencyScan/DependencyScan.h +++ b/include/swift-c/DependencyScan/DependencyScan.h @@ -337,6 +337,30 @@ swiftscan_batch_scan_result_create(swiftscan_scanner_t scanner, SWIFTSCAN_PUBLIC swiftscan_import_set_t swiftscan_import_set_create( swiftscan_scanner_t scanner, swiftscan_scan_invocation_t invocation); +//=== Scanner Cache Operations --------------------------------------------===// +// The following operations expose an implementation detail of the dependency +// scanner: its module dependencies cache. This is done in order +// to allow clients to perform incremental dependency scans by having the +// scanner's state be serializable and re-usable. + +/// For the specified \c scanner instance, serialize its state to the specified file-system \c path . +SWIFTSCAN_PUBLIC void +swiftscan_scanner_cache_serialize(swiftscan_scanner_t scanner, + const char * path); + +/// For the specified \c scanner instance, load in scanner state from a file at +/// the specified file-system \c path . +SWIFTSCAN_PUBLIC bool +swiftscan_scanner_cache_load(swiftscan_scanner_t scanner, + const char * path); + +/// For the specified \c scanner instance, reset its internal state, ensuring subsequent +/// scanning queries are done "from-scratch". +SWIFTSCAN_PUBLIC void +swiftscan_scanner_cache_reset(swiftscan_scanner_t scanner); + +//===----------------------------------------------------------------------===// + SWIFTSCAN_END_DECLS #endif // SWIFT_C_DEPENDENCY_SCAN_H diff --git a/include/swift/AST/DiagnosticsCommon.def b/include/swift/AST/DiagnosticsCommon.def index d79dfbc137669..da90df5d3d5d9 100644 --- a/include/swift/AST/DiagnosticsCommon.def +++ b/include/swift/AST/DiagnosticsCommon.def @@ -192,6 +192,15 @@ ERROR(scanner_find_cycle, none, ERROR(scanner_arguments_invalid, none, "dependencies scanner cannot be configured with arguments: '%0'", (StringRef)) +WARNING(warn_scaner_deserialize_failed, none, + "Failed to load module scanning dependency cache from: '%0', re-building scanner cache from scratch.", (StringRef)) + +REMARK(remark_reuse_cache, none, + "Re-using serialized module scanning dependency cache from: '%0'.", (StringRef)) + +REMARK(remark_save_cache, none, + "Serializing module scanning dependency cache to: '%0'.", (StringRef)) + //------------------------------------------------------------------------------ // MARK: custom attribute diagnostics //------------------------------------------------------------------------------ diff --git a/include/swift/DependencyScan/DependencyScanningTool.h b/include/swift/DependencyScan/DependencyScanningTool.h index d18d5bf09167d..75430ebb791c4 100644 --- a/include/swift/DependencyScan/DependencyScanningTool.h +++ b/include/swift/DependencyScan/DependencyScanningTool.h @@ -57,6 +57,14 @@ class DependencyScanningTool { const std::vector &BatchInput, const llvm::StringSet<> &PlaceholderModules); + /// Writes the current `SharedCache` instance to a specified FileSystem path. + void serializeCache(llvm::StringRef path); + /// Loads an instance of a `ModuleDependenciesCache` to serve as the `SharedCache` + /// from a specified FileSystem path. + bool loadCache(llvm::StringRef path); + /// Discard the tool's current `SharedCache` and start anew. + void resetCache(); + private: /// Using the specified invocation command, instantiate a CompilerInstance /// that will be used for this scan. diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h index 319c9d97d4d0f..8f0b345f22059 100644 --- a/include/swift/Frontend/FrontendOptions.h +++ b/include/swift/Frontend/FrontendOptions.h @@ -313,6 +313,9 @@ class FrontendOptions { /// The path at which to either serialize or deserialize the dependency scanner cache. std::string SerializedDependencyScannerCachePath; + /// Emit remarks indicating use of the serialized module dependency scanning cache + bool EmitDependencyScannerCacheRemarks = false; + /// After performing a dependency scanning action, serialize and deserialize the /// dependency cache before producing the result. bool TestDependencyScannerCacheSerialization = false; diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index 732f1872ab890..3b4ae78276136 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -194,6 +194,9 @@ def reuse_dependency_scan_cache : Flag<["-"], "load-dependency-scan-cache">, def dependency_scan_cache_path : Separate<["-"], "dependency-scan-cache-path">, HelpText<"The path to output the dependency scanner's internal state.">; +def dependency_scan_cache_remarks : Flag<["-"], "Rdependency-scan-cache">, + HelpText<"Emit remarks indicating use of the serialized module dependency scanning cache.">; + def enable_copy_propagation : Flag<["-"], "enable-copy-propagation">, HelpText<"Run SIL copy propagation to shorten object lifetime.">; def disable_copy_propagation : Flag<["-"], "disable-copy-propagation">, diff --git a/lib/DependencyScan/DependencyScanningTool.cpp b/lib/DependencyScan/DependencyScanningTool.cpp index 26c12e5c03205..c69c597f40c11 100644 --- a/lib/DependencyScan/DependencyScanningTool.cpp +++ b/lib/DependencyScan/DependencyScanningTool.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "swift/DependencyScan/DependencyScanningTool.h" +#include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h" #include "swift/AST/DiagnosticEngine.h" #include "swift/AST/DiagnosticsFrontend.h" #include "swift/Basic/LLVMInitialize.h" @@ -77,11 +78,37 @@ DependencyScanningTool::getDependencies( BatchInput.size(), std::make_error_code(std::errc::invalid_argument)); auto Instance = std::move(*InstanceOrErr); - auto batchScanResults = performBatchModuleScan( + auto BatchScanResults = performBatchModuleScan( *Instance.get(), *SharedCache, VersionedPCMInstanceCacheCache.get(), Saver, BatchInput); - return batchScanResults; + return BatchScanResults; +} + +void DependencyScanningTool::serializeCache(llvm::StringRef path) { + SourceManager SM; + DiagnosticEngine Diags(SM); + Diags.addConsumer(PDC); + module_dependency_cache_serialization::writeInterModuleDependenciesCache( + Diags, path, *SharedCache); +} + +bool DependencyScanningTool::loadCache(llvm::StringRef path) { + SourceManager SM; + DiagnosticEngine Diags(SM); + Diags.addConsumer(PDC); + SharedCache = std::make_unique(); + bool Success = + module_dependency_cache_serialization::readInterModuleDependenciesCache( + path, *SharedCache); + if (!Success) { + Diags.diagnose(SourceLoc(), diag::warn_scaner_deserialize_failed, path); + } + return Success; +} + +void DependencyScanningTool::resetCache() { + SharedCache.reset(new ModuleDependenciesCache()); } llvm::ErrorOr> diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index da99ac609f7fa..142de49e16985 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -472,7 +472,7 @@ bool swift::dependencies::module_dependency_cache_serialization:: "loading inter-module dependency graph", path); auto buffer = llvm::MemoryBuffer::getFile(path); if (!buffer) - return false; + return true; return readInterModuleDependenciesCache(*buffer.get(), cache); } diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 3a66d4e2aebe3..7baa5f813444f 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -282,7 +282,16 @@ static void discoverCrosssImportOverlayDependencies( // Record the dummy main module's direct dependencies. The dummy main module // only directly depend on these newly discovered overlay modules. - cache.recordDependencies(dummyMainName, dummyMainDependencies); + if (cache.findDependencies(dummyMainName, + ModuleDependenciesKind::SwiftTextual)) { + cache.updateDependencies( + std::make_pair(dummyMainName.str(), + ModuleDependenciesKind::SwiftTextual), + dummyMainDependencies); + } else { + cache.recordDependencies(dummyMainName, dummyMainDependencies); + } + llvm::SetVector, std::set> allModules; @@ -1100,17 +1109,30 @@ identifyMainModuleDependencies(CompilerInstance &instance) { } // namespace -static void serializeDependencyCache(DiagnosticEngine &diags, - const std::string &path, +static void serializeDependencyCache(CompilerInstance &instance, const ModuleDependenciesCache &cache) { + const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); + ASTContext &Context = instance.getASTContext(); + auto savePath = opts.SerializedDependencyScannerCachePath; module_dependency_cache_serialization::writeInterModuleDependenciesCache( - diags, path, cache); + Context.Diags, savePath, cache); + if (opts.EmitDependencyScannerCacheRemarks) { + Context.Diags.diagnose(SourceLoc(), diag::remark_save_cache, savePath); + } } -static void deserializeDependencyCache(const std::string &path, +static void deserializeDependencyCache(CompilerInstance &instance, ModuleDependenciesCache &cache) { - module_dependency_cache_serialization::readInterModuleDependenciesCache( - path, cache); + const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); + ASTContext &Context = instance.getASTContext(); + auto loadPath = opts.SerializedDependencyScannerCachePath; + if (module_dependency_cache_serialization::readInterModuleDependenciesCache( + loadPath, cache)) { + Context.Diags.diagnose(SourceLoc(), diag::warn_scaner_deserialize_failed, + loadPath); + } else if (opts.EmitDependencyScannerCacheRemarks) { + Context.Diags.diagnose(SourceLoc(), diag::remark_reuse_cache, loadPath); + } } bool swift::dependencies::scanDependencies(CompilerInstance &instance) { @@ -1119,10 +1141,6 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); std::error_code EC; llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::F_None); - - // `-scan-dependencies` invocations use a single new instance - // of a module cache - ModuleDependenciesCache cache; if (out.has_error() || EC) { Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, EC.message()); @@ -1130,11 +1148,25 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { return true; } - // Execute scan, and write JSON output to the output stream + // `-scan-dependencies` invocations use a single new instance + // of a module cache + ModuleDependenciesCache cache; + + if (opts.ReuseDependencyScannerCache) + deserializeDependencyCache(instance, cache); + + // Execute scan auto dependenciesOrErr = performModuleScan(instance, cache); + + // Serialize the dependency cache if -serialize-dependency-scan-cache + // is specified + if (opts.SerializeDependencyScannerCache) + serializeDependencyCache(instance, cache); + if (dependenciesOrErr.getError()) return true; auto dependencies = std::move(*dependenciesOrErr); + // Write out the JSON description. writeJSON(out, dependencies); // This process succeeds regardless of whether any errors occurred. @@ -1253,7 +1285,19 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, std::set> allModules; allModules.insert({mainModuleName.str(), mainDependencies.getKind()}); - cache.recordDependencies(mainModuleName, std::move(mainDependencies)); + + // We may be re-using an instance of the cache which already contains + // an entry for this module. + if (cache.findDependencies(mainModuleName, + ModuleDependenciesKind::SwiftTextual)) { + cache.updateDependencies( + std::make_pair(mainModuleName.str(), + ModuleDependenciesKind::SwiftTextual), + std::move(mainDependencies)); + } else { + cache.recordDependencies(mainModuleName, std::move(mainDependencies)); + } + auto &ctx = instance.getASTContext(); auto ModuleCachePath = getModuleCachePathFromClang( ctx.getClangModuleLoader()->getClangInstance()); @@ -1295,15 +1339,17 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, ModuleDependenciesCache loadedCache; if (FEOpts.TestDependencyScannerCacheSerialization) { llvm::SmallString<128> buffer; - auto EC = llvm::sys::fs::createTemporaryFile("depscan_cache", "moddepcache", buffer); + auto EC = llvm::sys::fs::createTemporaryFile("depscan_cache", "moddepcache", + buffer); if (EC) { - instance.getDiags().diagnose(SourceLoc(), - diag::error_unable_to_make_temporary_file, - EC.message()); + instance.getDiags().diagnose( + SourceLoc(), diag::error_unable_to_make_temporary_file, EC.message()); } else { std::string path = buffer.str().str(); - serializeDependencyCache(instance.getDiags(), path, cache); - deserializeDependencyCache(path, loadedCache); + module_dependency_cache_serialization::writeInterModuleDependenciesCache( + instance.getDiags(), path, cache); + module_dependency_cache_serialization::readInterModuleDependenciesCache( + path, loadedCache); resultCache = &loadedCache; } } diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index a71c44bc751e5..2829027234c31 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -119,6 +119,7 @@ bool ArgsToFrontendOptionsConverter::convert( Opts.SerializeDependencyScannerCache |= Args.hasArg(OPT_serialize_dependency_scan_cache); Opts.ReuseDependencyScannerCache |= Args.hasArg(OPT_reuse_dependency_scan_cache); + Opts.EmitDependencyScannerCacheRemarks |= Args.hasArg(OPT_dependency_scan_cache_remarks); Opts.TestDependencyScannerCacheSerialization |= Args.hasArg(OPT_debug_test_dependency_scan_cache_serialization); if (const Arg *A = Args.getLastArg(OPT_dependency_scan_cache_path)) { Opts.SerializedDependencyScannerCachePath = A->getValue(); diff --git a/test/ScanDependencies/module_deps_cache_reuse.swift b/test/ScanDependencies/module_deps_cache_reuse.swift new file mode 100644 index 0000000000000..ce74581081b76 --- /dev/null +++ b/test/ScanDependencies/module_deps_cache_reuse.swift @@ -0,0 +1,195 @@ +// RUN: %empty-directory(%t) +// RUN: mkdir -p %t/clang-module-cache + +// Run the scanner once, emitting the serialized scanner cache +// RUN: %target-swift-frontend -scan-dependencies -Rdependency-scan-cache -serialize-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/clang-module-cache %s -o %t/deps_initial.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 2>&1 | %FileCheck %s -check-prefix CHECK-REMARK-SAVE + +// Run the scanner again, but now re-using previously-serialized cache +// RUN: %target-swift-frontend -scan-dependencies -Rdependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 2>&1 | %FileCheck %s -check-prefix CHECK-REMARK-LOAD + +// Check the contents of the JSON output +// RUN: %FileCheck %s < %t/deps.json + +// REQUIRES: executable_test +// REQUIRES: objc_interop + +import C +import E +import G +import SubE + +// CHECK-REMARK-SAVE: remark: Serializing module scanning dependency cache to: +// CHECK-REMARK-LOAD: remark: Re-using serialized module scanning dependency cache from: + +// CHECK: "mainModuleName": "deps" + +/// --------Main module +// CHECK-LABEL: "modulePath": "deps.swiftmodule", +// CHECK-NEXT: sourceFiles +// CHECK-NEXT: module_deps_cache_reuse.swift +// CHECK-NEXT: ], +// CHECK-NEXT: "directDependencies": [ +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "A" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "C" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "E" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "F" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "G" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "SubE" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "SwiftOnoneSupport" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "_Concurrency" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "_cross_import_E" +// CHECK-NEXT: } +// CHECK-NEXT: ], + +// CHECK: "extraPcmArgs": [ +// CHECK-NEXT: "-Xcc", +// CHECK-NEXT: "-target", +// CHECK-NEXT: "-Xcc", +// CHECK: "-fapinotes-swift-version=4" +// CHECK-NOT: "error: cannot open Swift placeholder dependency module map from" +// CHECK: "bridgingHeader": +// CHECK-NEXT: "path": +// CHECK-SAME: Bridging.h + +// CHECK-NEXT: "sourceFiles": +// CHECK-NEXT: Bridging.h +// CHECK-NEXT: BridgingOther.h + +// CHECK: "moduleDependencies": [ +// CHECK-NEXT: "F" +// CHECK-NEXT: ] + +/// --------Swift module A +// CHECK-LABEL: "modulePath": "A.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "A" +// CHECK-NEXT: } +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" +// CHECK-NEXT: }, + +/// --------Clang module C +// CHECK-LABEL: "modulePath": "C.pcm", + +// CHECK: "sourceFiles": [ +// CHECK-DAG: module.modulemap +// CHECK-DAG: C.h + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "B" + +// CHECK: "moduleMapPath" +// CHECK-SAME: module.modulemap + +// CHECK: "contextHash" +// CHECK-SAME: "{{.*}}" + +// CHECK: "commandLine": [ +// CHECK-NEXT: "-frontend" +// CHECK-NEXT: "-only-use-extra-clang-opts" +// CHECK-NEXT: "-Xcc" +// CHECK-NEXT: "clang" +// CHECK: "-fsystem-module", +// CHECK-NEXT: "-emit-pcm", +// CHECK-NEXT: "-module-name", +// CHECK-NEXT: "C" + +/// --------Swift module E +// CHECK: "swift": "E" +// CHECK-LABEL: modulePath": "E.swiftmodule" +// CHECK: "directDependencies" +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" + +// CHECK: "moduleInterfacePath" +// CHECK-SAME: E.swiftinterface + +/// --------Swift module F +// CHECK: "modulePath": "F.swiftmodule", +// CHECK-NEXT: "sourceFiles": [ +// CHECK-NEXT: ], +// CHECK-NEXT: "directDependencies": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "F" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "SwiftOnoneSupport" +// CHECK-NEXT: } +// CHECK-NEXT: ], + +/// --------Swift module G +// CHECK-LABEL: "modulePath": "G.swiftmodule" +// CHECK: "directDependencies" +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "G" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "SwiftOnoneSupport" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "details": { + +// CHECK: "contextHash": "{{.*}}", +// CHECK: "commandLine": [ +// CHECK: "-compile-module-from-interface" +// CHECK: "-target" +// CHECK: "-module-name" +// CHECK: "G" +// CHECK: "-swift-version" +// CHECK: "5" +// CHECK: ], +// CHECK" "extraPcmArgs": [ +// CHECK" "-target", +// CHECK" "-fapinotes-swift-version=5" +// CHECK" ] + +/// --------Swift module Swift +// CHECK-LABEL: "modulePath": "Swift.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "SwiftShims" + +/// --------Clang module B +// CHECK-LABEL: "modulePath": "B.pcm" + +// CHECK-NEXT: sourceFiles +// CHECK-DAG: module.modulemap +// CHECK-DAG: B.h + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "A" +// CHECK-NEXT: } + +/// --------Clang module SwiftShims +// CHECK-LABEL: "modulePath": "SwiftShims.pcm", diff --git a/tools/libSwiftScan/libSwiftScan.cpp b/tools/libSwiftScan/libSwiftScan.cpp index e8c423515b5fd..842f05839aa6d 100644 --- a/tools/libSwiftScan/libSwiftScan.cpp +++ b/tools/libSwiftScan/libSwiftScan.cpp @@ -95,6 +95,25 @@ void swiftscan_dependency_set_dispose(swiftscan_dependency_set_t *set) { delete set; } +//=== Scanner Cache Operations --------------------------------------------===// + +void swiftscan_scanner_cache_serialize(swiftscan_scanner_t scanner, + const char * path) { + DependencyScanningTool *ScanningTool = unwrap(scanner); + ScanningTool->serializeCache(path); +} + +bool swiftscan_scanner_cache_load(swiftscan_scanner_t scanner, + const char * path) { + DependencyScanningTool *ScanningTool = unwrap(scanner); + return ScanningTool->loadCache(path); +} + +void swiftscan_scanner_cache_reset(swiftscan_scanner_t scanner) { + DependencyScanningTool *ScanningTool = unwrap(scanner); + ScanningTool->resetCache(); +} + //=== Scanner Functions ---------------------------------------------------===// swiftscan_scanner_t swiftscan_scanner_create(void) { diff --git a/tools/libSwiftScan/libSwiftScan.exports b/tools/libSwiftScan/libSwiftScan.exports index e2c62ef1faa25..b16110ae98e6d 100644 --- a/tools/libSwiftScan/libSwiftScan.exports +++ b/tools/libSwiftScan/libSwiftScan.exports @@ -54,3 +54,6 @@ swiftscan_import_set_dispose swiftscan_scanner_dispose swiftscan_compiler_supported_arguments_query swiftscan_compiler_supported_features_query +swiftscan_scanner_cache_serialize +swiftscan_scanner_cache_load +swiftscan_scanner_cache_reset