From fb2532c0c1a24afaa0e3b380a425ee3991453c64 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 23 Jan 2023 15:25:50 -0800 Subject: [PATCH 01/11] Virtualize swift-frontend outputs Using a virutal output backend to capture all the outputs from swift-frontend invocation. This allows future virtualization of swift compiler outputs. --- include/swift/AST/ASTContext.h | 18 ++++ .../AST/AbstractSourceFileDepGraphFactory.h | 7 +- include/swift/AST/FileSystem.h | 33 +++--- include/swift/AST/FineGrainedDependencies.h | 9 +- .../swift/AST/FineGrainedDependencyFormat.h | 4 + include/swift/AST/ModuleLoader.h | 4 + .../SerializedModuleDependencyCacheFormat.h | 4 + .../Driver/FineGrainedDependencyDriverGraph.h | 10 ++ include/swift/Frontend/Frontend.h | 13 +++ .../swift/Frontend/ModuleInterfaceLoader.h | 1 + include/swift/Subsystems.h | 5 + lib/AST/ASTContext.cpp | 19 +++- lib/AST/AbstractSourceFileDepGraphFactory.cpp | 13 +-- lib/AST/FineGrainedDependencies.cpp | 12 ++- lib/AST/FineGrainedDependencyFormat.cpp | 5 +- lib/AST/FrontendSourceFileDepGraphFactory.cpp | 26 +++-- lib/AST/FrontendSourceFileDepGraphFactory.h | 8 +- lib/DependencyScan/DependencyScanningTool.cpp | 4 +- .../ModuleDependencyCacheSerialization.cpp | 7 +- lib/DependencyScan/ScanDependencies.cpp | 2 +- .../FineGrainedDependencyDriverGraph.cpp | 6 +- lib/DriverTool/modulewrap_main.cpp | 5 +- lib/Frontend/Frontend.cpp | 13 ++- lib/Frontend/ModuleInterfaceBuilder.cpp | 3 +- lib/Frontend/ModuleInterfaceLoader.cpp | 40 ++++--- lib/Frontend/Serialization.cpp | 82 +++++++------- lib/FrontendTool/FrontendTool.cpp | 102 ++++++++++-------- lib/IRGen/IRGen.cpp | 72 +++++++------ lib/IRGen/IRGenModule.cpp | 1 - lib/Immediate/Immediate.cpp | 3 +- lib/SymbolGraphGen/SymbolGraphGen.cpp | 12 ++- tools/sil-llvm-gen/SILLLVMGen.cpp | 27 +++-- .../swift-dependency-tool.cpp | 6 +- .../MockingFineGrainedDependencyGraphs.cpp | 4 +- .../UnitTestSourceFileDepGraphFactory.h | 5 +- 35 files changed, 372 insertions(+), 213 deletions(-) diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h index 3c763b23a1e49..c9775a1126dfb 100644 --- a/include/swift/AST/ASTContext.h +++ b/include/swift/AST/ASTContext.h @@ -44,6 +44,7 @@ #include "llvm/ADT/TinyPtrVector.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/DataTypes.h" +#include "llvm/Support/VirtualOutputBackend.h" #include #include #include @@ -231,6 +232,7 @@ class ASTContext final { ClangImporterOptions &ClangImporterOpts, symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts, SourceManager &SourceMgr, DiagnosticEngine &Diags, + llvm::IntrusiveRefCntPtr OutBackend = nullptr, std::function PreModuleImportCallback = {}); public: @@ -248,6 +250,7 @@ class ASTContext final { ClangImporterOptions &ClangImporterOpts, symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts, SourceManager &SourceMgr, DiagnosticEngine &Diags, + llvm::IntrusiveRefCntPtr OutBackend = nullptr, std::function PreModuleImportCallback = {}); ~ASTContext(); @@ -281,6 +284,9 @@ class ASTContext final { /// Diags - The diagnostics engine. DiagnosticEngine &Diags; + /// OutputBackend for writing outputs. + llvm::IntrusiveRefCntPtr Backend; + /// If the shared pointer is not a \c nullptr and the pointee is \c true, /// all operations working on this ASTContext should be aborted at the next /// possible opportunity. @@ -1516,6 +1522,18 @@ class ASTContext final { const llvm::StringSet<> &getLoadedPluginLibraryPaths() const; + /// Get the output backend. The output backend needs to be initialized via + /// constructor or `setOutputBackend`. + llvm::vfs::OutputBackend &getOutputBackend() const { + assert(Backend && "OutputBackend is not setup"); + return *Backend; + } + /// Set output backend for virtualized outputs. + void setOutputBackend( + llvm::IntrusiveRefCntPtr OutputBackend) { + Backend = std::move(OutputBackend); + } + private: friend Decl; diff --git a/include/swift/AST/AbstractSourceFileDepGraphFactory.h b/include/swift/AST/AbstractSourceFileDepGraphFactory.h index 6858627dfb5cf..1f44f3f3106cd 100644 --- a/include/swift/AST/AbstractSourceFileDepGraphFactory.h +++ b/include/swift/AST/AbstractSourceFileDepGraphFactory.h @@ -16,6 +16,7 @@ #include "swift/AST/Decl.h" #include "swift/AST/DeclContext.h" #include "swift/AST/FineGrainedDependencies.h" +#include "llvm/Support/VirtualOutputBackend.h" namespace swift { class DiagnosticEngine; @@ -39,6 +40,9 @@ class AbstractSourceFileDepGraphFactory { DiagnosticEngine &diags; + /// OutputBackend. + llvm::vfs::OutputBackend &backend; + /// Graph under construction SourceFileDepGraph g; @@ -49,7 +53,8 @@ class AbstractSourceFileDepGraphFactory { StringRef swiftDeps, Fingerprint fileFingerprint, bool emitDotFileAfterConstruction, - DiagnosticEngine &diags); + DiagnosticEngine &diags, + llvm::vfs::OutputBackend &outputBackend); virtual ~AbstractSourceFileDepGraphFactory() = default; diff --git a/include/swift/AST/FileSystem.h b/include/swift/AST/FileSystem.h index 15641164c3e08..c2d5158c0f7bb 100644 --- a/include/swift/AST/FileSystem.h +++ b/include/swift/AST/FileSystem.h @@ -16,29 +16,38 @@ #include "swift/Basic/FileSystem.h" #include "swift/AST/DiagnosticEngine.h" #include "swift/AST/DiagnosticsCommon.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputConfig.h" namespace swift { - -/// A wrapper around swift::atomicallyWritingToFile that handles diagnosing any -/// filesystem errors and asserts the output path is nonempty. +/// A wrapper around llvm::vfs::OutputBackend to handle diagnosing any file +/// system errors during output creation. /// /// \returns true if there were any errors, either from the filesystem /// operations or from \p action returning true. inline bool -withOutputFile(DiagnosticEngine &diags, StringRef outputPath, +withOutputFile(DiagnosticEngine &diags, llvm::vfs::OutputBackend &Backend, + StringRef outputPath, llvm::function_ref action) { assert(!outputPath.empty()); + llvm::vfs::OutputConfig config; + config.setAtomicWrite().setOnlyIfDifferent(); + + auto outputFile = Backend.createFile(outputPath, config); + if (!outputFile) { + diags.diagnose(SourceLoc(), diag::error_opening_output, outputPath, + toString(outputFile.takeError())); + return false; + } - bool actionFailed = false; - std::error_code EC = swift::atomicallyWritingToFile( - outputPath, - [&](llvm::raw_pwrite_stream &out) { actionFailed = action(out); }); - if (EC) { + bool failed = action(*outputFile); + // If there is an error, discard output. Otherwise keep the output file. + if (auto error = failed ? outputFile->discard() : outputFile->keep()) { diags.diagnose(SourceLoc(), diag::error_opening_output, outputPath, - EC.message()); - return true; + toString(std::move(error))); + return false; } - return actionFailed; + return failed; } } // end namespace swift diff --git a/include/swift/AST/FineGrainedDependencies.h b/include/swift/AST/FineGrainedDependencies.h index 254d3a0d79002..ffac726152c42 100644 --- a/include/swift/AST/FineGrainedDependencies.h +++ b/include/swift/AST/FineGrainedDependencies.h @@ -24,6 +24,7 @@ #include "llvm/ADT/SetVector.h" #include "llvm/Support/MD5.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" #include "llvm/Support/raw_ostream.h" @@ -357,8 +358,9 @@ class BiIndexedTwoStageMap { /// \Note The returned graph should not be escaped from the callback. bool withReferenceDependencies( llvm::PointerUnion MSF, - const DependencyTracker &depTracker, StringRef outputPath, - bool alsoEmitDotFile, llvm::function_ref); + const DependencyTracker &depTracker, llvm::vfs::OutputBackend &backend, + StringRef outputPath, bool alsoEmitDotFile, + llvm::function_ref); //============================================================================== // MARK: Enums @@ -895,7 +897,8 @@ class SourceFileDepGraph { bool verifySequenceNumber() const; - void emitDotFile(StringRef outputPath, DiagnosticEngine &diags); + void emitDotFile(llvm::vfs::OutputBackend &outputBackend, + StringRef outputPath, DiagnosticEngine &diags); void addNode(SourceFileDepGraphNode *n) { n->setSequenceNumber(allNodes.size()); diff --git a/include/swift/AST/FineGrainedDependencyFormat.h b/include/swift/AST/FineGrainedDependencyFormat.h index 0957d03d98353..086de7e60b2c7 100644 --- a/include/swift/AST/FineGrainedDependencyFormat.h +++ b/include/swift/AST/FineGrainedDependencyFormat.h @@ -18,6 +18,9 @@ namespace llvm { class MemoryBuffer; +namespace vfs { +class OutputBackend; +} } namespace swift { @@ -132,6 +135,7 @@ bool readFineGrainedDependencyGraph(llvm::StringRef path, /// Tries to write the dependency graph to the given path name. /// Returns true if there was an error. bool writeFineGrainedDependencyGraphToPath(DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend, llvm::StringRef path, const SourceFileDepGraph &g); diff --git a/include/swift/AST/ModuleLoader.h b/include/swift/AST/ModuleLoader.h index 60ef5ac69693f..0072984634251 100644 --- a/include/swift/AST/ModuleLoader.h +++ b/include/swift/AST/ModuleLoader.h @@ -32,6 +32,9 @@ namespace llvm { class FileCollectorBase; +namespace vfs { +class OutputBackend; +} } namespace clang { @@ -156,6 +159,7 @@ class ModuleInterfaceChecker { virtual bool tryEmitForwardingModule(StringRef moduleName, StringRef interfacePath, ArrayRef candidates, + llvm::vfs::OutputBackend &backend, StringRef outPath) = 0; virtual ~ModuleInterfaceChecker() = default; }; diff --git a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h index a8fa5a316140b..254c985fbc85b 100644 --- a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h +++ b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h @@ -18,6 +18,9 @@ namespace llvm { class MemoryBuffer; +namespace vfs{ +class OutputBackend; +} } namespace swift { @@ -187,6 +190,7 @@ bool readInterModuleDependenciesCache(llvm::StringRef path, /// Tries to write the dependency graph to the given path name. /// Returns true if there was an error. bool writeInterModuleDependenciesCache(DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend, llvm::StringRef path, const SwiftDependencyScanningService &cache); diff --git a/include/swift/Driver/FineGrainedDependencyDriverGraph.h b/include/swift/Driver/FineGrainedDependencyDriverGraph.h index e2266ae03a13c..f36f0c269b65c 100644 --- a/include/swift/Driver/FineGrainedDependencyDriverGraph.h +++ b/include/swift/Driver/FineGrainedDependencyDriverGraph.h @@ -20,11 +20,14 @@ #include "swift/Basic/OptionSet.h" #include "swift/Driver/Job.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Support/Path.h" #include "llvm/Support/PointerLikeTypeTraits.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputBackends.h" #include #include #include @@ -201,6 +204,9 @@ class ModuleDepGraph { std::vector> dependencyPathsToJobs; + /// VirtualOutputBackend for emitting graphs. + llvm::IntrusiveRefCntPtr backend; + /// For helping with performance tuning, may be null: UnifiedStatsReporter *stats; @@ -306,6 +312,10 @@ class ModuleDepGraph { : None), stats(stats) { assert(verify() && "ModuleDepGraph should be fine when created"); + + // Create a OnDiskOutputBackend for emitting graphs. Currently, this is + // only used by driver so the backend is not shared with a CompilerInstance. + backend = llvm::makeIntrusiveRefCnt(); } /// For unit tests. diff --git a/include/swift/Frontend/Frontend.h b/include/swift/Frontend/Frontend.h index 30785440fa5fe..b179557dc4883 100644 --- a/include/swift/Frontend/Frontend.h +++ b/include/swift/Frontend/Frontend.h @@ -47,6 +47,7 @@ #include "llvm/Option/ArgList.h" #include "llvm/Support/Host.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackend.h" #include @@ -457,6 +458,9 @@ class CompilerInstance { /// instance has completed its setup, this will be null. std::unique_ptr Stats; + /// Virtual OutputBackend. + llvm::IntrusiveRefCntPtr TheOutputBackend = nullptr; + mutable ModuleDecl *MainModule = nullptr; SerializedModuleLoaderBase *DefaultSerializedLoader = nullptr; MemoryBufferSerializedModuleLoader *MemoryBufferLoader = nullptr; @@ -507,6 +511,14 @@ class CompilerInstance { SourceMgr.setFileSystem(FS); } + llvm::vfs::OutputBackend &getOutputBackend() const { + return *TheOutputBackend; + } + void + setOutputBackend(llvm::IntrusiveRefCntPtr Backend) { + TheOutputBackend = std::move(Backend); + } + ASTContext &getASTContext() { return *Context; } const ASTContext &getASTContext() const { return *Context; } @@ -629,6 +641,7 @@ class CompilerInstance { bool setUpASTContextIfNeeded(); void setupStatsReporter(); void setupDependencyTrackerIfNeeded(); + void setupOutputBackend(); /// \return false if successful, true on error. bool setupDiagnosticVerifierIfNeeded(); diff --git a/include/swift/Frontend/ModuleInterfaceLoader.h b/include/swift/Frontend/ModuleInterfaceLoader.h index 1f7a6d9fa1cdd..871cdb7cb313a 100644 --- a/include/swift/Frontend/ModuleInterfaceLoader.h +++ b/include/swift/Frontend/ModuleInterfaceLoader.h @@ -447,6 +447,7 @@ class ModuleInterfaceCheckerImpl: public ModuleInterfaceChecker { bool tryEmitForwardingModule(StringRef moduleName, StringRef interfacePath, ArrayRef candidates, + llvm::vfs::OutputBackend &backend, StringRef outPath) override; bool isCached(StringRef DepPath); }; diff --git a/include/swift/Subsystems.h b/include/swift/Subsystems.h index dc3b11ac6210e..b6fcda2b7bd20 100644 --- a/include/swift/Subsystems.h +++ b/include/swift/Subsystems.h @@ -35,6 +35,9 @@ namespace llvm { class Module; class TargetOptions; class TargetMachine; + namespace vfs { + class OutputBackend; + } } namespace swift { @@ -276,6 +279,7 @@ namespace swift { /// \param Module LLVM module to code gen, required. /// \param TargetMachine target of code gen, required. /// \param OutputFilename Filename for output. + /// \param Backend OutputBackend for writing output. bool performLLVM(const IRGenOptions &Opts, DiagnosticEngine &Diags, llvm::sys::Mutex *DiagMutex, @@ -283,6 +287,7 @@ namespace swift { llvm::Module *Module, llvm::TargetMachine *TargetMachine, StringRef OutputFilename, + llvm::vfs::OutputBackend &Backend, UnifiedStatsReporter *Stats); /// Dump YAML describing all fixed-size types imported from the given module. diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp index 9b495cdfeb8cc..5d3dad9765902 100644 --- a/lib/AST/ASTContext.cpp +++ b/lib/AST/ASTContext.cpp @@ -72,6 +72,8 @@ #include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputBackends.h" #include #include #include @@ -635,6 +637,7 @@ ASTContext *ASTContext::get( ClangImporterOptions &ClangImporterOpts, symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts, SourceManager &SourceMgr, DiagnosticEngine &Diags, + llvm::IntrusiveRefCntPtr OutputBackend, std::function PreModuleImportCallback) { // If more than two data structures are concatentated, then the aggregate // size math needs to become more complicated due to per-struct alignment @@ -646,9 +649,10 @@ ASTContext *ASTContext::get( impl = reinterpret_cast( llvm::alignAddr(impl, llvm::Align(alignof(Implementation)))); new (impl) Implementation(); - return new (mem) ASTContext(langOpts, typecheckOpts, silOpts, SearchPathOpts, - ClangImporterOpts, SymbolGraphOpts, SourceMgr, - Diags, PreModuleImportCallback); + return new (mem) + ASTContext(langOpts, typecheckOpts, silOpts, SearchPathOpts, + ClangImporterOpts, SymbolGraphOpts, SourceMgr, Diags, + std::move(OutputBackend), PreModuleImportCallback); } ASTContext::ASTContext( @@ -657,17 +661,19 @@ ASTContext::ASTContext( ClangImporterOptions &ClangImporterOpts, symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts, SourceManager &SourceMgr, DiagnosticEngine &Diags, + llvm::IntrusiveRefCntPtr OutputBackend, std::function PreModuleImportCallback) : LangOpts(langOpts), TypeCheckerOpts(typecheckOpts), SILOpts(silOpts), SearchPathOpts(SearchPathOpts), ClangImporterOpts(ClangImporterOpts), SymbolGraphOpts(SymbolGraphOpts), SourceMgr(SourceMgr), Diags(Diags), - evaluator(Diags, langOpts), TheBuiltinModule(createBuiltinModule(*this)), + Backend(std::move(OutputBackend)), evaluator(Diags, langOpts), + TheBuiltinModule(createBuiltinModule(*this)), StdlibModuleName(getIdentifier(STDLIB_NAME)), SwiftShimsModuleName(getIdentifier(SWIFT_SHIMS_NAME)), PreModuleImportCallback(PreModuleImportCallback), TheErrorType(new (*this, AllocationArena::Permanent) ErrorType( *this, Type(), RecursiveTypeProperties::HasError)), - TheUnresolvedType(new (*this, AllocationArena::Permanent) + TheUnresolvedType(new(*this, AllocationArena::Permanent) UnresolvedType(*this)), TheEmptyTupleType(TupleType::get(ArrayRef(), *this)), TheEmptyPackType(PackType::get(*this, {})), @@ -707,6 +713,9 @@ ASTContext::ASTContext( registerNameLookupRequestFunctions(evaluator); createModuleToExecutablePluginMap(); + // Provide a default OnDiskOutputBackend if user didn't supply one. + if (!Backend) + Backend = llvm::makeIntrusiveRefCnt(); } ASTContext::~ASTContext() { diff --git a/lib/AST/AbstractSourceFileDepGraphFactory.cpp b/lib/AST/AbstractSourceFileDepGraphFactory.cpp index 1b73a66b6a42b..78bdec1b0b6ad 100644 --- a/lib/AST/AbstractSourceFileDepGraphFactory.cpp +++ b/lib/AST/AbstractSourceFileDepGraphFactory.cpp @@ -23,6 +23,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" using namespace swift; @@ -33,13 +34,13 @@ using namespace fine_grained_dependencies; //============================================================================== AbstractSourceFileDepGraphFactory::AbstractSourceFileDepGraphFactory( - bool hadCompilationError, StringRef swiftDeps, - Fingerprint fileFingerprint, bool emitDotFileAfterConstruction, - DiagnosticEngine &diags) + bool hadCompilationError, StringRef swiftDeps, Fingerprint fileFingerprint, + bool emitDotFileAfterConstruction, DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend) : hadCompilationError(hadCompilationError), swiftDeps(swiftDeps.str()), fileFingerprint(fileFingerprint), - emitDotFileAfterConstruction(emitDotFileAfterConstruction), diags(diags) { -} + emitDotFileAfterConstruction(emitDotFileAfterConstruction), diags(diags), + backend(backend) {} SourceFileDepGraph AbstractSourceFileDepGraphFactory::construct() { addSourceFileNodesToGraph(); @@ -49,7 +50,7 @@ SourceFileDepGraph AbstractSourceFileDepGraphFactory::construct() { } assert(g.verify()); if (emitDotFileAfterConstruction) - g.emitDotFile(swiftDeps, diags); + g.emitDotFile(backend, swiftDeps, diags); return std::move(g); } diff --git a/lib/AST/FineGrainedDependencies.cpp b/lib/AST/FineGrainedDependencies.cpp index 609a03512b058..3a909dbcdbe38 100644 --- a/lib/AST/FineGrainedDependencies.cpp +++ b/lib/AST/FineGrainedDependencies.cpp @@ -373,11 +373,13 @@ void SourceFileDepGraph::verifySame(const SourceFileDepGraph &other) const { #endif } -void SourceFileDepGraph::emitDotFile(StringRef outputPath, +void SourceFileDepGraph::emitDotFile(llvm::vfs::OutputBackend &outputBackend, + StringRef outputPath, DiagnosticEngine &diags) { std::string dotFileName = outputPath.str() + ".dot"; - withOutputFile(diags, dotFileName, [&](llvm::raw_pwrite_stream &out) { - DotFileEmitter(out, *this, false, false).emit(); - return false; - }); + withOutputFile( + diags, outputBackend, dotFileName, [&](llvm::raw_pwrite_stream &out) { + DotFileEmitter(out, *this, false, false).emit(); + return false; + }); } diff --git a/lib/AST/FineGrainedDependencyFormat.cpp b/lib/AST/FineGrainedDependencyFormat.cpp index bab3cdf60a98f..02c04fe270bb1 100644 --- a/lib/AST/FineGrainedDependencyFormat.cpp +++ b/lib/AST/FineGrainedDependencyFormat.cpp @@ -21,6 +21,7 @@ #include "llvm/Bitstream/BitstreamWriter.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackend.h" using namespace swift; using namespace fine_grained_dependencies; @@ -498,10 +499,10 @@ void swift::fine_grained_dependencies::writeFineGrainedDependencyGraph( } bool swift::fine_grained_dependencies::writeFineGrainedDependencyGraphToPath( - DiagnosticEngine &diags, StringRef path, + DiagnosticEngine &diags, llvm::vfs::OutputBackend &backend, StringRef path, const SourceFileDepGraph &g) { PrettyStackTraceStringAction stackTrace("saving fine-grained dependency graph", path); - return withOutputFile(diags, path, [&](llvm::raw_ostream &out) { + return withOutputFile(diags, backend, path, [&](llvm::raw_ostream &out) { SmallVector Buffer; llvm::BitstreamWriter Writer{Buffer}; writeFineGrainedDependencyGraph(Writer, g, Purpose::ForSwiftDeps); diff --git a/lib/AST/FrontendSourceFileDepGraphFactory.cpp b/lib/AST/FrontendSourceFileDepGraphFactory.cpp index 74afe1a7f810c..7b50752032919 100644 --- a/lib/AST/FrontendSourceFileDepGraphFactory.cpp +++ b/lib/AST/FrontendSourceFileDepGraphFactory.cpp @@ -43,6 +43,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" using namespace swift; @@ -232,18 +233,19 @@ StringRef DependencyKey::Builder::getTopLevelName(const Decl *decl) { bool fine_grained_dependencies::withReferenceDependencies( llvm::PointerUnion MSF, - const DependencyTracker &depTracker, StringRef outputPath, - bool alsoEmitDotFile, + const DependencyTracker &depTracker, llvm::vfs::OutputBackend &backend, + StringRef outputPath, bool alsoEmitDotFile, llvm::function_ref cont) { if (auto *MD = MSF.dyn_cast()) { SourceFileDepGraph g = - ModuleDepGraphFactory(MD, alsoEmitDotFile).construct(); + ModuleDepGraphFactory(backend, MD, alsoEmitDotFile).construct(); return cont(std::move(g)); } else { auto *SF = MSF.get(); - SourceFileDepGraph g = FrontendSourceFileDepGraphFactory( - SF, outputPath, depTracker, alsoEmitDotFile) - .construct(); + SourceFileDepGraph g = + FrontendSourceFileDepGraphFactory(SF, backend, outputPath, depTracker, + alsoEmitDotFile) + .construct(); return cont(std::move(g)); } } @@ -253,11 +255,12 @@ bool fine_grained_dependencies::withReferenceDependencies( //============================================================================== FrontendSourceFileDepGraphFactory::FrontendSourceFileDepGraphFactory( - const SourceFile *SF, StringRef outputPath, - const DependencyTracker &depTracker, const bool alsoEmitDotFile) + const SourceFile *SF, llvm::vfs::OutputBackend &backend, + StringRef outputPath, const DependencyTracker &depTracker, + const bool alsoEmitDotFile) : AbstractSourceFileDepGraphFactory( SF->getASTContext().hadError(), outputPath, SF->getInterfaceHash(), - alsoEmitDotFile, SF->getASTContext().Diags), + alsoEmitDotFile, SF->getASTContext().Diags, backend), SF(SF), depTracker(depTracker) {} //============================================================================== @@ -577,11 +580,12 @@ void FrontendSourceFileDepGraphFactory::addAllUsedDecls() { // MARK: ModuleDepGraphFactory //============================================================================== -ModuleDepGraphFactory::ModuleDepGraphFactory(const ModuleDecl *Mod, +ModuleDepGraphFactory::ModuleDepGraphFactory(llvm::vfs::OutputBackend &backend, + const ModuleDecl *Mod, bool emitDot) : AbstractSourceFileDepGraphFactory( Mod->getASTContext().hadError(), Mod->getNameStr(), - Mod->getFingerprint(), emitDot, Mod->getASTContext().Diags), + Mod->getFingerprint(), emitDot, Mod->getASTContext().Diags, backend), Mod(Mod) {} void ModuleDepGraphFactory::addAllDefinedDecls() { diff --git a/lib/AST/FrontendSourceFileDepGraphFactory.h b/lib/AST/FrontendSourceFileDepGraphFactory.h index d46105e45bc28..9fa24035527f8 100644 --- a/lib/AST/FrontendSourceFileDepGraphFactory.h +++ b/lib/AST/FrontendSourceFileDepGraphFactory.h @@ -14,6 +14,7 @@ #define FrontendSourceFileDepGraphFactory_h #include "swift/AST/AbstractSourceFileDepGraphFactory.h" +#include "llvm/Support/VirtualOutputBackend.h" namespace swift { namespace fine_grained_dependencies { @@ -27,7 +28,9 @@ class FrontendSourceFileDepGraphFactory const DependencyTracker &depTracker; public: - FrontendSourceFileDepGraphFactory(const SourceFile *SF, StringRef outputPath, + FrontendSourceFileDepGraphFactory(const SourceFile *SF, + llvm::vfs::OutputBackend &backend, + StringRef outputPath, const DependencyTracker &depTracker, bool alsoEmitDotFile); @@ -42,7 +45,8 @@ class ModuleDepGraphFactory : public AbstractSourceFileDepGraphFactory { const ModuleDecl *Mod; public: - ModuleDepGraphFactory(const ModuleDecl *Mod, bool emitDot); + ModuleDepGraphFactory(llvm::vfs::OutputBackend &backend, + const ModuleDecl *Mod, bool emitDot); ~ModuleDepGraphFactory() override = default; diff --git a/lib/DependencyScan/DependencyScanningTool.cpp b/lib/DependencyScan/DependencyScanningTool.cpp index cc812e4c3a954..d91c9053ec968 100644 --- a/lib/DependencyScan/DependencyScanningTool.cpp +++ b/lib/DependencyScan/DependencyScanningTool.cpp @@ -20,6 +20,7 @@ #include "swift/DependencyScan/DependencyScanImpl.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/VirtualOutputBackends.h" #include @@ -187,8 +188,9 @@ void DependencyScanningTool::serializeCache(llvm::StringRef path) { SourceManager SM; DiagnosticEngine Diags(SM); Diags.addConsumer(CDC); + llvm::vfs::OnDiskOutputBackend Backend; module_dependency_cache_serialization::writeInterModuleDependenciesCache( - Diags, path, *ScanningService); + Diags, Backend, path, *ScanningService); } bool DependencyScanningTool::loadCache(llvm::StringRef path) { diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index e69b3ae5f2462..2963920d3f76a 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -16,6 +16,7 @@ #include "swift/Basic/Version.h" #include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/Support/VirtualOutputBackend.h" #include using namespace swift; @@ -1146,11 +1147,11 @@ void swift::dependencies::module_dependency_cache_serialization:: bool swift::dependencies::module_dependency_cache_serialization:: writeInterModuleDependenciesCache( - DiagnosticEngine &diags, StringRef path, - const SwiftDependencyScanningService &cache) { + DiagnosticEngine &diags, llvm::vfs::OutputBackend &backend, + StringRef path, const SwiftDependencyScanningService &cache) { PrettyStackTraceStringAction stackTrace( "saving inter-module dependency graph", path); - return withOutputFile(diags, path, [&](llvm::raw_ostream &out) { + return withOutputFile(diags, backend, path, [&](llvm::raw_ostream &out) { SmallVector Buffer; llvm::BitstreamWriter Writer{Buffer}; writeInterModuleDependenciesCache(Writer, cache); diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 84d35f09779cf..9011c428777fa 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -1369,7 +1369,7 @@ static void serializeDependencyCache(CompilerInstance &instance, ASTContext &Context = instance.getASTContext(); auto savePath = opts.SerializedDependencyScannerCachePath; module_dependency_cache_serialization::writeInterModuleDependenciesCache( - Context.Diags, savePath, service); + Context.Diags, instance.getOutputBackend(), savePath, service); if (opts.EmitDependencyScannerCacheRemarks) { Context.Diags.diagnose(SourceLoc(), diag::remark_save_cache, savePath); } diff --git a/lib/Driver/FineGrainedDependencyDriverGraph.cpp b/lib/Driver/FineGrainedDependencyDriverGraph.cpp index 1b0c20a5fc849..bd097169ac8b3 100644 --- a/lib/Driver/FineGrainedDependencyDriverGraph.cpp +++ b/lib/Driver/FineGrainedDependencyDriverGraph.cpp @@ -29,6 +29,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/SourceMgr.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/raw_ostream.h" #include @@ -529,11 +530,12 @@ void ModuleDepGraph::emitDotFileForJob(DiagnosticEngine &diags, emitDotFile(diags, getSwiftDeps(job)); } -void ModuleDepGraph::emitDotFile(DiagnosticEngine &diags, StringRef baseName) { +void ModuleDepGraph::emitDotFile(DiagnosticEngine &diags, + StringRef baseName) { unsigned seqNo = dotFileSequenceNumber[baseName.str()]++; std::string fullName = baseName.str() + "-post-integration." + std::to_string(seqNo) + ".dot"; - withOutputFile(diags, fullName, [&](llvm::raw_ostream &out) { + withOutputFile(diags, *backend, fullName, [&](llvm::raw_ostream &out) { emitDotFile(out); return false; }); diff --git a/lib/DriverTool/modulewrap_main.cpp b/lib/DriverTool/modulewrap_main.cpp index eff93f616cc5d..676b78e720ae4 100644 --- a/lib/DriverTool/modulewrap_main.cpp +++ b/lib/DriverTool/modulewrap_main.cpp @@ -28,6 +28,7 @@ #include "swift/Subsystems.h" #include "swift/SymbolGraphGen/SymbolGraphOptions.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/Bitstream/BitstreamReader.h" #include "llvm/Option/ArgList.h" #include "llvm/Option/Option.h" @@ -35,6 +36,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/TargetSelect.h" +#include "llvm/Support/VirtualOutputBackends.h" using namespace llvm::opt; using namespace swift; @@ -183,7 +185,8 @@ int modulewrap_main(ArrayRef Args, const char *Argv0, LangOpts.Target = Invocation.getTargetTriple(); ASTContext &ASTCtx = *ASTContext::get( LangOpts, TypeCheckOpts, SILOpts, SearchPathOpts, ClangImporterOpts, - SymbolGraphOpts, SrcMgr, Instance.getDiags()); + SymbolGraphOpts, SrcMgr, Instance.getDiags(), + llvm::makeIntrusiveRefCnt()); registerParseRequestFunctions(ASTCtx.evaluator); registerTypeCheckerRequestFunctions(ASTCtx.evaluator); diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index 5a8ddacecae36..a7f138682becc 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -38,6 +38,7 @@ #include "swift/Subsystems.h" #include "clang/AST/ASTContext.h" #include "llvm/ADT/Hashing.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Triple.h" #include "llvm/Support/CommandLine.h" @@ -45,6 +46,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/Process.h" +#include "llvm/Support/VirtualOutputBackends.h" #include using namespace swift; @@ -245,11 +247,12 @@ bool CompilerInstance::setUpASTContextIfNeeded() { Invocation.getLangOptions().RecordRequestReferences = !isWholeModuleCompilation(); + // Make sure the output backend is initialized. Context.reset(ASTContext::get( Invocation.getLangOptions(), Invocation.getTypeCheckerOptions(), Invocation.getSILOptions(), Invocation.getSearchPathOptions(), Invocation.getClangImporterOptions(), Invocation.getSymbolGraphOptions(), - SourceMgr, Diagnostics)); + SourceMgr, Diagnostics, TheOutputBackend)); if (!Invocation.getFrontendOptions().ModuleAliasMap.empty()) Context->setModuleAliases(Invocation.getFrontendOptions().ModuleAliasMap); @@ -393,11 +396,19 @@ void CompilerInstance::setupDependencyTrackerIfNeeded() { DepTracker->addDependency(path, /*isSystem=*/false); } +void CompilerInstance::setupOutputBackend() { + // Skip if output backend is not setup, default to OnDiskOutputBackend. + if (!TheOutputBackend) + TheOutputBackend = + llvm::makeIntrusiveRefCnt(); +} + bool CompilerInstance::setup(const CompilerInvocation &Invoke, std::string &Error) { Invocation = Invoke; setupDependencyTrackerIfNeeded(); + setupOutputBackend(); // If initializing the overlay file system fails there's no sense in // continuing because the compiler will read the wrong files. diff --git a/lib/Frontend/ModuleInterfaceBuilder.cpp b/lib/Frontend/ModuleInterfaceBuilder.cpp index a898ae822d6ca..1d872b6b3b24d 100644 --- a/lib/Frontend/ModuleInterfaceBuilder.cpp +++ b/lib/Frontend/ModuleInterfaceBuilder.cpp @@ -190,7 +190,8 @@ std::error_code ExplicitModuleInterfaceBuilder::buildSwiftModuleFromInterface( if (Instance.getASTContext() .getModuleInterfaceChecker() ->tryEmitForwardingModule(Invocation.getModuleName(), InterfacePath, - CompiledCandidates, OutputPath)) { + CompiledCandidates, + Instance.getOutputBackend(), OutputPath)) { return std::error_code(); } FrontendOptions &FEOpts = Invocation.getFrontendOptions(); diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index acc34930c72e4..2aeb3d179068f 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -40,6 +40,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/Errc.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" #include "llvm/Support/xxhash.h" @@ -901,8 +902,8 @@ class ModuleInterfaceLoaderImpl { /// this. If the write was successful, it also updates the /// list of dependencies to match what was written to the forwarding file. bool writeForwardingModuleAndUpdateDeps( - const DiscoveredModule &mod, StringRef outputPath, - SmallVectorImpl &deps) { + const DiscoveredModule &mod, llvm::vfs::OutputBackend &backend, + StringRef outputPath, SmallVectorImpl &deps) { assert(mod.isPrebuilt() && "cannot write forwarding file for non-prebuilt module"); ForwardingModule fwd(mod.path); @@ -938,16 +939,12 @@ class ModuleInterfaceLoaderImpl { depsAdjustedToMTime.push_back(adjustedDep); } - // Create the module cache if we haven't created it yet. - StringRef parentDir = path::parent_path(outputPath); - (void)llvm::sys::fs::create_directories(parentDir); - - auto hadError = withOutputFile(diags, outputPath, - [&](llvm::raw_pwrite_stream &out) { - llvm::yaml::Output yamlWriter(out); - yamlWriter << fwd; - return false; - }); + auto hadError = withOutputFile(diags, backend, outputPath, + [&](llvm::raw_pwrite_stream &out) { + llvm::yaml::Output yamlWriter(out); + yamlWriter << fwd; + return false; + }); if (hadError) return true; @@ -1006,8 +1003,8 @@ class ModuleInterfaceLoaderImpl { // If it's prebuilt, use this time to generate a forwarding module and // update the dependencies to use modification times. if (module.isPrebuilt()) - if (writeForwardingModuleAndUpdateDeps(module, cachedOutputPath, - allDeps)) + if (writeForwardingModuleAndUpdateDeps(module, ctx.getOutputBackend(), + cachedOutputPath, allDeps)) return std::make_error_code(std::errc::not_supported); // Report the module's dependencies to the dependencyTracker @@ -1251,7 +1248,8 @@ ModuleInterfaceCheckerImpl::getCompiledModuleCandidatesForInterface(StringRef mo bool ModuleInterfaceCheckerImpl::tryEmitForwardingModule( StringRef moduleName, StringRef interfacePath, - ArrayRef candidates, StringRef outputPath) { + ArrayRef candidates, llvm::vfs::OutputBackend &backend, + StringRef outputPath) { // Derive .swiftmodule path from the .swiftinterface path. auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile); llvm::SmallString<32> modulePath = interfacePath; @@ -1269,12 +1267,12 @@ bool ModuleInterfaceCheckerImpl::tryEmitForwardingModule( deps, moduleBuffer)) { // If so, emit a forwarding module to the candidate. ForwardingModule FM(mod); - auto hadError = withOutputFile(Ctx.Diags, outputPath, - [&](llvm::raw_pwrite_stream &out) { - llvm::yaml::Output yamlWriter(out); - yamlWriter << FM; - return false; - }); + auto hadError = withOutputFile(Ctx.Diags, backend, outputPath, + [&](llvm::raw_pwrite_stream &out) { + llvm::yaml::Output yamlWriter(out); + yamlWriter << FM; + return false; + }); if (!hadError) return true; } diff --git a/lib/Frontend/Serialization.cpp b/lib/Frontend/Serialization.cpp index 3af22d3bf2df0..68429aa707ca9 100644 --- a/lib/Frontend/Serialization.cpp +++ b/lib/Frontend/Serialization.cpp @@ -17,6 +17,7 @@ #include "swift/SymbolGraphGen/SymbolGraphGen.h" #include "swift/SymbolGraphGen/SymbolGraphOptions.h" #include "llvm/Support/SmallVectorMemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackend.h" using namespace swift; @@ -56,11 +57,12 @@ void swift::serializeToBuffers( llvm::raw_svector_ostream stream(buf); serialization::writeToStream(stream, DC, M, options, /*dependency info*/ nullptr); - bool hadError = withOutputFile(getContext(DC).Diags, options.OutputPath, - [&](raw_ostream &out) { - out << stream.str(); - return false; - }); + bool hadError = withOutputFile( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.OutputPath, [&](raw_ostream &out) { + out << stream.str(); + return false; + }); if (hadError) return; @@ -77,11 +79,12 @@ void swift::serializeToBuffers( llvm::SmallString<1024> buf; llvm::raw_svector_ostream stream(buf); serialization::writeDocToStream(stream, DC, options.GroupInfoPath); - (void)withOutputFile(getContext(DC).Diags, options.DocOutputPath, - [&](raw_ostream &out) { - out << stream.str(); - return false; - }); + (void)withOutputFile(getContext(DC).Diags, + getContext(DC).getOutputBackend(), + options.DocOutputPath, [&](raw_ostream &out) { + out << stream.str(); + return false; + }); if (moduleDocBuffer) *moduleDocBuffer = std::make_unique( std::move(buf), options.DocOutputPath, @@ -94,11 +97,12 @@ void swift::serializeToBuffers( llvm::SmallString<1024> buf; llvm::raw_svector_ostream stream(buf); serialization::writeSourceInfoToStream(stream, DC); - (void)withOutputFile(getContext(DC).Diags, options.SourceInfoOutputPath, - [&](raw_ostream &out) { - out << stream.str(); - return false; - }); + (void)withOutputFile( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.SourceInfoOutputPath, [&](raw_ostream &out) { + out << stream.str(); + return false; + }); if (moduleSourceInfoBuffer) *moduleSourceInfoBuffer = std::make_unique( std::move(buf), options.SourceInfoOutputPath, @@ -120,37 +124,37 @@ void swift::serialize( return; } - bool hadError = withOutputFile(getContext(DC).Diags, - options.OutputPath, - [&](raw_ostream &out) { - FrontendStatsTracer tracer(getContext(DC).Stats, - "Serialization, swiftmodule"); - serialization::writeToStream(out, DC, M, options, DG); - return false; - }); + bool hadError = withOutputFile( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.OutputPath, [&](raw_ostream &out) { + FrontendStatsTracer tracer(getContext(DC).Stats, + "Serialization, swiftmodule"); + serialization::writeToStream(out, DC, M, options, DG); + return false; + }); if (hadError) return; if (!options.DocOutputPath.empty()) { - (void)withOutputFile(getContext(DC).Diags, - options.DocOutputPath, - [&](raw_ostream &out) { - FrontendStatsTracer tracer(getContext(DC).Stats, - "Serialization, swiftdoc"); - serialization::writeDocToStream(out, DC, options.GroupInfoPath); - return false; - }); + (void)withOutputFile( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.DocOutputPath, [&](raw_ostream &out) { + FrontendStatsTracer tracer(getContext(DC).Stats, + "Serialization, swiftdoc"); + serialization::writeDocToStream(out, DC, options.GroupInfoPath); + return false; + }); } if (!options.SourceInfoOutputPath.empty()) { - (void)withOutputFile(getContext(DC).Diags, - options.SourceInfoOutputPath, - [&](raw_ostream &out) { - FrontendStatsTracer tracer(getContext(DC).Stats, - "Serialization, swiftsourceinfo"); - serialization::writeSourceInfoToStream(out, DC); - return false; - }); + (void)withOutputFile( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.SourceInfoOutputPath, [&](raw_ostream &out) { + FrontendStatsTracer tracer(getContext(DC).Stats, + "Serialization, swiftsourceinfo"); + serialization::writeSourceInfoToStream(out, DC); + return false; + }); } if (!symbolGraphOptions.OutputDir.empty()) { diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index 12ce64135d364..b49207a7bf36b 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -81,6 +81,7 @@ #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Support/FileSystem.h" @@ -123,37 +124,36 @@ emitLoadedModuleTraceForAllPrimariesIfNeeded(ModuleDecl *mainModule, }); } -/// Gets an output stream for the provided output filename, or diagnoses to the -/// provided AST Context and returns null if there was an error getting the -/// stream. -static std::unique_ptr -getFileOutputStream(StringRef OutputFilename, ASTContext &Ctx) { - std::error_code errorCode; - auto os = std::make_unique( - OutputFilename, errorCode, llvm::sys::fs::OF_None); - if (errorCode) { - Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_output, - OutputFilename, errorCode.message()); - return nullptr; - } - return os; +/// Diagnose output file error to ASTContext. +void diagnoseOutputError(ASTContext &Ctx, StringRef OutputFilename, + llvm::Error Err) { + Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_output, OutputFilename, + toString(std::move(Err))); } /// Writes SIL out to the given file. static bool writeSIL(SILModule &SM, ModuleDecl *M, const SILOptions &Opts, - StringRef OutputFilename) { - auto OS = getFileOutputStream(OutputFilename, M->getASTContext()); - if (!OS) return true; - SM.print(*OS, M, Opts); - + StringRef OutputFilename, + llvm::vfs::OutputBackend &Backend) { + auto OutFile = Backend.createFile(OutputFilename); + if (!OutFile) { + diagnoseOutputError(M->getASTContext(), OutputFilename, + OutFile.takeError()); + return true; + } + SM.print(*OutFile, M, Opts); + if (auto E = OutFile->keep()) { + diagnoseOutputError(M->getASTContext(), OutputFilename, std::move(E)); + return true; + } return M->getASTContext().hadError(); } static bool writeSIL(SILModule &SM, const PrimarySpecificPaths &PSPs, - const CompilerInstance &Instance, + CompilerInstance &Instance, const SILOptions &Opts) { return writeSIL(SM, Instance.getMainModule(), Opts, - PSPs.OutputFilename); + PSPs.OutputFilename, Instance.getOutputBackend()); } /// Prints the Objective-C "generated header" interface for \p M to \p @@ -166,14 +166,14 @@ static bool writeSIL(SILModule &SM, const PrimarySpecificPaths &PSPs, /// \returns true if there were any errors /// /// \see swift::printAsClangHeader -static bool printAsClangHeaderIfNeeded( +static bool printAsClangHeaderIfNeeded(llvm::vfs::OutputBackend &outputBackend, StringRef outputPath, ModuleDecl *M, StringRef bridgingHeader, const FrontendOptions &frontendOpts, const IRGenOptions &irGenOpts, clang::HeaderSearch &clangHeaderSearchInfo) { if (outputPath.empty()) return false; return withOutputFile( - M->getDiags(), outputPath, [&](raw_ostream &out) -> bool { + M->getDiags(), outputBackend, outputPath, [&](raw_ostream &out) -> bool { return printAsClangHeader(out, M, bridgingHeader, frontendOpts, irGenOpts, clangHeaderSearchInfo); }); @@ -187,7 +187,8 @@ static bool printAsClangHeaderIfNeeded( /// /// \see swift::emitSwiftInterface static bool -printModuleInterfaceIfNeeded(StringRef outputPath, +printModuleInterfaceIfNeeded(llvm::vfs::OutputBackend &outputBackend, + StringRef outputPath, ModuleInterfaceOptions const &Opts, LangOptions const &LangOpts, ModuleDecl *M) { @@ -205,10 +206,10 @@ printModuleInterfaceIfNeeded(StringRef outputPath, diags.diagnose(SourceLoc(), diag::warn_unsupported_module_interface_library_evolution); } - return withOutputFile(diags, outputPath, + return withOutputFile(diags, outputBackend, outputPath, [M, Opts](raw_ostream &out) -> bool { - return swift::emitSwiftInterface(out, Opts, M); - }); + return swift::emitSwiftInterface(out, Opts, M); + }); } namespace { @@ -517,8 +518,18 @@ static bool dumpAST(CompilerInstance &Instance) { for (SourceFile *sourceFile: primaryFiles) { auto PSPs = Instance.getPrimarySpecificPathsForSourceFile(*sourceFile); auto OutputFilename = PSPs.OutputFilename; - auto OS = getFileOutputStream(OutputFilename, Instance.getASTContext()); - sourceFile->dump(*OS, /*parseIfNeeded*/ true); + auto OutFile = Instance.getOutputBackend().createFile(OutputFilename); + if (!OutFile) { + diagnoseOutputError(Instance.getASTContext(), OutputFilename, + OutFile.takeError()); + return true; + } + sourceFile->dump(*OutFile, /*parseIfNeeded*/ true); + if (auto E = OutFile->keep()) { + diagnoseOutputError(Instance.getASTContext(), OutputFilename, + std::move(E)); + return true; + } } } else { // Some invocations don't have primary files. In that case, we default to @@ -543,17 +554,19 @@ static bool emitReferenceDependencies(CompilerInstance &Instance, using SourceFileDepGraph = fine_grained_dependencies::SourceFileDepGraph; return fine_grained_dependencies::withReferenceDependencies( - SF, *Instance.getDependencyTracker(), outputPath, alsoEmitDotFile, - [&](SourceFileDepGraph &&g) -> bool { + SF, *Instance.getDependencyTracker(), Instance.getOutputBackend(), + outputPath, alsoEmitDotFile, [&](SourceFileDepGraph &&g) -> bool { const bool hadError = fine_grained_dependencies::writeFineGrainedDependencyGraphToPath( - Instance.getDiags(), outputPath, g); + Instance.getDiags(), Instance.getOutputBackend(), outputPath, + g); // If path is stdout, cannot read it back, so check for "-" assert(outputPath == "-" || g.verifyReadsWhatIsWritten(outputPath)); if (alsoEmitDotFile) - g.emitDotFile(outputPath, Instance.getDiags()); + g.emitDotFile(Instance.getOutputBackend(), outputPath, + Instance.getDiags()); return hadError; }); } @@ -925,6 +938,7 @@ static bool emitAnyWholeModulePostTypeCheckSupplementaryOutputs( } } hadAnyError |= printAsClangHeaderIfNeeded( + Instance.getOutputBackend(), Invocation.getClangHeaderOutputPathForAtMostOnePrimary(), Instance.getMainModule(), BridgingHeaderPathForPrint, opts, Invocation.getIRGenOptions(), @@ -940,6 +954,7 @@ static bool emitAnyWholeModulePostTypeCheckSupplementaryOutputs( if (opts.InputsAndOutputs.hasModuleInterfaceOutputPath()) { hadAnyError |= printModuleInterfaceIfNeeded( + Instance.getOutputBackend(), Invocation.getModuleInterfaceOutputPathForWholeModule(), Invocation.getModuleInterfaceOptions(), Invocation.getLangOptions(), @@ -953,6 +968,7 @@ static bool emitAnyWholeModulePostTypeCheckSupplementaryOutputs( privOpts.ModulesToSkipInPublicInterface.clear(); hadAnyError |= printModuleInterfaceIfNeeded( + Instance.getOutputBackend(), Invocation.getPrivateModuleInterfaceOutputPathForWholeModule(), privOpts, Invocation.getLangOptions(), @@ -1437,11 +1453,11 @@ static bool serializeModuleSummary(SILModule *SM, const PrimarySpecificPaths &PSPs, const ASTContext &Context) { auto summaryOutputPath = PSPs.SupplementaryOutputs.ModuleSummaryOutputPath; - return withOutputFile(Context.Diags, summaryOutputPath, - [&](llvm::raw_ostream &out) { - out << "Some stuff"; - return false; - }); + return withOutputFile(Context.Diags, Context.getOutputBackend(), + summaryOutputPath, [&](llvm::raw_ostream &out) { + out << "Some stuff"; + return false; + }); } static GeneratedModule @@ -1618,6 +1634,7 @@ static bool generateCode(CompilerInstance &Instance, StringRef OutputFilename, // Now that we have a single IR Module, hand it over to performLLVM. return performLLVM(opts, Instance.getDiags(), nullptr, HashGlobal, IRModule, TargetMachine.get(), OutputFilename, + Instance.getOutputBackend(), Instance.getStatsReporter()); } @@ -1696,7 +1713,8 @@ static bool performCompileStepsPostSILGen(CompilerInstance &Instance, using SourceFileDepGraph = fine_grained_dependencies::SourceFileDepGraph; auto *Mod = MSF.get(); fine_grained_dependencies::withReferenceDependencies( - Mod, *Instance.getDependencyTracker(), Mod->getModuleFilename(), + Mod, *Instance.getDependencyTracker(), + Instance.getOutputBackend(), Mod->getModuleFilename(), alsoEmitDotFile, [&](SourceFileDepGraph &&g) { serialize(MSF, serializationOpts, Invocation.getSymbolGraphOptions(), SM.get(), &g); return false; @@ -1780,9 +1798,9 @@ static bool performCompileStepsPostSILGen(CompilerInstance &Instance, std::vector ParallelOutputFilenames = opts.InputsAndOutputs.copyOutputFilenames(); llvm::GlobalVariable *HashGlobal; - auto IRModule = generateIR( - IRGenOpts, Invocation.getTBDGenOptions(), std::move(SM), PSPs, - OutputFilename, MSF, HashGlobal, ParallelOutputFilenames); + auto IRModule = + generateIR(IRGenOpts, Invocation.getTBDGenOptions(), std::move(SM), PSPs, + OutputFilename, MSF, HashGlobal, ParallelOutputFilenames); // Cancellation check after IRGen. if (Instance.isCancellationRequested()) diff --git a/lib/IRGen/IRGen.cpp b/lib/IRGen/IRGen.cpp index 2df7a065bf92e..2d546c3bbd859 100644 --- a/lib/IRGen/IRGen.cpp +++ b/lib/IRGen/IRGen.cpp @@ -46,6 +46,7 @@ #include "swift/Subsystems.h" #include "clang/Basic/TargetInfo.h" #include "clang/Frontend/CompilerInstance.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/StringSet.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Bitcode/BitcodeWriter.h" @@ -74,6 +75,8 @@ #include "llvm/Support/FormattedStream.h" #include "llvm/Support/Mutex.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputConfig.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" @@ -534,6 +537,7 @@ bool swift::performLLVM(const IRGenOptions &Opts, llvm::Module *Module, llvm::TargetMachine *TargetMachine, StringRef OutputFilename, + llvm::vfs::OutputBackend &Backend, UnifiedStatsReporter *Stats) { if (Opts.UseIncrementalLLVMCodeGen && HashGlobal) { @@ -564,23 +568,28 @@ bool swift::performLLVM(const IRGenOptions &Opts, HashGlobal->setInitializer(HashConstant); } - Optional RawOS; + llvm::Optional OutputFile; + auto CloseOutputFile = llvm::make_scope_exit([&]() { + if (!OutputFile) + return; + if (auto E = OutputFile->keep()) { + diagnoseSync(Diags, DiagMutex, SourceLoc(), diag::error_opening_output, + OutputFilename, toString(std::move(E))); + } + }); if (!OutputFilename.empty()) { // Try to open the output file. Clobbering an existing file is fine. // Open in binary mode if we're doing binary output. - llvm::sys::fs::OpenFlags OSFlags = llvm::sys::fs::OF_None; - std::error_code EC; - RawOS.emplace(OutputFilename, EC, OSFlags); - - if (RawOS->has_error() || EC) { - diagnoseSync(Diags, DiagMutex, - SourceLoc(), diag::error_opening_output, - OutputFilename, EC.message()); - RawOS->clear_error(); + llvm::vfs::OutputConfig Config; + if (auto E = + Backend.createFile(OutputFilename, Config).moveInto(OutputFile)) { + diagnoseSync(Diags, DiagMutex, SourceLoc(), diag::error_opening_output, + OutputFilename, toString(std::move(E))); return true; } + if (Opts.OutputKind == IRGenOutputKind::LLVMAssemblyBeforeOptimization) { - Module->print(RawOS.value(), nullptr); + Module->print(*OutputFile, nullptr); return false; } } else { @@ -588,7 +597,7 @@ bool swift::performLLVM(const IRGenOptions &Opts, } performLLVMOptimizations(Opts, Module, TargetMachine, - RawOS ? &*RawOS : nullptr); + OutputFile ? &OutputFile->getOS() : nullptr); if (Stats) { if (DiagMutex) @@ -598,11 +607,11 @@ bool swift::performLLVM(const IRGenOptions &Opts, DiagMutex->unlock(); } - if (!RawOS) + if (OutputFilename.empty()) return false; - return compileAndWriteLLVM(Module, TargetMachine, Opts, Stats, Diags, *RawOS, - DiagMutex); + return compileAndWriteLLVM(Module, TargetMachine, Opts, Stats, Diags, + *OutputFile, DiagMutex); } bool swift::compileAndWriteLLVM(llvm::Module *module, @@ -1241,7 +1250,8 @@ struct LLVMCodeGenThreads { embedBitcode(IGM->getModule(), parent.irgen->Opts); performLLVM(parent.irgen->Opts, IGM->Context.Diags, diagMutex, IGM->ModuleHash, IGM->getModule(), IGM->TargetMachine.get(), - IGM->OutputFilename, IGM->Context.Stats); + IGM->OutputFilename, IGM->Context.getOutputBackend(), + IGM->Context.Stats); if (IGM->Context.Diags.hadAnyError()) return; } @@ -1359,10 +1369,9 @@ static void performParallelIRGeneration(IRGenDescriptor desc) { if (!targetMachine) continue; // Create the IR emitter. - IRGenModule *IGM = - new IRGenModule(irgen, std::move(targetMachine), nextSF, - desc.ModuleName, *OutputIter++, nextSF->getFilename(), - nextSF->getPrivateDiscriminator().str()); + IRGenModule *IGM = new IRGenModule( + irgen, std::move(targetMachine), nextSF, desc.ModuleName, *OutputIter++, + nextSF->getFilename(), nextSF->getPrivateDiscriminator().str()); IGMcreated = true; initLLVMModule(*IGM, *SILMod); @@ -1535,9 +1544,9 @@ GeneratedModule swift::performIRGeneration( const auto *SILModPtr = SILMod.get(); const auto &SILOpts = SILModPtr->getOptions(); auto desc = IRGenDescriptor::forWholeModule( - M, Opts, TBDOpts, SILOpts, SILModPtr->Types, std::move(SILMod), - ModuleName, PSPs, /*symsToEmit*/ None, parallelOutputFilenames, - outModuleHash); + M, Opts, TBDOpts, SILOpts, SILModPtr->Types, + std::move(SILMod), ModuleName, PSPs, /*symsToEmit*/ None, + parallelOutputFilenames, outModuleHash); if (Opts.shouldPerformIRGenerationInParallel() && !parallelOutputFilenames.empty() && @@ -1561,15 +1570,14 @@ performIRGeneration(FileUnit *file, const IRGenOptions &Opts, const auto *SILModPtr = SILMod.get(); const auto &SILOpts = SILModPtr->getOptions(); auto desc = IRGenDescriptor::forFile( - file, Opts, TBDOpts, SILOpts, SILModPtr->Types, std::move(SILMod), - ModuleName, PSPs, PrivateDiscriminator, /*symsToEmit*/ None, - outModuleHash); + file, Opts, TBDOpts, SILOpts, SILModPtr->Types, + std::move(SILMod), ModuleName, PSPs, PrivateDiscriminator, + /*symsToEmit*/ None, outModuleHash); return llvm::cantFail(file->getASTContext().evaluator(IRGenRequest{desc})); } -void -swift::createSwiftModuleObjectFile(SILModule &SILMod, StringRef Buffer, - StringRef OutputPath) { +void swift::createSwiftModuleObjectFile(SILModule &SILMod, StringRef Buffer, + StringRef OutputPath) { auto &Ctx = SILMod.getASTContext(); assert(!Ctx.hadError()); @@ -1620,7 +1628,7 @@ swift::createSwiftModuleObjectFile(SILModule &SILMod, StringRef Buffer, ASTSym->setAlignment(llvm::MaybeAlign(serialization::SWIFTMODULE_ALIGNMENT)); ::performLLVM(Opts, Ctx.Diags, nullptr, nullptr, IGM.getModule(), IGM.TargetMachine.get(), - OutputPath, Ctx.Stats); + OutputPath, Ctx.getOutputBackend(), Ctx.Stats); } bool swift::performLLVM(const IRGenOptions &Opts, ASTContext &Ctx, @@ -1636,8 +1644,8 @@ bool swift::performLLVM(const IRGenOptions &Opts, ASTContext &Ctx, embedBitcode(Module, Opts); if (::performLLVM(Opts, Ctx.Diags, nullptr, nullptr, Module, - TargetMachine.get(), - OutputFilename, Ctx.Stats)) + TargetMachine.get(), OutputFilename, Ctx.getOutputBackend(), + Ctx.Stats)) return true; return false; } diff --git a/lib/IRGen/IRGenModule.cpp b/lib/IRGen/IRGenModule.cpp index 5537135d163ba..75ee76b427684 100644 --- a/lib/IRGen/IRGenModule.cpp +++ b/lib/IRGen/IRGenModule.cpp @@ -2034,7 +2034,6 @@ bool swift::writeEmptyOutputFilesFor( const ASTContext &Context, std::vector& ParallelOutputFilenames, const IRGenOptions &IRGenOpts) { - for (auto fileName : ParallelOutputFilenames) { // The first output file, was use for genuine output. if (fileName == ParallelOutputFilenames[0]) diff --git a/lib/Immediate/Immediate.cpp b/lib/Immediate/Immediate.cpp index 74195af2dff94..5e20fe1485b25 100644 --- a/lib/Immediate/Immediate.cpp +++ b/lib/Immediate/Immediate.cpp @@ -248,7 +248,8 @@ int swift::RunImmediately(CompilerInstance &CI, performLLVM(IRGenOpts, Context.Diags, /*diagMutex*/ nullptr, /*hash*/ nullptr, GenModule.getModule(), GenModule.getTargetMachine(), - PSPs.OutputFilename, Context.Stats); + PSPs.OutputFilename, CI.getOutputBackend(), + Context.Stats); if (Context.hadError()) return -1; diff --git a/lib/SymbolGraphGen/SymbolGraphGen.cpp b/lib/SymbolGraphGen/SymbolGraphGen.cpp index 8256c84d89291..ef7850f84e26f 100644 --- a/lib/SymbolGraphGen/SymbolGraphGen.cpp +++ b/lib/SymbolGraphGen/SymbolGraphGen.cpp @@ -40,11 +40,13 @@ int serializeSymbolGraph(SymbolGraph &SG, SmallString<1024> OutputPath(Options.OutputDir); llvm::sys::path::append(OutputPath, FileName); - return withOutputFile(SG.M.getASTContext().Diags, OutputPath, [&](raw_ostream &OS) { - llvm::json::OStream J(OS, Options.PrettyPrint ? 2 : 0); - SG.serialize(J); - return false; - }); + return withOutputFile( + SG.M.getASTContext().Diags, SG.M.getASTContext().getOutputBackend(), + OutputPath, [&](raw_ostream &OS) { + llvm::json::OStream J(OS, Options.PrettyPrint ? 2 : 0); + SG.serialize(J); + return false; + }); } } // end anonymous namespace diff --git a/tools/sil-llvm-gen/SILLLVMGen.cpp b/tools/sil-llvm-gen/SILLLVMGen.cpp index 9b9ff651f456a..b2ca3a7e6093a 100644 --- a/tools/sil-llvm-gen/SILLLVMGen.cpp +++ b/tools/sil-llvm-gen/SILLLVMGen.cpp @@ -34,6 +34,7 @@ #include "swift/Serialization/SerializedSILLoader.h" #include "swift/Strings.h" #include "swift/Subsystems.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/Statistic.h" #include "llvm/IR/Module.h" #include "llvm/Support/CommandLine.h" @@ -43,6 +44,7 @@ #include "llvm/Support/Path.h" #include "llvm/Support/Signals.h" #include "llvm/Support/TargetSelect.h" +#include "llvm/Support/VirtualOutputBackends.h" #include using namespace swift; @@ -181,14 +183,19 @@ int main(int argc, char **argv) { return 1; } - std::error_code EC; - llvm::raw_fd_ostream outStream(OutputFilename, EC, llvm::sys::fs::OF_None); - if (outStream.has_error() || EC) { + llvm::vfs::OnDiskOutputBackend Backend; + auto outFile = Backend.createFile(OutputFilename); + if (!outFile) { CI.getDiags().diagnose(SourceLoc(), diag::error_opening_output, - OutputFilename, EC.message()); - outStream.clear_error(); + OutputFilename, toString(outFile.takeError())); return 1; } + auto closeFile = llvm::make_scope_exit([&]() { + if (auto E = outFile->keep()) { + CI.getDiags().diagnose(SourceLoc(), diag::error_opening_output, + OutputFilename, toString(std::move(E))); + } + }); auto *mod = CI.getMainModule(); assert(mod->getFiles().size() == 1); @@ -205,20 +212,20 @@ int main(int argc, char **argv) { mod, Opts, TBDOpts, SILOpts, SILTypes, /*SILMod*/ nullptr, moduleName, PSPs); } else { - return IRGenDescriptor::forFile(mod->getFiles()[0], Opts, TBDOpts, - SILOpts, SILTypes, /*SILMod*/ nullptr, - moduleName, PSPs, /*discriminator*/ ""); + return IRGenDescriptor::forFile( + mod->getFiles()[0], Opts, TBDOpts, SILOpts, SILTypes, + /*SILMod*/ nullptr, moduleName, PSPs, /*discriminator*/ ""); } }; auto &eval = CI.getASTContext().evaluator; auto desc = getDescriptor(); - desc.out = &outStream; + desc.out = &outFile->getOS(); auto generatedMod = llvm::cantFail(eval(OptimizedIRRequest{desc})); if (!generatedMod) return 1; return compileAndWriteLLVM(generatedMod.getModule(), generatedMod.getTargetMachine(), Opts, - CI.getStatsReporter(), CI.getDiags(), outStream); + CI.getStatsReporter(), CI.getDiags(), *outFile); } diff --git a/tools/swift-dependency-tool/swift-dependency-tool.cpp b/tools/swift-dependency-tool/swift-dependency-tool.cpp index cd4854b2fc9df..5f935f73409d7 100644 --- a/tools/swift-dependency-tool/swift-dependency-tool.cpp +++ b/tools/swift-dependency-tool/swift-dependency-tool.cpp @@ -18,6 +18,7 @@ #include "swift/Basic/LLVMInitialize.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackends.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" @@ -208,6 +209,7 @@ int main(int argc, char *argv[]) { SourceManager sourceMgr; DiagnosticEngine diags(sourceMgr); + llvm::vfs::OnDiskOutputBackend outputBackend; switch (options::Action) { case ActionType::None: { @@ -224,7 +226,7 @@ int main(int argc, char *argv[]) { } bool hadError = - withOutputFile(diags, options::OutputFilename, + withOutputFile(diags, outputBackend, options::OutputFilename, [&](llvm::raw_pwrite_stream &out) { out << "# Fine-grained v0\n"; llvm::yaml::Output yamlWriter(out); @@ -256,7 +258,7 @@ int main(int argc, char *argv[]) { } if (writeFineGrainedDependencyGraphToPath( - diags, options::OutputFilename, fg)) { + diags, outputBackend, options::OutputFilename, fg)) { llvm::errs() << "Failed to write binary swiftdeps\n"; return 1; } diff --git a/unittests/Driver/MockingFineGrainedDependencyGraphs.cpp b/unittests/Driver/MockingFineGrainedDependencyGraphs.cpp index 327f871af8c50..1ce0f2fe3d534 100644 --- a/unittests/Driver/MockingFineGrainedDependencyGraphs.cpp +++ b/unittests/Driver/MockingFineGrainedDependencyGraphs.cpp @@ -15,6 +15,7 @@ #include "swift/AST/DiagnosticEngine.h" #include "swift/Basic/ReferenceDependencyKeys.h" #include "swift/Basic/SourceManager.h" +#include "llvm/Support/VirtualOutputBackends.h" #include #include @@ -48,12 +49,13 @@ mocking_fine_grained_dependency_graphs::getChangesForSimulatedLoad( SourceManager sm; DiagnosticEngine diags(sm); + llvm::vfs::OnDiskOutputBackend backend; auto sfdg = UnitTestSourceFileDepGraphFactory( hadCompilationError, swiftDeps, interfaceHash, g.emitFineGrainedDependencyDotFileAfterEveryImport, - dependencyDescriptions, diags) + dependencyDescriptions, diags, backend) .construct(); return g.loadFromSourceFileDepGraph(cmd, sfdg, diags); diff --git a/unittests/Driver/UnitTestSourceFileDepGraphFactory.h b/unittests/Driver/UnitTestSourceFileDepGraphFactory.h index 2a53bf89521c2..90f32a1239273 100644 --- a/unittests/Driver/UnitTestSourceFileDepGraphFactory.h +++ b/unittests/Driver/UnitTestSourceFileDepGraphFactory.h @@ -13,6 +13,7 @@ #define UnitTestSourceFileDepGraphFactory_h #include "swift/AST/AbstractSourceFileDepGraphFactory.h" +#include "llvm/Support/VirtualOutputBackend.h" namespace swift { namespace fine_grained_dependencies { @@ -30,10 +31,10 @@ class UnitTestSourceFileDepGraphFactory bool hadCompilationError, StringRef swiftDeps, Fingerprint fileFingerprint, bool emitDotFileAfterConstruction, const DependencyDescriptions &dependencyDescriptions, - DiagnosticEngine &diags) + DiagnosticEngine &diags, llvm::vfs::OutputBackend &backend) : AbstractSourceFileDepGraphFactory( hadCompilationError, swiftDeps, fileFingerprint, - emitDotFileAfterConstruction, diags), + emitDotFileAfterConstruction, diags, backend), dependencyDescriptions(dependencyDescriptions) {} ~UnitTestSourceFileDepGraphFactory() override = default; From f463a3cdb2e900ac3c9887e4c980b3afe4b22bfb Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 24 Jan 2023 15:02:57 -0800 Subject: [PATCH 02/11] Add an option to verify the deterministic output of swift compiler Add option to test swift compiler determinism by running the job twice and compare the output produced. --- include/swift/AST/DiagnosticsFrontend.def | 10 ++++ include/swift/Frontend/Frontend.h | 8 +++ include/swift/Frontend/FrontendOptions.h | 3 + include/swift/Option/FrontendOptions.td | 4 ++ .../ArgsToFrontendOptionsConverter.cpp | 1 + lib/Frontend/Frontend.cpp | 16 ++++- lib/FrontendTool/FrontendTool.cpp | 59 +++++++++++++++++++ test/Frontend/output_determinism_check.swift | 12 ++++ 8 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 test/Frontend/output_determinism_check.swift diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def index 12b108310ad12..2389925a42f24 100644 --- a/include/swift/AST/DiagnosticsFrontend.def +++ b/include/swift/AST/DiagnosticsFrontend.def @@ -475,6 +475,16 @@ REMARK(interface_file_backup_used,none, WARNING(warn_flag_deprecated,none, "flag '%0' is deprecated", (StringRef)) +// Output deterministic check +ERROR(error_nondeterministic_output,none, + "output file '%0' is not deterministic: hash value '%1' vs '%2' between two compilations", + (StringRef, StringRef, StringRef)) +ERROR(error_output_missing,none, + "output file '%0' is missing from %select{first|second}1 compilation for deterministic check", + (StringRef, /*SecondRun=*/bool)) +REMARK(matching_output_produced,none, + "produced matching output file '%0' for deterministic check: hash '%1'", (StringRef, StringRef)) + // Dependency Verifier Diagnostics ERROR(missing_member_dependency,none, "expected " diff --git a/include/swift/Frontend/Frontend.h b/include/swift/Frontend/Frontend.h index b179557dc4883..47cc2f9f37d17 100644 --- a/include/swift/Frontend/Frontend.h +++ b/include/swift/Frontend/Frontend.h @@ -45,6 +45,8 @@ #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/SetVector.h" #include "llvm/Option/ArgList.h" +#include "llvm/Support/BLAKE3.h" +#include "llvm/Support/HashingOutputBackend.h" #include "llvm/Support/Host.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/VirtualOutputBackend.h" @@ -461,6 +463,10 @@ class CompilerInstance { /// Virtual OutputBackend. llvm::IntrusiveRefCntPtr TheOutputBackend = nullptr; + /// The verification output backend. + using HashBackendTy = llvm::vfs::HashingOutputBackend; + llvm::IntrusiveRefCntPtr HashBackend; + mutable ModuleDecl *MainModule = nullptr; SerializedModuleLoaderBase *DefaultSerializedLoader = nullptr; MemoryBufferSerializedModuleLoader *MemoryBufferLoader = nullptr; @@ -518,6 +524,8 @@ class CompilerInstance { setOutputBackend(llvm::IntrusiveRefCntPtr Backend) { TheOutputBackend = std::move(Backend); } + using HashingBackendPtrTy = llvm::IntrusiveRefCntPtr; + HashingBackendPtrTy getHashingBackend() { return HashBackend; } ASTContext &getASTContext() { return *Context; } const ASTContext &getASTContext() const { return *Context; } diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h index b1aa4e33db151..76fda8039434e 100644 --- a/include/swift/Frontend/FrontendOptions.h +++ b/include/swift/Frontend/FrontendOptions.h @@ -498,6 +498,9 @@ class FrontendOptions { /// to encode the actual paths into the .swiftmodule file. PathObfuscator serializedPathObfuscator; + /// Whether to run the job twice to check determinism. + bool DeterministicCheck = false; + /// Avoid printing actual module content into the ABI descriptor file. /// This should only be used as a workaround when emitting ABI descriptor files /// crashes the compiler. diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index 0f2713531c53e..449123a44dabf 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -1176,6 +1176,10 @@ def enable_emit_generic_class_ro_t_list : HelpText<"Enable emission of a section with references to class_ro_t of " "generic class patterns">; +def enable_swift_deterministic_check : + Flag<["-"], "enable-swift-deterministic-check">, + HelpText<"Check swift compiler output determinisim by run it twice">; + def experimental_spi_only_imports : Flag<["-"], "experimental-spi-only-imports">, HelpText<"Enable use of @_spiOnly imports">; diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index 64fa849fb6ee2..e345a5e105ae0 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -345,6 +345,7 @@ bool ArgsToFrontendOptionsConverter::convert( Opts.serializedPathObfuscator.addMapping(SplitMap.first, SplitMap.second); } Opts.emptyABIDescriptor = Args.hasArg(OPT_empty_abi_descriptor); + Opts.DeterministicCheck = Args.hasArg(OPT_enable_swift_deterministic_check); return false; } diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index a7f138682becc..7e6d3c0df393d 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -398,9 +398,21 @@ void CompilerInstance::setupDependencyTrackerIfNeeded() { void CompilerInstance::setupOutputBackend() { // Skip if output backend is not setup, default to OnDiskOutputBackend. - if (!TheOutputBackend) + if (TheOutputBackend) + return; + + TheOutputBackend = + llvm::makeIntrusiveRefCnt(); + + // Setup verification backend. + // Create a mirroring outputbackend to produce hash for output files. + // We cannot skip disk here since swift compiler is expecting to read back + // some output file in later stages. + if (Invocation.getFrontendOptions().DeterministicCheck) { + HashBackend = llvm::makeIntrusiveRefCnt(); TheOutputBackend = - llvm::makeIntrusiveRefCnt(); + llvm::vfs::makeMirroringOutputBackend(TheOutputBackend, HashBackend); + } } bool CompilerInstance::setup(const CompilerInvocation &Invoke, diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index b49207a7bf36b..1737e1a237b27 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -71,7 +71,10 @@ #include "clang/Lex/Preprocessor.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" @@ -82,6 +85,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Path.h" #include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputBackends.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Support/FileSystem.h" @@ -2342,6 +2346,23 @@ int swift::performFrontend(ArrayRef Args, PDC.setSuppressOutput(true); } + CompilerInstance::HashingBackendPtrTy HashBackend = nullptr; + if (Invocation.getFrontendOptions().DeterministicCheck) { + // Setup a verfication instance to run. + std::unique_ptr VerifyInstance = + std::make_unique(); + std::string InstanceSetupError; + // This should not fail because it passed already. + (void)VerifyInstance->setup(Invocation, InstanceSetupError); + + // Run the first time without observer and discard return value; + int ReturnValueTest = 0; + (void)performCompile(*VerifyInstance, ReturnValueTest, + /*observer*/ nullptr); + // Get the hashing output backend and free the compiler instance. + HashBackend = VerifyInstance->getHashingBackend(); + } + int ReturnValue = 0; bool HadError = performCompile(*Instance, ReturnValue, observer); @@ -2356,6 +2377,44 @@ int swift::performFrontend(ArrayRef Args, } } + if (Invocation.getFrontendOptions().DeterministicCheck) { + // Collect all output files. + auto ReHashBackend = Instance->getHashingBackend(); + std::set AllOutputs; + llvm::for_each(HashBackend->outputFiles(), [&](StringRef F) { + AllOutputs.insert(F.str()); + }); + llvm::for_each(ReHashBackend->outputFiles(), [&](StringRef F) { + AllOutputs.insert(F.str()); + }); + + DiagnosticEngine &diags = Instance->getDiags(); + for (auto &Filename : AllOutputs) { + auto O1 = HashBackend->getHashValueForFile(Filename); + if (!O1) { + diags.diagnose(SourceLoc(), diag::error_output_missing, Filename, + /*SecondRun=*/false); + HadError = true; + continue; + } + auto O2 = ReHashBackend->getHashValueForFile(Filename); + if (!O2) { + diags.diagnose(SourceLoc(), diag::error_output_missing, Filename, + /*SecondRun=*/true); + HadError = true; + continue; + } + if (*O1 != *O2) { + diags.diagnose(SourceLoc(), diag::error_nondeterministic_output, + Filename, *O1, *O2); + HadError = true; + continue; + } + diags.diagnose(SourceLoc(), diag::matching_output_produced, Filename, + *O1); + } + } + auto r = finishDiagProcessing(HadError ? 1 : ReturnValue, verifierEnabled); if (auto *StatsReporter = Instance->getStatsReporter()) StatsReporter->noteCurrentProcessExitStatus(r); diff --git a/test/Frontend/output_determinism_check.swift b/test/Frontend/output_determinism_check.swift new file mode 100644 index 0000000000000..f20196645fe80 --- /dev/null +++ b/test/Frontend/output_determinism_check.swift @@ -0,0 +1,12 @@ +// RUN: %empty-directory(%t) +// RUN: %target-swift-frontend -module-name test -emit-module -o %t/test.swiftmodule -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=MODULE_OUTPUT +// RUN: %target-swift-frontend -module-name test -emit-sib -o %t/test.sib -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=SIB_OUTPUT + +/// object files are "not" deterministic because the second run going to match the mod hash and skip code generation. +// RUN: not %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT + +// MODULE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftmodule' +// SIB_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.sib' +// OBJECT_OUTPUT: error: output file '{{.*}}{{/|\\}}test.o' is missing from second run + +public var x = 1 From 65dd0b60fad4cb133202292a7636e3e1f7b8d629 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 26 Jan 2023 13:18:10 -0800 Subject: [PATCH 03/11] Add an option to force compiling object file output Add an option to skip module hash checking that checks if swift compiler can skip object file generation because it is going to produce the same result. Always generation object file output when flag is used so we can check for output determinism. --- include/swift/AST/IRGenOptions.h | 5 ++++- include/swift/Option/FrontendOptions.td | 3 +++ lib/Frontend/CompilerInvocation.cpp | 1 + lib/IRGen/IRGen.cpp | 2 +- test/Frontend/output_determinism_check.swift | 8 ++++++-- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/include/swift/AST/IRGenOptions.h b/include/swift/AST/IRGenOptions.h index 0c1e8a67c32eb..406e4c9f0f593 100644 --- a/include/swift/AST/IRGenOptions.h +++ b/include/swift/AST/IRGenOptions.h @@ -318,6 +318,9 @@ class IRGenOptions { /// Print the LLVM inline tree at the end of the LLVM pass pipeline. unsigned PrintInlineTree : 1; + /// Always recompile the output even the module hash might match. + unsigned AlwaysCompile : 1; + /// Whether we should embed the bitcode file. IRGenEmbedMode EmbedMode : 2; @@ -481,7 +484,7 @@ class IRGenOptions { DisableLLVMOptzns(false), DisableSwiftSpecificLLVMOptzns(false), Playground(false), EmitStackPromotionChecks(false), UseSingleModuleLLVMEmission(false), - FunctionSections(false), PrintInlineTree(false), + FunctionSections(false), PrintInlineTree(false), AlwaysCompile(false), EmbedMode(IRGenEmbedMode::None), LLVMLTOKind(IRGenLLVMLTOKind::None), SwiftAsyncFramePointer(SwiftAsyncFramePointerKind::Auto), HasValueNamesSetting(false), ValueNames(false), diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index 449123a44dabf..988b38d9224b3 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -1179,6 +1179,9 @@ def enable_emit_generic_class_ro_t_list : def enable_swift_deterministic_check : Flag<["-"], "enable-swift-deterministic-check">, HelpText<"Check swift compiler output determinisim by run it twice">; +def always_compile_output_files : + Flag<["-"], "always-compile-output-files">, + HelpText<"Always compile output files even it might not change the results">; def experimental_spi_only_imports : Flag<["-"], "experimental-spi-only-imports">, diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index e464b57dc052f..3e934851d9f7d 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -2364,6 +2364,7 @@ static bool ParseIRGenArgs(IRGenOptions &Opts, ArgList &Args, Opts.UseProfile = ProfileUse ? ProfileUse->getValue() : ""; Opts.PrintInlineTree |= Args.hasArg(OPT_print_llvm_inline_tree); + Opts.AlwaysCompile |= Args.hasArg(OPT_always_compile_output_files); Opts.EnableDynamicReplacementChaining |= Args.hasArg(OPT_enable_dynamic_replacement_chaining); diff --git a/lib/IRGen/IRGen.cpp b/lib/IRGen/IRGen.cpp index 2d546c3bbd859..1ee21934b1d3f 100644 --- a/lib/IRGen/IRGen.cpp +++ b/lib/IRGen/IRGen.cpp @@ -556,7 +556,7 @@ bool swift::performLLVM(const IRGenOptions &Opts, ArrayRef HashData(reinterpret_cast(&hash), sizeof(hash)); if (Opts.OutputKind == IRGenOutputKind::ObjectFile && - !Opts.PrintInlineTree && + !Opts.PrintInlineTree && !Opts.AlwaysCompile && !needsRecompile(OutputFilename, HashData, HashGlobal, DiagMutex)) { // The llvm IR did not change. We don't need to re-create the object file. return false; diff --git a/test/Frontend/output_determinism_check.swift b/test/Frontend/output_determinism_check.swift index f20196645fe80..7bc0297a2a7ac 100644 --- a/test/Frontend/output_determinism_check.swift +++ b/test/Frontend/output_determinism_check.swift @@ -3,10 +3,14 @@ // RUN: %target-swift-frontend -module-name test -emit-sib -o %t/test.sib -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=SIB_OUTPUT /// object files are "not" deterministic because the second run going to match the mod hash and skip code generation. -// RUN: not %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT +// RUN: not %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=OBJECT_MISMATCH +/// object files should match when forcing object generation. +// RUN: %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check -always-compile-output-files 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT // MODULE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftmodule' // SIB_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.sib' -// OBJECT_OUTPUT: error: output file '{{.*}}{{/|\\}}test.o' is missing from second run +// OBJECT_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.o' +// OBJECT_MISMATCH: error: output file '{{.*}}{{/|\\}}test.o' is missing from second compilation for deterministic check public var x = 1 +public func test() {} From 928839841dd0e6247aaf261f595105f02efbede3 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 31 Jan 2023 16:31:23 -0800 Subject: [PATCH 04/11] Support deterministic checking in explicit module build --- lib/Frontend/ModuleInterfaceLoader.cpp | 30 ++++++++++++-------- test/Frontend/output_determinism_check.swift | 11 +++++++ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index 2aeb3d179068f..b1ea5f75212b7 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -1400,19 +1400,25 @@ bool ModuleInterfaceLoader::buildExplicitSwiftModuleFromSwiftInterface( ArrayRef CompiledCandidates, DependencyTracker *tracker) { - // First, check if the expected output already exists and possibly up-to-date w.r.t. - // all of the dependencies it was built with. If so, early exit. - UpToDateModuleCheker checker(Instance.getASTContext(), - RequireOSSAModules_t(Instance.getSILOptions())); - ModuleRebuildInfo rebuildInfo; - SmallVector allDeps; - std::unique_ptr moduleBuffer; - if (checker.swiftModuleIsUpToDate(outputPath, rebuildInfo, allDeps, moduleBuffer)) { - if (Instance.getASTContext().LangOpts.EnableSkipExplicitInterfaceModuleBuildRemarks) { - Instance.getDiags().diagnose(SourceLoc(), - diag::explicit_interface_build_skipped, outputPath); + if (!Instance.getInvocation().getIRGenOptions().AlwaysCompile) { + // First, check if the expected output already exists and possibly + // up-to-date w.r.t. all of the dependencies it was built with. If so, early + // exit. + UpToDateModuleCheker checker( + Instance.getASTContext(), + RequireOSSAModules_t(Instance.getSILOptions())); + ModuleRebuildInfo rebuildInfo; + SmallVector allDeps; + std::unique_ptr moduleBuffer; + if (checker.swiftModuleIsUpToDate(outputPath, rebuildInfo, allDeps, + moduleBuffer)) { + if (Instance.getASTContext() + .LangOpts.EnableSkipExplicitInterfaceModuleBuildRemarks) { + Instance.getDiags().diagnose( + SourceLoc(), diag::explicit_interface_build_skipped, outputPath); + } + return false; } - return false; } // Read out the compiler version. diff --git a/test/Frontend/output_determinism_check.swift b/test/Frontend/output_determinism_check.swift index 7bc0297a2a7ac..b9d2eecbf5e80 100644 --- a/test/Frontend/output_determinism_check.swift +++ b/test/Frontend/output_determinism_check.swift @@ -7,10 +7,21 @@ /// object files should match when forcing object generation. // RUN: %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check -always-compile-output-files 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT +/// Explicit module build. Check building swiftmodule from interface file. +/// TODO: Implicit module build use a different compiler instance so it doesn't support checking yet. +// RUN: %target-swift-frontend -typecheck -emit-module-interface-path %t/test.swiftinterface %s -O -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=INTERFACE_OUTPUT +/// Hit cache and not emit the second time. +// RUN: rm %t/test.swiftmodule +// RUN: not %target-swift-frontend -compile-module-from-interface %t/test.swiftinterface -explicit-interface-module-build -o %t/test.swiftmodule -enable-swift-deterministic-check 2>&1 | %FileCheck --check-prefix=MODULE_MISMATCH %s +/// Force swiftmodule generation. +// RUN: %target-swift-frontend -compile-module-from-interface %t/test.swiftinterface -explicit-interface-module-build -o %t/test.swiftmodule -enable-swift-deterministic-check -always-compile-output-files 2>&1 | %FileCheck --check-prefix=MODULE_OUTPUT %s + // MODULE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftmodule' // SIB_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.sib' // OBJECT_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.o' // OBJECT_MISMATCH: error: output file '{{.*}}{{/|\\}}test.o' is missing from second compilation for deterministic check +// INTERFACE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftinterface' +// MODULE_MISMATCH: error: output file '{{.*}}{{/|\\}}test.swiftmodule' is missing from second compilation for deterministic check public var x = 1 public func test() {} From a98f7d91efb67de822e07a35ad2ba5e121b6d65a Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 2 Feb 2023 10:54:15 -0800 Subject: [PATCH 05/11] Virtualize depscan output to allow deterministic check Batch mode depscan also gets diagnostics for output errors. --- lib/DependencyScan/ScanDependencies.cpp | 56 +++++++++++--------- test/Frontend/output_determinism_check.swift | 2 + 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 9011c428777fa..b9d27a2ee5c27 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -20,6 +20,7 @@ #include "swift/AST/DiagnosticsFrontend.h" #include "swift/AST/DiagnosticsSema.h" #include "swift/AST/DiagnosticsDriver.h" +#include "swift/AST/FileSystem.h" #include "swift/AST/Module.h" #include "swift/AST/ModuleDependencies.h" #include "swift/AST/ModuleLoader.h" @@ -45,6 +46,7 @@ #include "llvm/Support/StringSaver.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" +#include "llvm/Support/raw_ostream.h" #include #include #include @@ -1393,15 +1395,6 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { ASTContext &Context = instance.getASTContext(); const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); - std::error_code EC; - llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::OF_None); - if (out.has_error() || EC) { - Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, - EC.message()); - out.clear_error(); - return true; - } - // `-scan-dependencies` invocations use a single new instance // of a module cache SwiftDependencyScanningService service; @@ -1427,8 +1420,13 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { return true; auto dependencies = std::move(*dependenciesOrErr); - // Write out the JSON description. - writeJSON(out, dependencies); + if (withOutputFile(Context.Diags, instance.getOutputBackend(), path, + [&](llvm::raw_pwrite_stream &os) { + writeJSON(os, dependencies); + return false; + })) + return true; + // This process succeeds regardless of whether any errors occurred. // FIXME: We shouldn't need this, but it's masking bugs in our scanning // logic where we don't create a fresh context when scanning Swift interfaces @@ -1441,20 +1439,12 @@ bool swift::dependencies::prescanDependencies(CompilerInstance &instance) { ASTContext &Context = instance.getASTContext(); const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); - std::error_code EC; - llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::OF_None); // `-scan-dependencies` invocations use a single new instance // of a module cache SwiftDependencyScanningService singleUseService; ModuleDependenciesCache cache(singleUseService, instance.getMainModule()->getNameStr().str(), instance.getInvocation().getModuleScanningHash()); - if (out.has_error() || EC) { - Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, - EC.message()); - out.clear_error(); - return true; - } // Execute import prescan, and write JSON output to the output stream auto importSetOrErr = performModulePrescan(instance); @@ -1463,7 +1453,13 @@ bool swift::dependencies::prescanDependencies(CompilerInstance &instance) { auto importSet = std::move(*importSetOrErr); // Serialize and output main module dependencies only and exit. - writePrescanJSON(out, importSet); + if (withOutputFile(Context.Diags, instance.getOutputBackend(), path, + [&](llvm::raw_pwrite_stream &os) { + writePrescanJSON(os, importSet); + return false; + })) + return true; + // This process succeeds regardless of whether any errors occurred. // FIXME: We shouldn't need this, but it's masking bugs in our scanning // logic where we don't create a fresh context when scanning Swift interfaces @@ -1499,12 +1495,16 @@ bool swift::dependencies::batchScanDependencies( auto iresults = batchScanResults.cbegin(); for (; ientries != batchInput->end() and iresults != batchScanResults.end(); ++ientries, ++iresults) { - std::error_code EC; - llvm::raw_fd_ostream out((*ientries).outputPath, EC, llvm::sys::fs::OF_None); if ((*iresults).getError()) return true; - writeJSON(out, **iresults); + if (withOutputFile(instance.getASTContext().Diags, + instance.getOutputBackend(), (*ientries).outputPath, + [&](llvm::raw_pwrite_stream &os) { + writeJSON(os, **iresults); + return false; + })) + return true; } return false; } @@ -1534,12 +1534,16 @@ bool swift::dependencies::batchPrescanDependencies( for (; ientries != batchInput->end() and iresults != batchPrescanResults.end(); ++ientries, ++iresults) { - std::error_code EC; - llvm::raw_fd_ostream out((*ientries).outputPath, EC, llvm::sys::fs::OF_None); if ((*iresults).getError()) return true; - writePrescanJSON(out, **iresults); + if (withOutputFile(instance.getASTContext().Diags, + instance.getOutputBackend(), (*ientries).outputPath, + [&](llvm::raw_pwrite_stream &os) { + writePrescanJSON(os, **iresults); + return false; + })) + return true; } return false; } diff --git a/test/Frontend/output_determinism_check.swift b/test/Frontend/output_determinism_check.swift index b9d2eecbf5e80..184c703a15c9d 100644 --- a/test/Frontend/output_determinism_check.swift +++ b/test/Frontend/output_determinism_check.swift @@ -8,6 +8,7 @@ // RUN: %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check -always-compile-output-files 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT /// Explicit module build. Check building swiftmodule from interface file. +// RUN: %target-swift-frontend -scan-dependencies -module-name test -o %t/test.json %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=DEPSCAN_OUTPUT /// TODO: Implicit module build use a different compiler instance so it doesn't support checking yet. // RUN: %target-swift-frontend -typecheck -emit-module-interface-path %t/test.swiftinterface %s -O -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=INTERFACE_OUTPUT /// Hit cache and not emit the second time. @@ -20,6 +21,7 @@ // SIB_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.sib' // OBJECT_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.o' // OBJECT_MISMATCH: error: output file '{{.*}}{{/|\\}}test.o' is missing from second compilation for deterministic check +// DEPSCAN_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.json' // INTERFACE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftinterface' // MODULE_MISMATCH: error: output file '{{.*}}{{/|\\}}test.swiftmodule' is missing from second compilation for deterministic check From 60b06d5470edd396e66901ddd9c3d37b5d68c053 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 7 Feb 2023 16:15:37 -0800 Subject: [PATCH 06/11] Capture clang importer output in virtual output --- include/swift/ClangImporter/ClangImporter.h | 22 ++++++--- lib/ClangImporter/ClangImporter.cpp | 46 ++++++++++++------- lib/DriverTool/swift_api_digester_main.cpp | 7 ++- lib/FrontendTool/FrontendTool.cpp | 7 ++- .../ClangImporter/ClangImporterTests.cpp | 5 +- 5 files changed, 60 insertions(+), 27 deletions(-) diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h index 3bbeb8f4d1e2e..919aff75a2cb5 100644 --- a/include/swift/ClangImporter/ClangImporter.h +++ b/include/swift/ClangImporter/ClangImporter.h @@ -28,6 +28,7 @@ namespace llvm { template class function_ref; namespace vfs { class FileSystem; + class OutputBackend; } } @@ -394,8 +395,9 @@ class ClangImporter final : public ClangModuleLoader { /// replica. /// /// \sa clang::GeneratePCHAction - bool emitBridgingPCH(StringRef headerPath, - StringRef outputPCHPath); + bool + emitBridgingPCH(llvm::IntrusiveRefCntPtr backend, + StringRef headerPath, StringRef outputPCHPath); /// Returns true if a clang CompilerInstance can successfully read in a PCH, /// assuming it exists, with the current options. This can be used to find out @@ -406,16 +408,21 @@ class ClangImporter final : public ClangModuleLoader { /// module map into the replica and emits a PCM file for one of the modules it /// declares. Delegates to clang for everything except construction of the /// replica. - bool emitPrecompiledModule(StringRef moduleMapPath, StringRef moduleName, - StringRef outputPath); + bool emitPrecompiledModule( + llvm::IntrusiveRefCntPtr backend, + StringRef moduleMapPath, StringRef moduleName, StringRef outputPath); /// Makes a temporary replica of the ClangImporter's CompilerInstance and /// dumps information about a PCM file (assumed to be generated by -emit-pcm /// or in the Swift module cache). Delegates to clang for everything except /// construction of the replica. - bool dumpPrecompiledModule(StringRef modulePath, StringRef outputPath); + bool dumpPrecompiledModule( + llvm::IntrusiveRefCntPtr backend, + StringRef modulePath, StringRef outputPath); - bool runPreprocessor(StringRef inputPath, StringRef outputPath); + bool + runPreprocessor(llvm::IntrusiveRefCntPtr backend, + StringRef inputPath, StringRef outputPath); const clang::Module *getClangOwningModule(ClangNode Node) const; bool hasTypedef(const clang::Decl *typeDecl) const; @@ -503,7 +510,8 @@ class ClangImporter final : public ClangModuleLoader { Optional getOrCreatePCH(const ClangImporterOptions &ImporterOptions, - StringRef SwiftPCHHash); + StringRef SwiftPCHHash, + llvm::IntrusiveRefCntPtr Backend); Optional /// \param isExplicit true if the PCH filename was passed directly /// with -import-objc-header option. diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index 395b2a0d2d58b..4a9f9e641012e 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -66,6 +66,7 @@ #include "clang/Sema/Sema.h" #include "clang/Serialization/ASTReader.h" #include "clang/Serialization/ASTWriter.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Support/CrashRecoveryContext.h" @@ -74,6 +75,7 @@ #include "llvm/Support/Memory.h" #include "llvm/Support/Path.h" #include "llvm/Support/VirtualFileSystem.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include #include @@ -192,7 +194,8 @@ namespace { Importer.addSearchPath(path, /*isFramework*/false, /*isSystem=*/false); } - auto PCH = Importer.getOrCreatePCH(ImporterOpts, SwiftPCHHash); + auto PCH = Importer.getOrCreatePCH(ImporterOpts, SwiftPCHHash, + Ctx.getOutputBackend().clone()); if (PCH.has_value()) { Impl.getClangInstance()->getPreprocessorOpts().ImplicitPCHInclude = PCH.value(); @@ -940,10 +943,9 @@ ClangImporter::getPCHFilename(const ClangImporterOptions &ImporterOptions, return PCHFilename.str().str(); } - -Optional -ClangImporter::getOrCreatePCH(const ClangImporterOptions &ImporterOptions, - StringRef SwiftPCHHash) { +Optional ClangImporter::getOrCreatePCH( + const ClangImporterOptions &ImporterOptions, StringRef SwiftPCHHash, + llvm::IntrusiveRefCntPtr Backend) { bool isExplicit; auto PCHFilename = getPCHFilename(ImporterOptions, SwiftPCHHash, isExplicit); @@ -959,8 +961,9 @@ ClangImporter::getOrCreatePCH(const ClangImporterOptions &ImporterOptions, << EC.message(); return None; } - auto FailedToEmit = emitBridgingPCH(ImporterOptions.BridgingHeader, - PCHFilename.value()); + auto FailedToEmit = + emitBridgingPCH(std::move(Backend), ImporterOptions.BridgingHeader, + PCHFilename.value()); if (FailedToEmit) { return None; } @@ -1697,9 +1700,9 @@ ClangImporter::cloneCompilerInstanceForPrecompiling() { return clonedInstance; } -bool -ClangImporter::emitBridgingPCH(StringRef headerPath, - StringRef outputPCHPath) { +bool ClangImporter::emitBridgingPCH( + llvm::IntrusiveRefCntPtr backend, + StringRef headerPath, StringRef outputPCHPath) { auto emitInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = emitInstance->getInvocation(); @@ -1715,6 +1718,8 @@ ClangImporter::emitBridgingPCH(StringRef headerPath, FrontendOpts.OutputFile = outputPCHPath.str(); FrontendOpts.ProgramAction = clang::frontend::GeneratePCH; + emitInstance->setOutputBackend(std::move(backend)); + auto action = wrapActionForIndexingIfEnabled( FrontendOpts, std::make_unique()); emitInstance->ExecuteAction(*action); @@ -1728,7 +1733,9 @@ ClangImporter::emitBridgingPCH(StringRef headerPath, return false; } -bool ClangImporter::runPreprocessor(StringRef inputPath, StringRef outputPath) { +bool ClangImporter::runPreprocessor( + llvm::IntrusiveRefCntPtr backend, + StringRef inputPath, StringRef outputPath) { auto emitInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = emitInstance->getInvocation(); auto LangOpts = invocation.getLangOpts(); @@ -1746,15 +1753,17 @@ bool ClangImporter::runPreprocessor(StringRef inputPath, StringRef outputPath) { FrontendOpts.OutputFile = outputPath.str(); FrontendOpts.ProgramAction = clang::frontend::PrintPreprocessedInput; + emitInstance->setOutputBackend(std::move(backend)); + auto action = wrapActionForIndexingIfEnabled( FrontendOpts, std::make_unique()); emitInstance->ExecuteAction(*action); return emitInstance->getDiagnostics().hasErrorOccurred(); } -bool ClangImporter::emitPrecompiledModule(StringRef moduleMapPath, - StringRef moduleName, - StringRef outputPath) { +bool ClangImporter::emitPrecompiledModule( + llvm::IntrusiveRefCntPtr backend, + StringRef moduleMapPath, StringRef moduleName, StringRef outputPath) { auto emitInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = emitInstance->getInvocation(); @@ -1775,6 +1784,8 @@ bool ClangImporter::emitPrecompiledModule(StringRef moduleMapPath, FrontendOpts.OutputFile = outputPath.str(); FrontendOpts.ProgramAction = clang::frontend::GenerateModule; + emitInstance->setOutputBackend(std::move(backend)); + auto action = wrapActionForIndexingIfEnabled( FrontendOpts, std::make_unique()); @@ -1788,8 +1799,9 @@ bool ClangImporter::emitPrecompiledModule(StringRef moduleMapPath, return false; } -bool ClangImporter::dumpPrecompiledModule(StringRef modulePath, - StringRef outputPath) { +bool ClangImporter::dumpPrecompiledModule( + llvm::IntrusiveRefCntPtr backend, + StringRef modulePath, StringRef outputPath) { auto dumpInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = dumpInstance->getInvocation(); @@ -1801,6 +1813,8 @@ bool ClangImporter::dumpPrecompiledModule(StringRef modulePath, FrontendOpts.Inputs = {inputFile}; FrontendOpts.OutputFile = outputPath.str(); + dumpInstance->setOutputBackend(std::move(backend)); + auto action = std::make_unique(); dumpInstance->ExecuteAction(*action); diff --git a/lib/DriverTool/swift_api_digester_main.cpp b/lib/DriverTool/swift_api_digester_main.cpp index a2b32b4a03e13..984ab507e8fc4 100644 --- a/lib/DriverTool/swift_api_digester_main.cpp +++ b/lib/DriverTool/swift_api_digester_main.cpp @@ -36,6 +36,8 @@ #include "swift/IDE/APIDigesterData.h" #include "swift/Option/Options.h" #include "swift/Parse/ParseVersion.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/Support/VirtualOutputBackends.h" #include using namespace swift; @@ -1895,8 +1897,9 @@ static bool readBreakageAllowlist(SDKContext &Ctx, llvm::StringSet<> &lines, "breakage-allowlist-", "txt", preprocessedFilePath)) { return 1; } - if (importer->runPreprocessor(BreakageAllowlistPath, - preprocessedFilePath.str())) { + if (importer->runPreprocessor( + llvm::makeIntrusiveRefCnt(), + BreakageAllowlistPath, preprocessedFilePath.str())) { return 1; } return readFileLineByLine(preprocessedFilePath, lines); diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index 1737e1a237b27..58bc699d1358d 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -377,13 +377,16 @@ static bool precompileBridgingHeader(const CompilerInstance &Instance) { Instance.getASTContext().getClangModuleLoader()); auto &ImporterOpts = Invocation.getClangImporterOptions(); auto &PCHOutDir = ImporterOpts.PrecompiledHeaderOutputDir; + auto OutputBackend = Instance.getOutputBackend().clone(); if (!PCHOutDir.empty()) { // Create or validate a persistent PCH. auto SwiftPCHHash = Invocation.getPCHHash(); - auto PCH = clangImporter->getOrCreatePCH(ImporterOpts, SwiftPCHHash); + auto PCH = clangImporter->getOrCreatePCH(ImporterOpts, SwiftPCHHash, + OutputBackend); return !PCH.has_value(); } return clangImporter->emitBridgingPCH( + OutputBackend, opts.InputsAndOutputs.getFilenameOfFirstInput(), opts.InputsAndOutputs.getSingleOutputFilename()); } @@ -393,6 +396,7 @@ static bool precompileClangModule(const CompilerInstance &Instance) { auto clangImporter = static_cast( Instance.getASTContext().getClangModuleLoader()); return clangImporter->emitPrecompiledModule( + Instance.getOutputBackend().clone(), opts.InputsAndOutputs.getFilenameOfFirstInput(), opts.ModuleName, opts.InputsAndOutputs.getSingleOutputFilename()); } @@ -402,6 +406,7 @@ static bool dumpPrecompiledClangModule(const CompilerInstance &Instance) { auto clangImporter = static_cast( Instance.getASTContext().getClangModuleLoader()); return clangImporter->dumpPrecompiledModule( + Instance.getOutputBackend().clone(), opts.InputsAndOutputs.getFilenameOfFirstInput(), opts.InputsAndOutputs.getSingleOutputFilename()); } diff --git a/unittests/ClangImporter/ClangImporterTests.cpp b/unittests/ClangImporter/ClangImporterTests.cpp index 074e660f97f8a..9ce31da9b3f0a 100644 --- a/unittests/ClangImporter/ClangImporterTests.cpp +++ b/unittests/ClangImporter/ClangImporterTests.cpp @@ -7,8 +7,10 @@ #include "swift/Basic/SourceManager.h" #include "swift/ClangImporter/ClangImporter.h" #include "swift/SymbolGraphGen/SymbolGraphOptions.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackends.h" #include "llvm/Support/raw_ostream.h" #include "gtest/gtest.h" @@ -81,10 +83,11 @@ TEST(ClangImporterTest, emitPCHInMemory) { ASTContext::get(langOpts, typecheckOpts, silOpts, searchPathOpts, options, symbolGraphOpts, sourceMgr, diags)); auto importer = ClangImporter::create(*context); + auto backend = llvm::makeIntrusiveRefCnt(); std::string PCH = createFilename(cache, "bridging.h.pch"); ASSERT_FALSE(importer->canReadPCH(PCH)); - ASSERT_FALSE(importer->emitBridgingPCH(options.BridgingHeader, PCH)); + ASSERT_FALSE(importer->emitBridgingPCH(backend, options.BridgingHeader, PCH)); ASSERT_TRUE(importer->canReadPCH(PCH)); // Overwrite the PCH with garbage. We should still be able to read it from From af6c5157a4b56251452bb58f7a3ffe345592553e Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 23 Feb 2023 13:39:45 -0800 Subject: [PATCH 07/11] Handle dependencies output Support makefile style dependency file and imported modules. Need improvements to virtual backend to allow emit loaded module trace file. --- lib/FrontendTool/Dependencies.h | 12 +++++++++-- lib/FrontendTool/FrontendTool.cpp | 12 +++++++---- lib/FrontendTool/ImportedModules.cpp | 24 ++++++++++++++-------- lib/FrontendTool/MakeStyleDependencies.cpp | 21 +++++++++++-------- 4 files changed, 46 insertions(+), 23 deletions(-) diff --git a/lib/FrontendTool/Dependencies.h b/lib/FrontendTool/Dependencies.h index 54291ebe03f63..91397e642fae7 100644 --- a/lib/FrontendTool/Dependencies.h +++ b/lib/FrontendTool/Dependencies.h @@ -13,6 +13,12 @@ #ifndef SWIFT_FRONTENDTOOL_DEPENDENCIES_H #define SWIFT_FRONTENDTOOL_DEPENDENCIES_H +namespace llvm { +namespace vfs { +class OutputBackend; +} +} // namespace llvm + namespace swift { class ASTContext; @@ -23,11 +29,13 @@ class InputFile; class ModuleDecl; /// Emit the names of the modules imported by \c mainModule. -bool emitImportedModules(ModuleDecl *mainModule, const FrontendOptions &opts); +bool emitImportedModules(ModuleDecl *mainModule, const FrontendOptions &opts, + llvm::vfs::OutputBackend &backend); bool emitMakeDependenciesIfNeeded(DiagnosticEngine &diags, DependencyTracker *depTracker, const FrontendOptions &opts, - const InputFile &input); + const InputFile &input, + llvm::vfs::OutputBackend &backend); bool emitLoadedModuleTraceIfNeeded(ModuleDecl *mainModule, DependencyTracker *depTracker, const FrontendOptions &opts, diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index 58bc699d1358d..b591a9e1b7734 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -110,10 +110,12 @@ static std::string displayName(StringRef MainExecutablePath) { static void emitMakeDependenciesIfNeeded(DiagnosticEngine &diags, DependencyTracker *depTracker, - const FrontendOptions &opts) { + const FrontendOptions &opts, + llvm::vfs::OutputBackend &backend) { opts.InputsAndOutputs.forEachInputProducingSupplementaryOutput( [&](const InputFile &f) -> bool { - return swift::emitMakeDependenciesIfNeeded(diags, depTracker, opts, f); + return swift::emitMakeDependenciesIfNeeded(diags, depTracker, opts, f, + backend); }); } @@ -1164,7 +1166,8 @@ static void performEndOfPipelineActions(CompilerInstance &Instance) { // Emit Make-style dependencies. emitMakeDependenciesIfNeeded(Instance.getDiags(), - Instance.getDependencyTracker(), opts); + Instance.getDependencyTracker(), opts, + Instance.getOutputBackend()); // Emit extracted constant values for every file in the batch emitConstValuesForAllPrimaryInputsIfNeeded(Instance); @@ -1358,7 +1361,8 @@ static bool performAction(CompilerInstance &Instance, getPrimaryOrMainSourceFile(Instance).dumpInterfaceHash(llvm::errs()); return Instance.getASTContext().hadError(); case FrontendOptions::ActionType::EmitImportedModules: - return emitImportedModules(Instance.getMainModule(), opts); + return emitImportedModules(Instance.getMainModule(), opts, + Instance.getOutputBackend()); // MARK: Dependency Scanning Actions case FrontendOptions::ActionType::ScanDependencies: diff --git a/lib/FrontendTool/ImportedModules.cpp b/lib/FrontendTool/ImportedModules.cpp index 595c9351a3b03..883b0b5186d20 100644 --- a/lib/FrontendTool/ImportedModules.cpp +++ b/lib/FrontendTool/ImportedModules.cpp @@ -24,6 +24,7 @@ #include "llvm/ADT/SetVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/VirtualOutputBackend.h" using namespace swift; @@ -43,16 +44,15 @@ static void findAllClangImports(const clang::Module *module, } bool swift::emitImportedModules(ModuleDecl *mainModule, - const FrontendOptions &opts) { + const FrontendOptions &opts, + llvm::vfs::OutputBackend &backend) { auto &Context = mainModule->getASTContext(); std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); - std::error_code EC; - llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::OF_None); - - if (out.has_error() || EC) { - Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, - EC.message()); - out.clear_error(); + auto &diags = Context.Diags; + auto out = backend.createFile(path); + if (!out) { + diags.diagnose(SourceLoc(), diag::error_opening_output, + path, toString(out.takeError())); return true; } @@ -110,7 +110,13 @@ bool swift::emitImportedModules(ModuleDecl *mainModule, } for (auto name : Modules) { - out << name << "\n"; + *out << name << "\n"; + } + + if (auto error = out->keep()) { + diags.diagnose(SourceLoc(), diag::error_opening_output, + path, toString(std::move(error))); + return true; } return false; diff --git a/lib/FrontendTool/MakeStyleDependencies.cpp b/lib/FrontendTool/MakeStyleDependencies.cpp index 38898bbae791f..54ecc5f9f391f 100644 --- a/lib/FrontendTool/MakeStyleDependencies.cpp +++ b/lib/FrontendTool/MakeStyleDependencies.cpp @@ -20,6 +20,7 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Support/VirtualOutputBackend.h" using namespace swift; @@ -91,18 +92,16 @@ reversePathSortedFilenames(const Container &elts) { bool swift::emitMakeDependenciesIfNeeded(DiagnosticEngine &diags, DependencyTracker *depTracker, const FrontendOptions &opts, - const InputFile &input) { + const InputFile &input, + llvm::vfs::OutputBackend &backend) { auto dependenciesFilePath = input.getDependenciesFilePath(); if (dependenciesFilePath.empty()) return false; - std::error_code EC; - llvm::raw_fd_ostream out(dependenciesFilePath, EC, llvm::sys::fs::OF_None); - - if (out.has_error() || EC) { + auto out = backend.createFile(dependenciesFilePath); + if (!out) { diags.diagnose(SourceLoc(), diag::error_opening_output, - dependenciesFilePath, EC.message()); - out.clear_error(); + dependenciesFilePath, toString(out.takeError())); return true; } @@ -138,8 +137,14 @@ bool swift::emitMakeDependenciesIfNeeded(DiagnosticEngine &diags, // dependency line. opts.forAllOutputPaths(input, [&](const StringRef targetName) { auto targetNameEscaped = frontend::utils::escapeForMake(targetName, buffer); - out << targetNameEscaped << " :" << dependencyString << '\n'; + *out << targetNameEscaped << " :" << dependencyString << '\n'; }); + if (auto error = out->keep()) { + diags.diagnose(SourceLoc(), diag::error_opening_output, + dependenciesFilePath, toString(std::move(error))); + return true; + } + return false; } From 2aaeb37eb068ae6424a47acaf9dfb21002286786 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 20 Feb 2023 14:30:49 -0800 Subject: [PATCH 08/11] Virtualize swift abi description file --- .../swift/APIDigester/ModuleAnalyzerNodes.h | 13 ++++---- lib/APIDigester/ModuleAnalyzerNodes.cpp | 31 +++++++++---------- lib/DriverTool/swift_api_digester_main.cpp | 24 ++++++++++---- lib/Frontend/Serialization.cpp | 17 +++++++++- 4 files changed, 56 insertions(+), 29 deletions(-) diff --git a/include/swift/APIDigester/ModuleAnalyzerNodes.h b/include/swift/APIDigester/ModuleAnalyzerNodes.h index 367763d5f4870..13b54ac41a405 100644 --- a/include/swift/APIDigester/ModuleAnalyzerNodes.h +++ b/include/swift/APIDigester/ModuleAnalyzerNodes.h @@ -784,8 +784,8 @@ class SwiftDeclCollector: public VisibleDeclConsumer { void deSerialize(StringRef Filename); // Serialize the content of all roots to a given file using JSON format. - void serialize(StringRef Filename); - static void serialize(StringRef Filename, SDKNode *Root, PayLoad otherInfo); + void serialize(llvm::raw_ostream &os); + static void serialize(llvm::raw_ostream &os, SDKNode *Root, PayLoad otherInfo); // After collecting decls, either from imported modules or from a previously // serialized JSON file, using this function to get the root of the SDK. @@ -835,14 +835,15 @@ SDKNodeRoot *getSDKNodeRoot(SDKContext &SDKCtx, SDKNodeRoot *getEmptySDKNodeRoot(SDKContext &SDKCtx); -void dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, StringRef OutputFile); -void dumpSDKRoot(SDKNodeRoot *Root, StringRef OutputFile); +void dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, llvm::raw_ostream &os); +void dumpSDKRoot(SDKNodeRoot *Root, llvm::raw_ostream &os); int dumpSDKContent(const CompilerInvocation &InitInvoke, const llvm::StringSet<> &ModuleNames, - StringRef OutputFile, CheckerOptions Opts); + llvm::raw_ostream &os, CheckerOptions Opts); -void dumpModuleContent(ModuleDecl *MD, StringRef OutputFile, bool ABI, bool Empty); +void dumpModuleContent(ModuleDecl *MD, llvm::raw_ostream &os, bool ABI, + bool Empty); /// Mostly for testing purposes, this function de-serializes the SDK dump in /// dumpPath and re-serialize them to OutputPath. If the tool performs correctly, diff --git a/lib/APIDigester/ModuleAnalyzerNodes.cpp b/lib/APIDigester/ModuleAnalyzerNodes.cpp index c3d8caacc2be9..753402f6466d1 100644 --- a/lib/APIDigester/ModuleAnalyzerNodes.cpp +++ b/lib/APIDigester/ModuleAnalyzerNodes.cpp @@ -2470,11 +2470,10 @@ void SwiftDeclCollector::deSerialize(StringRef Filename) { } // Serialize the content of all roots to a given file using JSON format. -void SwiftDeclCollector::serialize(StringRef Filename, SDKNode *Root, +void SwiftDeclCollector::serialize(llvm::raw_ostream &os, SDKNode *Root, PayLoad OtherInfo) { std::error_code EC; - llvm::raw_fd_ostream fs(Filename, EC, llvm::sys::fs::OF_None); - json::Output yout(fs); + json::Output yout(os); assert(Root->getKind() == SDKNodeKind::Root); SDKNodeRoot &root = *static_cast(Root); yout.beginObject(); @@ -2486,8 +2485,8 @@ void SwiftDeclCollector::serialize(StringRef Filename, SDKNode *Root, } // Serialize the content of all roots to a given file using JSON format. -void SwiftDeclCollector::serialize(StringRef Filename) { - SwiftDeclCollector::serialize(Filename, RootNode, PayLoad()); +void SwiftDeclCollector::serialize(llvm::raw_ostream &os) { + SwiftDeclCollector::serialize(os, RootNode, PayLoad()); } SDKNodeRoot * @@ -2550,28 +2549,28 @@ swift::ide::api::getSDKNodeRoot(SDKContext &SDKCtx, } void swift::ide::api::dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, - StringRef OutputFile) { + llvm::raw_ostream &os) { assert(Root); auto Opts = Root->getSDKContext().getOpts(); if (Opts.Verbose) llvm::errs() << "Dumping SDK...\n"; - SwiftDeclCollector::serialize(OutputFile, Root, load); - if (Opts.Verbose) - llvm::errs() << "Dumped to "<< OutputFile << "\n"; + SwiftDeclCollector::serialize(os, Root, load); + // if (Opts.Verbose) + // llvm::errs() << "Dumped to "<< OutputFile << "\n"; } -void swift::ide::api::dumpSDKRoot(SDKNodeRoot *Root, StringRef OutputFile) { - dumpSDKRoot(Root, PayLoad(), OutputFile); +void swift::ide::api::dumpSDKRoot(SDKNodeRoot *Root, llvm::raw_ostream &os) { + dumpSDKRoot(Root, PayLoad(), os); } int swift::ide::api::dumpSDKContent(const CompilerInvocation &InitInvoke, const llvm::StringSet<> &ModuleNames, - StringRef OutputFile, CheckerOptions Opts) { + llvm::raw_ostream &os, CheckerOptions Opts) { SDKContext SDKCtx(Opts); SDKNodeRoot *Root = getSDKNodeRoot(SDKCtx, InitInvoke, ModuleNames); if (!Root) return 1; - dumpSDKRoot(Root, OutputFile); + dumpSDKRoot(Root, os); return 0; } @@ -2589,11 +2588,11 @@ int swift::ide::api::deserializeSDKDump(StringRef dumpPath, StringRef OutputPath SwiftDeclCollector Collector(Ctx); Collector.deSerialize(dumpPath); - Collector.serialize(OutputPath); + Collector.serialize(FS); return 0; } -void swift::ide::api::dumpModuleContent(ModuleDecl *MD, StringRef OutputFile, +void swift::ide::api::dumpModuleContent(ModuleDecl *MD, llvm::raw_ostream &os, bool ABI, bool Empty) { CheckerOptions opts; opts.ABI = ABI; @@ -2610,7 +2609,7 @@ void swift::ide::api::dumpModuleContent(ModuleDecl *MD, StringRef OutputFile, PayLoad payload; SWIFT_DEFER { payload.allContsValues = &extractor.getAllConstValues(); - dumpSDKRoot(collector.getSDKRoot(), payload, OutputFile); + dumpSDKRoot(collector.getSDKRoot(), payload, os); }; if (Empty) { return; diff --git a/lib/DriverTool/swift_api_digester_main.cpp b/lib/DriverTool/swift_api_digester_main.cpp index 984ab507e8fc4..4bf0b760aeb4d 100644 --- a/lib/DriverTool/swift_api_digester_main.cpp +++ b/lib/DriverTool/swift_api_digester_main.cpp @@ -38,6 +38,7 @@ #include "swift/Parse/ParseVersion.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/Support/VirtualOutputBackends.h" +#include "llvm/Support/raw_ostream.h" #include using namespace swift; @@ -2539,13 +2540,18 @@ class SwiftAPIDigesterInvocation { switch (Action) { case ActionType::DumpSDK: { llvm::StringSet<> Modules; + auto JsonOut = + getJsonOutputFilePath(InitInvoke.getLangOptions().Target, + CheckerOpts.ABI, OutputFile, OutputDir); + std::error_code EC; + llvm::raw_fd_ostream fs(JsonOut, EC); + if (EC) { + llvm::errs() << "Cannot open JSON output file: " << JsonOut << "\n"; + return 1; + } return (prepareForDump(InitInvoke, Modules)) ? 1 - : dumpSDKContent(InitInvoke, Modules, - getJsonOutputFilePath( - InitInvoke.getLangOptions().Target, - CheckerOpts.ABI, OutputFile, OutputDir), - CheckerOpts); + : dumpSDKContent(InitInvoke, Modules, fs, CheckerOpts); } case ActionType::MigratorGen: case ActionType::DiagnoseSDKs: { @@ -2609,7 +2615,13 @@ class SwiftAPIDigesterInvocation { } case ActionType::GenerateEmptyBaseline: { SDKContext Ctx(CheckerOpts); - dumpSDKRoot(getEmptySDKNodeRoot(Ctx), OutputFile); + std::error_code EC; + llvm::raw_fd_ostream fs(OutputFile, EC); + if (EC) { + llvm::errs() << "Cannot open output file: " << OutputFile << "\n"; + return 1; + } + dumpSDKRoot(getEmptySDKNodeRoot(Ctx), fs); return 0; } case ActionType::FindUsr: { diff --git a/lib/Frontend/Serialization.cpp b/lib/Frontend/Serialization.cpp index 68429aa707ca9..c6c32022a7a6a 100644 --- a/lib/Frontend/Serialization.cpp +++ b/lib/Frontend/Serialization.cpp @@ -12,6 +12,7 @@ #include "swift/Serialization/Serialization.h" #include "swift/APIDigester/ModuleAnalyzerNodes.h" +#include "swift/AST/DiagnosticsFrontend.h" #include "swift/AST/FileSystem.h" #include "swift/Subsystems.h" #include "swift/SymbolGraphGen/SymbolGraphGen.h" @@ -36,8 +37,22 @@ static void emitABIDescriptor(ModuleOrSourceFile DC, using namespace swift::ide::api; if (!options.ABIDescriptorPath.empty()) { if (DC.is()) { - dumpModuleContent(DC.get(), options.ABIDescriptorPath, true, + auto &OutputBackend = getContext(DC).getOutputBackend(); + auto ABIDesFile = OutputBackend.createFile(options.ABIDescriptorPath); + if (!ABIDesFile) { + getContext(DC).Diags.diagnose(SourceLoc(), diag::cannot_open_file, + options.ABIDescriptorPath, + toString(ABIDesFile.takeError())); + return; + } + dumpModuleContent(DC.get(), *ABIDesFile, true, options.emptyABIDescriptor); + if (auto E = ABIDesFile->keep()) { + getContext(DC).Diags.diagnose(SourceLoc(), diag::cannot_open_file, + options.ABIDescriptorPath, + toString(std::move(E))); + return; + } } } } From f70d2bb313bd026001f13923109e36a253504a95 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 7 Mar 2023 14:54:38 -0800 Subject: [PATCH 09/11] Add test cases for virtualized output types Add unit-tests for already virtualized outputs --- test/Frontend/output_determinism_check.swift | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test/Frontend/output_determinism_check.swift b/test/Frontend/output_determinism_check.swift index 184c703a15c9d..251aae367425e 100644 --- a/test/Frontend/output_determinism_check.swift +++ b/test/Frontend/output_determinism_check.swift @@ -1,11 +1,14 @@ // RUN: %empty-directory(%t) -// RUN: %target-swift-frontend -module-name test -emit-module -o %t/test.swiftmodule -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=MODULE_OUTPUT +// RUN: %target-swift-frontend -module-name test -emit-module -o %t/test.swiftmodule -primary-file %s -emit-module-doc-path %t/test.docc -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=MODULE_OUTPUT --check-prefix=DOCC_OUTPUT // RUN: %target-swift-frontend -module-name test -emit-sib -o %t/test.sib -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=SIB_OUTPUT /// object files are "not" deterministic because the second run going to match the mod hash and skip code generation. // RUN: not %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=OBJECT_MISMATCH /// object files should match when forcing object generation. -// RUN: %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check -always-compile-output-files 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT +// RUN: %target-swift-frontend -module-name test -emit-dependencies -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check -always-compile-output-files 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT --check-prefix=DEPS_OUTPUT + +/// FIXME: Fine-grain dependencies graph is not deterministics. +/// FAIL: %target-swift-frontend -module-name test -emit-reference-dependencies-path %t/test.swiftdeps -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check -always-compile-output-files /// Explicit module build. Check building swiftmodule from interface file. // RUN: %target-swift-frontend -scan-dependencies -module-name test -o %t/test.json %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=DEPSCAN_OUTPUT @@ -17,13 +20,21 @@ /// Force swiftmodule generation. // RUN: %target-swift-frontend -compile-module-from-interface %t/test.swiftinterface -explicit-interface-module-build -o %t/test.swiftmodule -enable-swift-deterministic-check -always-compile-output-files 2>&1 | %FileCheck --check-prefix=MODULE_OUTPUT %s +// RUN: %target-swift-frontend -scan-dependencies -module-name test %s -o %t/test.deps.json -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=DEPS_JSON_OUTPUT + +// RUN: %target-swift-frontend -emit-pcm -module-name UserClangModule -o %t/test.pcm %S/Inputs/dependencies/module.modulemap -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=PCM_OUTPUT + +// DOCC_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.docc' // MODULE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftmodule' // SIB_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.sib' +// DEPS_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.d' // OBJECT_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.o' // OBJECT_MISMATCH: error: output file '{{.*}}{{/|\\}}test.o' is missing from second compilation for deterministic check // DEPSCAN_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.json' // INTERFACE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftinterface' // MODULE_MISMATCH: error: output file '{{.*}}{{/|\\}}test.swiftmodule' is missing from second compilation for deterministic check +// DEPS_JSON_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.deps.json' +// PCM_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.pcm' public var x = 1 public func test() {} From 9be198f3abf94f0dff8f95352b8215d4675ac6be Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 3 Apr 2023 20:20:39 -0700 Subject: [PATCH 10/11] Address review feedback --- include/swift/AST/ASTContext.h | 10 ++-- include/swift/AST/DiagnosticsCommon.def | 3 ++ include/swift/AST/FileSystem.h | 14 +++-- include/swift/AST/IRGenOptions.h | 2 +- include/swift/ClangImporter/ClangImporter.h | 20 +++---- include/swift/Frontend/Frontend.h | 6 +-- include/swift/Option/FrontendOptions.td | 6 +-- lib/AST/ASTContext.cpp | 8 +-- lib/AST/FineGrainedDependencies.cpp | 2 +- lib/AST/FineGrainedDependencyFormat.cpp | 2 +- lib/ClangImporter/ClangImporter.cpp | 22 ++------ .../ModuleDependencyCacheSerialization.cpp | 2 +- lib/DependencyScan/ScanDependencies.cpp | 52 ++++++++++-------- .../FineGrainedDependencyDriverGraph.cpp | 2 +- lib/DriverTool/swift_api_digester_main.cpp | 5 +- .../ArgsToFrontendOptionsConverter.cpp | 2 +- lib/Frontend/Frontend.cpp | 11 ++-- lib/Frontend/ModuleInterfaceLoader.cpp | 4 +- lib/Frontend/Serialization.cpp | 28 +++++----- lib/FrontendTool/FrontendTool.cpp | 53 ++++++------------- lib/FrontendTool/ImportedModules.cpp | 2 +- lib/FrontendTool/MakeStyleDependencies.cpp | 2 +- lib/IRGen/IRGen.cpp | 6 +-- lib/SymbolGraphGen/SymbolGraphGen.cpp | 2 +- test/Frontend/output_determinism_check.swift | 22 ++++---- tools/sil-llvm-gen/SILLLVMGen.cpp | 2 +- .../swift-dependency-tool.cpp | 2 +- .../ClangImporter/ClangImporterTests.cpp | 3 +- 28 files changed, 132 insertions(+), 163 deletions(-) diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h index c9775a1126dfb..341a115155748 100644 --- a/include/swift/AST/ASTContext.h +++ b/include/swift/AST/ASTContext.h @@ -285,7 +285,7 @@ class ASTContext final { DiagnosticEngine &Diags; /// OutputBackend for writing outputs. - llvm::IntrusiveRefCntPtr Backend; + llvm::IntrusiveRefCntPtr OutputBackend; /// If the shared pointer is not a \c nullptr and the pointee is \c true, /// all operations working on this ASTContext should be aborted at the next @@ -1525,13 +1525,13 @@ class ASTContext final { /// Get the output backend. The output backend needs to be initialized via /// constructor or `setOutputBackend`. llvm::vfs::OutputBackend &getOutputBackend() const { - assert(Backend && "OutputBackend is not setup"); - return *Backend; + assert(OutputBackend && "OutputBackend is not setup"); + return *OutputBackend; } /// Set output backend for virtualized outputs. void setOutputBackend( - llvm::IntrusiveRefCntPtr OutputBackend) { - Backend = std::move(OutputBackend); + llvm::IntrusiveRefCntPtr OutBackend) { + OutputBackend = std::move(OutBackend); } private: diff --git a/include/swift/AST/DiagnosticsCommon.def b/include/swift/AST/DiagnosticsCommon.def index 535dbd475eabb..f438e7a93b31e 100644 --- a/include/swift/AST/DiagnosticsCommon.def +++ b/include/swift/AST/DiagnosticsCommon.def @@ -29,6 +29,9 @@ ERROR(not_implemented,none, ERROR(error_opening_output,none, "error opening '%0' for output: %1", (StringRef, StringRef)) +ERROR(error_closing_output,none, + "error closing '%0' for output: %1", (StringRef, StringRef)) + ERROR(cannot_find_group_info_file,none, "cannot find group info file at path: '%0'", (StringRef)) diff --git a/include/swift/AST/FileSystem.h b/include/swift/AST/FileSystem.h index c2d5158c0f7bb..69f4c5dd7ffce 100644 --- a/include/swift/AST/FileSystem.h +++ b/include/swift/AST/FileSystem.h @@ -26,7 +26,7 @@ namespace swift { /// \returns true if there were any errors, either from the filesystem /// operations or from \p action returning true. inline bool -withOutputFile(DiagnosticEngine &diags, llvm::vfs::OutputBackend &Backend, +withOutputPath(DiagnosticEngine &diags, llvm::vfs::OutputBackend &Backend, StringRef outputPath, llvm::function_ref action) { assert(!outputPath.empty()); @@ -37,15 +37,19 @@ withOutputFile(DiagnosticEngine &diags, llvm::vfs::OutputBackend &Backend, if (!outputFile) { diags.diagnose(SourceLoc(), diag::error_opening_output, outputPath, toString(outputFile.takeError())); - return false; + return true; } bool failed = action(*outputFile); // If there is an error, discard output. Otherwise keep the output file. if (auto error = failed ? outputFile->discard() : outputFile->keep()) { - diags.diagnose(SourceLoc(), diag::error_opening_output, outputPath, - toString(std::move(error))); - return false; + // Don't diagnose discard error. + if (failed) + consumeError(std::move(error)); + else + diags.diagnose(SourceLoc(), diag::error_closing_output, outputPath, + toString(std::move(error))); + return true; } return failed; } diff --git a/include/swift/AST/IRGenOptions.h b/include/swift/AST/IRGenOptions.h index 406e4c9f0f593..bfb6f6efd413a 100644 --- a/include/swift/AST/IRGenOptions.h +++ b/include/swift/AST/IRGenOptions.h @@ -318,7 +318,7 @@ class IRGenOptions { /// Print the LLVM inline tree at the end of the LLVM pass pipeline. unsigned PrintInlineTree : 1; - /// Always recompile the output even the module hash might match. + /// Always recompile the output even if the module hash might match. unsigned AlwaysCompile : 1; /// Whether we should embed the bitcode file. diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h index 919aff75a2cb5..5ad232c279a6b 100644 --- a/include/swift/ClangImporter/ClangImporter.h +++ b/include/swift/ClangImporter/ClangImporter.h @@ -395,9 +395,7 @@ class ClangImporter final : public ClangModuleLoader { /// replica. /// /// \sa clang::GeneratePCHAction - bool - emitBridgingPCH(llvm::IntrusiveRefCntPtr backend, - StringRef headerPath, StringRef outputPCHPath); + bool emitBridgingPCH(StringRef headerPath, StringRef outputPCHPath); /// Returns true if a clang CompilerInstance can successfully read in a PCH, /// assuming it exists, with the current options. This can be used to find out @@ -408,21 +406,16 @@ class ClangImporter final : public ClangModuleLoader { /// module map into the replica and emits a PCM file for one of the modules it /// declares. Delegates to clang for everything except construction of the /// replica. - bool emitPrecompiledModule( - llvm::IntrusiveRefCntPtr backend, - StringRef moduleMapPath, StringRef moduleName, StringRef outputPath); + bool emitPrecompiledModule(StringRef moduleMapPath, StringRef moduleName, + StringRef outputPath); /// Makes a temporary replica of the ClangImporter's CompilerInstance and /// dumps information about a PCM file (assumed to be generated by -emit-pcm /// or in the Swift module cache). Delegates to clang for everything except /// construction of the replica. - bool dumpPrecompiledModule( - llvm::IntrusiveRefCntPtr backend, - StringRef modulePath, StringRef outputPath); + bool dumpPrecompiledModule(StringRef modulePath, StringRef outputPath); - bool - runPreprocessor(llvm::IntrusiveRefCntPtr backend, - StringRef inputPath, StringRef outputPath); + bool runPreprocessor(StringRef inputPath, StringRef outputPath); const clang::Module *getClangOwningModule(ClangNode Node) const; bool hasTypedef(const clang::Decl *typeDecl) const; @@ -510,8 +503,7 @@ class ClangImporter final : public ClangModuleLoader { Optional getOrCreatePCH(const ClangImporterOptions &ImporterOptions, - StringRef SwiftPCHHash, - llvm::IntrusiveRefCntPtr Backend); + StringRef SwiftPCHHash); Optional /// \param isExplicit true if the PCH filename was passed directly /// with -import-objc-header option. diff --git a/include/swift/Frontend/Frontend.h b/include/swift/Frontend/Frontend.h index 47cc2f9f37d17..9669e98d8488b 100644 --- a/include/swift/Frontend/Frontend.h +++ b/include/swift/Frontend/Frontend.h @@ -461,7 +461,7 @@ class CompilerInstance { std::unique_ptr Stats; /// Virtual OutputBackend. - llvm::IntrusiveRefCntPtr TheOutputBackend = nullptr; + llvm::IntrusiveRefCntPtr OutputBackend = nullptr; /// The verification output backend. using HashBackendTy = llvm::vfs::HashingOutputBackend; @@ -518,11 +518,11 @@ class CompilerInstance { } llvm::vfs::OutputBackend &getOutputBackend() const { - return *TheOutputBackend; + return *OutputBackend; } void setOutputBackend(llvm::IntrusiveRefCntPtr Backend) { - TheOutputBackend = std::move(Backend); + OutputBackend = std::move(Backend); } using HashingBackendPtrTy = llvm::IntrusiveRefCntPtr; HashingBackendPtrTy getHashingBackend() { return HashBackend; } diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index 988b38d9224b3..8a3ff3da4c9e2 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -1176,9 +1176,9 @@ def enable_emit_generic_class_ro_t_list : HelpText<"Enable emission of a section with references to class_ro_t of " "generic class patterns">; -def enable_swift_deterministic_check : - Flag<["-"], "enable-swift-deterministic-check">, - HelpText<"Check swift compiler output determinisim by run it twice">; +def enable_deterministic_check : + Flag<["-"], "enable-deterministic-check">, + HelpText<"Check compiler output determinisim by running it twice">; def always_compile_output_files : Flag<["-"], "always-compile-output-files">, HelpText<"Always compile output files even it might not change the results">; diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp index 5d3dad9765902..da794196dbffe 100644 --- a/lib/AST/ASTContext.cpp +++ b/lib/AST/ASTContext.cpp @@ -661,12 +661,12 @@ ASTContext::ASTContext( ClangImporterOptions &ClangImporterOpts, symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts, SourceManager &SourceMgr, DiagnosticEngine &Diags, - llvm::IntrusiveRefCntPtr OutputBackend, + llvm::IntrusiveRefCntPtr OutBackend, std::function PreModuleImportCallback) : LangOpts(langOpts), TypeCheckerOpts(typecheckOpts), SILOpts(silOpts), SearchPathOpts(SearchPathOpts), ClangImporterOpts(ClangImporterOpts), SymbolGraphOpts(SymbolGraphOpts), SourceMgr(SourceMgr), Diags(Diags), - Backend(std::move(OutputBackend)), evaluator(Diags, langOpts), + OutputBackend(std::move(OutBackend)), evaluator(Diags, langOpts), TheBuiltinModule(createBuiltinModule(*this)), StdlibModuleName(getIdentifier(STDLIB_NAME)), SwiftShimsModuleName(getIdentifier(SWIFT_SHIMS_NAME)), @@ -714,8 +714,8 @@ ASTContext::ASTContext( createModuleToExecutablePluginMap(); // Provide a default OnDiskOutputBackend if user didn't supply one. - if (!Backend) - Backend = llvm::makeIntrusiveRefCnt(); + if (!OutputBackend) + OutputBackend = llvm::makeIntrusiveRefCnt(); } ASTContext::~ASTContext() { diff --git a/lib/AST/FineGrainedDependencies.cpp b/lib/AST/FineGrainedDependencies.cpp index 3a909dbcdbe38..39d362ce6d8da 100644 --- a/lib/AST/FineGrainedDependencies.cpp +++ b/lib/AST/FineGrainedDependencies.cpp @@ -377,7 +377,7 @@ void SourceFileDepGraph::emitDotFile(llvm::vfs::OutputBackend &outputBackend, StringRef outputPath, DiagnosticEngine &diags) { std::string dotFileName = outputPath.str() + ".dot"; - withOutputFile( + withOutputPath( diags, outputBackend, dotFileName, [&](llvm::raw_pwrite_stream &out) { DotFileEmitter(out, *this, false, false).emit(); return false; diff --git a/lib/AST/FineGrainedDependencyFormat.cpp b/lib/AST/FineGrainedDependencyFormat.cpp index 02c04fe270bb1..770c8745d8634 100644 --- a/lib/AST/FineGrainedDependencyFormat.cpp +++ b/lib/AST/FineGrainedDependencyFormat.cpp @@ -502,7 +502,7 @@ bool swift::fine_grained_dependencies::writeFineGrainedDependencyGraphToPath( DiagnosticEngine &diags, llvm::vfs::OutputBackend &backend, StringRef path, const SourceFileDepGraph &g) { PrettyStackTraceStringAction stackTrace("saving fine-grained dependency graph", path); - return withOutputFile(diags, backend, path, [&](llvm::raw_ostream &out) { + return withOutputPath(diags, backend, path, [&](llvm::raw_ostream &out) { SmallVector Buffer; llvm::BitstreamWriter Writer{Buffer}; writeFineGrainedDependencyGraph(Writer, g, Purpose::ForSwiftDeps); diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index 4a9f9e641012e..a02374ae6b241 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -194,8 +194,7 @@ namespace { Importer.addSearchPath(path, /*isFramework*/false, /*isSystem=*/false); } - auto PCH = Importer.getOrCreatePCH(ImporterOpts, SwiftPCHHash, - Ctx.getOutputBackend().clone()); + auto PCH = Importer.getOrCreatePCH(ImporterOpts, SwiftPCHHash); if (PCH.has_value()) { Impl.getClangInstance()->getPreprocessorOpts().ImplicitPCHInclude = PCH.value(); @@ -944,8 +943,7 @@ ClangImporter::getPCHFilename(const ClangImporterOptions &ImporterOptions, } Optional ClangImporter::getOrCreatePCH( - const ClangImporterOptions &ImporterOptions, StringRef SwiftPCHHash, - llvm::IntrusiveRefCntPtr Backend) { + const ClangImporterOptions &ImporterOptions, StringRef SwiftPCHHash) { bool isExplicit; auto PCHFilename = getPCHFilename(ImporterOptions, SwiftPCHHash, isExplicit); @@ -962,8 +960,7 @@ Optional ClangImporter::getOrCreatePCH( return None; } auto FailedToEmit = - emitBridgingPCH(std::move(Backend), ImporterOptions.BridgingHeader, - PCHFilename.value()); + emitBridgingPCH(ImporterOptions.BridgingHeader, PCHFilename.value()); if (FailedToEmit) { return None; } @@ -1696,12 +1693,12 @@ ClangImporter::cloneCompilerInstanceForPrecompiling() { clonedInstance->setFileManager(&fileManager); clonedInstance->createSourceManager(fileManager); clonedInstance->setTarget(&Impl.Instance->getTarget()); + clonedInstance->setOutputBackend(Impl.SwiftContext.OutputBackend); return clonedInstance; } bool ClangImporter::emitBridgingPCH( - llvm::IntrusiveRefCntPtr backend, StringRef headerPath, StringRef outputPCHPath) { auto emitInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = emitInstance->getInvocation(); @@ -1718,8 +1715,6 @@ bool ClangImporter::emitBridgingPCH( FrontendOpts.OutputFile = outputPCHPath.str(); FrontendOpts.ProgramAction = clang::frontend::GeneratePCH; - emitInstance->setOutputBackend(std::move(backend)); - auto action = wrapActionForIndexingIfEnabled( FrontendOpts, std::make_unique()); emitInstance->ExecuteAction(*action); @@ -1734,7 +1729,6 @@ bool ClangImporter::emitBridgingPCH( } bool ClangImporter::runPreprocessor( - llvm::IntrusiveRefCntPtr backend, StringRef inputPath, StringRef outputPath) { auto emitInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = emitInstance->getInvocation(); @@ -1753,8 +1747,6 @@ bool ClangImporter::runPreprocessor( FrontendOpts.OutputFile = outputPath.str(); FrontendOpts.ProgramAction = clang::frontend::PrintPreprocessedInput; - emitInstance->setOutputBackend(std::move(backend)); - auto action = wrapActionForIndexingIfEnabled( FrontendOpts, std::make_unique()); emitInstance->ExecuteAction(*action); @@ -1762,7 +1754,6 @@ bool ClangImporter::runPreprocessor( } bool ClangImporter::emitPrecompiledModule( - llvm::IntrusiveRefCntPtr backend, StringRef moduleMapPath, StringRef moduleName, StringRef outputPath) { auto emitInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = emitInstance->getInvocation(); @@ -1784,8 +1775,6 @@ bool ClangImporter::emitPrecompiledModule( FrontendOpts.OutputFile = outputPath.str(); FrontendOpts.ProgramAction = clang::frontend::GenerateModule; - emitInstance->setOutputBackend(std::move(backend)); - auto action = wrapActionForIndexingIfEnabled( FrontendOpts, std::make_unique()); @@ -1800,7 +1789,6 @@ bool ClangImporter::emitPrecompiledModule( } bool ClangImporter::dumpPrecompiledModule( - llvm::IntrusiveRefCntPtr backend, StringRef modulePath, StringRef outputPath) { auto dumpInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = dumpInstance->getInvocation(); @@ -1813,8 +1801,6 @@ bool ClangImporter::dumpPrecompiledModule( FrontendOpts.Inputs = {inputFile}; FrontendOpts.OutputFile = outputPath.str(); - dumpInstance->setOutputBackend(std::move(backend)); - auto action = std::make_unique(); dumpInstance->ExecuteAction(*action); diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index 2963920d3f76a..075666ddc5dbc 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -1151,7 +1151,7 @@ bool swift::dependencies::module_dependency_cache_serialization:: StringRef path, const SwiftDependencyScanningService &cache) { PrettyStackTraceStringAction stackTrace( "saving inter-module dependency graph", path); - return withOutputFile(diags, backend, path, [&](llvm::raw_ostream &out) { + return withOutputPath(diags, backend, path, [&](llvm::raw_ostream &out) { SmallVector Buffer; llvm::BitstreamWriter Writer{Buffer}; writeInterModuleDependenciesCache(Writer, cache); diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index b9d27a2ee5c27..3e37c7345ba1b 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -44,6 +44,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/StringSaver.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" #include "llvm/Support/raw_ostream.h" @@ -935,6 +936,25 @@ static void writeJSON(llvm::raw_ostream &out, } } +static bool writePrescanJSONToOutput(DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend, + StringRef path, + const swiftscan_import_set_t importSet) { + return withOutputPath(diags, backend, path, [&](llvm::raw_pwrite_stream &os) { + writePrescanJSON(os, importSet); + return false; + }); +} + +static bool writeJSONToOutput(DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend, StringRef path, + const swiftscan_dependency_graph_t dependencies) { + return withOutputPath(diags, backend, path, [&](llvm::raw_pwrite_stream &os) { + writeJSON(os, dependencies); + return false; + }); +} + static swiftscan_dependency_graph_t generateFullDependencyGraph(CompilerInstance &instance, ModuleDependenciesCache &cache, @@ -1420,11 +1440,8 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { return true; auto dependencies = std::move(*dependenciesOrErr); - if (withOutputFile(Context.Diags, instance.getOutputBackend(), path, - [&](llvm::raw_pwrite_stream &os) { - writeJSON(os, dependencies); - return false; - })) + if (writeJSONToOutput(Context.Diags, instance.getOutputBackend(), path, + dependencies)) return true; // This process succeeds regardless of whether any errors occurred. @@ -1453,11 +1470,8 @@ bool swift::dependencies::prescanDependencies(CompilerInstance &instance) { auto importSet = std::move(*importSetOrErr); // Serialize and output main module dependencies only and exit. - if (withOutputFile(Context.Diags, instance.getOutputBackend(), path, - [&](llvm::raw_pwrite_stream &os) { - writePrescanJSON(os, importSet); - return false; - })) + if (writePrescanJSONToOutput(Context.Diags, instance.getOutputBackend(), path, + importSet)) return true; // This process succeeds regardless of whether any errors occurred. @@ -1498,12 +1512,9 @@ bool swift::dependencies::batchScanDependencies( if ((*iresults).getError()) return true; - if (withOutputFile(instance.getASTContext().Diags, - instance.getOutputBackend(), (*ientries).outputPath, - [&](llvm::raw_pwrite_stream &os) { - writeJSON(os, **iresults); - return false; - })) + if (writeJSONToOutput(instance.getASTContext().Diags, + instance.getOutputBackend(), (*ientries).outputPath, + **iresults)) return true; } return false; @@ -1537,12 +1548,9 @@ bool swift::dependencies::batchPrescanDependencies( if ((*iresults).getError()) return true; - if (withOutputFile(instance.getASTContext().Diags, - instance.getOutputBackend(), (*ientries).outputPath, - [&](llvm::raw_pwrite_stream &os) { - writePrescanJSON(os, **iresults); - return false; - })) + if (writePrescanJSONToOutput(instance.getASTContext().Diags, + instance.getOutputBackend(), + (*ientries).outputPath, **iresults)) return true; } return false; diff --git a/lib/Driver/FineGrainedDependencyDriverGraph.cpp b/lib/Driver/FineGrainedDependencyDriverGraph.cpp index bd097169ac8b3..9e2b36c359570 100644 --- a/lib/Driver/FineGrainedDependencyDriverGraph.cpp +++ b/lib/Driver/FineGrainedDependencyDriverGraph.cpp @@ -535,7 +535,7 @@ void ModuleDepGraph::emitDotFile(DiagnosticEngine &diags, unsigned seqNo = dotFileSequenceNumber[baseName.str()]++; std::string fullName = baseName.str() + "-post-integration." + std::to_string(seqNo) + ".dot"; - withOutputFile(diags, *backend, fullName, [&](llvm::raw_ostream &out) { + withOutputPath(diags, *backend, fullName, [&](llvm::raw_ostream &out) { emitDotFile(out); return false; }); diff --git a/lib/DriverTool/swift_api_digester_main.cpp b/lib/DriverTool/swift_api_digester_main.cpp index 4bf0b760aeb4d..34e457b041362 100644 --- a/lib/DriverTool/swift_api_digester_main.cpp +++ b/lib/DriverTool/swift_api_digester_main.cpp @@ -1898,9 +1898,8 @@ static bool readBreakageAllowlist(SDKContext &Ctx, llvm::StringSet<> &lines, "breakage-allowlist-", "txt", preprocessedFilePath)) { return 1; } - if (importer->runPreprocessor( - llvm::makeIntrusiveRefCnt(), - BreakageAllowlistPath, preprocessedFilePath.str())) { + if (importer->runPreprocessor(BreakageAllowlistPath, + preprocessedFilePath.str())) { return 1; } return readFileLineByLine(preprocessedFilePath, lines); diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index e345a5e105ae0..78cf036605abe 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -345,7 +345,7 @@ bool ArgsToFrontendOptionsConverter::convert( Opts.serializedPathObfuscator.addMapping(SplitMap.first, SplitMap.second); } Opts.emptyABIDescriptor = Args.hasArg(OPT_empty_abi_descriptor); - Opts.DeterministicCheck = Args.hasArg(OPT_enable_swift_deterministic_check); + Opts.DeterministicCheck = Args.hasArg(OPT_enable_deterministic_check); return false; } diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index 7e6d3c0df393d..e228ca5db8966 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -247,12 +247,11 @@ bool CompilerInstance::setUpASTContextIfNeeded() { Invocation.getLangOptions().RecordRequestReferences = !isWholeModuleCompilation(); - // Make sure the output backend is initialized. Context.reset(ASTContext::get( Invocation.getLangOptions(), Invocation.getTypeCheckerOptions(), Invocation.getSILOptions(), Invocation.getSearchPathOptions(), Invocation.getClangImporterOptions(), Invocation.getSymbolGraphOptions(), - SourceMgr, Diagnostics, TheOutputBackend)); + SourceMgr, Diagnostics, OutputBackend)); if (!Invocation.getFrontendOptions().ModuleAliasMap.empty()) Context->setModuleAliases(Invocation.getFrontendOptions().ModuleAliasMap); @@ -398,10 +397,10 @@ void CompilerInstance::setupDependencyTrackerIfNeeded() { void CompilerInstance::setupOutputBackend() { // Skip if output backend is not setup, default to OnDiskOutputBackend. - if (TheOutputBackend) + if (OutputBackend) return; - TheOutputBackend = + OutputBackend = llvm::makeIntrusiveRefCnt(); // Setup verification backend. @@ -410,8 +409,8 @@ void CompilerInstance::setupOutputBackend() { // some output file in later stages. if (Invocation.getFrontendOptions().DeterministicCheck) { HashBackend = llvm::makeIntrusiveRefCnt(); - TheOutputBackend = - llvm::vfs::makeMirroringOutputBackend(TheOutputBackend, HashBackend); + OutputBackend = + llvm::vfs::makeMirroringOutputBackend(OutputBackend, HashBackend); } } diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index b1ea5f75212b7..65946488be1d1 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -939,7 +939,7 @@ class ModuleInterfaceLoaderImpl { depsAdjustedToMTime.push_back(adjustedDep); } - auto hadError = withOutputFile(diags, backend, outputPath, + auto hadError = withOutputPath(diags, backend, outputPath, [&](llvm::raw_pwrite_stream &out) { llvm::yaml::Output yamlWriter(out); yamlWriter << fwd; @@ -1267,7 +1267,7 @@ bool ModuleInterfaceCheckerImpl::tryEmitForwardingModule( deps, moduleBuffer)) { // If so, emit a forwarding module to the candidate. ForwardingModule FM(mod); - auto hadError = withOutputFile(Ctx.Diags, backend, outputPath, + auto hadError = withOutputPath(Ctx.Diags, backend, outputPath, [&](llvm::raw_pwrite_stream &out) { llvm::yaml::Output yamlWriter(out); yamlWriter << FM; diff --git a/lib/Frontend/Serialization.cpp b/lib/Frontend/Serialization.cpp index c6c32022a7a6a..88ad917fab610 100644 --- a/lib/Frontend/Serialization.cpp +++ b/lib/Frontend/Serialization.cpp @@ -40,19 +40,21 @@ static void emitABIDescriptor(ModuleOrSourceFile DC, auto &OutputBackend = getContext(DC).getOutputBackend(); auto ABIDesFile = OutputBackend.createFile(options.ABIDescriptorPath); if (!ABIDesFile) { - getContext(DC).Diags.diagnose(SourceLoc(), diag::cannot_open_file, + getContext(DC).Diags.diagnose(SourceLoc(), diag::error_opening_output, options.ABIDescriptorPath, toString(ABIDesFile.takeError())); return; } + SWIFT_DEFER { + if (auto E = ABIDesFile->keep()) { + getContext(DC).Diags.diagnose(SourceLoc(), diag::error_closing_output, + options.ABIDescriptorPath, + toString(std::move(E))); + return; + } + }; dumpModuleContent(DC.get(), *ABIDesFile, true, options.emptyABIDescriptor); - if (auto E = ABIDesFile->keep()) { - getContext(DC).Diags.diagnose(SourceLoc(), diag::cannot_open_file, - options.ABIDescriptorPath, - toString(std::move(E))); - return; - } } } } @@ -72,7 +74,7 @@ void swift::serializeToBuffers( llvm::raw_svector_ostream stream(buf); serialization::writeToStream(stream, DC, M, options, /*dependency info*/ nullptr); - bool hadError = withOutputFile( + bool hadError = withOutputPath( getContext(DC).Diags, getContext(DC).getOutputBackend(), options.OutputPath, [&](raw_ostream &out) { out << stream.str(); @@ -94,7 +96,7 @@ void swift::serializeToBuffers( llvm::SmallString<1024> buf; llvm::raw_svector_ostream stream(buf); serialization::writeDocToStream(stream, DC, options.GroupInfoPath); - (void)withOutputFile(getContext(DC).Diags, + (void)withOutputPath(getContext(DC).Diags, getContext(DC).getOutputBackend(), options.DocOutputPath, [&](raw_ostream &out) { out << stream.str(); @@ -112,7 +114,7 @@ void swift::serializeToBuffers( llvm::SmallString<1024> buf; llvm::raw_svector_ostream stream(buf); serialization::writeSourceInfoToStream(stream, DC); - (void)withOutputFile( + (void)withOutputPath( getContext(DC).Diags, getContext(DC).getOutputBackend(), options.SourceInfoOutputPath, [&](raw_ostream &out) { out << stream.str(); @@ -139,7 +141,7 @@ void swift::serialize( return; } - bool hadError = withOutputFile( + bool hadError = withOutputPath( getContext(DC).Diags, getContext(DC).getOutputBackend(), options.OutputPath, [&](raw_ostream &out) { FrontendStatsTracer tracer(getContext(DC).Stats, @@ -151,7 +153,7 @@ void swift::serialize( return; if (!options.DocOutputPath.empty()) { - (void)withOutputFile( + (void)withOutputPath( getContext(DC).Diags, getContext(DC).getOutputBackend(), options.DocOutputPath, [&](raw_ostream &out) { FrontendStatsTracer tracer(getContext(DC).Stats, @@ -162,7 +164,7 @@ void swift::serialize( } if (!options.SourceInfoOutputPath.empty()) { - (void)withOutputFile( + (void)withOutputPath( getContext(DC).Diags, getContext(DC).getOutputBackend(), options.SourceInfoOutputPath, [&](raw_ostream &out) { FrontendStatsTracer tracer(getContext(DC).Stats, diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index b591a9e1b7734..cad2a25345dbf 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -130,29 +130,15 @@ emitLoadedModuleTraceForAllPrimariesIfNeeded(ModuleDecl *mainModule, }); } -/// Diagnose output file error to ASTContext. -void diagnoseOutputError(ASTContext &Ctx, StringRef OutputFilename, - llvm::Error Err) { - Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_output, OutputFilename, - toString(std::move(Err))); -} - /// Writes SIL out to the given file. static bool writeSIL(SILModule &SM, ModuleDecl *M, const SILOptions &Opts, StringRef OutputFilename, llvm::vfs::OutputBackend &Backend) { - auto OutFile = Backend.createFile(OutputFilename); - if (!OutFile) { - diagnoseOutputError(M->getASTContext(), OutputFilename, - OutFile.takeError()); - return true; - } - SM.print(*OutFile, M, Opts); - if (auto E = OutFile->keep()) { - diagnoseOutputError(M->getASTContext(), OutputFilename, std::move(E)); - return true; - } - return M->getASTContext().hadError(); + return withOutputPath(M->getDiags(), Backend, OutputFilename, + [&](raw_ostream &out) -> bool { + SM.print(out, M, Opts); + return M->getASTContext().hadError(); + }); } static bool writeSIL(SILModule &SM, const PrimarySpecificPaths &PSPs, @@ -178,7 +164,7 @@ static bool printAsClangHeaderIfNeeded(llvm::vfs::OutputBackend &outputBackend, clang::HeaderSearch &clangHeaderSearchInfo) { if (outputPath.empty()) return false; - return withOutputFile( + return withOutputPath( M->getDiags(), outputBackend, outputPath, [&](raw_ostream &out) -> bool { return printAsClangHeader(out, M, bridgingHeader, frontendOpts, irGenOpts, clangHeaderSearchInfo); @@ -212,7 +198,7 @@ printModuleInterfaceIfNeeded(llvm::vfs::OutputBackend &outputBackend, diags.diagnose(SourceLoc(), diag::warn_unsupported_module_interface_library_evolution); } - return withOutputFile(diags, outputBackend, outputPath, + return withOutputPath(diags, outputBackend, outputPath, [M, Opts](raw_ostream &out) -> bool { return swift::emitSwiftInterface(out, Opts, M); }); @@ -383,12 +369,10 @@ static bool precompileBridgingHeader(const CompilerInstance &Instance) { if (!PCHOutDir.empty()) { // Create or validate a persistent PCH. auto SwiftPCHHash = Invocation.getPCHHash(); - auto PCH = clangImporter->getOrCreatePCH(ImporterOpts, SwiftPCHHash, - OutputBackend); + auto PCH = clangImporter->getOrCreatePCH(ImporterOpts, SwiftPCHHash); return !PCH.has_value(); } return clangImporter->emitBridgingPCH( - OutputBackend, opts.InputsAndOutputs.getFilenameOfFirstInput(), opts.InputsAndOutputs.getSingleOutputFilename()); } @@ -398,7 +382,6 @@ static bool precompileClangModule(const CompilerInstance &Instance) { auto clangImporter = static_cast( Instance.getASTContext().getClangModuleLoader()); return clangImporter->emitPrecompiledModule( - Instance.getOutputBackend().clone(), opts.InputsAndOutputs.getFilenameOfFirstInput(), opts.ModuleName, opts.InputsAndOutputs.getSingleOutputFilename()); } @@ -408,7 +391,6 @@ static bool dumpPrecompiledClangModule(const CompilerInstance &Instance) { auto clangImporter = static_cast( Instance.getASTContext().getClangModuleLoader()); return clangImporter->dumpPrecompiledModule( - Instance.getOutputBackend().clone(), opts.InputsAndOutputs.getFilenameOfFirstInput(), opts.InputsAndOutputs.getSingleOutputFilename()); } @@ -529,18 +511,13 @@ static bool dumpAST(CompilerInstance &Instance) { for (SourceFile *sourceFile: primaryFiles) { auto PSPs = Instance.getPrimarySpecificPathsForSourceFile(*sourceFile); auto OutputFilename = PSPs.OutputFilename; - auto OutFile = Instance.getOutputBackend().createFile(OutputFilename); - if (!OutFile) { - diagnoseOutputError(Instance.getASTContext(), OutputFilename, - OutFile.takeError()); + if (withOutputPath(Instance.getASTContext().Diags, + Instance.getOutputBackend(), OutputFilename, + [&](raw_ostream &out) -> bool { + sourceFile->dump(out, /*parseIfNeeded*/ true); + return false; + })) return true; - } - sourceFile->dump(*OutFile, /*parseIfNeeded*/ true); - if (auto E = OutFile->keep()) { - diagnoseOutputError(Instance.getASTContext(), OutputFilename, - std::move(E)); - return true; - } } } else { // Some invocations don't have primary files. In that case, we default to @@ -1466,7 +1443,7 @@ static bool serializeModuleSummary(SILModule *SM, const PrimarySpecificPaths &PSPs, const ASTContext &Context) { auto summaryOutputPath = PSPs.SupplementaryOutputs.ModuleSummaryOutputPath; - return withOutputFile(Context.Diags, Context.getOutputBackend(), + return withOutputPath(Context.Diags, Context.getOutputBackend(), summaryOutputPath, [&](llvm::raw_ostream &out) { out << "Some stuff"; return false; diff --git a/lib/FrontendTool/ImportedModules.cpp b/lib/FrontendTool/ImportedModules.cpp index 883b0b5186d20..39d9cbf58c4bd 100644 --- a/lib/FrontendTool/ImportedModules.cpp +++ b/lib/FrontendTool/ImportedModules.cpp @@ -114,7 +114,7 @@ bool swift::emitImportedModules(ModuleDecl *mainModule, } if (auto error = out->keep()) { - diags.diagnose(SourceLoc(), diag::error_opening_output, + diags.diagnose(SourceLoc(), diag::error_closing_output, path, toString(std::move(error))); return true; } diff --git a/lib/FrontendTool/MakeStyleDependencies.cpp b/lib/FrontendTool/MakeStyleDependencies.cpp index 54ecc5f9f391f..4aeea9701b47b 100644 --- a/lib/FrontendTool/MakeStyleDependencies.cpp +++ b/lib/FrontendTool/MakeStyleDependencies.cpp @@ -141,7 +141,7 @@ bool swift::emitMakeDependenciesIfNeeded(DiagnosticEngine &diags, }); if (auto error = out->keep()) { - diags.diagnose(SourceLoc(), diag::error_opening_output, + diags.diagnose(SourceLoc(), diag::error_closing_output, dependenciesFilePath, toString(std::move(error))); return true; } diff --git a/lib/IRGen/IRGen.cpp b/lib/IRGen/IRGen.cpp index 1ee21934b1d3f..e653747e52546 100644 --- a/lib/IRGen/IRGen.cpp +++ b/lib/IRGen/IRGen.cpp @@ -569,14 +569,14 @@ bool swift::performLLVM(const IRGenOptions &Opts, } llvm::Optional OutputFile; - auto CloseOutputFile = llvm::make_scope_exit([&]() { + SWIFT_DEFER { if (!OutputFile) return; if (auto E = OutputFile->keep()) { - diagnoseSync(Diags, DiagMutex, SourceLoc(), diag::error_opening_output, + diagnoseSync(Diags, DiagMutex, SourceLoc(), diag::error_closing_output, OutputFilename, toString(std::move(E))); } - }); + }; if (!OutputFilename.empty()) { // Try to open the output file. Clobbering an existing file is fine. // Open in binary mode if we're doing binary output. diff --git a/lib/SymbolGraphGen/SymbolGraphGen.cpp b/lib/SymbolGraphGen/SymbolGraphGen.cpp index ef7850f84e26f..101c34b1b2b11 100644 --- a/lib/SymbolGraphGen/SymbolGraphGen.cpp +++ b/lib/SymbolGraphGen/SymbolGraphGen.cpp @@ -40,7 +40,7 @@ int serializeSymbolGraph(SymbolGraph &SG, SmallString<1024> OutputPath(Options.OutputDir); llvm::sys::path::append(OutputPath, FileName); - return withOutputFile( + return withOutputPath( SG.M.getASTContext().Diags, SG.M.getASTContext().getOutputBackend(), OutputPath, [&](raw_ostream &OS) { llvm::json::OStream J(OS, Options.PrettyPrint ? 2 : 0); diff --git a/test/Frontend/output_determinism_check.swift b/test/Frontend/output_determinism_check.swift index 251aae367425e..318ffaaead8f5 100644 --- a/test/Frontend/output_determinism_check.swift +++ b/test/Frontend/output_determinism_check.swift @@ -1,28 +1,28 @@ // RUN: %empty-directory(%t) -// RUN: %target-swift-frontend -module-name test -emit-module -o %t/test.swiftmodule -primary-file %s -emit-module-doc-path %t/test.docc -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=MODULE_OUTPUT --check-prefix=DOCC_OUTPUT -// RUN: %target-swift-frontend -module-name test -emit-sib -o %t/test.sib -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=SIB_OUTPUT +// RUN: %target-swift-frontend -module-name test -emit-module -o %t/test.swiftmodule -primary-file %s -emit-module-doc-path %t/test.docc -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=MODULE_OUTPUT --check-prefix=DOCC_OUTPUT +// RUN: %target-swift-frontend -module-name test -emit-sib -o %t/test.sib -primary-file %s -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=SIB_OUTPUT /// object files are "not" deterministic because the second run going to match the mod hash and skip code generation. -// RUN: not %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=OBJECT_MISMATCH +// RUN: not %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=OBJECT_MISMATCH /// object files should match when forcing object generation. -// RUN: %target-swift-frontend -module-name test -emit-dependencies -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check -always-compile-output-files 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT --check-prefix=DEPS_OUTPUT +// RUN: %target-swift-frontend -module-name test -emit-dependencies -c -o %t/test.o -primary-file %s -enable-deterministic-check -always-compile-output-files 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT --check-prefix=DEPS_OUTPUT /// FIXME: Fine-grain dependencies graph is not deterministics. -/// FAIL: %target-swift-frontend -module-name test -emit-reference-dependencies-path %t/test.swiftdeps -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check -always-compile-output-files +/// FAIL: %target-swift-frontend -module-name test -emit-reference-dependencies-path %t/test.swiftdeps -c -o %t/test.o -primary-file %s -enable-deterministic-check -always-compile-output-files /// Explicit module build. Check building swiftmodule from interface file. -// RUN: %target-swift-frontend -scan-dependencies -module-name test -o %t/test.json %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=DEPSCAN_OUTPUT +// RUN: %target-swift-frontend -scan-dependencies -module-name test -o %t/test.json %s -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=DEPSCAN_OUTPUT /// TODO: Implicit module build use a different compiler instance so it doesn't support checking yet. -// RUN: %target-swift-frontend -typecheck -emit-module-interface-path %t/test.swiftinterface %s -O -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=INTERFACE_OUTPUT +// RUN: %target-swift-frontend -typecheck -emit-module-interface-path %t/test.swiftinterface %s -O -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=INTERFACE_OUTPUT /// Hit cache and not emit the second time. // RUN: rm %t/test.swiftmodule -// RUN: not %target-swift-frontend -compile-module-from-interface %t/test.swiftinterface -explicit-interface-module-build -o %t/test.swiftmodule -enable-swift-deterministic-check 2>&1 | %FileCheck --check-prefix=MODULE_MISMATCH %s +// RUN: not %target-swift-frontend -compile-module-from-interface %t/test.swiftinterface -explicit-interface-module-build -o %t/test.swiftmodule -enable-deterministic-check 2>&1 | %FileCheck --check-prefix=MODULE_MISMATCH %s /// Force swiftmodule generation. -// RUN: %target-swift-frontend -compile-module-from-interface %t/test.swiftinterface -explicit-interface-module-build -o %t/test.swiftmodule -enable-swift-deterministic-check -always-compile-output-files 2>&1 | %FileCheck --check-prefix=MODULE_OUTPUT %s +// RUN: %target-swift-frontend -compile-module-from-interface %t/test.swiftinterface -explicit-interface-module-build -o %t/test.swiftmodule -enable-deterministic-check -always-compile-output-files 2>&1 | %FileCheck --check-prefix=MODULE_OUTPUT %s -// RUN: %target-swift-frontend -scan-dependencies -module-name test %s -o %t/test.deps.json -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=DEPS_JSON_OUTPUT +// RUN: %target-swift-frontend -scan-dependencies -module-name test %s -o %t/test.deps.json -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=DEPS_JSON_OUTPUT -// RUN: %target-swift-frontend -emit-pcm -module-name UserClangModule -o %t/test.pcm %S/Inputs/dependencies/module.modulemap -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=PCM_OUTPUT +// RUN: %target-swift-frontend -emit-pcm -module-name UserClangModule -o %t/test.pcm %S/Inputs/dependencies/module.modulemap -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=PCM_OUTPUT // DOCC_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.docc' // MODULE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftmodule' diff --git a/tools/sil-llvm-gen/SILLLVMGen.cpp b/tools/sil-llvm-gen/SILLLVMGen.cpp index b2ca3a7e6093a..b5e47878b7430 100644 --- a/tools/sil-llvm-gen/SILLLVMGen.cpp +++ b/tools/sil-llvm-gen/SILLLVMGen.cpp @@ -192,7 +192,7 @@ int main(int argc, char **argv) { } auto closeFile = llvm::make_scope_exit([&]() { if (auto E = outFile->keep()) { - CI.getDiags().diagnose(SourceLoc(), diag::error_opening_output, + CI.getDiags().diagnose(SourceLoc(), diag::error_closing_output, OutputFilename, toString(std::move(E))); } }); diff --git a/tools/swift-dependency-tool/swift-dependency-tool.cpp b/tools/swift-dependency-tool/swift-dependency-tool.cpp index 5f935f73409d7..35737d832d63e 100644 --- a/tools/swift-dependency-tool/swift-dependency-tool.cpp +++ b/tools/swift-dependency-tool/swift-dependency-tool.cpp @@ -226,7 +226,7 @@ int main(int argc, char *argv[]) { } bool hadError = - withOutputFile(diags, outputBackend, options::OutputFilename, + withOutputPath(diags, outputBackend, options::OutputFilename, [&](llvm::raw_pwrite_stream &out) { out << "# Fine-grained v0\n"; llvm::yaml::Output yamlWriter(out); diff --git a/unittests/ClangImporter/ClangImporterTests.cpp b/unittests/ClangImporter/ClangImporterTests.cpp index 9ce31da9b3f0a..f5f631dc0af12 100644 --- a/unittests/ClangImporter/ClangImporterTests.cpp +++ b/unittests/ClangImporter/ClangImporterTests.cpp @@ -83,11 +83,10 @@ TEST(ClangImporterTest, emitPCHInMemory) { ASTContext::get(langOpts, typecheckOpts, silOpts, searchPathOpts, options, symbolGraphOpts, sourceMgr, diags)); auto importer = ClangImporter::create(*context); - auto backend = llvm::makeIntrusiveRefCnt(); std::string PCH = createFilename(cache, "bridging.h.pch"); ASSERT_FALSE(importer->canReadPCH(PCH)); - ASSERT_FALSE(importer->emitBridgingPCH(backend, options.BridgingHeader, PCH)); + ASSERT_FALSE(importer->emitBridgingPCH(options.BridgingHeader, PCH)); ASSERT_TRUE(importer->canReadPCH(PCH)); // Overwrite the PCH with garbage. We should still be able to read it from From ab9975f34d37cb94abdced13106b1b08cb4245fc Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 5 Apr 2023 03:52:39 -0700 Subject: [PATCH 11/11] Fix abi-digester for non-macos platforms Fix broken test cases on non-macos platforms. --- lib/APIDigester/ModuleAnalyzerNodes.cpp | 2 -- lib/DriverTool/swift_api_digester_main.cpp | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/APIDigester/ModuleAnalyzerNodes.cpp b/lib/APIDigester/ModuleAnalyzerNodes.cpp index 753402f6466d1..090db50ab61d6 100644 --- a/lib/APIDigester/ModuleAnalyzerNodes.cpp +++ b/lib/APIDigester/ModuleAnalyzerNodes.cpp @@ -2555,8 +2555,6 @@ void swift::ide::api::dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, if (Opts.Verbose) llvm::errs() << "Dumping SDK...\n"; SwiftDeclCollector::serialize(os, Root, load); - // if (Opts.Verbose) - // llvm::errs() << "Dumped to "<< OutputFile << "\n"; } void swift::ide::api::dumpSDKRoot(SDKNodeRoot *Root, llvm::raw_ostream &os) { diff --git a/lib/DriverTool/swift_api_digester_main.cpp b/lib/DriverTool/swift_api_digester_main.cpp index 34e457b041362..7265b48b447e8 100644 --- a/lib/DriverTool/swift_api_digester_main.cpp +++ b/lib/DriverTool/swift_api_digester_main.cpp @@ -2539,6 +2539,8 @@ class SwiftAPIDigesterInvocation { switch (Action) { case ActionType::DumpSDK: { llvm::StringSet<> Modules; + if (prepareForDump(InitInvoke, Modules)) + return 1; auto JsonOut = getJsonOutputFilePath(InitInvoke.getLangOptions().Target, CheckerOpts.ABI, OutputFile, OutputDir); @@ -2548,9 +2550,7 @@ class SwiftAPIDigesterInvocation { llvm::errs() << "Cannot open JSON output file: " << JsonOut << "\n"; return 1; } - return (prepareForDump(InitInvoke, Modules)) - ? 1 - : dumpSDKContent(InitInvoke, Modules, fs, CheckerOpts); + return dumpSDKContent(InitInvoke, Modules, fs, CheckerOpts); } case ActionType::MigratorGen: case ActionType::DiagnoseSDKs: {