From f4da34f363e04274b59a6e36bec19cd2a2d9c06e Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Thu, 7 Mar 2019 16:52:53 -0800 Subject: [PATCH 1/3] [ParseableInterface] Only open module buffers once while loading In addition to being wasteful, this is a correctness issue -- the compiler should only ever have one view of this file, and it should not read a potentially different file after validating dependencies. rdar://48654608 --- .../Serialization/SerializedModuleLoader.h | 5 + include/swift/Subsystems.h | 12 ++ .../ParseableInterfaceModuleLoader.cpp | 141 +++++++++++------- lib/Serialization/Serialization.cpp | 43 ++++++ lib/Serialization/SerializedModuleLoader.cpp | 37 +++-- 5 files changed, 171 insertions(+), 67 deletions(-) diff --git a/include/swift/Serialization/SerializedModuleLoader.h b/include/swift/Serialization/SerializedModuleLoader.h index a76a1b1a61b31..3af6cfcbd2f9a 100644 --- a/include/swift/Serialization/SerializedModuleLoader.h +++ b/include/swift/Serialization/SerializedModuleLoader.h @@ -63,6 +63,11 @@ class SerializedModuleLoaderBase : public ModuleLoader { std::unique_ptr *ModuleBuffer, std::unique_ptr *ModuleDocBuffer); + std::error_code + openModuleDocFile(AccessPathElem ModuleID, + StringRef ModuleDocPath, + std::unique_ptr *ModuleDocBuffer); + /// If the module loader subclass knows that all options have been tried for /// loading an architecture-specific file out of a swiftmodule bundle, try /// to list the architectures that \e are present. diff --git a/include/swift/Subsystems.h b/include/swift/Subsystems.h index 673e8e6994fc0..c245a44de1f92 100644 --- a/include/swift/Subsystems.h +++ b/include/swift/Subsystems.h @@ -264,6 +264,18 @@ namespace swift { void serialize(ModuleOrSourceFile DC, const SerializationOptions &options, const SILModule *M = nullptr); + /// Serializes a module or single source file to the given output file and + /// returns back the file's contents as a memory buffer. + /// + /// Use this if you intend to immediately load the serialized module, as that + /// will both avoid extra filesystem traffic and will ensure you read back + /// exactly what was written. + void serializeToBuffers(ModuleOrSourceFile DC, + const SerializationOptions &opts, + std::unique_ptr *moduleBuffer, + std::unique_ptr *moduleDocBuffer, + const SILModule *M = nullptr); + /// Get the CPU, subtarget feature options, and triple to use when emitting code. std::tuple, std::string> diff --git a/lib/Frontend/ParseableInterfaceModuleLoader.cpp b/lib/Frontend/ParseableInterfaceModuleLoader.cpp index 238e527727625..f252469a2b252 100644 --- a/lib/Frontend/ParseableInterfaceModuleLoader.cpp +++ b/lib/Frontend/ParseableInterfaceModuleLoader.cpp @@ -173,30 +173,40 @@ class DiscoveredModule { /// The kind of module that's been discovered. const Kind kind; - DiscoveredModule(StringRef path, Kind kind): kind(kind), path(path) {} + DiscoveredModule(StringRef path, Kind kind, + std::unique_ptr moduleBuffer) + : kind(kind), moduleBuffer(std::move(moduleBuffer)), path(path) {} + public: + /// The contents of the .swiftmodule, if we've read it while validating + /// dependencies. + std::unique_ptr moduleBuffer; + /// The path to the discovered serialized .swiftmodule on disk. const std::string path; /// Creates a \c Normal discovered module. - static DiscoveredModule normal(StringRef path) { - return { path, Kind::Normal }; + static DiscoveredModule normal(StringRef path, + std::unique_ptr moduleBuffer) { + return { path, Kind::Normal, std::move(moduleBuffer) }; } /// Creates a \c Prebuilt discovered module. - static DiscoveredModule prebuilt(StringRef path) { - return { path, Kind::Prebuilt }; + static DiscoveredModule prebuilt( + StringRef path, std::unique_ptr moduleBuffer) { + return { path, Kind::Prebuilt, std::move(moduleBuffer) }; } /// Creates a \c Forwarded discovered module, whose dependencies have been /// externally validated by a \c ForwardingModule. - static DiscoveredModule forwarded(StringRef path) { - return { path, Kind::Forwarded }; + static DiscoveredModule forwarded( + StringRef path, std::unique_ptr moduleBuffer) { + return { path, Kind::Forwarded, std::move(moduleBuffer) }; } bool isNormal() const { return kind == Kind::Normal; } bool isPrebuilt() const { return kind == Kind::Prebuilt; } - bool isForwarding() const { return kind == Kind::Forwarded; } + bool isForwarded() const { return kind == Kind::Forwarded; } }; } // end anonymous namespace @@ -453,7 +463,8 @@ class swift::ParseableInterfaceBuilder { return subInvocation; } - bool buildSwiftModule(StringRef OutPath, bool ShouldSerializeDeps) { + bool buildSwiftModule(StringRef OutPath, bool ShouldSerializeDeps, + std::unique_ptr *ModuleBuffer) { bool SubError = false; bool RunSuccess = llvm::CrashRecoveryContext().RunSafelyOnThread([&] { // Note that we don't assume cachePath is the same as the Clang @@ -559,7 +570,10 @@ class swift::ParseableInterfaceBuilder { if (ShouldSerializeDeps) SerializationOpts.Dependencies = Deps; SILMod->setSerializeSILAction([&]() { - serialize(Mod, SerializationOpts, SILMod.get()); + // We don't want to serialize module docs in the cache -- they + // will be serialized beside the interface file. + serializeToBuffers(Mod, SerializationOpts, ModuleBuffer, + /*ModuleDocBuffer*/nullptr, SILMod.get()); }); LLVM_DEBUG(llvm::dbgs() << "Running SIL processing passes\n"); @@ -717,22 +731,23 @@ class ParseableInterfaceModuleLoaderImpl { // Check that the output .swiftmodule file is at least as new as all the // dependencies it read when it was built last time. - bool swiftModuleIsUpToDate(StringRef modulePath, - SmallVectorImpl &AllDeps) { + bool swiftModuleIsUpToDate( + StringRef modulePath, SmallVectorImpl &AllDeps, + std::unique_ptr &moduleBuffer) { auto OutBuf = fs.getBufferForFile(modulePath); if (!OutBuf) return false; - return serializedASTBufferIsUpToDate(*OutBuf.get(), AllDeps); + moduleBuffer = std::move(*OutBuf); + return serializedASTBufferIsUpToDate(*moduleBuffer, AllDeps); } // Check that a "forwarding" .swiftmodule file is at least as new as all the // dependencies it read when it was built last time. Requires that the // forwarding module has been loaded from disk. - bool forwardingModuleIsUpToDate(const ForwardingModule &fwd, - SmallVectorImpl &deps) { + bool forwardingModuleIsUpToDate( + const ForwardingModule &fwd, SmallVectorImpl &deps, + std::unique_ptr &moduleBuffer) { // First, make sure the underlying module path exists and is valid. - // FIXME: We should preserve this buffer, rather than opening it again - // when loading the module. auto modBuf = fs.getBufferForFile(fwd.underlyingModulePath); if (!modBuf || !serializedASTLooksValid(*modBuf.get())) return false; @@ -743,7 +758,11 @@ class ParseableInterfaceModuleLoaderImpl { FileDependency::modTimeBased( dep.path, dep.size, dep.lastModificationTime)); } - return dependenciesAreUpToDate(deps); + if (!dependenciesAreUpToDate(deps)) + return false; + + moduleBuffer = std::move(*modBuf); + return true; } Optional @@ -809,22 +828,25 @@ class ParseableInterfaceModuleLoaderImpl { // First, check the cached module path. Whatever's in this cache represents // the most up-to-date knowledge we have about the module. if (auto cachedBufOrError = fs.getBufferForFile(cachedOutputPath)) { - auto &buf = *cachedBufOrError.get(); + auto buf = std::move(*cachedBufOrError); // Check to see if the module is a serialized AST. If it's not, then we're // probably dealing with a Forwarding Module, which is a YAML file. bool isForwardingModule = - !serialization::isSerializedAST(buf.getBuffer()); + !serialization::isSerializedAST(buf->getBuffer()); // If it's a forwarding module, load the YAML file from disk and check // if it's up-to-date. if (isForwardingModule) { - auto modOrErr = ForwardingModule::load(buf); - if (modOrErr && forwardingModuleIsUpToDate(*modOrErr, deps)) - return DiscoveredModule::forwarded(modOrErr->underlyingModulePath); + if (auto forwardingModule = ForwardingModule::load(*buf)) { + std::unique_ptr moduleBuffer; + if (forwardingModuleIsUpToDate(*forwardingModule, deps, moduleBuffer)) + return DiscoveredModule::forwarded( + forwardingModule->underlyingModulePath, std::move(moduleBuffer)); + } // Otherwise, check if the AST buffer itself is up to date. - } else if (serializedASTBufferIsUpToDate(buf, deps)) { - return DiscoveredModule::normal(cachedOutputPath); + } else if (serializedASTBufferIsUpToDate(*buf, deps)) { + return DiscoveredModule::normal(cachedOutputPath, std::move(buf)); } } @@ -835,9 +857,10 @@ class ParseableInterfaceModuleLoaderImpl { // from the SDK. if (!prebuiltCacheDir.empty()) { llvm::SmallString<256> scratch; + std::unique_ptr moduleBuffer; auto path = computePrebuiltModulePath(scratch); - if (path && swiftModuleIsUpToDate(*path, deps)) - return DiscoveredModule::prebuilt(*path); + if (path && swiftModuleIsUpToDate(*path, deps, moduleBuffer)) + return DiscoveredModule::prebuilt(*path, std::move(moduleBuffer)); } // Finally, if there's a module adjacent to the .swiftinterface that we can @@ -899,10 +922,12 @@ class ParseableInterfaceModuleLoaderImpl { }); } - /// Looks up the best module to load for a given interface. See the main - /// comment in \c ParseableInterfaceModuleLoader.h for an explanation of - /// the module loading strategy. - llvm::ErrorOr findOrBuildLoadableModule() { + /// Looks up the best module to load for a given interface, and returns a + /// buffer of the module's contents. See the main comment in + /// \c ParseableInterfaceModuleLoader.h for an explanation of the module + /// loading strategy. + llvm::ErrorOr> + findOrBuildLoadableModule() { // Set up a builder if we need to build the module. It'll also set up // the subinvocation we'll need to use to compute the cache paths. @@ -927,24 +952,27 @@ class ParseableInterfaceModuleLoaderImpl { moduleOrErr.getError() != std::errc::no_such_file_or_directory) return moduleOrErr.getError(); - // We discovered a module! Return the module's path so we know what to load. + // We discovered a module! Return that module's buffer so we can load it. if (moduleOrErr) { - auto &module = *moduleOrErr; + auto module = std::move(moduleOrErr.get()); + // If it's prebuilt, use this time to generate a forwarding module. if (module.isPrebuilt()) if (writeForwardingModule(module, cachedOutputPath, allDeps)) return std::make_error_code(std::errc::not_supported); - // FIXME: return and load module buffer directly - return module.path; + return std::move(module.moduleBuffer); } + std::unique_ptr moduleBuffer; // We didn't discover a module corresponding to this interface. Build one. - if (builder.buildSwiftModule(cachedOutputPath, /*shouldSerializeDeps*/true)) + if (builder.buildSwiftModule(cachedOutputPath, /*shouldSerializeDeps*/true, + &moduleBuffer)) return std::make_error_code(std::errc::invalid_argument); - // FIXME: return and load module buffer directly - return cachedOutputPath.str(); + assert(moduleBuffer && + "failed to write module buffer but returned success?"); + return std::move(moduleBuffer); } }; @@ -984,26 +1012,24 @@ std::error_code ParseableInterfaceModuleLoader::findModuleFilesInDirectory( // Ask the impl to find us a module that we can load or give us an error // telling us that we couldn't load it. - auto PathOrErr = Impl.findOrBuildLoadableModule(); - if (!PathOrErr) - return PathOrErr.getError(); - std::string FinalPath = std::move(*PathOrErr); - - // Finish off by delegating back up to the SerializedModuleLoaderBase - // routine that can load the recently-manufactured serialized module. - LLVM_DEBUG(llvm::dbgs() << "Loading " << FinalPath - << " via normal module loader\n"); + auto ModuleBufferOrErr = Impl.findOrBuildLoadableModule(); + if (!ModuleBufferOrErr) + return ModuleBufferOrErr.getError(); + + if (ModuleBuffer) { + *ModuleBuffer = std::move(*ModuleBufferOrErr); + } + + // Delegate back to the serialized module loader to load the module doc. llvm::SmallString<256> DocPath{DirPath}; path::append(DocPath, ModuleDocFilename); - auto ErrorCode = SerializedModuleLoaderBase::openModuleFiles( - ModuleID, FinalPath, DocPath, ModuleBuffer, ModuleDocBuffer); - LLVM_DEBUG(llvm::dbgs() << "Loaded " << FinalPath - << " via normal module loader"); - if (ErrorCode) { - LLVM_DEBUG(llvm::dbgs() << " with error: " << ErrorCode.message()); - } - LLVM_DEBUG(llvm::dbgs() << "\n"); - return ErrorCode; + auto DocLoadErr = + SerializedModuleLoaderBase::openModuleDocFile(ModuleID, DocPath, + ModuleDocBuffer); + if (DocLoadErr) + return DocLoadErr; + + return std::error_code(); } @@ -1018,5 +1044,6 @@ bool ParseableInterfaceModuleLoader::buildSwiftModuleFromSwiftInterface( // make them relocatable (SDK-relative) if we want to ship the built // swiftmodules to another machine. Just track them as absolute paths // for now, so we can test the dependency tracking locally. - return builder.buildSwiftModule(OutPath, /*shouldSerializeDeps*/true); + return builder.buildSwiftModule(OutPath, /*shouldSerializeDeps*/true, + /*ModuleBuffer*/nullptr); } diff --git a/lib/Serialization/Serialization.cpp b/lib/Serialization/Serialization.cpp index 7f21057684bc5..41b2997319423 100644 --- a/lib/Serialization/Serialization.cpp +++ b/lib/Serialization/Serialization.cpp @@ -51,6 +51,7 @@ #include "llvm/Support/OnDiskHashTable.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" #include @@ -4809,6 +4810,48 @@ void Serializer::writeToStream(raw_ostream &os, ModuleOrSourceFile DC, S.writeToStream(os); } +void swift::serializeToBuffers( + ModuleOrSourceFile DC, const SerializationOptions &options, + std::unique_ptr *moduleBuffer, + std::unique_ptr *moduleDocBuffer, + const SILModule *M) { + + assert(options.OutputPath && options.OutputPath[0] != '\0'); + { + SharedTimer timer("Serialization, swiftmodule, to buffer"); + llvm::SmallString<1024> buf; + llvm::raw_svector_ostream stream(buf); + Serializer::writeToStream(stream, DC, M, options); + bool hadError = withOutputFile(getContext(DC).Diags, + options.OutputPath, + [&](raw_ostream &out) { + out << stream.str(); + return false; + }); + if (hadError) + return; + if (moduleBuffer) + *moduleBuffer = llvm::make_unique( + std::move(buf), options.OutputPath); + } + + if (options.DocOutputPath && options.DocOutputPath[0] != '\0') { + SharedTimer timer("Serialization, swiftdoc, to buffer"); + llvm::SmallString<1024> buf; + llvm::raw_svector_ostream stream(buf); + writeDocToStream(stream, DC, options.GroupInfoPath); + (void)withOutputFile(getContext(DC).Diags, + options.DocOutputPath, + [&](raw_ostream &out) { + out << stream.str(); + return false; + }); + if (moduleDocBuffer) + *moduleDocBuffer = llvm::make_unique( + std::move(buf), options.DocOutputPath); + } +} + void swift::serialize(ModuleOrSourceFile DC, const SerializationOptions &options, const SILModule *M) { diff --git a/lib/Serialization/SerializedModuleLoader.cpp b/lib/Serialization/SerializedModuleLoader.cpp index 7c229a2e5f198..1db71e68fd54a 100644 --- a/lib/Serialization/SerializedModuleLoader.cpp +++ b/lib/Serialization/SerializedModuleLoader.cpp @@ -43,6 +43,29 @@ SerializedModuleLoaderBase::~SerializedModuleLoaderBase() = default; SerializedModuleLoader::~SerializedModuleLoader() = default; +std::error_code SerializedModuleLoaderBase::openModuleDocFile( + AccessPathElem ModuleID, StringRef ModuleDocPath, + std::unique_ptr *ModuleDocBuffer) { + + if (!ModuleDocBuffer) + return std::error_code(); + + llvm::vfs::FileSystem &FS = *Ctx.SourceMgr.getFileSystem(); + + // Try to open the module documentation file. If it does not exist, ignore + // the error. However, pass though all other errors. + llvm::ErrorOr> ModuleDocOrErr = + FS.getBufferForFile(ModuleDocPath); + if (ModuleDocOrErr) { + *ModuleDocBuffer = std::move(*ModuleDocOrErr); + } else if (ModuleDocOrErr.getError() != + std::errc::no_such_file_or_directory) { + return ModuleDocOrErr.getError(); + } + + return std::error_code(); +} + std::error_code SerializedModuleLoaderBase::openModuleFiles( AccessPathElem ModuleID, StringRef ModulePath, StringRef ModuleDocPath, std::unique_ptr *ModuleBuffer, @@ -74,18 +97,12 @@ std::error_code SerializedModuleLoaderBase::openModuleFiles( if (!ModuleOrErr) return ModuleOrErr.getError(); - // Try to open the module documentation file. If it does not exist, ignore - // the error. However, pass though all other errors. - llvm::ErrorOr> ModuleDocOrErr = - FS.getBufferForFile(ModuleDocPath); - if (!ModuleDocOrErr && - ModuleDocOrErr.getError() != std::errc::no_such_file_or_directory) { - return ModuleDocOrErr.getError(); - } + auto ModuleDocErr = + openModuleDocFile(ModuleID, ModuleDocPath, ModuleDocBuffer); + if (ModuleDocErr) + return ModuleDocErr; *ModuleBuffer = std::move(ModuleOrErr.get()); - if (ModuleDocOrErr) - *ModuleDocBuffer = std::move(ModuleDocOrErr.get()); return std::error_code(); } From 9ef4b5f358e42ace3b4971624f45e47c14e2d818 Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Thu, 21 Mar 2019 18:16:52 -0700 Subject: [PATCH 2/3] [ParseableInterface] Add test for writing modules to a buffer --- .../Frontend/ParseableInterfaceModuleLoader.h | 6 +- unittests/FrontendTool/CMakeLists.txt | 9 +- unittests/FrontendTool/ModuleLoadingTests.cpp | 145 ++++++++++++++++++ 3 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 unittests/FrontendTool/ModuleLoadingTests.cpp diff --git a/include/swift/Frontend/ParseableInterfaceModuleLoader.h b/include/swift/Frontend/ParseableInterfaceModuleLoader.h index 757f169593f33..e9392342fbb81 100644 --- a/include/swift/Frontend/ParseableInterfaceModuleLoader.h +++ b/include/swift/Frontend/ParseableInterfaceModuleLoader.h @@ -110,6 +110,10 @@ namespace clang { class CompilerInstance; } +namespace unittest { + class ParseableInterfaceModuleLoaderTest; +} + namespace swift { /// A ModuleLoader that runs a subordinate \c CompilerInvocation and @@ -117,6 +121,7 @@ namespace swift { /// files on the fly, caching the resulting .swiftmodules in the module cache /// directory, and loading the serialized .swiftmodules from there. class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase { + friend class unittest::ParseableInterfaceModuleLoaderTest; explicit ParseableInterfaceModuleLoader(ASTContext &ctx, StringRef cacheDir, StringRef prebuiltCacheDir, DependencyTracker *tracker, @@ -134,7 +139,6 @@ class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase { std::unique_ptr *ModuleBuffer, std::unique_ptr *ModuleDocBuffer) override; - public: static std::unique_ptr create(ASTContext &ctx, StringRef cacheDir, StringRef prebuiltCacheDir, diff --git a/unittests/FrontendTool/CMakeLists.txt b/unittests/FrontendTool/CMakeLists.txt index a5abc9d3ceb7c..e726099af6b86 100644 --- a/unittests/FrontendTool/CMakeLists.txt +++ b/unittests/FrontendTool/CMakeLists.txt @@ -1,3 +1,8 @@ add_swift_unittest(SwiftFrontendTests - FrontendToolTests.cpp) -target_link_libraries(SwiftFrontendTests PRIVATE swiftFrontendTool) + FrontendToolTests.cpp + ModuleLoadingTests.cpp) + +target_link_libraries(SwiftFrontendTests + PRIVATE + swiftFrontend + swiftFrontendTool) diff --git a/unittests/FrontendTool/ModuleLoadingTests.cpp b/unittests/FrontendTool/ModuleLoadingTests.cpp new file mode 100644 index 0000000000000..81337b1ac6e13 --- /dev/null +++ b/unittests/FrontendTool/ModuleLoadingTests.cpp @@ -0,0 +1,145 @@ +//===--- ModuleLoadingTests.cpp -------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "gtest/gtest.h" +#include "swift/AST/ASTContext.h" +#include "swift/Frontend/Frontend.h" +#include "swift/Frontend/ParseableInterfaceModuleLoader.h" +#include "swift/Frontend/PrintingDiagnosticConsumer.h" +#include "swift/Serialization/Validation.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/Support/VirtualFileSystem.h" + +using namespace swift; + +static std::string createFilename(StringRef base, StringRef name) { + SmallString<256> path = base; + llvm::sys::path::append(path, name); + return llvm::Twine(path).str(); +} + +static bool emitFileWithContents(StringRef path, StringRef contents, + std::string *pathOut = nullptr) { + int fd; + if (llvm::sys::fs::openFileForWrite(path, fd)) + return true; + if (pathOut) + *pathOut = path; + llvm::raw_fd_ostream file(fd, /*shouldClose=*/true); + file << contents; + return false; +} + +static bool emitFileWithContents(StringRef base, StringRef name, + StringRef contents, + std::string *pathOut = nullptr) { + return emitFileWithContents(createFilename(base, name), contents, pathOut); +} + +namespace unittest { + +class OpenTrackingFileSystem : public llvm::vfs::ProxyFileSystem { + llvm::StringMap numberOfOpensPerFile; +public: + OpenTrackingFileSystem(llvm::IntrusiveRefCntPtr fs) + : llvm::vfs::ProxyFileSystem(fs) {} + + llvm::ErrorOr> + openFileForRead(const Twine &Path) override { + numberOfOpensPerFile[Path.str()] += 1; + return ProxyFileSystem::openFileForRead(Path); + } + + unsigned numberOfOpens(StringRef path) { + return numberOfOpensPerFile[path]; + } +}; + +class ParseableInterfaceModuleLoaderTest : public testing::Test { +protected: + void setupAndLoadParseableModule() { + SmallString<256> tempDir; + ASSERT_FALSE(llvm::sys::fs::createUniqueDirectory( + "ParseableModuleBufferTests.emitModuleInMemory", tempDir)); + SWIFT_DEFER { llvm::sys::fs::remove_directories(tempDir); }; + + auto cacheDir = createFilename(tempDir, "ModuleCache"); + ASSERT_FALSE(llvm::sys::fs::create_directory(cacheDir)); + + auto prebuiltCacheDir = createFilename(tempDir, "PrebuiltModuleCache"); + ASSERT_FALSE(llvm::sys::fs::create_directory(prebuiltCacheDir)); + + // Emit an interface file that we can attempt to compile. + ASSERT_FALSE(emitFileWithContents(tempDir, "Library.swiftinterface", + "// swift-interface-format-version: 1.0\n" + "// swift-module-flags: -module-name TestModule -parse-stdlib\n" + "public func foo()\n")); + + SourceManager sourceMgr; + + // Create a file system that tracks how many times a file has been opened. + llvm::IntrusiveRefCntPtr fs( + new OpenTrackingFileSystem(sourceMgr.getFileSystem())); + + sourceMgr.setFileSystem(fs); + PrintingDiagnosticConsumer printingConsumer; + DiagnosticEngine diags(sourceMgr); + diags.addConsumer(printingConsumer); + LangOptions langOpts; + langOpts.Target = llvm::Triple(llvm::sys::getDefaultTargetTriple()); + SearchPathOptions searchPathOpts; + auto ctx = ASTContext::get(langOpts, searchPathOpts, sourceMgr, diags); + + auto loader = ParseableInterfaceModuleLoader::create( + *ctx, cacheDir, prebuiltCacheDir, + /*dependencyTracker*/nullptr, + ModuleLoadingMode::PreferSerialized); + + Identifier moduleName = ctx->getIdentifier("TestModule"); + + std::unique_ptr moduleBuffer; + std::unique_ptr moduleDocBuffer; + + auto error = + loader->findModuleFilesInDirectory({moduleName, SourceLoc()}, tempDir, + "Library.swiftmodule", "Library.swiftdoc", + &moduleBuffer, &moduleDocBuffer); + ASSERT_FALSE(error); + ASSERT_FALSE(diags.hadAnyError()); + + ASSERT_NE(nullptr, moduleBuffer); + + // We should not have written a module doc file. + ASSERT_EQ(nullptr, moduleDocBuffer); + + // Make sure the buffer identifier points to the written module. + StringRef cachedModulePath = moduleBuffer->getBufferIdentifier(); + ASSERT_TRUE(fs->exists(cachedModulePath)); + + // Assert that we've only opened this file once, to write it. + ASSERT_EQ((unsigned)1, fs->numberOfOpens(cachedModulePath)); + + auto bufOrErr = fs->getBufferForFile(cachedModulePath); + ASSERT_TRUE(bufOrErr); + + auto bufData = (*bufOrErr)->getBuffer(); + auto validationInfo = serialization::validateSerializedAST(bufData); + ASSERT_EQ(serialization::Status::Valid, validationInfo.status); + ASSERT_EQ(bufData, moduleBuffer->getBuffer()); + } +}; + +TEST_F(ParseableInterfaceModuleLoaderTest, LoadModuleFromBuffer) { + setupAndLoadParseableModule(); +} + +} // end namespace unittest From 5a29cdc118288d27018e28ccd28106d0cb5e00d9 Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Fri, 22 Mar 2019 11:12:54 -0700 Subject: [PATCH 3/3] [ClangImporter] Allow importing without a resource dir on non-Darwin Previously, we would assert that there's a runtime resource dir available, but then accept the possibility of no glibc.modulemap. We should just do the same thing for 'no resource dir' as 'no module map'. --- lib/ClangImporter/ClangImporter.cpp | 53 +++++++++++++++++------------ 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index a88196a6c5380..76630992cc0b7 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -407,6 +407,35 @@ void ClangImporter::clearTypeResolver() { #pragma mark Module loading +/// Finds the glibc.modulemap file relative to the provided resource dir. +/// +/// Note that the module map used for Glibc depends on the target we're +/// compiling for, and is not included in the resource directory with the other +/// implicit module maps. It's at {freebsd|linux}/{arch}/glibc.modulemap. +static Optional +getGlibcModuleMapPath(StringRef resourceDir, llvm::Triple triple, + SmallVectorImpl &scratch) { + if (resourceDir.empty()) + return None; + + scratch.append(resourceDir.begin(), resourceDir.end()); + llvm::sys::path::append( + scratch, + swift::getPlatformNameForTriple(triple), + swift::getMajorArchitectureName(triple), + "glibc.modulemap"); + + // Only specify the module map if that file actually exists. + // It may not--for example in the case that + // `swiftc -target x86_64-unknown-linux-gnu -emit-ir` is invoked using + // a Swift compiler not built for Linux targets. + if (llvm::sys::fs::exists(scratch)) { + return StringRef(scratch.data(), scratch.size()); + } else { + return None; + } +} + static void getNormalInvocationArguments(std::vector &invocationArgStrs, ASTContext &ctx, @@ -571,28 +600,10 @@ getNormalInvocationArguments(std::vector &invocationArgStrs, } } - // The module map used for Glibc depends on the target we're compiling for, - // and is not included in the resource directory with the other implicit - // module maps. It's at {freebsd|linux}/{arch}/glibc.modulemap. SmallString<128> GlibcModuleMapPath; - GlibcModuleMapPath = searchPathOpts.RuntimeResourcePath; - - // Running without a resource directory is not a supported configuration. - assert(!GlibcModuleMapPath.empty()); - - llvm::sys::path::append( - GlibcModuleMapPath, - swift::getPlatformNameForTriple(triple), - swift::getMajorArchitectureName(triple), - "glibc.modulemap"); - - // Only specify the module map if that file actually exists. - // It may not--for example in the case that - // `swiftc -target x86_64-unknown-linux-gnu -emit-ir` is invoked using - // a Swift compiler not built for Linux targets. - if (llvm::sys::fs::exists(GlibcModuleMapPath)) { - invocationArgStrs.push_back( - (Twine("-fmodule-map-file=") + GlibcModuleMapPath).str()); + if (auto path = getGlibcModuleMapPath(searchPathOpts.RuntimeResourcePath, + triple, GlibcModuleMapPath)) { + invocationArgStrs.push_back((Twine("-fmodule-map-file=") + *path).str()); } else { // FIXME: Emit a warning of some kind. }