From 75da6cff5bde36ca5cd4b25870c673cb550d7296 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 15 Mar 2023 11:16:55 -0700 Subject: [PATCH 01/38] [clang][cas] Factor IncludeTreeBuilder out of IncludeTreeActionConsumer The IncludeTreeBuilder is responsible for building the include-tree for a specific TU. Factor it out of IncludeTreeActionConsumer to prepare for building modules, which require their own builder. (cherry picked from commit 062b023007b3b28e707a74c3bbdffb27b3d8b528) (cherry picked from commit 30c89b9f767156267c9973fd928069150a935498) --- .../IncludeTreeActionController.cpp | 192 ++++++++++++------ 1 file changed, 132 insertions(+), 60 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 5d8a2ee36b334..3182cf4ce41e9 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -20,6 +20,8 @@ using namespace dependencies; using llvm::Error; namespace { +class IncludeTreeBuilder; + class IncludeTreeActionController : public CallbackActionController { public: IncludeTreeActionController(cas::ObjectStore &DB, @@ -47,12 +49,41 @@ class IncludeTreeActionController : public CallbackActionController { Error finalize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) override; - Expected getObjectForFile(Preprocessor &PP, FileID FID); - Expected - getObjectForFileNonCached(FileManager &FM, const SrcMgr::FileInfo &FI); - Expected getObjectForBuffer(const SrcMgr::FileInfo &FI); - Expected addToFileList(FileManager &FM, const FileEntry *FE); +private: + IncludeTreeBuilder ¤t() { + assert(!BuilderStack.empty()); + return *BuilderStack.back(); + } + +private: + cas::ObjectStore &DB; + CASOptions CASOpts; + DepscanPrefixMapping PrefixMapping; + llvm::PrefixMapper PrefixMapper; + SmallVector> BuilderStack; + std::optional IncludeTreeResult; +}; +/// Callbacks for building an include-tree for a given translation unit or +/// module. The \c IncludeTreeActionController is responsiblee for pushing and +/// popping builders from the stack as modules are required. +class IncludeTreeBuilder { +public: + IncludeTreeBuilder(cas::ObjectStore &DB, llvm::PrefixMapper &PrefixMapper) + : DB(DB), PrefixMapper(PrefixMapper) {} + + Expected + finishIncludeTree(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation); + + void enteredInclude(Preprocessor &PP, FileID FID); + + void exitedInclude(Preprocessor &PP, FileID IncludedBy, FileID Include, + SourceLocation ExitLoc); + + void handleHasIncludeCheck(Preprocessor &PP, bool Result); + +private: struct FilePPState { SrcMgr::CharacteristicKind FileCharacteristic; cas::ObjectRef File; @@ -60,8 +91,12 @@ class IncludeTreeActionController : public CallbackActionController { llvm::SmallBitVector HasIncludeChecks; }; + Expected getObjectForFile(Preprocessor &PP, FileID FID); + Expected + getObjectForFileNonCached(FileManager &FM, const SrcMgr::FileInfo &FI); + Expected getObjectForBuffer(const SrcMgr::FileInfo &FI); + Expected addToFileList(FileManager &FM, const FileEntry *FE); Expected getCASTreeForFileIncludes(FilePPState &&PPState); - Expected createIncludeFile(StringRef Filename, cas::ObjectRef Contents); @@ -75,10 +110,10 @@ class IncludeTreeActionController : public CallbackActionController { return *E; } +private: cas::ObjectStore &DB; - CASOptions CASOpts; - DepscanPrefixMapping PrefixMapping; - llvm::PrefixMapper PrefixMapper; + llvm::PrefixMapper &PrefixMapper; + Optional PCHRef; bool StartedEnteringIncludes = false; // When a PCH is used this lists the filenames of the included files as they @@ -87,6 +122,10 @@ class IncludeTreeActionController : public CallbackActionController { llvm::BitVector SeenIncludeFiles; SmallVector IncludedFiles; Optional PredefinesBufferRef; + Optional ModuleIncludesBufferRef; + Optional ModuleMapFileRef; + /// When the builder is created from an existing tree, the main include tree. + Optional MainIncludeTreeRef; SmallVector IncludeStack; llvm::DenseMap> ObjectForFile; Optional ErrorToReport; @@ -152,6 +191,13 @@ void dependencies::addReversePrefixMappingFileSystem( ScanInstance.getFileManager().setVirtualFileSystem(std::move(FS)); } +Expected IncludeTreeActionController::getIncludeTree() { + if (IncludeTreeResult) + return *IncludeTreeResult; + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "failed to produce include-tree"); +} + Error IncludeTreeActionController::initialize( CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { if (Error E = @@ -184,10 +230,50 @@ Error IncludeTreeActionController::initialize( CASOpts = ScanInstance.getCASOpts(); + BuilderStack.push_back( + std::make_unique(DB, PrefixMapper)); + return Error::success(); } void IncludeTreeActionController::enteredInclude(Preprocessor &PP, FileID FID) { + current().enteredInclude(PP, FID); +} + +void IncludeTreeActionController::exitedInclude(Preprocessor &PP, + FileID IncludedBy, + FileID Include, + SourceLocation ExitLoc) { + current().exitedInclude(PP, IncludedBy, Include, ExitLoc); +} + +void IncludeTreeActionController::handleHasIncludeCheck(Preprocessor &PP, + bool Result) { + current().handleHasIncludeCheck(PP, Result); +} + +Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation) { + assert(!IncludeTreeResult); + assert(BuilderStack.size() == 1); + auto Builder = BuilderStack.pop_back_val(); + Error E = Builder->finishIncludeTree(ScanInstance, NewInvocation) + .moveInto(IncludeTreeResult); + if (E) + return E; + + configureInvocationForCaching(NewInvocation, CASOpts, + IncludeTreeResult->getID().toString(), + // FIXME: working dir? + /*CASFSWorkingDir=*/"", + /*ProduceIncludeTree=*/true); + + DepscanPrefixMapping::remapInvocationPaths(NewInvocation, PrefixMapper); + + return Error::success(); +} + +void IncludeTreeBuilder::enteredInclude(Preprocessor &PP, FileID FID) { if (hasErrorOccurred()) return; @@ -212,10 +298,8 @@ void IncludeTreeActionController::enteredInclude(Preprocessor &PP, FileID FID) { IncludeStack.push_back({FI.getFileCharacteristic(), *FileRef, {}, {}}); } -void IncludeTreeActionController::exitedInclude(Preprocessor &PP, - FileID IncludedBy, - FileID Include, - SourceLocation ExitLoc) { +void IncludeTreeBuilder::exitedInclude(Preprocessor &PP, FileID IncludedBy, + FileID Include, SourceLocation ExitLoc) { if (hasErrorOccurred()) return; @@ -231,16 +315,19 @@ void IncludeTreeActionController::exitedInclude(Preprocessor &PP, {IncludeTree->getRef(), LocInfo.second}); } -void IncludeTreeActionController::handleHasIncludeCheck(Preprocessor &PP, - bool Result) { +void IncludeTreeBuilder::handleHasIncludeCheck(Preprocessor &PP, bool Result) { if (hasErrorOccurred()) return; IncludeStack.back().HasIncludeChecks.push_back(Result); } -Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, - CompilerInvocation &NewInvocation) { +Expected +IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation) { + if (ErrorToReport) + return std::move(*ErrorToReport); + FileManager &FM = ScanInstance.getFileManager(); auto addFile = [&](StringRef FilePath, @@ -257,7 +344,7 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, for (StringRef FilePath : NewInvocation.getLangOpts()->NoSanitizeFiles) { if (Error E = addFile(FilePath)) - return E; + return std::move(E); } // Add profile files. // FIXME: Do not have the logic here to determine which path should be set @@ -265,13 +352,13 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, // checked the file needed exists. Just try load and ignore errors. if (Error E = addFile(NewInvocation.getCodeGenOpts().ProfileInstrumentUsePath, /*IgnoreFileError=*/true)) - return E; + return std::move(E); if (Error E = addFile(NewInvocation.getCodeGenOpts().SampleProfileFile, /*IgnoreFileError=*/true)) - return E; + return std::move(E); if (Error E = addFile(NewInvocation.getCodeGenOpts().ProfileRemappingFile, /*IgnoreFileError=*/true)) - return E; + return std::move(E); StringRef Sysroot = NewInvocation.getHeaderSearchOpts().Sysroot; if (!Sysroot.empty()) { @@ -280,7 +367,7 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, llvm::SmallString<256> FilePath = Sysroot; llvm::sys::path::append(FilePath, "SDKSettings.json"); if (Error E = addFile(FilePath, /*IgnoreFileError*/ true)) - return E; + return std::move(E); } auto FinishIncludeTree = [&]() -> Error { @@ -319,24 +406,26 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, }; if (Error E = FinishIncludeTree()) - return E; - - auto IncludeTreeRoot = getIncludeTree(); - if (!IncludeTreeRoot) - return IncludeTreeRoot.takeError(); + return std::move(E); - configureInvocationForCaching(NewInvocation, CASOpts, - IncludeTreeRoot->getID().toString(), - /*CASFSWorkingDir=*/"", - /*ProduceIncludeTree=*/true); + if (ErrorToReport) + return std::move(*ErrorToReport); - DepscanPrefixMapping::remapInvocationPaths(NewInvocation, PrefixMapper); + assert(IncludeStack.size() == 1); + Expected MainIncludeTree = + getCASTreeForFileIncludes(IncludeStack.pop_back_val()); + if (!MainIncludeTree) + return MainIncludeTree.takeError(); + auto FileList = cas::IncludeFileList::create(DB, IncludedFiles); + if (!FileList) + return FileList.takeError(); - return Error::success(); + return cas::IncludeTreeRoot::create(DB, MainIncludeTree->getRef(), + FileList->getRef(), PCHRef); } -Expected -IncludeTreeActionController::getObjectForFile(Preprocessor &PP, FileID FID) { +Expected IncludeTreeBuilder::getObjectForFile(Preprocessor &PP, + FileID FID) { SourceManager &SM = PP.getSourceManager(); const SrcMgr::FileInfo &FI = SM.getSLocEntry(FID).getFile(); if (PP.getPredefinesFileID() == FID) { @@ -359,8 +448,9 @@ IncludeTreeActionController::getObjectForFile(Preprocessor &PP, FileID FID) { return *FileRef; } -Expected IncludeTreeActionController::getObjectForFileNonCached( - FileManager &FM, const SrcMgr::FileInfo &FI) { +Expected +IncludeTreeBuilder::getObjectForFileNonCached(FileManager &FM, + const SrcMgr::FileInfo &FI) { const FileEntry *FE = FI.getContentCache().OrigEntry; assert(FE); @@ -373,7 +463,7 @@ Expected IncludeTreeActionController::getObjectForFileNonCached( } Expected -IncludeTreeActionController::getObjectForBuffer(const SrcMgr::FileInfo &FI) { +IncludeTreeBuilder::getObjectForBuffer(const SrcMgr::FileInfo &FI) { // This is a non-file buffer, like the predefines. auto Ref = DB.storeFromString( {}, FI.getContentCache().getBufferIfLoaded()->getBuffer()); @@ -386,8 +476,7 @@ IncludeTreeActionController::getObjectForBuffer(const SrcMgr::FileInfo &FI) { } Expected -IncludeTreeActionController::addToFileList(FileManager &FM, - const FileEntry *FE) { +IncludeTreeBuilder::addToFileList(FileManager &FM, const FileEntry *FE) { StringRef Filename = FE->getName(); llvm::ErrorOr> CASContents = FM.getObjectRefForFileContent(Filename); @@ -420,14 +509,14 @@ IncludeTreeActionController::addToFileList(FileManager &FM, } Expected -IncludeTreeActionController::getCASTreeForFileIncludes(FilePPState &&PPState) { +IncludeTreeBuilder::getCASTreeForFileIncludes(FilePPState &&PPState) { return cas::IncludeTree::create(DB, PPState.FileCharacteristic, PPState.File, PPState.Includes, PPState.HasIncludeChecks); } Expected -IncludeTreeActionController::createIncludeFile(StringRef Filename, - cas::ObjectRef Contents) { +IncludeTreeBuilder::createIncludeFile(StringRef Filename, + cas::ObjectRef Contents) { SmallString<256> MappedPath; if (!PrefixMapper.empty()) { PrefixMapper.map(Filename, MappedPath); @@ -436,23 +525,6 @@ IncludeTreeActionController::createIncludeFile(StringRef Filename, return cas::IncludeFile::create(DB, Filename, std::move(Contents)); } -Expected IncludeTreeActionController::getIncludeTree() { - if (ErrorToReport) - return std::move(*ErrorToReport); - - assert(IncludeStack.size() == 1); - Expected MainIncludeTree = - getCASTreeForFileIncludes(IncludeStack.pop_back_val()); - if (!MainIncludeTree) - return MainIncludeTree.takeError(); - auto FileList = cas::IncludeFileList::create(DB, IncludedFiles); - if (!FileList) - return FileList.takeError(); - - return cas::IncludeTreeRoot::create(DB, MainIncludeTree->getRef(), - FileList->getRef(), PCHRef); -} - std::unique_ptr dependencies::createIncludeTreeActionController( cas::ObjectStore &DB, DepscanPrefixMapping PrefixMapping) { From e34a755d544881569d0fe0e195a214231801ccbf Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 15 Mar 2023 11:25:45 -0700 Subject: [PATCH 02/38] [clang][cas] Remove PPCallbacks from DependencyActionController These were only used by include-tree, and we now have a mechanism to inject PPCallbacks without going through DependencyActionController. Remove them and simplify the IncludeTreeActionController. (cherry picked from commit c603c65426e0fafe33cfdf3d062f927cad442cdd) (cherry picked from commit ca4eed9ef66a23c49df71b33dc588b42ebce41c6) --- .../DependencyScanningWorker.h | 7 --- .../IncludeTreeActionController.cpp | 48 +++++-------------- 2 files changed, 13 insertions(+), 42 deletions(-) diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h index 1c2182811da40..a0df3970696c7 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -97,13 +97,6 @@ class DependencyActionController { return llvm::Error::success(); } - virtual void enteredInclude(Preprocessor &PP, FileID FID) {} - - virtual void exitedInclude(Preprocessor &PP, FileID IncludedBy, - FileID Include, SourceLocation ExitLoc) {} - - virtual void handleHasIncludeCheck(Preprocessor &PP, bool Result) {} - /// FIXME: This is temporary until we eliminate the split between consumers in /// \p DependencyScanningTool and collectors in \p DependencyScanningWorker /// and have them both in the same file. see FIXME in \p diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 3182cf4ce41e9..9800692475901 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -35,13 +35,6 @@ class IncludeTreeActionController : public CallbackActionController { Error initialize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) override; - void enteredInclude(Preprocessor &PP, FileID FID) override; - - void exitedInclude(Preprocessor &PP, FileID IncludedBy, FileID Include, - SourceLocation ExitLoc) override; - - void handleHasIncludeCheck(Preprocessor &PP, bool Result) override; - const DepscanPrefixMapping *getPrefixMapping() override { return &PrefixMapping; } @@ -60,6 +53,8 @@ class IncludeTreeActionController : public CallbackActionController { CASOptions CASOpts; DepscanPrefixMapping PrefixMapping; llvm::PrefixMapper PrefixMapper; + // IncludeTreePPCallbacks keeps a pointer to the current builder, so use a + // pointer so the builder cannot move when resizing. SmallVector> BuilderStack; std::optional IncludeTreeResult; }; @@ -146,23 +141,22 @@ struct PPCallbacksDependencyCollector : public DependencyCollector { }; struct IncludeTreePPCallbacks : public PPCallbacks { - DependencyActionController &Controller; + IncludeTreeBuilder &Builder; Preprocessor &PP; public: - IncludeTreePPCallbacks(DependencyActionController &Controller, - Preprocessor &PP) - : Controller(Controller), PP(PP) {} + IncludeTreePPCallbacks(IncludeTreeBuilder &Builder, Preprocessor &PP) + : Builder(Builder), PP(PP) {} void LexedFileChanged(FileID FID, LexedFileChangeReason Reason, SrcMgr::CharacteristicKind FileType, FileID PrevFID, SourceLocation Loc) override { switch (Reason) { case LexedFileChangeReason::EnterFile: - Controller.enteredInclude(PP, FID); + Builder.enteredInclude(PP, FID); break; case LexedFileChangeReason::ExitFile: { - Controller.exitedInclude(PP, FID, PrevFID, Loc); + Builder.exitedInclude(PP, FID, PrevFID, Loc); break; } } @@ -171,7 +165,7 @@ struct IncludeTreePPCallbacks : public PPCallbacks { void HasInclude(SourceLocation Loc, StringRef FileName, bool IsAngled, Optional File, SrcMgr::CharacteristicKind FileType) override { - Controller.handleHasIncludeCheck(PP, File.has_value()); + Builder.handleHasIncludeCheck(PP, File.has_value()); } }; } // namespace @@ -220,38 +214,22 @@ Error IncludeTreeActionController::initialize( }; ensurePathRemapping(); + BuilderStack.push_back( + std::make_unique(DB, PrefixMapper)); + // Attach callbacks for the IncludeTree of the TU. The preprocessor // does not exist yet, so we need to indirect this via DependencyCollector. auto DC = std::make_shared( - [this](Preprocessor &PP) { - return std::make_unique(*this, PP); + [&Builder = current()](Preprocessor &PP) { + return std::make_unique(Builder, PP); }); ScanInstance.addDependencyCollector(std::move(DC)); CASOpts = ScanInstance.getCASOpts(); - BuilderStack.push_back( - std::make_unique(DB, PrefixMapper)); - return Error::success(); } -void IncludeTreeActionController::enteredInclude(Preprocessor &PP, FileID FID) { - current().enteredInclude(PP, FID); -} - -void IncludeTreeActionController::exitedInclude(Preprocessor &PP, - FileID IncludedBy, - FileID Include, - SourceLocation ExitLoc) { - current().exitedInclude(PP, IncludedBy, Include, ExitLoc); -} - -void IncludeTreeActionController::handleHasIncludeCheck(Preprocessor &PP, - bool Result) { - current().handleHasIncludeCheck(PP, Result); -} - Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { assert(!IncludeTreeResult); From a450905f7acffc25531c84c8133bfac7648f12e9 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 15 Mar 2023 15:53:41 -0700 Subject: [PATCH 03/38] [clang][cas] Scaffolding for modules with include-tree This adds the scaffolding needed for include-tree to push IncludeTreeBuilder stacks, to compute an include-tree for a module and to get fmodule-file-cache-key options for dependencies. Note: this does not yet support building or importing modules, but it lets us compute dependencies in the right way. (cherry picked from commit 3b87a2091b1055f6a7acf55cd4fa237d794ac65b) (cherry picked from commit 322105695fc18d0ce00dd0542be0fb64a3565a05) --- .../include/clang/Basic/DiagnosticCASKinds.td | 4 + clang/include/clang/Basic/Module.h | 13 ++ .../include/clang/Serialization/ASTBitCodes.h | 6 +- clang/include/clang/Serialization/ASTReader.h | 6 + .../include/clang/Serialization/ModuleFile.h | 4 + .../DependencyScanningTool.h | 12 +- .../DependencyScanning/ModuleDepCollector.h | 3 + clang/lib/Frontend/CompilerInstance.cpp | 31 ++- clang/lib/Serialization/ASTReader.cpp | 16 +- clang/lib/Serialization/ASTWriter.cpp | 10 + .../DependencyScanning/CachingActions.h | 3 +- .../DependencyScanningTool.cpp | 18 +- .../DependencyScanningWorker.cpp | 15 +- .../IncludeTreeActionController.cpp | 76 ++++++- .../DependencyScanning/ModuleDepCollector.cpp | 1 + .../DependencyScanning/ScanAndUpdateArgs.cpp | 13 +- .../test/ClangScanDeps/modules-include-tree.c | 202 ++++++++++++++++++ clang/tools/clang-scan-deps/ClangScanDeps.cpp | 12 +- clang/unittests/CAS/IncludeTreeTest.cpp | 3 +- 19 files changed, 407 insertions(+), 41 deletions(-) create mode 100644 clang/test/ClangScanDeps/modules-include-tree.c diff --git a/clang/include/clang/Basic/DiagnosticCASKinds.td b/clang/include/clang/Basic/DiagnosticCASKinds.td index c173f434be6fe..d08b034b98d6f 100644 --- a/clang/include/clang/Basic/DiagnosticCASKinds.td +++ b/clang/include/clang/Basic/DiagnosticCASKinds.td @@ -36,6 +36,10 @@ def err_cas_missing_root_id : Error< "CAS missing expected root-id '%0'">, DefaultFatal; def err_cas_cannot_parse_root_id_for_module : Error< "CAS cannot parse root-id '%0' for module '%1'">, DefaultFatal; +def err_cas_cannot_parse_include_tree_id : Error< + "CAS cannot parse include-tree-id '%0'">, DefaultFatal; +def err_cas_missing_include_tree_id : Error< + "CAS missing expected include-tree '%0'">, DefaultFatal; def warn_clang_cache_disabled_caching: Warning< "caching disabled because %0">, diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index 3218e33bb88ff..8b9a93c6b0e7d 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -196,6 +196,10 @@ class alignas(8) Module { /// scanner, if any. Optional CASFileSystemRootID; + /// The include-tree root ID for implicit modules built with the dependency + /// scanner, if any. + Optional IncludeTreeID; + /// The top-level headers associated with this module. llvm::SmallSetVector TopHeaders; @@ -645,6 +649,15 @@ class alignas(8) Module { getTopLevelModule()->CASFileSystemRootID = std::move(ID); } + Optional getIncludeTreeID() const { + return getTopLevelModule()->IncludeTreeID; + } + + void setIncludeTreeID(std::string ID) { + assert(!getIncludeTreeID() || *getIncludeTreeID() == ID); + getTopLevelModule()->IncludeTreeID = std::move(ID); + } + /// Retrieve the directory for which this module serves as the /// umbrella. DirectoryName getUmbrellaDir() const; diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h index 34792dae0e36e..1cfeaf5bc81c5 100644 --- a/clang/include/clang/Serialization/ASTBitCodes.h +++ b/clang/include/clang/Serialization/ASTBitCodes.h @@ -41,7 +41,7 @@ namespace serialization { /// Version 4 of AST files also requires that the version control branch and /// revision match exactly, since there is no backward compatibility of /// AST files at this time. -const unsigned VERSION_MAJOR = 25; +const unsigned VERSION_MAJOR = 26; /// AST file minor version number supported by this version of /// Clang. @@ -367,6 +367,10 @@ enum ControlRecordTypes { /// Record code for the (optional) CAS filesystem root ID for implicit modules /// built with the dependency scanner. CASFS_ROOT_ID, + + /// Record code for the (optional) include-tree ID for implicit modules + /// built with the dependency scanner. + CAS_INCLUDE_TREE_ID, }; /// Record types that occur within the options block inside diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index cad25a360c553..191372471c5ff 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -246,6 +246,11 @@ class ASTReaderListener { return false; } + /// Called for each CAS include-tree root ID. + /// + /// \returns true to indicate \p RootID is invalid, or false otherwise. + virtual bool readIncludeTreeID(StringRef ID, bool Complain) { return false; } + /// Called for each module cache key. /// /// \returns true to indicate the key cannot be loaded. @@ -300,6 +305,7 @@ class ChainedASTReaderListener : public ASTReaderListener { bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden, bool isExplicitModule) override; bool readCASFileSystemRootID(StringRef RootID, bool Complain) override; + bool readIncludeTreeID(StringRef ID, bool Complain) override; bool readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) override; void readModuleFileExtension( diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 2d0be293d93d3..c94b089bc7936 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -143,6 +143,10 @@ class ModuleFile { /// scanner, or empty. std::string CASFileSystemRootID; + /// The include-tree root ID for implicit modules built with the dependency + /// scanner, or empty. + std::string IncludeTreeID; + /// The name of the module. std::string ModuleName; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h index c9f96805f5a4e..87d56443a9e33 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -71,6 +71,9 @@ struct TranslationUnitDeps { /// The CASID for input file dependency tree. llvm::Optional CASFileSystemRootID; + /// The include-tree for input file dependency tree. + llvm::Optional IncludeTreeID; + /// The sequence of commands required to build the translation unit. Commands /// should be executed in order. /// @@ -120,6 +123,7 @@ class DependencyScanningTool { Expected getIncludeTree(cas::ObjectStore &DB, const std::vector &CommandLine, StringRef CWD, + LookupModuleOutputCallback LookupModuleOutput, const DepscanPrefixMapping &PrefixMapping); /// If \p DiagGenerationAsCompilation is true it will generate error @@ -128,7 +132,8 @@ class DependencyScanningTool { /// \p DiagOpts.DiagnosticSerializationFile setting is set for the invocation. Expected getIncludeTreeFromCompilerInvocation( cas::ObjectStore &DB, std::shared_ptr Invocation, - StringRef CWD, const DepscanPrefixMapping &PrefixMapping, + StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, + const DepscanPrefixMapping &PrefixMapping, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, bool DiagGenerationAsCompilation); @@ -225,6 +230,10 @@ class FullDependencyConsumer : public DependencyConsumer { CASFileSystemRootID = std::move(ID); } + void handleIncludeTreeID(std::string ID) override { + IncludeTreeID = std::move(ID); + } + TranslationUnitDeps takeTranslationUnitDeps(); ModuleDepsGraph takeModuleGraphDeps(); @@ -236,6 +245,7 @@ class FullDependencyConsumer : public DependencyConsumer { std::vector Commands; std::string ContextHash; Optional CASFileSystemRootID; + Optional IncludeTreeID; std::vector OutputPaths; const llvm::StringSet<> &AlreadySeen; }; diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h index b4b8b1206098c..809abf722e5ec 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -121,6 +121,9 @@ struct ModuleDeps { /// The CASID for the module input dependency tree, if any. llvm::Optional CASFileSystemRootID; + /// The CASID for the module include-tree, if any. + llvm::Optional IncludeTreeID; + /// The \c ActionCache key for this module, if any. llvm::Optional ModuleCacheKey; diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 2ee4509d7eb42..f53c34d9bff87 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -640,10 +640,14 @@ class CompileCacheASTReaderHelper : public ASTReaderListener { : CAS(CAS), Cache(Cache), ModuleCache(ModuleCache), Diags(Diags) {} bool readCASFileSystemRootID(StringRef RootID, bool Complain) override; + bool readIncludeTreeID(StringRef ID, bool Complain) override; bool readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) override; private: + bool checkCASID(bool Complain, StringRef RootID, unsigned ParseDiagID, + unsigned MissingDiagID); + cas::ObjectStore &CAS; cas::ActionCache &Cache; InMemoryModuleCache &ModuleCache; @@ -2431,21 +2435,36 @@ bool CompileCacheASTReaderHelper::readModuleCacheKey(StringRef ModuleName, Filename, CacheKey, "imported module", CAS, Cache, ModuleCache, Diags); } -bool CompileCacheASTReaderHelper::readCASFileSystemRootID(StringRef RootID, - bool Complain) { - // Verify that RootID is in the CAS. Otherwise the module cache probably was - // created with a different CAS. +/// Verify that ID is in the CAS. Otherwise the module cache probably was +/// created with a different CAS. +bool CompileCacheASTReaderHelper::checkCASID(bool Complain, StringRef RootID, + unsigned ParseDiagID, + unsigned MissingDiagID) { std::optional ID; if (errorToBool(CAS.parseID(RootID).moveInto(ID))) { if (Complain) - Diags.Report(diag::err_cas_cannot_parse_root_id) << RootID; + Diags.Report(ParseDiagID) << RootID; return true; } if (errorToBool(CAS.getProxy(*ID).takeError())) { if (Complain) { - Diags.Report(diag::err_cas_missing_root_id) << RootID; + Diags.Report(MissingDiagID) << RootID; } return true; } return false; } + +bool CompileCacheASTReaderHelper::readCASFileSystemRootID(StringRef RootID, + bool Complain) { + return checkCASID(Complain, RootID, diag::err_cas_cannot_parse_root_id, + diag::err_cas_missing_root_id); +} + +bool CompileCacheASTReaderHelper::readIncludeTreeID(StringRef ID, + bool Complain) { + // Verify that ID is in the CAS. Otherwise the module cache probably was + // created with a different CAS. + return checkCASID(Complain, ID, diag::err_cas_cannot_parse_include_tree_id, + diag::err_cas_missing_include_tree_id); +} diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index eb7333b167a72..61e17c51af70f 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -257,7 +257,10 @@ bool ChainedASTReaderListener::readCASFileSystemRootID(StringRef RootID, return First->readCASFileSystemRootID(RootID, Complain) || Second->readCASFileSystemRootID(RootID, Complain); } - +bool ChainedASTReaderListener::readIncludeTreeID(StringRef ID, bool Complain) { + return First->readIncludeTreeID(ID, Complain) || + Second->readIncludeTreeID(ID, Complain); +} bool ChainedASTReaderListener::readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) { @@ -2984,6 +2987,15 @@ ASTReader::ReadControlBlock(ModuleFile &F, return OutOfDate; } break; + case CAS_INCLUDE_TREE_ID: + F.IncludeTreeID = Blob.str(); + if (Listener) { + bool Complain = + !canRecoverFromOutOfDate(F.FileName, ClientLoadCapabilities); + if (Listener->readIncludeTreeID(F.IncludeTreeID, Complain)) + return OutOfDate; + } + break; } } } @@ -5626,6 +5638,8 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F, CurrentModule->setModuleCacheKey(F.ModuleCacheKey); if (!F.CASFileSystemRootID.empty()) CurrentModule->setCASFileSystemRootID(F.CASFileSystemRootID); + if (!F.IncludeTreeID.empty()) + CurrentModule->setIncludeTreeID(F.IncludeTreeID); } CurrentModule->Kind = Kind; diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 419b7c109ecfd..a79ae19a10317 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -797,6 +797,7 @@ void ASTWriter::WriteBlockInfoBlock() { RECORD(INPUT_FILE_OFFSETS); RECORD(MODULE_CACHE_KEY); RECORD(CASFS_ROOT_ID); + RECORD(CAS_INCLUDE_TREE_ID); BLOCK(OPTIONS_BLOCK); RECORD(LANGUAGE_OPTIONS); @@ -1361,6 +1362,15 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context, RecordData::value_type Record[] = {CASFS_ROOT_ID}; Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); } + // CAS include-tree id, for the scanner. + if (auto ID = WritingModule->getIncludeTreeID()) { + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(CAS_INCLUDE_TREE_ID)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + RecordData::value_type Record[] = {CAS_INCLUDE_TREE_ID}; + Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); + } } // Imports diff --git a/clang/lib/Tooling/DependencyScanning/CachingActions.h b/clang/lib/Tooling/DependencyScanning/CachingActions.h index 4cbcdd87ddbdc..8dc3e33b61553 100644 --- a/clang/lib/Tooling/DependencyScanning/CachingActions.h +++ b/clang/lib/Tooling/DependencyScanning/CachingActions.h @@ -19,7 +19,8 @@ class CachingOnDiskFileSystem; namespace clang::tooling::dependencies { std::unique_ptr -createIncludeTreeActionController(cas::ObjectStore &DB, +createIncludeTreeActionController(LookupModuleOutputCallback LookupModuleOutput, + cas::ObjectStore &DB, DepscanPrefixMapping PrefixMapping); std::unique_ptr diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp index 5845ca5bd3f96..90b92d3790cc2 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -190,10 +190,11 @@ DependencyScanningTool::getDependencyTreeFromCompilerInvocation( Expected DependencyScanningTool::getIncludeTree( cas::ObjectStore &DB, const std::vector &CommandLine, - StringRef CWD, const DepscanPrefixMapping &PrefixMapping) { + StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, + const DepscanPrefixMapping &PrefixMapping) { GetIncludeTree Consumer(DB); - auto Controller = - createIncludeTreeActionController(DB, std::move(PrefixMapping)); + auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB, + std::move(PrefixMapping)); llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller); if (Result) @@ -204,12 +205,13 @@ Expected DependencyScanningTool::getIncludeTree( Expected DependencyScanningTool::getIncludeTreeFromCompilerInvocation( cas::ObjectStore &DB, std::shared_ptr Invocation, - StringRef CWD, const DepscanPrefixMapping &PrefixMapping, + StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, + const DepscanPrefixMapping &PrefixMapping, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, bool DiagGenerationAsCompilation) { GetIncludeTree Consumer(DB); - auto Controller = - createIncludeTreeActionController(DB, std::move(PrefixMapping)); + auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB, + std::move(PrefixMapping)); Worker.computeDependenciesFromCompilerInvocation( std::move(Invocation), CWD, Consumer, *Controller, DiagsConsumer, VerboseOS, DiagGenerationAsCompilation); @@ -255,6 +257,7 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() { TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps); TU.Commands = std::move(Commands); TU.CASFileSystemRootID = std::move(CASFileSystemRootID); + TU.IncludeTreeID = std::move(IncludeTreeID); for (auto &&M : ClangModuleDeps) { auto &MD = M.second; @@ -293,7 +296,8 @@ DependencyScanningTool::createActionController( LookupModuleOutputCallback LookupModuleOutput, DepscanPrefixMapping PrefixMapping) { if (Worker.getScanningFormat() == ScanningOutputFormat::FullIncludeTree) - return createIncludeTreeActionController(*Worker.getCAS(), std::move(PrefixMapping)); + return createIncludeTreeActionController( + LookupModuleOutput, *Worker.getCAS(), std::move(PrefixMapping)); if (auto CacheFS = Worker.getCASFS()) return createCASFSActionController(LookupModuleOutput, *CacheFS, std::move(PrefixMapping)); diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index d08e41e4b2fc8..5d5cd6c8326f5 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -435,15 +435,12 @@ class DependencyScanningAction : public tooling::ToolAction { std::move(Opts), ScanInstance, Consumer, Controller, OriginalInvocation, OptimizeArgs, EagerLoadModules); ScanInstance.addDependencyCollector(MDC); - if (CacheFS) { - ScanInstance.setGenModuleActionWrapper( - [CacheFS = CacheFS, &Controller = Controller]( - const FrontendOptions &Opts, - std::unique_ptr Wrapped) { - return std::make_unique( - std::move(Wrapped), Controller); - }); - } + ScanInstance.setGenModuleActionWrapper( + [&Controller = Controller](const FrontendOptions &Opts, + std::unique_ptr Wrapped) { + return std::make_unique( + std::move(Wrapped), Controller); + }); break; } diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 9800692475901..14a860a7fe587 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -25,8 +25,9 @@ class IncludeTreeBuilder; class IncludeTreeActionController : public CallbackActionController { public: IncludeTreeActionController(cas::ObjectStore &DB, - DepscanPrefixMapping PrefixMapping) - : CallbackActionController(nullptr), DB(DB), + DepscanPrefixMapping PrefixMapping, + LookupModuleOutputCallback LookupOutput) + : CallbackActionController(LookupOutput), DB(DB), PrefixMapping(std::move(PrefixMapping)) {} Expected getIncludeTree(); @@ -34,14 +35,18 @@ class IncludeTreeActionController : public CallbackActionController { private: Error initialize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) override; + Error finalize(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation) override; + + Error initializeModuleBuild(CompilerInstance &ModuleScanInstance) override; + Error finalizeModuleBuild(CompilerInstance &ModuleScanInstance) override; + Error finalizeModuleInvocation(CompilerInvocation &CI, + const ModuleDeps &MD) override; const DepscanPrefixMapping *getPrefixMapping() override { return &PrefixMapping; } - Error finalize(CompilerInstance &ScanInstance, - CompilerInvocation &NewInvocation) override; - private: IncludeTreeBuilder ¤t() { assert(!BuilderStack.empty()); @@ -225,6 +230,8 @@ Error IncludeTreeActionController::initialize( }); ScanInstance.addDependencyCollector(std::move(DC)); + // Enable caching in the resulting commands. + ScanInstance.getFrontendOpts().CacheCompileJob = true; CASOpts = ScanInstance.getCASOpts(); return Error::success(); @@ -251,6 +258,51 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, return Error::success(); } +Error IncludeTreeActionController::initializeModuleBuild( + CompilerInstance &ModuleScanInstance) { + BuilderStack.push_back( + std::make_unique(DB, PrefixMapper)); + + // Attach callbacks for the IncludeTree of the module. The preprocessor + // does not exist yet, so we need to indirect this via DependencyCollector. + auto DC = std::make_shared( + [&Builder = current()](Preprocessor &PP) { + return std::make_unique(Builder, PP); + }); + ModuleScanInstance.addDependencyCollector(std::move(DC)); + + return Error::success(); +} + +Error IncludeTreeActionController::finalizeModuleBuild( + CompilerInstance &ModuleScanInstance) { + // FIXME: the scan invocation is incorrect here; we need the `NewInvocation` + // from `finalizeModuleInvocation` to finish the tree. + auto Builder = BuilderStack.pop_back_val(); + auto Tree = Builder->finishIncludeTree(ModuleScanInstance, + ModuleScanInstance.getInvocation()); + if (!Tree) + return Tree.takeError(); + + Module *M = ModuleScanInstance.getPreprocessor().getCurrentModule(); + assert(M && "finalizing without a module"); + M->setIncludeTreeID(Tree->getID().toString()); + + return Error::success(); +} + +Error IncludeTreeActionController::finalizeModuleInvocation( + CompilerInvocation &CI, const ModuleDeps &MD) { + if (auto ID = MD.IncludeTreeID) { + configureInvocationForCaching(CI, CASOpts, std::move(*ID), + /*CASFSWorkingDir=*/"", + /*ProduceIncludeTree=*/true); + } + + DepscanPrefixMapping::remapInvocationPaths(CI, PrefixMapper); + return Error::success(); +} + void IncludeTreeBuilder::enteredInclude(Preprocessor &PP, FileID FID) { if (hasErrorOccurred()) return; @@ -415,6 +467,15 @@ Expected IncludeTreeBuilder::getObjectForFile(Preprocessor &PP, } return *PredefinesBufferRef; } + if (!FI.getContentCache().OrigEntry && + FI.getName() == Module::getModuleInputBufferName()) { + // Virtual buffer + if (!ModuleIncludesBufferRef) { + if (Error E = getObjectForBuffer(FI).moveInto(ModuleIncludesBufferRef)) + return std::move(E); + } + return *ModuleIncludesBufferRef; + } assert(FI.getContentCache().OrigEntry); auto &FileRef = ObjectForFile[FI.getContentCache().OrigEntry]; if (!FileRef) { @@ -505,7 +566,8 @@ IncludeTreeBuilder::createIncludeFile(StringRef Filename, std::unique_ptr dependencies::createIncludeTreeActionController( - cas::ObjectStore &DB, DepscanPrefixMapping PrefixMapping) { + LookupModuleOutputCallback LookupModuleOutput, cas::ObjectStore &DB, + DepscanPrefixMapping PrefixMapping) { return std::make_unique( - DB, std::move(PrefixMapping)); + DB, std::move(PrefixMapping), LookupModuleOutput); } diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index 984582c08350d..b3e692cec9962 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -449,6 +449,7 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { << *ID << MD.ID.ModuleName; } } + MD.IncludeTreeID = M->getIncludeTreeID(); ModuleMap &ModMapInfo = MDC.ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap(); diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index 43272f80290e1..e1234095a95e0 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -36,10 +36,12 @@ void tooling::dependencies::configureInvocationForCaching( FrontendOpts.CASIncludeTreeID = std::move(RootID); FrontendOpts.Inputs.clear(); // Preserve sysroot path to accommodate lookup for 'SDKSettings.json' during - // availability checking. - std::string OriginalSysroot = CI.getHeaderSearchOpts().Sysroot; - CI.getHeaderSearchOpts() = HeaderSearchOptions(); - CI.getHeaderSearchOpts().Sysroot = OriginalSysroot; + // availability checking, and module files. + HeaderSearchOptions &HSOpts = CI.getHeaderSearchOpts(); + HeaderSearchOptions OriginalHSOpts; + std::swap(HSOpts, OriginalHSOpts); + HSOpts.Sysroot = std::move(OriginalHSOpts.Sysroot); + HSOpts.PrebuiltModuleFiles = std::move(OriginalHSOpts.PrebuiltModuleFiles); auto &PPOpts = CI.getPreprocessorOpts(); // We don't need this because we save the contents of the PCH file in the // include tree root. @@ -254,7 +256,8 @@ Expected clang::scanAndUpdateCC1InlineWithTool( if (ProduceIncludeTree) { if (Error E = Tool.getIncludeTreeFromCompilerInvocation( DB, std::move(ScanInvocation), WorkingDirectory, - PrefixMapping, DiagsConsumer, VerboseOS, + /*LookupModuleOutput=*/nullptr, PrefixMapping, + DiagsConsumer, VerboseOS, /*DiagGenerationAsCompilation*/ true) .moveInto(Root)) return std::move(E); diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c new file mode 100644 index 0000000000000..26792c21724df --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -0,0 +1,202 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: FileCheck %s -input-file %t/deps.json -DPREFIX=%/t + +// CHECK: { +// CHECK-NEXT "modules": [ +// CHECK-NEXT { +// CHECK: "cas-include-tree-id": "[[LEFT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK: { +// CHECK: "module-name": "Top" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[LEFT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Left" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Left.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK: ] +// CHECK: "name": "Left" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Top" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[RIGHT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file-cache-key +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Right" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Right.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Right" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[TOP_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[TOP_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Top" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Top.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Top" +// CHECK: } +// CHECK: ] +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "commands": [ +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[TU_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Left" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "module-name": "Right" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK: "-disable-free" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[TU_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-fsyntax-only" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodule-file=Left=[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/tu.c" +// CHECK-NEXT: ] +// CHECK: "input-file": "[[PREFIX]]/tu.c" +// CHECK: } +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } + +//--- cdb.json.template +[{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- module.modulemap +module Top { header "Top.h" export *} +module Left { header "Left.h" export *} +module Right { header "Right.h" export *} + +//--- Top.h +#pragma once +struct Top { + int x; +}; +void top(void); + +//--- Left.h +#pragma once +#include "Top.h" +void left(void); + +//--- Right.h +#pragma once +#include "Top.h" +void right(void); + +//--- tu.c +#include "Left.h" +#include "Right.h" + +void tu(void) { + left(); + right(); + top(); +} diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index cfe6d22aef6f1..ecb3d04f80adf 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -544,6 +544,7 @@ class FullDeps { ID.FileDeps = std::move(TUDeps.FileDeps); ID.ModuleDeps = std::move(TUDeps.ClangModuleDeps); ID.CASFileSystemRootID = std::move(TUDeps.CASFileSystemRootID); + ID.IncludeTreeID = std::move(TUDeps.IncludeTreeID); ID.DriverCommandLine = std::move(TUDeps.DriverCommandLine); ID.Commands = std::move(TUDeps.Commands); @@ -624,6 +625,8 @@ class FullDeps { }; if (MD.CASFileSystemRootID) O.try_emplace("casfs-root-id", MD.CASFileSystemRootID->toString()); + if (MD.IncludeTreeID) + O.try_emplace("cas-include-tree-id", MD.IncludeTreeID); OutModules.push_back(std::move(O)); } @@ -642,6 +645,8 @@ class FullDeps { }; if (I.CASFileSystemRootID) O.try_emplace("casfs-root-id", I.CASFileSystemRootID); + if (I.IncludeTreeID) + O.try_emplace("cas-include-tree-id", I.IncludeTreeID); Commands.push_back(std::move(O)); } } else { @@ -655,6 +660,8 @@ class FullDeps { }; if (I.CASFileSystemRootID) O.try_emplace("casfs-root-id", I.CASFileSystemRootID); + if (I.IncludeTreeID) + O.try_emplace("cas-include-tree-id", I.IncludeTreeID); Commands.push_back(std::move(O)); } TUs.push_back(Object{ @@ -694,7 +701,8 @@ class FullDeps { std::string ContextHash; std::vector FileDeps; std::vector ModuleDeps; - llvm::Optional CASFileSystemRootID; + Optional CASFileSystemRootID; + Optional IncludeTreeID; std::vector DriverCommandLine; std::vector Commands; }; @@ -995,7 +1003,7 @@ int main(int argc, const char **argv) { std::move(MaybeTree)); } else if (Format == ScanningOutputFormat::IncludeTree) { auto MaybeTree = WorkerTools[I]->getIncludeTree( - *CAS, Input->CommandLine, CWD, PrefixMapping); + *CAS, Input->CommandLine, CWD, LookupOutput, PrefixMapping); std::unique_lock LockGuard(Lock); TreeResults.emplace_back(LocalIndex, std::move(Filename), std::move(MaybeTree)); diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index 6ddba05cde1ca..1c82d5abb40fa 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -55,7 +55,8 @@ TEST(IncludeTree, IncludeTreeScan) { Optional Root; DepscanPrefixMapping PrefixMapping; ASSERT_THAT_ERROR( - ScanTool.getIncludeTree(*DB, CommandLine, /*CWD*/ "", PrefixMapping) + ScanTool + .getIncludeTree(*DB, CommandLine, /*CWD*/ "", nullptr, PrefixMapping) .moveInto(Root), llvm::Succeeded()); From 737d65eab19f6a32e8cf5374a98214a72674554f Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 22 Mar 2023 10:23:20 -0700 Subject: [PATCH 04/38] [clang][cas] Move IncludeFile after IncludeTree in the header NFC Moves IncludeFile after IncludeTree in the header file in preparation for changing it to IncludeTree::File. Also moves FileInfo to IncludeTree to simplify declaration. (cherry picked from commit f36469e06536b6d08aaf384bfacfcadf01df77a0) (cherry picked from commit fa13d21f32dd79b28041c8567ce06ca5e40a9790) --- clang/include/clang/CAS/IncludeTree.h | 140 +++++++++++------------- clang/lib/CAS/IncludeTree.cpp | 14 +++ clang/unittests/CAS/IncludeTreeTest.cpp | 10 +- 3 files changed, 85 insertions(+), 79 deletions(-) diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 5519010160937..47f2d13dacd60 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -45,68 +45,7 @@ template class IncludeTreeBase : public ObjectProxy { friend class IncludeTreeRoot; }; -/// Represents a \p SourceManager file (or buffer in the case of preprocessor -/// predefines) that got included by the preprocessor. -class IncludeFile : public IncludeTreeBase { -public: - static constexpr StringRef getNodeKind() { return "File"; } - - ObjectRef getFilenameRef() const { return getReference(0); } - ObjectRef getContentsRef() const { return getReference(1); } - - Expected getFilename() { - return getCAS().getProxy(getFilenameRef()); - } - - Expected getContents() { - return getCAS().getProxy(getContentsRef()); - } - - struct FileInfo { - StringRef Filename; - StringRef Contents; - }; - - Expected getFileInfo() { - auto Filename = getFilename(); - if (!Filename) - return Filename.takeError(); - auto Contents = getContents(); - if (!Contents) - return Contents.takeError(); - return FileInfo{Filename->getData(), Contents->getData()}; - } - - static Expected create(ObjectStore &DB, StringRef Filename, - ObjectRef Contents); - - llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); - - static bool isValid(const ObjectProxy &Node) { - if (!IncludeTreeBase::isValid(Node)) - return false; - IncludeTreeBase Base(Node); - return Base.getNumReferences() == 2 && Base.getData().empty(); - } - static bool isValid(ObjectStore &DB, ObjectRef Ref) { - auto Node = DB.getProxy(Ref); - if (!Node) { - llvm::consumeError(Node.takeError()); - return false; - } - return isValid(*Node); - } - -private: - friend class IncludeTreeBase; - friend class IncludeFileList; - friend class IncludeTree; - friend class IncludeTreeRoot; - - explicit IncludeFile(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { - assert(isValid(*this)); - } -}; +class IncludeFile; /// Represents a DAG of included files by the preprocessor. /// Each node in the DAG represents a particular inclusion of a file that @@ -117,22 +56,17 @@ class IncludeTree : public IncludeTreeBase { public: static constexpr StringRef getNodeKind() { return "Tree"; } - Expected getBaseFile() { - auto Node = getCAS().getProxy(getBaseFileRef()); - if (!Node) - return Node.takeError(); - return IncludeFile(std::move(*Node)); - } + Expected getBaseFile(); /// The include file that resulted in this include-tree. ObjectRef getBaseFileRef() const { return getReference(0); } - Expected getBaseFileInfo() { - auto File = getBaseFile(); - if (!File) - return File.takeError(); - return File->getFileInfo(); - } + struct FileInfo { + StringRef Filename; + StringRef Contents; + }; + + Expected getBaseFileInfo(); SrcMgr::CharacteristicKind getFileCharacteristic() const { return (SrcMgr::CharacteristicKind)dataSkippingIncludes().front(); @@ -216,6 +150,64 @@ class IncludeTree : public IncludeTreeBase { } }; +/// Represents a \p SourceManager file (or buffer in the case of preprocessor +/// predefines) that got included by the preprocessor. +class IncludeFile : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "File"; } + + ObjectRef getFilenameRef() const { return getReference(0); } + ObjectRef getContentsRef() const { return getReference(1); } + + Expected getFilename() { + return getCAS().getProxy(getFilenameRef()); + } + + Expected getContents() { + return getCAS().getProxy(getContentsRef()); + } + + Expected getFileInfo() { + auto Filename = getFilename(); + if (!Filename) + return Filename.takeError(); + auto Contents = getContents(); + if (!Contents) + return Contents.takeError(); + return IncludeTree::FileInfo{Filename->getData(), Contents->getData()}; + } + + static Expected create(ObjectStore &DB, StringRef Filename, + ObjectRef Contents); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + return Base.getNumReferences() == 2 && Base.getData().empty(); + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + friend class IncludeTreeBase; + friend class IncludeFileList; + friend class IncludeTree; + friend class IncludeTreeRoot; + + explicit IncludeFile(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } +}; + /// A flat list of \p IncludeFile entries. This is used along with a simple /// implementation of a \p vfs::FileSystem produced via /// \p createIncludeTreeFileSystem(). diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index 435522869d7c8..b507661bc6a88 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -42,6 +42,20 @@ Expected IncludeFile::create(ObjectStore &DB, StringRef Filename, return IncludeTreeBase::create(DB, Refs, {}); } +Expected IncludeTree::getBaseFile() { + auto Node = getCAS().getProxy(getBaseFileRef()); + if (!Node) + return Node.takeError(); + return IncludeFile(std::move(*Node)); +} + +Expected IncludeTree::getBaseFileInfo() { + auto File = getBaseFile(); + if (!File) + return File.takeError(); + return File->getFileInfo(); +} + llvm::Error IncludeTree::forEachInclude( llvm::function_ref)> Callback) { diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index 1c82d5abb40fa..ba422a1e99006 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -71,7 +71,7 @@ TEST(IncludeTree, IncludeTreeScan) { ASSERT_THAT_ERROR(Main->getBaseFile().moveInto(MainFile), llvm::Succeeded()); EXPECT_EQ(Main->getFileCharacteristic(), SrcMgr::C_User); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(MainFile->getFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, "t.cpp"); EXPECT_EQ(FI.Contents, MainContents); @@ -83,7 +83,7 @@ TEST(IncludeTree, IncludeTreeScan) { EXPECT_EQ(Main->getIncludeOffset(0), uint32_t(0)); { EXPECT_EQ(Predef->getFileCharacteristic(), SrcMgr::C_User); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(Predef->getBaseFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, ""); @@ -95,7 +95,7 @@ TEST(IncludeTree, IncludeTreeScan) { { ASSERT_THAT_ERROR(A1->getBaseFile().moveInto(A1File), llvm::Succeeded()); EXPECT_EQ(A1->getFileCharacteristic(), SrcMgr::C_User); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(A1File->getFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, "./a1.h"); EXPECT_EQ(FI.Contents, A1Contents); @@ -109,7 +109,7 @@ TEST(IncludeTree, IncludeTreeScan) { { ASSERT_THAT_ERROR(B1->getBaseFile().moveInto(B1File), llvm::Succeeded()); EXPECT_EQ(B1->getFileCharacteristic(), SrcMgr::C_User); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(B1->getBaseFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, "./b1.h"); EXPECT_EQ(FI.Contents, ""); @@ -124,7 +124,7 @@ TEST(IncludeTree, IncludeTreeScan) { { ASSERT_THAT_ERROR(Sys->getBaseFile().moveInto(SysFile), llvm::Succeeded()); EXPECT_EQ(Sys->getFileCharacteristic(), SrcMgr::C_System); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(Sys->getBaseFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, "sys/sys.h"); EXPECT_EQ(FI.Contents, ""); From cb78dfa07790548a0d9dc991d6d001e8b2b97037 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 22 Mar 2023 10:42:40 -0700 Subject: [PATCH 05/38] [clang][cas] Move IncludeFile[List] to IncludeTree::File[List] NFC (cherry picked from commit 7caa9d961f963f8875af76424108cd309931332d) (cherry picked from commit 2c3a9496492cf40629283293eab2f301e7c58212) --- clang/include/clang/CAS/IncludeTree.h | 56 +++++++++---------- clang/lib/CAS/IncludeTree.cpp | 47 +++++++++------- .../IncludeTreeActionController.cpp | 17 +++--- clang/unittests/CAS/IncludeTreeTest.cpp | 22 ++++---- 4 files changed, 72 insertions(+), 70 deletions(-) diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 47f2d13dacd60..a2b9051b59220 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -39,14 +39,9 @@ template class IncludeTreeBase : public ObjectProxy { return Node.getData().startswith(NodeT::getNodeKind()); } - friend class IncludeFile; - friend class IncludeFileList; - friend class IncludeTree; - friend class IncludeTreeRoot; + friend NodeT; }; -class IncludeFile; - /// Represents a DAG of included files by the preprocessor. /// Each node in the DAG represents a particular inclusion of a file that /// encompasses inclusions of other files as sub-trees, along with all the @@ -56,7 +51,10 @@ class IncludeTree : public IncludeTreeBase { public: static constexpr StringRef getNodeKind() { return "Tree"; } - Expected getBaseFile(); + class File; + class FileList; + + Expected getBaseFile(); /// The include file that resulted in this include-tree. ObjectRef getBaseFileRef() const { return getReference(0); } @@ -152,7 +150,7 @@ class IncludeTree : public IncludeTreeBase { /// Represents a \p SourceManager file (or buffer in the case of preprocessor /// predefines) that got included by the preprocessor. -class IncludeFile : public IncludeTreeBase { +class IncludeTree::File : public IncludeTreeBase { public: static constexpr StringRef getNodeKind() { return "File"; } @@ -167,7 +165,7 @@ class IncludeFile : public IncludeTreeBase { return getCAS().getProxy(getContentsRef()); } - Expected getFileInfo() { + Expected getFileInfo() { auto Filename = getFilename(); if (!Filename) return Filename.takeError(); @@ -177,8 +175,8 @@ class IncludeFile : public IncludeTreeBase { return IncludeTree::FileInfo{Filename->getData(), Contents->getData()}; } - static Expected create(ObjectStore &DB, StringRef Filename, - ObjectRef Contents); + static Expected create(ObjectStore &DB, StringRef Filename, + ObjectRef Contents); llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); @@ -198,20 +196,20 @@ class IncludeFile : public IncludeTreeBase { } private: - friend class IncludeTreeBase; - friend class IncludeFileList; + friend class IncludeTreeBase; + friend class FileList; friend class IncludeTree; friend class IncludeTreeRoot; - explicit IncludeFile(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + explicit File(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { assert(isValid(*this)); } }; -/// A flat list of \p IncludeFile entries. This is used along with a simple +/// A flat list of \c File entries. This is used along with a simple /// implementation of a \p vfs::FileSystem produced via /// \p createIncludeTreeFileSystem(). -class IncludeFileList : public IncludeTreeBase { +class IncludeTree::FileList : public IncludeTreeBase { public: static constexpr StringRef getNodeKind() { return "List"; } @@ -224,12 +222,12 @@ class IncludeFileList : public IncludeTreeBase { return getReference(I); } - Expected getFile(size_t I) { return getFile(getFileRef(I)); } + Expected getFile(size_t I) { return getFile(getFileRef(I)); } FileSizeTy getFileSize(size_t I) const; - /// \returns each \p IncludeFile entry along with its file size. - llvm::Error forEachFile( - llvm::function_ref Callback); + /// \returns each \c File entry along with its file size. + llvm::Error + forEachFile(llvm::function_ref Callback); /// We record the file size as well to avoid needing to materialize the /// underlying buffer for the \p IncludeTreeFileSystem::status() @@ -238,27 +236,25 @@ class IncludeFileList : public IncludeTreeBase { ObjectRef FileRef; FileSizeTy Size; }; - static Expected create(ObjectStore &DB, - ArrayRef Files); + static Expected create(ObjectStore &DB, ArrayRef Files); - static Expected get(ObjectStore &CAS, ObjectRef Ref); + static Expected get(ObjectStore &CAS, ObjectRef Ref); llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); private: - friend class IncludeTreeBase; + friend class IncludeTreeBase; friend class IncludeTreeRoot; - explicit IncludeFileList(ObjectProxy Node) - : IncludeTreeBase(std::move(Node)) { + explicit FileList(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { assert(isValid(*this)); } - Expected getFile(ObjectRef Ref) { + Expected getFile(ObjectRef Ref) { auto Node = getCAS().getProxy(Ref); if (!Node) return Node.takeError(); - return IncludeFile(std::move(*Node)); + return File(std::move(*Node)); } static bool isValid(const ObjectProxy &Node); @@ -294,11 +290,11 @@ class IncludeTreeRoot : public IncludeTreeBase { return IncludeTree(std::move(*Node)); } - Expected getFileList() { + Expected getFileList() { auto Node = getCAS().getProxy(getFileListRef()); if (!Node) return Node.takeError(); - return IncludeFileList(std::move(*Node)); + return IncludeTree::FileList(std::move(*Node)); } Expected> getPCHBuffer() { diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index b507661bc6a88..4ab886eee568c 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -33,8 +33,9 @@ Expected IncludeTreeBase::create(ObjectStore &DB, return NodeT(*Proxy); } -Expected IncludeFile::create(ObjectStore &DB, StringRef Filename, - ObjectRef Contents) { +Expected IncludeTree::File::create(ObjectStore &DB, + StringRef Filename, + ObjectRef Contents) { auto PathRef = DB.storeFromString({}, Filename); if (!PathRef) return PathRef.takeError(); @@ -42,11 +43,11 @@ Expected IncludeFile::create(ObjectStore &DB, StringRef Filename, return IncludeTreeBase::create(DB, Refs, {}); } -Expected IncludeTree::getBaseFile() { +Expected IncludeTree::getBaseFile() { auto Node = getCAS().getProxy(getBaseFileRef()); if (!Node) return Node.takeError(); - return IncludeFile(std::move(*Node)); + return File(std::move(*Node)); } Expected IncludeTree::getBaseFileInfo() { @@ -85,7 +86,7 @@ Expected IncludeTree::create( char Kind = FileCharacteristic; assert(Kind == FileCharacteristic && "SrcMgr::CharacteristicKind too big!"); - assert(IncludeFile::isValid(DB, BaseFile)); + assert(File::isValid(DB, BaseFile)); SmallVector Refs; Refs.reserve(Includes.size() + 1); Refs.push_back(BaseFile); @@ -169,7 +170,8 @@ bool IncludeTree::isValid(const ObjectProxy &Node) { return Base.getData().size() >= NumIncludes * sizeof(uint32_t) + 1; } -IncludeFileList::FileSizeTy IncludeFileList::getFileSize(size_t I) const { +IncludeTree::FileList::FileSizeTy +IncludeTree::FileList::getFileSize(size_t I) const { assert(I < getNumFiles()); StringRef Data = getData(); assert(Data.size() >= (I + 1) * sizeof(FileSizeTy)); @@ -177,8 +179,8 @@ IncludeFileList::FileSizeTy IncludeFileList::getFileSize(size_t I) const { Data.data() + I * sizeof(FileSizeTy)); } -llvm::Error IncludeFileList::forEachFile( - llvm::function_ref Callback) { +llvm::Error IncludeTree::FileList::forEachFile( + llvm::function_ref Callback) { size_t I = 0; return forEachReference([&](ObjectRef Ref) -> llvm::Error { auto Include = getFile(Ref); @@ -188,8 +190,8 @@ llvm::Error IncludeFileList::forEachFile( }); } -Expected IncludeFileList::create(ObjectStore &DB, - ArrayRef Files) { +Expected +IncludeTree::FileList::create(ObjectStore &DB, ArrayRef Files) { SmallVector Refs; Refs.reserve(Files.size()); SmallString<256> Buffer; @@ -199,24 +201,25 @@ Expected IncludeFileList::create(ObjectStore &DB, llvm::support::endian::Writer Writer(BufOS, llvm::support::little); for (const FileEntry &Entry : Files) { - assert(IncludeFile::isValid(DB, Entry.FileRef)); + assert(File::isValid(DB, Entry.FileRef)); Refs.push_back(Entry.FileRef); Writer.write(Entry.Size); } return IncludeTreeBase::create(DB, Refs, Buffer); } -Expected IncludeFileList::get(ObjectStore &DB, ObjectRef Ref) { +Expected IncludeTree::FileList::get(ObjectStore &DB, + ObjectRef Ref) { auto Node = DB.getProxy(Ref); if (!Node) return Node.takeError(); if (!isValid(*Node)) return llvm::createStringError(llvm::inconvertibleErrorCode(), "not a IncludeFileList node kind"); - return IncludeFileList(std::move(*Node)); + return FileList(std::move(*Node)); } -bool IncludeFileList::isValid(const ObjectProxy &Node) { +bool IncludeTree::FileList::isValid(const ObjectProxy &Node) { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); @@ -230,7 +233,7 @@ Expected IncludeTreeRoot::create(ObjectStore &DB, ObjectRef FileList, Optional PCHRef) { assert(IncludeTree::isValid(DB, MainFileTree)); - assert(IncludeFileList::isValid(DB, FileList)); + assert(IncludeTree::FileList::isValid(DB, FileList)); if (PCHRef) { return IncludeTreeBase::create(DB, {MainFileTree, FileList, *PCHRef}, {}); } else { @@ -248,7 +251,7 @@ Expected IncludeTreeRoot::get(ObjectStore &DB, ObjectRef Ref) { return IncludeTreeRoot(std::move(*Node)); } -llvm::Error IncludeFile::print(llvm::raw_ostream &OS, unsigned Indent) { +llvm::Error IncludeTree::File::print(llvm::raw_ostream &OS, unsigned Indent) { auto Filename = getFilename(); if (!Filename) return Filename.takeError(); @@ -282,8 +285,9 @@ llvm::Error IncludeTree::print(llvm::raw_ostream &OS, unsigned Indent) { }); } -llvm::Error IncludeFileList::print(llvm::raw_ostream &OS, unsigned Indent) { - return forEachFile([&](cas::IncludeFile File, FileSizeTy) -> llvm::Error { +llvm::Error IncludeTree::FileList::print(llvm::raw_ostream &OS, + unsigned Indent) { + return forEachFile([&](File File, FileSizeTy) -> llvm::Error { return File.print(OS, Indent); }); } @@ -300,7 +304,7 @@ llvm::Error IncludeTreeRoot::print(llvm::raw_ostream &OS, unsigned Indent) { if (llvm::Error E = MainTree->print(OS.indent(Indent), Indent)) return E; OS.indent(Indent) << "Files:\n"; - Optional List; + Optional List; if (llvm::Error E = getFileList().moveInto(List)) return E; return List->print(OS, Indent); @@ -347,7 +351,7 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { struct FileEntry { cas::ObjectRef ContentsRef; - IncludeFileList::FileSizeTy Size; + IncludeTree::FileList::FileSizeTy Size; llvm::sys::fs::UniqueID UniqueID; }; @@ -450,7 +454,8 @@ cas::createIncludeTreeFileSystem(IncludeTreeRoot &Root) { IntrusiveRefCntPtr IncludeTreeFS = new IncludeTreeFileSystem(Root.getCAS()); llvm::Error E = FileList->forEachFile( - [&](IncludeFile File, IncludeFileList::FileSizeTy Size) -> llvm::Error { + [&](IncludeTree::File File, + IncludeTree::FileList::FileSizeTy Size) -> llvm::Error { auto FilenameBlob = File.getFilename(); if (!FilenameBlob) return FilenameBlob.takeError(); diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 14a860a7fe587..32ac0595f06e8 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -97,8 +97,8 @@ class IncludeTreeBuilder { Expected getObjectForBuffer(const SrcMgr::FileInfo &FI); Expected addToFileList(FileManager &FM, const FileEntry *FE); Expected getCASTreeForFileIncludes(FilePPState &&PPState); - Expected createIncludeFile(StringRef Filename, - cas::ObjectRef Contents); + Expected createIncludeFile(StringRef Filename, + cas::ObjectRef Contents); bool hasErrorOccurred() const { return ErrorToReport.has_value(); } @@ -120,7 +120,7 @@ class IncludeTreeBuilder { // are recorded in the PCH, ordered by \p FileEntry::UID index. SmallVector PreIncludedFileNames; llvm::BitVector SeenIncludeFiles; - SmallVector IncludedFiles; + SmallVector IncludedFiles; Optional PredefinesBufferRef; Optional ModuleIncludesBufferRef; Optional ModuleMapFileRef; @@ -446,7 +446,7 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, getCASTreeForFileIncludes(IncludeStack.pop_back_val()); if (!MainIncludeTree) return MainIncludeTree.takeError(); - auto FileList = cas::IncludeFileList::create(DB, IncludedFiles); + auto FileList = cas::IncludeTree::FileList::create(DB, IncludedFiles); if (!FileList) return FileList.takeError(); @@ -508,7 +508,8 @@ IncludeTreeBuilder::getObjectForBuffer(const SrcMgr::FileInfo &FI) { {}, FI.getContentCache().getBufferIfLoaded()->getBuffer()); if (!Ref) return Ref.takeError(); - Expected FileNode = createIncludeFile(FI.getName(), *Ref); + Expected FileNode = + createIncludeFile(FI.getName(), *Ref); if (!FileNode) return FileNode.takeError(); return FileNode->getRef(); @@ -530,7 +531,7 @@ IncludeTreeBuilder::addToFileList(FileManager &FM, const FileEntry *FE) { return FileNode.takeError(); IncludedFiles.push_back( {FileNode->getRef(), - static_cast(FE->getSize())}); + static_cast(FE->getSize())}); return FileNode->getRef(); }; @@ -553,7 +554,7 @@ IncludeTreeBuilder::getCASTreeForFileIncludes(FilePPState &&PPState) { PPState.Includes, PPState.HasIncludeChecks); } -Expected +Expected IncludeTreeBuilder::createIncludeFile(StringRef Filename, cas::ObjectRef Contents) { SmallString<256> MappedPath; @@ -561,7 +562,7 @@ IncludeTreeBuilder::createIncludeFile(StringRef Filename, PrefixMapper.map(Filename, MappedPath); Filename = MappedPath; } - return cas::IncludeFile::create(DB, Filename, std::move(Contents)); + return cas::IncludeTree::File::create(DB, Filename, std::move(Contents)); } std::unique_ptr diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index ba422a1e99006..f687ef57762d8 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -60,10 +60,10 @@ TEST(IncludeTree, IncludeTreeScan) { .moveInto(Root), llvm::Succeeded()); - Optional MainFile; - Optional A1File; - Optional B1File; - Optional SysFile; + Optional MainFile; + Optional A1File; + Optional B1File; + Optional SysFile; Optional Main; ASSERT_THAT_ERROR(Root->getMainFileTree().moveInto(Main), llvm::Succeeded()); @@ -132,31 +132,31 @@ TEST(IncludeTree, IncludeTreeScan) { ASSERT_EQ(Sys->getNumIncludes(), uint32_t(0)); } - Optional FileList; + Optional FileList; ASSERT_THAT_ERROR(Root->getFileList().moveInto(FileList), llvm::Succeeded()); ASSERT_EQ(FileList->getNumFiles(), size_t(4)); { - Optional File; + Optional File; ASSERT_THAT_ERROR(FileList->getFile(0).moveInto(File), llvm::Succeeded()); EXPECT_EQ(File->getRef(), MainFile->getRef()); EXPECT_EQ(FileList->getFileSize(0), MainContents.size()); } { - Optional File; + Optional File; ASSERT_THAT_ERROR(FileList->getFile(1).moveInto(File), llvm::Succeeded()); EXPECT_EQ(File->getRef(), A1File->getRef()); EXPECT_EQ(FileList->getFileSize(1), A1Contents.size()); } { - Optional File; + Optional File; ASSERT_THAT_ERROR(FileList->getFile(2).moveInto(File), llvm::Succeeded()); EXPECT_EQ(File->getRef(), B1File->getRef()); - EXPECT_EQ(FileList->getFileSize(2), IncludeFileList::FileSizeTy(0)); + EXPECT_EQ(FileList->getFileSize(2), IncludeTree::FileList::FileSizeTy(0)); } { - Optional File; + Optional File; ASSERT_THAT_ERROR(FileList->getFile(3).moveInto(File), llvm::Succeeded()); EXPECT_EQ(File->getRef(), SysFile->getRef()); - EXPECT_EQ(FileList->getFileSize(3), IncludeFileList::FileSizeTy(0)); + EXPECT_EQ(FileList->getFileSize(3), IncludeTree::FileList::FileSizeTy(0)); } } From c3fb291519b1a9efc7a68330c65b23b0a657f899 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 15 Mar 2023 16:30:23 -0700 Subject: [PATCH 06/38] [clang][cas] Basic building and importing modules with include-tree This handles building and importing simple modules and PCH with modules using include-tree, when modules are found via header path. * Extend IncludeTree to represent module imports and modulemap mainfile * Make IncludeTreeBuilder add the new nodes for #include * Make PPDirectives import modules from PPCachedActions in #include * Teach FrontendAction/CompilerInvocation how to setup the main input file modulemap and module includes buffer from IncludeTreeRoot. There are still lots of things missing, including @import syntax and submodule semantics. (cherry picked from commit 55b697db03fd0a78ba4e41371772460c939800bc) (cherry picked from commit a9db0557b4226710616892af5ec33be1dfc97e90) --- clang/include/clang/CAS/IncludeTree.h | 147 ++++++++++++++--- clang/include/clang/Lex/PPCachedActions.h | 18 +- clang/lib/CAS/IncludeTree.cpp | 139 ++++++++++++---- clang/lib/Frontend/CompilerInvocation.cpp | 23 ++- clang/lib/Frontend/FrontendAction.cpp | 52 ++++-- clang/lib/Frontend/IncludeTreePPActions.cpp | 44 +++-- clang/lib/Lex/PPDirectives.cpp | 58 ++++--- .../IncludeTreeActionController.cpp | 91 ++++++++-- .../DependencyScanning/ScanAndUpdateArgs.cpp | 8 +- .../modules-include-tree-with-pch.c | 156 ++++++++++++++++++ .../test/ClangScanDeps/modules-include-tree.c | 95 ++++++++++- clang/unittests/CAS/IncludeTreeTest.cpp | 9 +- 12 files changed, 701 insertions(+), 139 deletions(-) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-with-pch.c diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index a2b9051b59220..1917f6f1c54bb 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -42,9 +42,9 @@ template class IncludeTreeBase : public ObjectProxy { friend NodeT; }; -/// Represents a DAG of included files by the preprocessor. -/// Each node in the DAG represents a particular inclusion of a file that -/// encompasses inclusions of other files as sub-trees, along with all the +/// Represents a DAG of included files by the preprocessor and module imports. +/// Each node in the DAG represents a particular inclusion of a file or module +/// that encompasses inclusions of other files as sub-trees, along with all the /// \p __has_include() preprocessor checks that occurred during preprocessing /// of that file. class IncludeTree : public IncludeTreeBase { @@ -53,6 +53,8 @@ class IncludeTree : public IncludeTreeBase { class File; class FileList; + class Node; + class ModuleImport; Expected getBaseFile(); @@ -77,9 +79,21 @@ class IncludeTree : public IncludeTreeBase { return getReference(I + 1); } + enum class NodeKind : uint8_t { + Tree, + ModuleImport, + }; + + /// The kind of node included at the given index. + NodeKind getIncludeKind(size_t I) const; + + /// The sub-include-tree or module import for the given index. + Expected getIncludeNode(size_t I); + /// The sub-include-trees of included files, in the order that they occurred. - Expected getInclude(size_t I) { - return getInclude(getIncludeRef(I)); + Expected getIncludeTree(size_t I) { + assert(getIncludeKind(I) == NodeKind::Tree); + return getIncludeTree(getIncludeRef(I)); } /// The source byte offset for a particular include, pointing to the beginning @@ -104,14 +118,19 @@ class IncludeTree : public IncludeTreeBase { /// occur in the same order. bool getCheckResult(size_t I) const; - /// Passes pairs of (IncludeTree, include offset) to \p Callback. + /// Passes pairs of (Node, include offset) to \p Callback. llvm::Error forEachInclude( - llvm::function_ref)> - Callback); + llvm::function_ref)> Callback); + + struct IncludeInfo { + ObjectRef Ref; ///< IncludeTree or IncludeTreeModuleImport + uint32_t Offset; + NodeKind Kind; + }; static Expected create(ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, - ObjectRef BaseFile, ArrayRef> Includes, + ObjectRef BaseFile, ArrayRef Includes, llvm::SmallBitVector Checks); static Expected get(ObjectStore &DB, ObjectRef Ref); @@ -126,15 +145,17 @@ class IncludeTree : public IncludeTreeBase { assert(isValid(*this)); } - Expected getInclude(ObjectRef Ref) { + Expected getIncludeTree(ObjectRef Ref) { auto Node = getCAS().getProxy(Ref); if (!Node) return Node.takeError(); return IncludeTree(std::move(*Node)); } + Expected getIncludeNode(ObjectRef Ref, NodeKind Kind); + StringRef dataSkippingIncludes() const { - return getData().drop_front(getNumIncludes() * sizeof(uint32_t)); + return getData().drop_front(getNumIncludes() * (sizeof(uint32_t) + 1)); } static bool isValid(const ObjectProxy &Node); @@ -175,6 +196,16 @@ class IncludeTree::File : public IncludeTreeBase { return IncludeTree::FileInfo{Filename->getData(), Contents->getData()}; } + Expected> getMemoryBuffer() { + auto Filename = getFilename(); + if (!Filename) + return Filename.takeError(); + auto Contents = getContents(); + if (!Contents) + return Contents.takeError(); + return Contents->getMemoryBuffer(Filename->getData()); + } + static Expected create(ObjectStore &DB, StringRef Filename, ObjectRef Contents); @@ -268,6 +299,63 @@ class IncludeTree::FileList : public IncludeTreeBase { } }; +/// Represents a module imported by an IncludeTree. +class IncludeTree::ModuleImport : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "ModI"; } + + static Expected create(ObjectStore &DB, StringRef ModuleName); + + StringRef getModuleName() { return getData(); } + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + return Base.getNumReferences() == 0 && !Base.getData().empty(); + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + friend class IncludeTreeBase; + friend class Node; + + explicit ModuleImport(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } +}; + +/// Represents an \c IncludeTree or \c ModuleImport. +class IncludeTree::Node { +public: + IncludeTree getIncludeTree() const { + assert(K == NodeKind::Tree); + return IncludeTree(N); + } + ModuleImport getModuleImport() const { + assert(K == NodeKind::ModuleImport); + return ModuleImport(N); + } + NodeKind getKind() const { return K; } + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + +private: + friend class IncludeTree; + Node(ObjectProxy N, NodeKind K) : N(std::move(N)), K(K) {} + ObjectProxy N; + NodeKind K; +}; + /// Represents the include-tree result for a translation unit. class IncludeTreeRoot : public IncludeTreeBase { public: @@ -278,9 +366,15 @@ class IncludeTreeRoot : public IncludeTreeBase { ObjectRef getFileListRef() const { return getReference(1); } Optional getPCHRef() const { - if (getNumReferences() > 2) - return getReference(2); - return None; + if (auto Index = getPCHRefIndex()) + return getReference(*Index); + return std::nullopt; + } + + Optional getModuleMapRef() const { + if (auto Index = getModuleMapRefIndex()) + return getReference(*Index); + return std::nullopt; } Expected getMainFileTree() { @@ -307,10 +401,20 @@ class IncludeTreeRoot : public IncludeTreeBase { return None; } - static Expected create(ObjectStore &DB, - ObjectRef MainFileTree, - ObjectRef FileList, - Optional PCHRef); + Expected> getModuleMapFile() { + if (Optional Ref = getModuleMapRef()) { + auto Node = getCAS().getProxy(*Ref); + if (!Node) + return Node.takeError(); + return IncludeTree::File(*Node); + } + return std::nullopt; + } + + static Expected + create(ObjectStore &DB, ObjectRef MainFileTree, ObjectRef FileList, + Optional PCHRef, + Optional ModuleMapRef); static Expected get(ObjectStore &DB, ObjectRef Ref); @@ -320,8 +424,8 @@ class IncludeTreeRoot : public IncludeTreeBase { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); - return (Base.getNumReferences() == 2 || Base.getNumReferences() == 3) && - Base.getData().empty(); + return (Base.getNumReferences() >= 2 && Base.getNumReferences() <= 4) && + Base.getData().size() == 1; } static bool isValid(ObjectStore &DB, ObjectRef Ref) { auto Node = DB.getProxy(Ref); @@ -335,6 +439,9 @@ class IncludeTreeRoot : public IncludeTreeBase { private: friend class IncludeTreeBase; + std::optional getPCHRefIndex() const; + std::optional getModuleMapRefIndex() const; + explicit IncludeTreeRoot(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { assert(isValid(*this)); diff --git a/clang/include/clang/Lex/PPCachedActions.h b/clang/include/clang/Lex/PPCachedActions.h index d9ae64030d621..da00f7f41939c 100644 --- a/clang/include/clang/Lex/PPCachedActions.h +++ b/clang/include/clang/Lex/PPCachedActions.h @@ -15,9 +15,11 @@ #define LLVM_CLANG_LEX_PPCACHEDACTIONS_H #include "clang/Basic/SourceLocation.h" +#include "llvm/ADT/SmallVector.h" namespace clang { +class IdentifierInfo; class Preprocessor; /// This interface provides a way to override the actions of the preprocessor as @@ -28,6 +30,15 @@ class PPCachedActions { virtual void anchor(); public: + /// The file that is included by an \c #include directive. + struct IncludeFile { + FileID FID; + }; + /// The module that is imported by an \c #include directive or \c @import. + struct IncludeModule { + SmallVector, 2> ImportPath; + }; + virtual ~PPCachedActions() = default; /// \returns the \p FileID that should be used for predefines. @@ -37,9 +48,10 @@ class PPCachedActions { virtual bool evaluateHasInclude(Preprocessor &PP, SourceLocation Loc, bool IsIncludeNext) = 0; - /// \returns the \p FileID that should be entered for an include directive. - /// \p None indicates that the directive should be skipped. - virtual Optional + /// \returns the file that should be entered or module that should be imported + /// for an \c #include directive. \c {} indicates that the directive + /// should be skipped. + virtual std::variant handleIncludeDirective(Preprocessor &PP, SourceLocation IncludeLoc, SourceLocation AfterDirectiveLoc) = 0; diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index 4ab886eee568c..ed8ce36a76fda 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -58,8 +58,7 @@ Expected IncludeTree::getBaseFileInfo() { } llvm::Error IncludeTree::forEachInclude( - llvm::function_ref)> - Callback) { + llvm::function_ref)> Callback) { size_t RefI = 0; return forEachReference([&](ObjectRef Ref) -> llvm::Error { if (RefI == 0) { @@ -68,19 +67,20 @@ llvm::Error IncludeTree::forEachInclude( } size_t IncludeI = RefI - 1; ++RefI; - auto Include = getInclude(Ref); + auto Include = getIncludeNode(Ref, getIncludeKind(IncludeI)); if (!Include) return Include.takeError(); return Callback({*Include, getIncludeOffset(IncludeI)}); }); } -Expected IncludeTree::create( - ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, - ObjectRef BaseFile, ArrayRef> Includes, - llvm::SmallBitVector Checks) { +Expected +IncludeTree::create(ObjectStore &DB, + SrcMgr::CharacteristicKind FileCharacteristic, + ObjectRef BaseFile, ArrayRef Includes, + llvm::SmallBitVector Checks) { // The data buffer is composed of - // 1. `uint32_t` offsets of includes + // 1. `uint32_t` offset and `uint8_t` kind for each includes // 2. 1 byte for `CharacteristicKind` // 3. variable number of bitset bytes for `Checks`. @@ -97,11 +97,14 @@ Expected IncludeTree::create( llvm::support::endian::Writer Writer(BufOS, llvm::support::little); for (const auto &Include : Includes) { - ObjectRef FileRef = Include.first; - uint32_t Offset = Include.second; - assert(IncludeTree::isValid(DB, FileRef)); - Refs.push_back(FileRef); - Writer.write(Offset); + assert((Include.Kind == NodeKind::Tree && + IncludeTree::isValid(DB, Include.Ref)) || + (Include.Kind == NodeKind::ModuleImport && + ModuleImport::isValid(DB, Include.Ref))); + Refs.push_back(Include.Ref); + Writer.write(Include.Offset); + static_assert(sizeof(uint8_t) == sizeof(Kind)); + Writer.write(static_cast(Include.Kind)); } Buffer += Kind; @@ -141,13 +144,21 @@ Expected IncludeTree::get(ObjectStore &DB, ObjectRef Ref) { return IncludeTree(std::move(*Node)); } +IncludeTree::NodeKind IncludeTree::getIncludeKind(size_t I) const { + assert(I < getNumIncludes()); + StringRef Data = getData(); + assert(Data.size() >= (I + 1) * (sizeof(uint32_t) + 1)); + uint8_t K = *(Data.data() + I * (sizeof(uint32_t) + 1) + sizeof(uint32_t)); + return NodeKind(K); +} + uint32_t IncludeTree::getIncludeOffset(size_t I) const { assert(I < getNumIncludes()); StringRef Data = getData(); assert(Data.size() >= (I + 1) * sizeof(uint32_t)); uint32_t Offset = llvm::support::endian::read( - Data.data() + I * sizeof(uint32_t)); + Data.data() + I * (sizeof(uint32_t) + 1)); return Offset; } @@ -160,6 +171,18 @@ bool IncludeTree::getCheckResult(size_t I) const { return Bits & (1 << RemainingIndex); } +Expected IncludeTree::getIncludeNode(size_t I) { + return getIncludeNode(getIncludeRef(I), getIncludeKind(I)); +} + +Expected IncludeTree::getIncludeNode(ObjectRef Ref, + NodeKind K) { + auto N = getCAS().getProxy(Ref); + if (!N) + return N.takeError(); + return Node(std::move(*N), K); +} + bool IncludeTree::isValid(const ObjectProxy &Node) { if (!IncludeTreeBase::isValid(Node)) return false; @@ -167,7 +190,13 @@ bool IncludeTree::isValid(const ObjectProxy &Node) { if (Base.getNumReferences() == 0) return false; unsigned NumIncludes = Base.getNumReferences() - 1; - return Base.getData().size() >= NumIncludes * sizeof(uint32_t) + 1; + return Base.getData().size() >= NumIncludes * (sizeof(uint32_t) + 1) + 1; +} + +Expected +IncludeTree::ModuleImport::create(ObjectStore &DB, StringRef ModuleName) { + return IncludeTreeBase::create(DB, {}, + llvm::arrayRefFromStringRef(ModuleName)); } IncludeTree::FileList::FileSizeTy @@ -228,17 +257,30 @@ bool IncludeTree::FileList::isValid(const ObjectProxy &Node) { Base.getData().size() == NumFiles * sizeof(FileSizeTy); } -Expected IncludeTreeRoot::create(ObjectStore &DB, - ObjectRef MainFileTree, - ObjectRef FileList, - Optional PCHRef) { +static constexpr char HasPCH = 0x01; +static constexpr char HasModuleMap = 0x02; + +Expected +IncludeTreeRoot::create(ObjectStore &DB, ObjectRef MainFileTree, + ObjectRef FileList, Optional PCHRef, + Optional ModuleMapRef) { assert(IncludeTree::isValid(DB, MainFileTree)); assert(IncludeTree::FileList::isValid(DB, FileList)); - if (PCHRef) { - return IncludeTreeBase::create(DB, {MainFileTree, FileList, *PCHRef}, {}); - } else { - return IncludeTreeBase::create(DB, {MainFileTree, FileList}, {}); - } + assert(!ModuleMapRef || IncludeTree::File::isValid(DB, *ModuleMapRef)); + + std::array Data = {0}; + if (PCHRef) + Data[0] |= HasPCH; + if (ModuleMapRef) + Data[0] |= HasModuleMap; + + SmallVector Refs = {MainFileTree, FileList}; + if (PCHRef) + Refs.push_back(*PCHRef); + if (ModuleMapRef) + Refs.push_back(*ModuleMapRef); + + return IncludeTreeBase::create(DB, Refs, Data); } Expected IncludeTreeRoot::get(ObjectStore &DB, ObjectRef Ref) { @@ -251,6 +293,17 @@ Expected IncludeTreeRoot::get(ObjectStore &DB, ObjectRef Ref) { return IncludeTreeRoot(std::move(*Node)); } +std::optional IncludeTreeRoot::getPCHRefIndex() const { + if (getData()[0] & HasPCH) + return 2; + return std::nullopt; +} +std::optional IncludeTreeRoot::getModuleMapRefIndex() const { + if (getData()[0] & HasModuleMap) + return (getData()[0] & HasPCH) ? 3u : 2u; + return std::nullopt; +} + llvm::Error IncludeTree::File::print(llvm::raw_ostream &OS, unsigned Indent) { auto Filename = getFilename(); if (!Filename) @@ -275,14 +328,13 @@ llvm::Error IncludeTree::print(llvm::raw_ostream &OS, unsigned Indent) { auto MemBuf = llvm::MemoryBuffer::getMemBuffer(Blob->getData()); unsigned BufID = SM.AddNewSourceBuffer(std::move(MemBuf), llvm::SMLoc()); - return forEachInclude( - [&](std::pair Include) -> llvm::Error { - llvm::SMLoc Loc = llvm::SMLoc::getFromPointer( - SM.getMemoryBuffer(BufID)->getBufferStart() + Include.second); - auto LineCol = SM.getLineAndColumn(Loc); - OS.indent(Indent) << LineCol.first << ':' << LineCol.second << ' '; - return Include.first.print(OS, Indent + 2); - }); + return forEachInclude([&](std::pair Include) -> llvm::Error { + llvm::SMLoc Loc = llvm::SMLoc::getFromPointer( + SM.getMemoryBuffer(BufID)->getBufferStart() + Include.second); + auto LineCol = SM.getLineAndColumn(Loc); + OS.indent(Indent) << LineCol.first << ':' << LineCol.second << ' '; + return Include.first.print(OS, Indent + 2); + }); } llvm::Error IncludeTree::FileList::print(llvm::raw_ostream &OS, @@ -292,6 +344,21 @@ llvm::Error IncludeTree::FileList::print(llvm::raw_ostream &OS, }); } +llvm::Error IncludeTree::ModuleImport::print(llvm::raw_ostream &OS, + unsigned Indent) { + OS << "(Module) " << getModuleName() << '\n'; + return llvm::Error::success(); +} + +llvm::Error IncludeTree::Node::print(llvm::raw_ostream &OS, unsigned Indent) { + switch (K) { + case NodeKind::Tree: + return getIncludeTree().print(OS, Indent); + case NodeKind::ModuleImport: + return getModuleImport().print(OS, Indent); + } +} + llvm::Error IncludeTreeRoot::print(llvm::raw_ostream &OS, unsigned Indent) { if (Optional PCHRef = getPCHRef()) { OS.indent(Indent) << "(PCH) "; @@ -303,6 +370,14 @@ llvm::Error IncludeTreeRoot::print(llvm::raw_ostream &OS, unsigned Indent) { return E; if (llvm::Error E = MainTree->print(OS.indent(Indent), Indent)) return E; + Optional ModuleMap; + if (llvm::Error E = getModuleMapFile().moveInto(ModuleMap)) + return E; + if (ModuleMap) { + OS.indent(Indent) << "Module Map: "; + if (llvm::Error E = ModuleMap->print(OS, Indent)) + return E; + } OS.indent(Indent) << "Files:\n"; Optional List; if (llvm::Error E = getFileList().moveInto(List)) diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 1502cfef57d1b..ecf1848f86738 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -3037,12 +3037,23 @@ static void determineInputFromIncludeTree( auto Root = cas::IncludeTreeRoot::get(*CAS, *Object); if (!Root) return reportError(Root.takeError()); - auto MainTree = Root->getMainFileTree(); - if (!MainTree) - return reportError(MainTree.takeError()); - auto BaseFile = MainTree->getBaseFile(); - if (!BaseFile) - return reportError(BaseFile.takeError()); + + Optional BaseFile; + + auto MaybeModuleMap = Root->getModuleMapFile(); + if (!MaybeModuleMap) + return reportError(MaybeModuleMap.takeError()); + if (*MaybeModuleMap) { + // Building a module from a modulemap, the modulemap is the primary input. + BaseFile = *MaybeModuleMap; + } else { + auto MainTree = Root->getMainFileTree(); + if (!MainTree) + return reportError(MainTree.takeError()); + if (llvm::Error E = MainTree->getBaseFile().moveInto(BaseFile)) + return reportError(std::move(E)); + } + auto FilenameBlob = BaseFile->getFilename(); if (!FilenameBlob) return reportError(FilenameBlob.takeError()); diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index 2a03e521bd704..7694664195708 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -856,28 +856,31 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, CI.getLangOpts().CurrentModule = CI.getLangOpts().ModuleName; } + auto reportError = [&](llvm::Error &&E) -> bool { + std::string IncludeTreeID = + CI.getOrCreateObjectStore().getID(Input.getIncludeTree()).toString(); + CI.getDiagnostics().Report(diag::err_fe_unable_to_load_include_tree) + << IncludeTreeID << llvm::toString(std::move(E)); + return false; + }; + + Optional IncludeTreeRoot; Optional IncludeTreePCHBuffer; if (Input.isIncludeTree()) { - auto reportError = [&](llvm::Error &&E) -> bool { - std::string IncludeTreeID = - CI.getOrCreateObjectStore().getID(Input.getIncludeTree()).toString(); - CI.getDiagnostics().Report(diag::err_fe_unable_to_load_include_tree) - << IncludeTreeID << llvm::toString(std::move(E)); - return false; - }; - auto Root = cas::IncludeTreeRoot::get(CI.getOrCreateObjectStore(), - Input.getIncludeTree()); - if (!Root) - return reportError(Root.takeError()); + if (llvm::Error E = cas::IncludeTreeRoot::get(CI.getOrCreateObjectStore(), + Input.getIncludeTree()) + .moveInto(IncludeTreeRoot)) + return reportError(std::move(E)); Expected> PPCachedAct = - createPPActionsFromIncludeTree(*Root); + createPPActionsFromIncludeTree(*IncludeTreeRoot); if (!PPCachedAct) return reportError(PPCachedAct.takeError()); CI.getPreprocessor().setPPCachedActions(std::move(*PPCachedAct)); CI.getFrontendOpts().IncludeTimestamps = false; - if (llvm::Error E = Root->getPCHBuffer().moveInto(IncludeTreePCHBuffer)) + if (llvm::Error E = + IncludeTreeRoot->getPCHBuffer().moveInto(IncludeTreePCHBuffer)) return reportError(std::move(E)); } @@ -921,10 +924,25 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, // If the module contents are in the same file, skip to them. CI.getPreprocessor().setSkipMainFilePreamble(OffsetToContents, true); else { - // Otherwise, convert the module description to a suitable input buffer. - auto Buffer = getInputBufferForModule(CI, CurrentModule); - if (!Buffer) - return false; + std::unique_ptr Buffer; + if (Input.isIncludeTree()) { + // Get the existing module include buffer from the include-tree. + assert(IncludeTreeRoot); + auto MainTree = IncludeTreeRoot->getMainFileTree(); + if (!MainTree) + return reportError(MainTree.takeError()); + auto BaseFile = MainTree->getBaseFile(); + if (!BaseFile) + return reportError(BaseFile.takeError()); + if (auto E = BaseFile->getMemoryBuffer().moveInto(Buffer)) + return reportError(std::move(E)); + assert(Buffer); + } else { + // Otherwise, convert the module description to a suitable input buffer. + Buffer = getInputBufferForModule(CI, CurrentModule); + if (!Buffer) + return false; + } // Reinitialize the main file entry to refer to the new input. auto Kind = CurrentModule->IsSystem ? SrcMgr::C_System : SrcMgr::C_User; diff --git a/clang/lib/Frontend/IncludeTreePPActions.cpp b/clang/lib/Frontend/IncludeTreePPActions.cpp index d18908577677f..8c6c1cdad2838 100644 --- a/clang/lib/Frontend/IncludeTreePPActions.cpp +++ b/clang/lib/Frontend/IncludeTreePPActions.cpp @@ -61,7 +61,7 @@ class IncludeTreePPActions final : public PPCachedActions { IncludeStackInfo &IncludeInfo = IncludeStack.back(); Expected EnteredTree = - IncludeInfo.Tree.getInclude(IncludeInfo.CurIncludeIndex++); + IncludeInfo.Tree.getIncludeTree(IncludeInfo.CurIncludeIndex++); if (!EnteredTree) return reportError(EnteredTree.takeError()); auto FileInfo = EnteredTree->getBaseFileInfo(); @@ -84,33 +84,47 @@ class IncludeTreePPActions final : public PPCachedActions { return IncludeInfo.Tree.getCheckResult(Index); } - Optional + std::variant handleIncludeDirective(Preprocessor &PP, SourceLocation IncludeLoc, SourceLocation AfterDirectiveLoc) override { if (HasCASErrorOccurred) - return None; + return {}; IncludeStackInfo &IncludeInfo = IncludeStack.back(); if (IncludeInfo.CurIncludeIndex >= IncludeInfo.Tree.getNumIncludes()) - return None; + return {}; unsigned ExpectedOffset = IncludeInfo.Tree.getIncludeOffset(IncludeInfo.CurIncludeIndex); SourceLocation ExpectedLoc = IncludeInfo.FileStartLoc.getLocWithOffset(ExpectedOffset); if (ExpectedLoc != AfterDirectiveLoc) - return None; + return {}; - auto reportError = [&](llvm::Error &&E) -> Optional { + auto reportError = [&](llvm::Error &&E) -> std::monostate { this->reportError(PP, std::move(E)); - return None; + return {}; }; - Expected EnteredTree = - IncludeInfo.Tree.getInclude(IncludeInfo.CurIncludeIndex++); - if (!EnteredTree) - return reportError(EnteredTree.takeError()); - auto File = EnteredTree->getBaseFile(); + Expected Node = + IncludeInfo.Tree.getIncludeNode(IncludeInfo.CurIncludeIndex++); + if (!Node) + return reportError(Node.takeError()); + + if (Node->getKind() == cas::IncludeTree::NodeKind::ModuleImport) { + cas::IncludeTree::ModuleImport Import = Node->getModuleImport(); + SmallVector, 2> Path; + SmallVector ModuleComponents; + Import.getModuleName().split(ModuleComponents, '.'); + for (StringRef Component : ModuleComponents) + Path.emplace_back(PP.getIdentifierInfo(Component), IncludeLoc); + return IncludeModule{std::move(Path)}; + } + + assert(Node->getKind() == cas::IncludeTree::NodeKind::Tree); + + cas::IncludeTree EnteredTree = Node->getIncludeTree(); + auto File = EnteredTree.getBaseFile(); if (!File) return reportError(File.takeError()); auto FilenameBlob = File->getFilename(); @@ -124,11 +138,11 @@ class IncludeTreePPActions final : public PPCachedActions { if (!FE) return reportError(FE.takeError()); FileID FID = - SM.createFileID(*FE, IncludeLoc, EnteredTree->getFileCharacteristic()); + SM.createFileID(*FE, IncludeLoc, EnteredTree.getFileCharacteristic()); PP.markIncluded(*FE); IncludeStack.push_back( - {std::move(*EnteredTree), SM.getLocForStartOfFile(FID)}); - return FID; + {std::move(EnteredTree), SM.getLocForStartOfFile(FID)}); + return IncludeFile{FID}; } void exitedFile(Preprocessor &PP, FileID FID) override { diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index 29f3790325c19..5a4530f49339c 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -1973,39 +1973,59 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, return; } + // Verify that there is nothing after the filename, other than EOD. Note + // that we allow macros that expand to nothing after the filename, because + // this falls into the category of "#include pp-tokens new-line" specified + // in C99 6.10.2p4. + SourceLocation EndLoc = + CheckEndOfDirective(IncludeTok.getIdentifierInfo()->getNameStart(), true); + if (auto *CActions = getPPCachedActions()) { - DiscardUntilEndOfDirective(); SourceLocation IncludePos = FilenameTok.getLocation(); // If the filename string was the result of macro expansions, set the // include position on the file where it will be included and after the // expansions. if (IncludePos.isMacroID()) IncludePos = SourceMgr.getExpansionRange(IncludePos).getEnd(); - Optional FID = CActions->handleIncludeDirective( + auto Include = CActions->handleIncludeDirective( *this, IncludePos, CurLexer->getSourceLocation()); - if (!FID) { - // FIXME: Report \p Callbacks->FileSkipped? Note that it currently - // requires the resolved FileEntry for this particular #include. + + if (auto *File = std::get_if(&Include)) { + const FileEntry *FE = SourceMgr.getFileEntryForID(File->FID); + bool IsImport = + IncludeTok.getIdentifierInfo()->getPPKeywordID() == tok::pp_import; + if (FE && IsImport) { + HeaderInfo.getFileInfo(FE).isImport = true; + } + EnterSourceFile(File->FID, nullptr, FilenameTok.getLocation(), + /*IsFirstIncludeOfFile*/ true); return; } - const FileEntry *FE = SourceMgr.getFileEntryForID(*FID); - bool IsImport = - IncludeTok.getIdentifierInfo()->getPPKeywordID() == tok::pp_import; - if (FE && IsImport) { - HeaderInfo.getFileInfo(FE).isImport = true; + if (auto *Import = std::get_if(&Include)) { + ModuleLoadResult Imported = TheModuleLoader.loadModule( + IncludeTok.getLocation(), Import->ImportPath, Module::Hidden, + /*IsIncludeDirective=*/true); + if (!Imported) { + assert(hadModuleLoaderFatalFailure() && "unexpected failure kind"); + if (hadModuleLoaderFatalFailure()) { + IncludeTok.setKind(tok::eof); + CurLexer->cutOffLexing(); + } + return; + } + makeModuleVisible(Imported, EndLoc); + if (IncludeTok.getIdentifierInfo()->getPPKeywordID() != + tok::pp___include_macros) + EnterAnnotationToken(SourceRange(HashLoc, EndLoc), + tok::annot_module_include, Imported); + return; } - EnterSourceFile(*FID, nullptr, FilenameTok.getLocation(), - /*IsFirstIncludeOfFile*/ true); + assert(std::holds_alternative(Include)); + // FIXME: Report \p Callbacks->FileSkipped? Note that it currently + // requires the resolved FileEntry for this particular #include. return; } - // Verify that there is nothing after the filename, other than EOD. Note - // that we allow macros that expand to nothing after the filename, because - // this falls into the category of "#include pp-tokens new-line" specified - // in C99 6.10.2p4. - SourceLocation EndLoc = - CheckEndOfDirective(IncludeTok.getIdentifierInfo()->getNameStart(), true); - auto Action = HandleHeaderIncludeOrImport(HashLoc, IncludeTok, FilenameTok, EndLoc, LookupFrom, LookupFromFile); switch (Action.Kind) { diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 32ac0595f06e8..909b9b7d56084 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -83,11 +83,13 @@ class IncludeTreeBuilder { void handleHasIncludeCheck(Preprocessor &PP, bool Result); + void moduleImport(Preprocessor &PP, const Module *M, SourceLocation EndLoc); + private: struct FilePPState { SrcMgr::CharacteristicKind FileCharacteristic; cas::ObjectRef File; - SmallVector, 6> Includes; + SmallVector Includes; llvm::SmallBitVector HasIncludeChecks; }; @@ -172,6 +174,32 @@ struct IncludeTreePPCallbacks : public PPCallbacks { SrcMgr::CharacteristicKind FileType) override { Builder.handleHasIncludeCheck(PP, File.has_value()); } + + void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok, + StringRef FileName, bool IsAngled, + CharSourceRange FilenameRange, + Optional File, StringRef SearchPath, + StringRef RelativePath, const Module *Imported, + SrcMgr::CharacteristicKind FileType) override { + if (!Imported) + return; // File includes handled by LexedFileChanged. + + // Calculate EndLoc for the directive + // FIXME: pass EndLoc through PPCallbacks; it is already calculated + SourceManager &SM = PP.getSourceManager(); + std::pair LocInfo = SM.getDecomposedExpansionLoc(HashLoc); + StringRef Buffer = SM.getBufferData(LocInfo.first); + Lexer L(SM.getLocForStartOfFile(LocInfo.first), PP.getLangOpts(), + Buffer.begin(), Buffer.begin() + LocInfo.second, Buffer.end()); + L.setParsingPreprocessorDirective(true); + Token Tok; + do { + L.LexFromRawLexer(Tok); + } while (!Tok.isOneOf(tok::eod, tok::eof)); + SourceLocation EndLoc = L.getSourceLocation(); + + Builder.moduleImport(PP, Imported, EndLoc); + } }; } // namespace @@ -341,8 +369,8 @@ void IncludeTreeBuilder::exitedInclude(Preprocessor &PP, FileID IncludedBy, assert(*check(getObjectForFile(PP, IncludedBy)) == IncludeStack.back().File); SourceManager &SM = PP.getSourceManager(); std::pair LocInfo = SM.getDecomposedExpansionLoc(ExitLoc); - IncludeStack.back().Includes.push_back( - {IncludeTree->getRef(), LocInfo.second}); + IncludeStack.back().Includes.push_back({IncludeTree->getRef(), LocInfo.second, + cas::IncludeTree::NodeKind::Tree}); } void IncludeTreeBuilder::handleHasIncludeCheck(Preprocessor &PP, bool Result) { @@ -352,6 +380,20 @@ void IncludeTreeBuilder::handleHasIncludeCheck(Preprocessor &PP, bool Result) { IncludeStack.back().HasIncludeChecks.push_back(Result); } +void IncludeTreeBuilder::moduleImport(Preprocessor &PP, const Module *M, + SourceLocation EndLoc) { + auto Import = + check(cas::IncludeTree::ModuleImport::create(DB, M->getFullModuleName())); + if (!Import) + return; + + std::pair EndLocInfo = + PP.getSourceManager().getDecomposedExpansionLoc(EndLoc); + IncludeStack.back().Includes.push_back( + {Import->getRef(), EndLocInfo.second, + cas::IncludeTree::NodeKind::ModuleImport}); +} + Expected IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { @@ -400,20 +442,36 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, return std::move(E); } + auto &FrontendOpts = NewInvocation.getFrontendOpts(); + if (!FrontendOpts.Inputs.empty() && + FrontendOpts.Inputs[0].getKind().getFormat() == InputKind::ModuleMap) { + // FIXME: handle inferred module maps + Expected FE = FM.getFileRef(FrontendOpts.Inputs[0].getFile()); + if (!FE) + return FE.takeError(); + if (Error E = addToFileList(FM, *FE).moveInto(ModuleMapFileRef)) + return std::move(E); + } + auto FinishIncludeTree = [&]() -> Error { - PreprocessorOptions &PPOpts = NewInvocation.getPreprocessorOpts(); - if (PPOpts.ImplicitPCHInclude.empty()) + IntrusiveRefCntPtr Reader = ScanInstance.getASTReader(); + if (!Reader) return Error::success(); // no need for additional work. - // Go through all the recorded included files; we'll get additional files - // from the PCH that we need to include in the file list, in case they are - // referenced while replaying the include-tree. + // Go through all the recorded input files. SmallVector NotSeenIncludes; - for (const FileEntry *FE : - ScanInstance.getPreprocessor().getIncludedFiles()) { - if (FE->getUID() >= SeenIncludeFiles.size() || - !SeenIncludeFiles[FE->getUID()]) - NotSeenIncludes.push_back(FE); + for (serialization::ModuleFile &MF : Reader->getModuleManager()) { + if (hasErrorOccurred()) + break; + Reader->visitInputFiles( + MF, /*IncludeSystem=*/true, /*Complain=*/false, + [&](const serialization::InputFile &IF, bool isSystem) { + Optional FE = IF.getFile(); + assert(FE); + if (FE->getUID() >= SeenIncludeFiles.size() || + !SeenIncludeFiles[FE->getUID()]) + NotSeenIncludes.push_back(*FE); + }); } // Sort so we can visit the files in deterministic order. llvm::sort(NotSeenIncludes, [](const FileEntry *LHS, const FileEntry *RHS) { @@ -426,6 +484,10 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, return FileNode.takeError(); } + PreprocessorOptions &PPOpts = NewInvocation.getPreprocessorOpts(); + if (PPOpts.ImplicitPCHInclude.empty()) + return Error::success(); // no need for additional work. + llvm::ErrorOr> CASContents = FM.getObjectRefForFileContent(PPOpts.ImplicitPCHInclude); if (!CASContents) @@ -451,7 +513,8 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, return FileList.takeError(); return cas::IncludeTreeRoot::create(DB, MainIncludeTree->getRef(), - FileList->getRef(), PCHRef); + FileList->getRef(), PCHRef, + ModuleMapFileRef); } Expected IncludeTreeBuilder::getObjectForFile(Preprocessor &PP, diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index e1234095a95e0..af851c979f89c 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -35,12 +35,16 @@ void tooling::dependencies::configureInvocationForCaching( if (ProduceIncludeTree) { FrontendOpts.CASIncludeTreeID = std::move(RootID); FrontendOpts.Inputs.clear(); - // Preserve sysroot path to accommodate lookup for 'SDKSettings.json' during - // availability checking, and module files. HeaderSearchOptions &HSOpts = CI.getHeaderSearchOpts(); HeaderSearchOptions OriginalHSOpts; std::swap(HSOpts, OriginalHSOpts); + // Preserve sysroot path to accommodate lookup for 'SDKSettings.json' during + // availability checking. HSOpts.Sysroot = std::move(OriginalHSOpts.Sysroot); + // Preserve resource-dir, which is added back by cc1_main if missing, and + // affects the cache key. + HSOpts.ResourceDir = std::move(OriginalHSOpts.ResourceDir); + // Preserve fmodule-file options. HSOpts.PrebuiltModuleFiles = std::move(OriginalHSOpts.PrebuiltModuleFiles); auto &PPOpts = CI.getPreprocessorOpts(); // We don't need this because we save the contents of the PCH file in the diff --git a/clang/test/ClangScanDeps/modules-include-tree-with-pch.c b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c new file mode 100644 index 0000000000000..97168db219320 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c @@ -0,0 +1,156 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json + +// Scan PCH +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch.json + +// Build PCH +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Top > %t/Top.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Left > %t/Left.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/Top.rsp +// RUN: %clang @%t/Left.rsp +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/pch.rsp +// RUN: rm -rf %t/outputs + +// Scan TU with PCH +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// Build TU +// RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/Right.rsp +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/tu.rsp + +// RUN: FileCheck %s -input-file %t/deps.json -DPREFIX=%/t + +// CHECK: { +// CHECK-NEXT: "modules": [ +// CHECK-NEXT: { +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Right" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Right.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Right" +// CHECK: } +// CHECK-NOT: "clang-modulemap-file" +// CHECK: ] +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "commands": [ +// CHECK-NEXT: { +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Right" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK: "-disable-free" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fcache-compile-job" +// CHECK: "-fsyntax-only" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodule-file=Right=[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/tu.c" +// CHECK-NEXT: "[[PREFIX]]/prefix.h.pch" +// CHECK-NEXT: ] +// CHECK: "input-file": "[[PREFIX]]/tu.c" +// CHECK: } +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } + +//--- cdb_pch.json.template +[{ + "file": "DIR/prefix.h", + "directory": "DIR", + "command": "clang -x c-header DIR/prefix.h -o DIR/prefix.h.pch -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- cdb.json.template +[{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -include DIR/prefix.h -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- module.modulemap +module Top { header "Top.h" export *} +module Left { header "Left.h" export *} +module Right { header "Right.h" export *} + +//--- Top.h +#pragma once +struct Top { int x; }; + +//--- Left.h +#pragma once +#include "Top.h" +struct Left { struct Top top; }; + +//--- Right.h +#pragma once +#include "Top.h" +struct Right { struct Top top; }; + +//--- prefix.h +#include "Left.h" + +//--- tu.c +#include "Right.h" + +void tu(void) { + struct Left _left; + struct Right _right; + struct Top _top; +} diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c index 26792c21724df..75a45400f63f0 100644 --- a/clang/test/ClangScanDeps/modules-include-tree.c +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -9,7 +9,71 @@ // RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ // RUN: > %t/deps.json -// RUN: FileCheck %s -input-file %t/deps.json -DPREFIX=%/t +// Extract the include-tree commands +// RUN: %deps-to-rsp %t/deps.json --module-name Top > %t/Top.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Left > %t/Left.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp + +// Extract include-tree casids +// RUN: cat %t/Top.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Top.casid +// RUN: cat %t/Left.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Left.casid +// RUN: cat %t/Right.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Right.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE Top" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Top.casid >> %t/result.txt +// RUN: echo "MODULE Left" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Left.casid >> %t/result.txt +// RUN: echo "MODULE Right" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Right.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt +// RUN: cat %t/deps.json >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// CHECK-LABEL: MODULE Top +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Module Map: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: Files: +// CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: MODULE Left +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Top +// CHECK: Module Map: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: Files: +// CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: MODULE Right +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Top +// CHECK: Module Map: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: Files: +// CHECK: [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Left +// CHECK: 3:1 (Module) Right +// CHECK: Files: +// CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} // CHECK: { // CHECK-NEXT "modules": [ @@ -162,11 +226,30 @@ // CHECK-NEXT: ] // CHECK-NEXT: } +// Build the include-tree commands +// RUN: %clang @%t/Top.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// Ensure the pcm comes from the action cache +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/Left.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/Right.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/tu.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS + +// Check cache hits +// RUN: %clang @%t/Top.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/Left.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/Right.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/tu.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT + +// CACHE_MISS: compile job cache miss +// CACHE_HIT: compile job cache hit + //--- cdb.json.template [{ "file": "DIR/tu.c", "directory": "DIR", - "command": "clang -fsyntax-only DIR/tu.c -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" + "command": "clang -fsyntax-only DIR/tu.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" }] //--- module.modulemap @@ -182,21 +265,19 @@ struct Top { void top(void); //--- Left.h -#pragma once #include "Top.h" void left(void); //--- Right.h -#pragma once #include "Top.h" void right(void); //--- tu.c -#include "Left.h" -#include "Right.h" +#import "Left.h" +#import void tu(void) { + top(); left(); right(); - top(); } diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index f687ef57762d8..c7739611bfce3 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -79,7 +79,8 @@ TEST(IncludeTree, IncludeTreeScan) { ASSERT_EQ(Main->getNumIncludes(), uint32_t(3)); Optional Predef; - ASSERT_THAT_ERROR(Main->getInclude(0).moveInto(Predef), llvm::Succeeded()); + ASSERT_THAT_ERROR(Main->getIncludeTree(0).moveInto(Predef), + llvm::Succeeded()); EXPECT_EQ(Main->getIncludeOffset(0), uint32_t(0)); { EXPECT_EQ(Predef->getFileCharacteristic(), SrcMgr::C_User); @@ -90,7 +91,7 @@ TEST(IncludeTree, IncludeTreeScan) { } Optional A1; - ASSERT_THAT_ERROR(Main->getInclude(1).moveInto(A1), llvm::Succeeded()); + ASSERT_THAT_ERROR(Main->getIncludeTree(1).moveInto(A1), llvm::Succeeded()); EXPECT_EQ(Main->getIncludeOffset(1), uint32_t(21)); { ASSERT_THAT_ERROR(A1->getBaseFile().moveInto(A1File), llvm::Succeeded()); @@ -104,7 +105,7 @@ TEST(IncludeTree, IncludeTreeScan) { ASSERT_EQ(A1->getNumIncludes(), uint32_t(1)); Optional B1; - ASSERT_THAT_ERROR(A1->getInclude(0).moveInto(B1), llvm::Succeeded()); + ASSERT_THAT_ERROR(A1->getIncludeTree(0).moveInto(B1), llvm::Succeeded()); EXPECT_EQ(A1->getIncludeOffset(0), uint32_t(122)); { ASSERT_THAT_ERROR(B1->getBaseFile().moveInto(B1File), llvm::Succeeded()); @@ -119,7 +120,7 @@ TEST(IncludeTree, IncludeTreeScan) { } Optional Sys; - ASSERT_THAT_ERROR(Main->getInclude(2).moveInto(Sys), llvm::Succeeded()); + ASSERT_THAT_ERROR(Main->getIncludeTree(2).moveInto(Sys), llvm::Succeeded()); EXPECT_EQ(Main->getIncludeOffset(2), uint32_t(42)); { ASSERT_THAT_ERROR(Sys->getBaseFile().moveInto(SysFile), llvm::Succeeded()); From dd7e074c48403e65a96287ee1e75ebaddf876cec Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 16 Mar 2023 13:50:54 -0700 Subject: [PATCH 07/38] [clang][cas] include-tree support for submodule visibility Track the submodule of each modular header and enter the submodule via PPCachedActions. This fixes declaration and macro header visibility. (cherry picked from commit bd61c2c82c59f703f157d4129f05aa36b5837ab9) (cherry picked from commit d926cee9331681166732cd90f36363b28a79486e) --- clang/include/clang/CAS/IncludeTree.h | 33 +++- clang/include/clang/Lex/PPCachedActions.h | 2 + clang/lib/CAS/IncludeTree.cpp | 45 ++++-- clang/lib/Frontend/IncludeTreePPActions.cpp | 29 +++- clang/lib/Lex/PPDirectives.cpp | 9 ++ .../IncludeTreeActionController.cpp | 37 ++++- .../modules-include-tree-submodules.c | 147 ++++++++++++++++++ 7 files changed, 281 insertions(+), 21 deletions(-) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-submodules.c diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 1917f6f1c54bb..3d136304a5989 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -69,10 +69,31 @@ class IncludeTree : public IncludeTreeBase { Expected getBaseFileInfo(); SrcMgr::CharacteristicKind getFileCharacteristic() const { - return (SrcMgr::CharacteristicKind)dataSkippingIncludes().front(); + return (SrcMgr::CharacteristicKind)(getData().front() & ~IsSubmoduleBit); } - size_t getNumIncludes() const { return getNumReferences() - 1; } + bool isSubmodule() const { return (getData().front() & IsSubmoduleBit) != 0; } + + std::optional getSubmoduleNameRef() const { + if (isSubmodule()) + return getReference(getNumReferences() - 1); + return std::nullopt; + } + + /// If \c getBaseFile() is a modular header, get its submodule name. + Expected> getSubmoduleName() { + auto Ref = getSubmoduleNameRef(); + if (!Ref) + return std::nullopt; + auto Node = getCAS().getProxy(*Ref); + if (!Node) + return Node.takeError(); + return Node->getData(); + } + + size_t getNumIncludes() const { + return getNumReferences() - (isSubmodule() ? 2 : 1); + }; ObjectRef getIncludeRef(size_t I) const { assert(I < getNumIncludes()); @@ -131,7 +152,7 @@ class IncludeTree : public IncludeTreeBase { static Expected create(ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, ObjectRef BaseFile, ArrayRef Includes, - llvm::SmallBitVector Checks); + std::optional SubmoduleName, llvm::SmallBitVector Checks); static Expected get(ObjectStore &DB, ObjectRef Ref); @@ -141,6 +162,8 @@ class IncludeTree : public IncludeTreeBase { friend class IncludeTreeBase; friend class IncludeTreeRoot; + static constexpr char IsSubmoduleBit = 0x80; + explicit IncludeTree(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { assert(isValid(*this)); } @@ -154,8 +177,10 @@ class IncludeTree : public IncludeTreeBase { Expected getIncludeNode(ObjectRef Ref, NodeKind Kind); + StringRef dataSkippingFlags() const { return getData().drop_front(); } StringRef dataSkippingIncludes() const { - return getData().drop_front(getNumIncludes() * (sizeof(uint32_t) + 1)); + return dataSkippingFlags().drop_front(getNumIncludes() * + (sizeof(uint32_t) + 1)); } static bool isValid(const ObjectProxy &Node); diff --git a/clang/include/clang/Lex/PPCachedActions.h b/clang/include/clang/Lex/PPCachedActions.h index da00f7f41939c..dde2c6f30ed93 100644 --- a/clang/include/clang/Lex/PPCachedActions.h +++ b/clang/include/clang/Lex/PPCachedActions.h @@ -20,6 +20,7 @@ namespace clang { class IdentifierInfo; +class Module; class Preprocessor; /// This interface provides a way to override the actions of the preprocessor as @@ -33,6 +34,7 @@ class PPCachedActions { /// The file that is included by an \c #include directive. struct IncludeFile { FileID FID; + Module *Submodule; }; /// The module that is imported by an \c #include directive or \c @import. struct IncludeModule { diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index ed8ce36a76fda..956c53dc9e546 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -60,12 +60,15 @@ Expected IncludeTree::getBaseFileInfo() { llvm::Error IncludeTree::forEachInclude( llvm::function_ref)> Callback) { size_t RefI = 0; + const size_t IncludeEnd = getNumIncludes(); return forEachReference([&](ObjectRef Ref) -> llvm::Error { if (RefI == 0) { ++RefI; return llvm::Error::success(); } size_t IncludeI = RefI - 1; + if (IncludeI >= IncludeEnd) + return llvm::Error::success(); ++RefI; auto Include = getIncludeNode(Ref, getIncludeKind(IncludeI)); if (!Include) @@ -74,25 +77,30 @@ llvm::Error IncludeTree::forEachInclude( }); } -Expected -IncludeTree::create(ObjectStore &DB, - SrcMgr::CharacteristicKind FileCharacteristic, - ObjectRef BaseFile, ArrayRef Includes, - llvm::SmallBitVector Checks) { +Expected IncludeTree::create( + ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, + ObjectRef BaseFile, ArrayRef Includes, + std::optional SubmoduleName, llvm::SmallBitVector Checks) { // The data buffer is composed of - // 1. `uint32_t` offset and `uint8_t` kind for each includes - // 2. 1 byte for `CharacteristicKind` + // 1. 1 byte for `CharacteristicKind` and IsSubmodule + // 2. `uint32_t` offset and `uint8_t` kind for each includes // 3. variable number of bitset bytes for `Checks`. char Kind = FileCharacteristic; - assert(Kind == FileCharacteristic && "SrcMgr::CharacteristicKind too big!"); + assert(Kind == FileCharacteristic && (Kind & IsSubmoduleBit) == 0 && + "SrcMgr::CharacteristicKind too big!"); + if (SubmoduleName) + Kind |= IsSubmoduleBit; + assert(File::isValid(DB, BaseFile)); SmallVector Refs; - Refs.reserve(Includes.size() + 1); + Refs.reserve(Includes.size() + 2); Refs.push_back(BaseFile); SmallString<64> Buffer; Buffer.reserve(Includes.size() * sizeof(uint32_t) + 1); + Buffer += Kind; + llvm::raw_svector_ostream BufOS(Buffer); llvm::support::endian::Writer Writer(BufOS, llvm::support::little); @@ -107,7 +115,8 @@ IncludeTree::create(ObjectStore &DB, Writer.write(static_cast(Include.Kind)); } - Buffer += Kind; + if (SubmoduleName) + Refs.push_back(*SubmoduleName); uintptr_t Store; ArrayRef BitWords = Checks.getData(Store); @@ -146,7 +155,7 @@ Expected IncludeTree::get(ObjectStore &DB, ObjectRef Ref) { IncludeTree::NodeKind IncludeTree::getIncludeKind(size_t I) const { assert(I < getNumIncludes()); - StringRef Data = getData(); + StringRef Data = dataSkippingFlags(); assert(Data.size() >= (I + 1) * (sizeof(uint32_t) + 1)); uint8_t K = *(Data.data() + I * (sizeof(uint32_t) + 1) + sizeof(uint32_t)); return NodeKind(K); @@ -154,7 +163,7 @@ IncludeTree::NodeKind IncludeTree::getIncludeKind(size_t I) const { uint32_t IncludeTree::getIncludeOffset(size_t I) const { assert(I < getNumIncludes()); - StringRef Data = getData(); + StringRef Data = dataSkippingFlags(); assert(Data.size() >= (I + 1) * sizeof(uint32_t)); uint32_t Offset = llvm::support::endian::read( @@ -164,7 +173,7 @@ uint32_t IncludeTree::getIncludeOffset(size_t I) const { bool IncludeTree::getCheckResult(size_t I) const { // Skip include offsets and CharacteristicKind. - StringRef Data = dataSkippingIncludes().drop_front(); + StringRef Data = dataSkippingIncludes(); unsigned ByteIndex = I / CHAR_BIT; size_t RemainingIndex = I % CHAR_BIT; uint8_t Bits = Data[ByteIndex]; @@ -187,9 +196,11 @@ bool IncludeTree::isValid(const ObjectProxy &Node) { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); - if (Base.getNumReferences() == 0) + if (Base.getNumReferences() == 0 || Base.getData().empty()) return false; unsigned NumIncludes = Base.getNumReferences() - 1; + if (Base.getData().front() & IsSubmoduleBit) + NumIncludes -= 1; return Base.getData().size() >= NumIncludes * (sizeof(uint32_t) + 1) + 1; } @@ -321,6 +332,12 @@ llvm::Error IncludeTree::print(llvm::raw_ostream &OS, unsigned Indent) { if (llvm::Error E = IncludeBy->print(OS)) return E; + auto Submodule = getSubmoduleName(); + if (!Submodule) + return Submodule.takeError(); + if (*Submodule) + OS.indent(Indent) << "Submodule: " << **Submodule << '\n'; + llvm::SourceMgr SM; auto Blob = IncludeBy->getContents(); if (!Blob) diff --git a/clang/lib/Frontend/IncludeTreePPActions.cpp b/clang/lib/Frontend/IncludeTreePPActions.cpp index 8c6c1cdad2838..3531f94c6d737 100644 --- a/clang/lib/Frontend/IncludeTreePPActions.cpp +++ b/clang/lib/Frontend/IncludeTreePPActions.cpp @@ -106,6 +106,11 @@ class IncludeTreePPActions final : public PPCachedActions { return {}; }; + auto reportErrorTwine = [&](const llvm::Twine &T) -> std::monostate { + return reportError( + llvm::createStringError(llvm::inconvertibleErrorCode(), T)); + }; + Expected Node = IncludeInfo.Tree.getIncludeNode(IncludeInfo.CurIncludeIndex++); if (!Node) @@ -142,7 +147,29 @@ class IncludeTreePPActions final : public PPCachedActions { PP.markIncluded(*FE); IncludeStack.push_back( {std::move(EnteredTree), SM.getLocForStartOfFile(FID)}); - return IncludeFile{FID}; + + Module *M = nullptr; + auto SubmoduleName = EnteredTree.getSubmoduleName(); + if (!SubmoduleName) + return reportError(SubmoduleName.takeError()); + if (*SubmoduleName) { + SmallVector ModuleComponents; + (*SubmoduleName)->split(ModuleComponents, '.'); + M = PP.getHeaderSearchInfo().lookupModule( + ModuleComponents[0], IncludeLoc, + /*AllowSearch=*/false, /*AllowExtraModuleMapSearch=*/false); + if (!M) + return reportErrorTwine(llvm::Twine("failed to find module '") + + ModuleComponents[0] + "'"); + for (StringRef Sub : ArrayRef(ModuleComponents).drop_front()) { + M = M->findOrInferSubmodule(Sub); + if (!M) + return reportErrorTwine( + llvm::Twine("failed to find or infer submodule '") + Sub + "'"); + } + } + + return IncludeFile{FID, M}; } void exitedFile(Preprocessor &PP, FileID FID) override { diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index 5a4530f49339c..92b68ddf10a9c 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -1999,6 +1999,15 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, } EnterSourceFile(File->FID, nullptr, FilenameTok.getLocation(), /*IsFirstIncludeOfFile*/ true); + + if (Module *SM = File->Submodule) { + assert(!CurLexerSubmodule && + "should not have marked this as a module yet"); + CurLexerSubmodule = SM; + EnterSubmodule(SM, EndLoc, /*ForPragma=*/false); + EnterAnnotationToken(SourceRange(HashLoc, EndLoc), + tok::annot_module_begin, SM); + } return; } if (auto *Import = std::get_if(&Include)) { diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 909b9b7d56084..65813116730d6 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -85,11 +85,17 @@ class IncludeTreeBuilder { void moduleImport(Preprocessor &PP, const Module *M, SourceLocation EndLoc); + void enteredSubmodule(Preprocessor &PP, Module *M, SourceLocation ImportLoc, + bool ForPragma); + void exitedSubmodule(Preprocessor &PP, Module *M, SourceLocation ImportLoc, + bool ForPragma); + private: struct FilePPState { SrcMgr::CharacteristicKind FileCharacteristic; cas::ObjectRef File; SmallVector Includes; + std::optional SubmoduleName; llvm::SmallBitVector HasIncludeChecks; }; @@ -200,6 +206,15 @@ struct IncludeTreePPCallbacks : public PPCallbacks { Builder.moduleImport(PP, Imported, EndLoc); } + + void EnteredSubmodule(Module *M, SourceLocation ImportLoc, + bool ForPragma) override { + Builder.enteredSubmodule(PP, M, ImportLoc, ForPragma); + } + void LeftSubmodule(Module *M, SourceLocation ImportLoc, + bool ForPragma) override { + Builder.exitedSubmodule(PP, M, ImportLoc, ForPragma); + } }; } // namespace @@ -353,7 +368,7 @@ void IncludeTreeBuilder::enteredInclude(Preprocessor &PP, FileID FID) { return; const SrcMgr::FileInfo &FI = PP.getSourceManager().getSLocEntry(FID).getFile(); - IncludeStack.push_back({FI.getFileCharacteristic(), *FileRef, {}, {}}); + IncludeStack.push_back({FI.getFileCharacteristic(), *FileRef, {}, {}, {}}); } void IncludeTreeBuilder::exitedInclude(Preprocessor &PP, FileID IncludedBy, @@ -394,6 +409,23 @@ void IncludeTreeBuilder::moduleImport(Preprocessor &PP, const Module *M, cas::IncludeTree::NodeKind::ModuleImport}); } +void IncludeTreeBuilder::enteredSubmodule(Preprocessor &PP, Module *M, + SourceLocation ImportLoc, + bool ForPragma) { + if (ForPragma) + return; // Will be parsed as normal. + if (hasErrorOccurred()) + return; + assert(!IncludeStack.back().SubmoduleName && "repeated enteredSubmodule"); + auto Ref = check(DB.storeFromString({}, M->getFullModuleName())); + IncludeStack.back().SubmoduleName = Ref; +} +void IncludeTreeBuilder::exitedSubmodule(Preprocessor &PP, Module *M, + SourceLocation ImportLoc, + bool ForPragma) { + // Submodule exit is handled automatically when leaving a modular file. +} + Expected IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { @@ -614,7 +646,8 @@ IncludeTreeBuilder::addToFileList(FileManager &FM, const FileEntry *FE) { Expected IncludeTreeBuilder::getCASTreeForFileIncludes(FilePPState &&PPState) { return cas::IncludeTree::create(DB, PPState.FileCharacteristic, PPState.File, - PPState.Includes, PPState.HasIncludeChecks); + PPState.Includes, PPState.SubmoduleName, + PPState.HasIncludeChecks); } Expected diff --git a/clang/test/ClangScanDeps/modules-include-tree-submodules.c b/clang/test/ClangScanDeps/modules-include-tree-submodules.c new file mode 100644 index 0000000000000..91b92443258f3 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-submodules.c @@ -0,0 +1,147 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// Extract the include-tree commands +// RUN: %deps-to-rsp %t/deps.json --module-name TwoSubs > %t/TwoSubs.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name WithExplicit > %t/WithExplicit.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu1.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 1 > %t/tu2.rsp + +// Extract include-tree casids +// RUN: cat %t/TwoSubs.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/TwoSubs.casid +// RUN: cat %t/WithExplicit.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/WithExplicit.casid +// RUN: cat %t/tu1.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu1.casid +// RUN: cat %t/tu2.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu2.casid + +// RUN: echo "MODULE TwoSubs" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/TwoSubs.casid >> %t/result.txt +// RUN: echo "MODULE WithExplicit" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/WithExplicit.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT 1" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu1.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT 2" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu2.casid >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// Build the include-tree commands +// RUN: %clang @%t/TwoSubs.rsp +// RUN: %clang @%t/WithExplicit.rsp +// RUN: not %clang @%t/tu1.rsp 2>&1 | FileCheck %s -check-prefix=TU1 +// RUN: not %clang @%t/tu2.rsp 2>&1 | FileCheck %s -check-prefix=TU2 + +// CHECK: MODULE TwoSubs +// CHECK: 2:1 [[PREFIX]]/Sub1.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: TwoSubs.Sub1 +// CHECK: 3:1 [[PREFIX]]/Sub2.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: TwoSubs.Sub2 + +// CHECK: MODULE WithExplicit +// CHECK: 2:1 [[PREFIX]]/TopLevel.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: WithExplicit +// CHECK: 3:1 [[PREFIX]]/Implicit.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: WithExplicit.Implicit +// CHECK: 4:1 [[PREFIX]]/Explicit.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: WithExplicit.Explicit + +// CHECK: TRANSLATION UNIT 1 +// CHECK: [[PREFIX]]/tu1.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) TwoSubs.Sub1 +// CHECK: 3:1 (Module) WithExplicit + +// CHECK: TRANSLATION UNIT 2 +// CHECK: [[PREFIX]]/tu2.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) TwoSubs.Sub2 +// CHECK: 3:1 (Module) WithExplicit.Explicit + +//--- cdb.json.template +[ +{ + "file": "DIR/tu1.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu1.c -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +{ + "file": "DIR/tu2.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu2.c -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +] + +//--- module.modulemap +module TwoSubs { + module Sub1 { header "Sub1.h" } + module Sub2 { header "Sub2.h" } +} + +module WithExplicit { + header "TopLevel.h" + module Implicit { header "Implicit.h" } + explicit module Explicit { header "Explicit.h" } +} + +//--- Sub1.h +void sub1(void); + +//--- Sub2.h +void sub2(void); + +//--- TopLevel.h +void top(void); + +//--- Implicit.h +void implicit(void); + +//--- Explicit.h +void explicit(void); + +//--- tu1.c +#include "Sub1.h" +#include "TopLevel.h" + +void tu1(void) { + top(); + sub1(); + implicit(); + sub2(); +// TU1-NOT: error: +// TU1: error: call to undeclared function 'sub2' +// TU1: error: missing '#include "Sub2.h"' +// TU1: note: declaration here is not visible + explicit(); +// TU1: error: call to undeclared function 'explicit' +// TU1: error: missing '#include "Explicit.h"' +// TU1: Explicit.h:1:6: note: declaration here is not visible +} + +//--- tu2.c +#include "Sub2.h" +#include "Explicit.h" + +void tu2(void) { + sub2(); + explicit(); +// TU2-NOT: error: + top(); +// TU2: error: call to undeclared function 'top' +// TU2: error: missing '#include "TopLevel.h"' +// TU2: note: declaration here is not visible + sub1(); +// TU2: error: call to undeclared function 'sub1' +// TU2: error: missing '#include "Sub1.h"' +// TU2: note: declaration here is not visible + implicit(); +// TU2: error: call to undeclared function 'implicit' +// TU2: error: missing '#include "Implicit.h"' +// TU2: note: declaration here is not visible +} From c3eb50289c7c114f3aa39c6e41a746530e0ea68c Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 16 Mar 2023 15:37:00 -0700 Subject: [PATCH 08/38] [clang][cas] Handle include-tree for a file considered part of the module When building a non-header file with -fmodule-name set, add the requisite modulemap file even though it is not imported as a module, and test that we are getting the right semantics for the header include. (cherry picked from commit 43df61c6ba7fc0014bd72354502bb45ac4edbcb7) (cherry picked from commit 145905544e3a071b65de4cfa6aa5ed404d43e7d5) --- .../IncludeTreeActionController.cpp | 4 ++ .../modules-include-tree-implementation.c | 58 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-implementation.c diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 65813116730d6..14bce2cde1e8b 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -485,6 +485,10 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, return std::move(E); } + for (StringRef ModuleMap : FrontendOpts.ModuleMapFiles) + if (Error E = addFile(ModuleMap)) + return std::move(E); + auto FinishIncludeTree = [&]() -> Error { IntrusiveRefCntPtr Reader = ScanInstance.getASTReader(); if (!Reader) diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation.c b/clang/test/ClangScanDeps/modules-include-tree-implementation.c new file mode 100644 index 0000000000000..8d879d21236ed --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation.c @@ -0,0 +1,58 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: FileCheck %s -input-file %t/deps.json -check-prefix=NO_MODULES +// NO_MODULES: "modules": [] + +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid | FileCheck %s -DPREFIX=%/t +// RUN: %clang @%t/tu.rsp + +// CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} + +// Note: this is surprising, but correct: when building the implementation files +// of a module, the first include is textual but still uses the submodule +// machinery. The second include is treated as a module import (unless in a PCH) +// but will not actually import the module only trigger visibility changes. + +// CHECK: 2:1 [[PREFIX]]/Mod.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: Mod +// CHECK: 3:1 (Module) Mod + +// CHECK: Files: +// CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Mod.h llvmcas://{{[[:xdigit:]]+}} + +// Despite not importing the module, we need its modulemap for submodule info. +// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} + +//--- cdb.json.template +[{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -I DIR -fmodule-name=Mod -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- module.modulemap +module Mod { header "Mod.h" } + +//--- Mod.h +#pragma once +void top(void); + +//--- tu.c +#include "Mod.h" +#include "Mod.h" +void tu(void) { + top(); +} From 7f474cb4ff1a7a99192bb7c42dcbfe6f233aece4 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 16 Mar 2023 15:57:06 -0700 Subject: [PATCH 09/38] [clang][cas] Add a test for include-tree with explicit imports With explicit imports, we do not record the module import in the include tree, because the only thing that happens is to lookup the module by name, which works already via the command-line options that provide the module via the action cache. (cherry picked from commit a13b79f23e54a598003fcd9c3cf848f474c82c22) (cherry picked from commit 232e6623d4a440ae3c4872cdd6c0b6caa14aed48) --- .../test/ClangScanDeps/modules-include-tree.c | 103 +++++++++++++++--- 1 file changed, 89 insertions(+), 14 deletions(-) diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c index 75a45400f63f0..8ab36ff66ab02 100644 --- a/clang/test/ClangScanDeps/modules-include-tree.c +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -13,6 +13,8 @@ // RUN: %deps-to-rsp %t/deps.json --module-name Top > %t/Top.rsp // RUN: %deps-to-rsp %t/deps.json --module-name Left > %t/Left.rsp // RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ZAtImport > %t/ZAtImport.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ZPragmaImport > %t/ZPragmaImport.rsp // RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp // Extract include-tree casids @@ -65,10 +67,15 @@ // CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} // CHECK-LABEL: TRANSLATION UNIT -// CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/tu.m llvmcas://{{[[:xdigit:]]+}} // CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} // CHECK: 2:1 (Module) Left // CHECK: 3:1 (Module) Right + +// Note: the modules with explicit imports are imported via parser and are not +// recorded in the include-tree; it's handled entirely by fmodule-map-file, +// fmodule-file, and fmodule-file-cache-key options. + // CHECK: Files: // CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} @@ -101,8 +108,6 @@ // CHECK: "-fmodule-file-cache-key" // CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-x" -// CHECK-NEXT: "c" // CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Left" @@ -138,8 +143,6 @@ // CHECK: "-fmodule-file-cache-key // CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-x" -// CHECK-NEXT: "c" // CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Right" @@ -167,8 +170,6 @@ // CHECK-NEXT: "[[TOP_TREE]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" -// CHECK: "-x" -// CHECK-NEXT: "c" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Top" // CHECK: "-fno-implicit-modules" @@ -179,6 +180,58 @@ // CHECK-NEXT: ] // CHECK: "name": "Top" // CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[AT_IMPORT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/ZAtImport-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[AT_IMPORT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=ZAtImport" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/AtImport.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "ZAtImport" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[PRAGMA_IMPORT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/ZPragmaImport-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[PRAGMA_IMPORT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=ZPragmaImport" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/PragmaImport.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "ZPragmaImport" +// CHECK: } // CHECK: ] // CHECK-NEXT: "translation-units": [ // CHECK-NEXT: { @@ -192,6 +245,12 @@ // CHECK-NEXT: { // CHECK: "module-name": "Right" // CHECK: } +// CHECK-NEXT: { +// CHECK: "module-name": "ZAtImport" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "module-name": "ZPragmaImport" +// CHECK: } // CHECK-NEXT: ] // CHECK: "command-line": [ // CHECK-NEXT: "-cc1" @@ -209,17 +268,15 @@ // CHECK: "-fmodule-file-cache-key" // CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-x" -// CHECK-NEXT: "c" // CHECK: "-fmodule-file=Left=[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" // CHECK: "-fmodule-file=Right=[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fno-implicit-modules" // CHECK: ] // CHECK: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]/tu.c" +// CHECK-NEXT: "[[PREFIX]]/tu.m" // CHECK-NEXT: ] -// CHECK: "input-file": "[[PREFIX]]/tu.c" +// CHECK: "input-file": "[[PREFIX]]/tu.m" // CHECK: } // CHECK-NEXT: ] // CHECK-NEXT: } @@ -234,12 +291,18 @@ // RUN: rm -rf %t/outputs // RUN: %clang @%t/Right.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS // RUN: rm -rf %t/outputs +// RUN: %clang @%t/ZAtImport.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/ZPragmaImport.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: rm -rf %t/outputs // RUN: %clang @%t/tu.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS // Check cache hits // RUN: %clang @%t/Top.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT // RUN: %clang @%t/Left.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT // RUN: %clang @%t/Right.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/ZAtImport.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/ZPragmaImport.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT // RUN: %clang @%t/tu.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT // CACHE_MISS: compile job cache miss @@ -247,15 +310,17 @@ //--- cdb.json.template [{ - "file": "DIR/tu.c", + "file": "DIR/tu.m", "directory": "DIR", - "command": "clang -fsyntax-only DIR/tu.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" + "command": "clang -fsyntax-only DIR/tu.m -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" }] //--- module.modulemap module Top { header "Top.h" export *} module Left { header "Left.h" export *} module Right { header "Right.h" export *} +module ZAtImport { header "AtImport.h" } +module ZPragmaImport { header "PragmaImport.h" } //--- Top.h #pragma once @@ -272,12 +337,22 @@ void left(void); #include "Top.h" void right(void); -//--- tu.c +//--- AtImport.h +void at_import(void); + +//--- PragmaImport.h +void pragma_import(void); + +//--- tu.m #import "Left.h" #import +@import ZAtImport; +#pragma clang module import ZPragmaImport void tu(void) { top(); left(); right(); + at_import(); + pragma_import(); } From 56691d94e48af7ccbe6ff0eea08483296c573db3 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 23 Mar 2023 11:44:51 -0700 Subject: [PATCH 10/38] [clang][cas] Move include-tree and cas-fs IDs out of Module into ASTContext NFC Unlike the cache key string, these are not needed for multiple modules at once, and this allows us to store the include-tree ID for a PCH, which will be useful in the next commit. (cherry picked from commit d64e27abdd95a8d978a374ae82b14b7020eb2c4f) (cherry picked from commit 843daa725a29ba9ec45bd872e619e6276a764b4d) --- clang/include/clang/AST/ASTContext.h | 22 +++++++++++ clang/include/clang/Basic/Module.h | 25 ------------- clang/include/clang/CAS/IncludeTree.h | 2 +- clang/lib/CAS/IncludeTree.cpp | 2 +- clang/lib/Frontend/FrontendAction.cpp | 4 ++ clang/lib/Serialization/ASTReader.cpp | 4 -- clang/lib/Serialization/ASTWriter.cpp | 37 ++++++++++--------- .../CASFSActionController.cpp | 2 +- .../IncludeTreeActionController.cpp | 7 ++-- .../DependencyScanning/ModuleDepCollector.cpp | 23 +++++++----- 10 files changed, 64 insertions(+), 64 deletions(-) diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h index e6f4fece8a4ad..4f62b11a05db4 100644 --- a/clang/include/clang/AST/ASTContext.h +++ b/clang/include/clang/AST/ASTContext.h @@ -477,6 +477,14 @@ class ASTContext : public RefCountedBase { /// For module code-gen cases, this is the top-level module we are building. Module *TopLevelModule = nullptr; + /// The include tree that is being built, if any. + /// See \c FrontendOptions::CASIncludeTreeID. + std::optional CASIncludeTreeID; + + /// The cas-fs tree that is being built, if any. + /// See \c FileSystemOptions::CASFileSystemRootID. + std::optional CASFileSystemRootID; + static constexpr unsigned ConstantArrayTypesLog2InitSize = 8; static constexpr unsigned GeneralTypesLog2InitSize = 9; static constexpr unsigned FunctionProtoTypesLog2InitSize = 12; @@ -1086,6 +1094,20 @@ class ASTContext : public RefCountedBase { /// Get module under construction, nullptr if this is not a C++20 module. Module *getModuleForCodeGen() const { return TopLevelModule; } + std::optional getCASIncludeTreeID() const { + return CASIncludeTreeID; + } + void setCASIncludeTreeID(std::string ID) { + CASIncludeTreeID = std::move(ID); + } + + std::optional getCASFileSystemRootID() const { + return CASFileSystemRootID; + } + void setCASFileSystemRootID(std::string ID) { + CASFileSystemRootID = std::move(ID); + } + TranslationUnitDecl *getTranslationUnitDecl() const { return TUDecl->getMostRecentDecl(); } diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index 8b9a93c6b0e7d..e216117194974 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -192,14 +192,6 @@ class alignas(8) Module { /// The \c ActionCache key for this module, if any. Optional ModuleCacheKey; - /// The CAS filesystem root ID for implicit modules built with the dependency - /// scanner, if any. - Optional CASFileSystemRootID; - - /// The include-tree root ID for implicit modules built with the dependency - /// scanner, if any. - Optional IncludeTreeID; - /// The top-level headers associated with this module. llvm::SmallSetVector TopHeaders; @@ -640,23 +632,6 @@ class alignas(8) Module { getTopLevelModule()->ModuleCacheKey = std::move(Key); } - Optional getCASFileSystemRootID() const { - return getTopLevelModule()->CASFileSystemRootID; - } - - void setCASFileSystemRootID(std::string ID) { - assert(!getCASFileSystemRootID() || *getCASFileSystemRootID() == ID); - getTopLevelModule()->CASFileSystemRootID = std::move(ID); - } - - Optional getIncludeTreeID() const { - return getTopLevelModule()->IncludeTreeID; - } - - void setIncludeTreeID(std::string ID) { - assert(!getIncludeTreeID() || *getIncludeTreeID() == ID); - getTopLevelModule()->IncludeTreeID = std::move(ID); - } /// Retrieve the directory for which this module serves as the /// umbrella. diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 3d136304a5989..977ff57bbcb5a 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -152,7 +152,7 @@ class IncludeTree : public IncludeTreeBase { static Expected create(ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, ObjectRef BaseFile, ArrayRef Includes, - std::optional SubmoduleName, llvm::SmallBitVector Checks); + Optional SubmoduleName, llvm::SmallBitVector Checks); static Expected get(ObjectStore &DB, ObjectRef Ref); diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index 956c53dc9e546..ae6deffe2052d 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -80,7 +80,7 @@ llvm::Error IncludeTree::forEachInclude( Expected IncludeTree::create( ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, ObjectRef BaseFile, ArrayRef Includes, - std::optional SubmoduleName, llvm::SmallBitVector Checks) { + Optional SubmoduleName, llvm::SmallBitVector Checks) { // The data buffer is composed of // 1. 1 byte for `CharacteristicKind` and IsSubmodule // 2. `uint32_t` offset and `uint8_t` kind for each includes diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index 7694664195708..b0e8a724310bb 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -1067,6 +1067,10 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, } } + if (!CI.getFrontendOpts().CASIncludeTreeID.empty()) + CI.getASTContext().setCASIncludeTreeID( + CI.getFrontendOpts().CASIncludeTreeID); + CI.setASTConsumer(std::move(Consumer)); if (!CI.hasASTConsumer()) return false; diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 61e17c51af70f..cc53bbdfe2ada 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -5636,10 +5636,6 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F, CurrentModule->PresumedModuleMapFile = F.ModuleMapPath; if (!F.ModuleCacheKey.empty()) CurrentModule->setModuleCacheKey(F.ModuleCacheKey); - if (!F.CASFileSystemRootID.empty()) - CurrentModule->setCASFileSystemRootID(F.CASFileSystemRootID); - if (!F.IncludeTreeID.empty()) - CurrentModule->setIncludeTreeID(F.IncludeTreeID); } CurrentModule->Kind = Kind; diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index a79ae19a10317..fbb0f90804752 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -1353,24 +1353,25 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context, RecordData::value_type Record[] = {MODULE_CACHE_KEY}; Stream.EmitRecordWithBlob(AbbrevCode, Record, *Key); } - // CAS filesystem root id, for the scanner. - if (auto ID = WritingModule->getCASFileSystemRootID()) { - auto Abbrev = std::make_shared(); - Abbrev->Add(BitCodeAbbrevOp(CASFS_ROOT_ID)); - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); - unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); - RecordData::value_type Record[] = {CASFS_ROOT_ID}; - Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); - } - // CAS include-tree id, for the scanner. - if (auto ID = WritingModule->getIncludeTreeID()) { - auto Abbrev = std::make_shared(); - Abbrev->Add(BitCodeAbbrevOp(CAS_INCLUDE_TREE_ID)); - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); - unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); - RecordData::value_type Record[] = {CAS_INCLUDE_TREE_ID}; - Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); - } + } + + // CAS include-tree id, for the scanner. + if (auto ID = Context.getCASIncludeTreeID()) { + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(CAS_INCLUDE_TREE_ID)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + RecordData::value_type Record[] = {CAS_INCLUDE_TREE_ID}; + Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); + } + // CAS filesystem root id, for the scanner. + if (auto ID = Context.getCASFileSystemRootID()) { + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(CASFS_ROOT_ID)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + RecordData::value_type Record[] = {CASFS_ROOT_ID}; + Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); } // Imports diff --git a/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp b/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp index e6cd88115ceb8..e1a6167923be3 100644 --- a/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp @@ -176,7 +176,7 @@ Error CASFSActionController::finalizeModuleBuild( Module *M = ModuleScanInstance.getPreprocessor().getCurrentModule(); assert(M && "finalizing without a module"); - M->setCASFileSystemRootID(RootID->toString()); + ModuleScanInstance.getASTContext().setCASFileSystemRootID(RootID->toString()); return Error::success(); } diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 14bce2cde1e8b..ced5e93d70cec 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -95,7 +95,7 @@ class IncludeTreeBuilder { SrcMgr::CharacteristicKind FileCharacteristic; cas::ObjectRef File; SmallVector Includes; - std::optional SubmoduleName; + Optional SubmoduleName; llvm::SmallBitVector HasIncludeChecks; }; @@ -327,9 +327,8 @@ Error IncludeTreeActionController::finalizeModuleBuild( if (!Tree) return Tree.takeError(); - Module *M = ModuleScanInstance.getPreprocessor().getCurrentModule(); - assert(M && "finalizing without a module"); - M->setIncludeTreeID(Tree->getID().toString()); + ModuleScanInstance.getASTContext().setCASIncludeTreeID( + Tree->getID().toString()); return Error::success(); } diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index b3e692cec9962..d157350603a91 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -441,16 +441,6 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { MD.ImplicitModulePCMPath = std::string(M->getASTFile()->getName()); MD.IsSystem = M->IsSystem; - if (auto ID = M->getCASFileSystemRootID()) { - auto &CAS = MDC.ScanInstance.getOrCreateObjectStore(); - if (auto Err = CAS.parseID(*ID).moveInto(MD.CASFileSystemRootID)) { - MDC.ScanInstance.getDiagnostics().Report( - diag::err_cas_cannot_parse_root_id_for_module) - << *ID << MD.ID.ModuleName; - } - } - MD.IncludeTreeID = M->getIncludeTreeID(); - ModuleMap &ModMapInfo = MDC.ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap(); @@ -491,6 +481,19 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { MD.ModuleMapFileDeps.emplace_back(FE.getNameAsRequested()); }); + if (!MF->IncludeTreeID.empty()) + MD.IncludeTreeID = MF->IncludeTreeID; + + if (!MF->CASFileSystemRootID.empty()) { + auto &CAS = MDC.ScanInstance.getOrCreateObjectStore(); + if (auto Err = CAS.parseID(MF->CASFileSystemRootID) + .moveInto(MD.CASFileSystemRootID)) { + MDC.ScanInstance.getDiagnostics().Report( + diag::err_cas_cannot_parse_root_id_for_module) + << MF->CASFileSystemRootID << MD.ID.ModuleName; + } + } + CompilerInvocation CI = MDC.makeInvocationForModuleBuildWithoutOutputs( MD, [&](CompilerInvocation &BuildInvocation) { if (MDC.OptimizeArgs) From 6c5e1430a7615fd3011c8a0b31b2c19f180cb13d Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 23 Mar 2023 11:46:33 -0700 Subject: [PATCH 11/38] [clang][cas] IncludeTree filesystem merges from imported PCH/modules Instead of looking through the PCH or modules for input files and loading them into the current FileManager while scanning, merge them directly from the include-tree that was used to build that PCH and/or module. This makes it easier to understand where files come from in the include-tree, and eliminates certain issues where the scanner module may include more files than are necessary, or include them with different paths in the case of VFS/symlinks. It also avoids needing to re-stat these files if they are not used during scanning the importing TU. As part of this, we teach IncludeTreeFilesystem to handle ./ stripping in paths to match what ASTWriter does. (cherry picked from commit 4c3f73ec729e478d2ea2c097699576b50cb29279) (cherry picked from commit fcf503d9e583722751bdf666b11f94e470416040) --- clang/include/clang/CAS/IncludeTree.h | 22 ++++++ clang/lib/CAS/IncludeTree.cpp | 24 +++++-- .../IncludeTreeActionController.cpp | 67 +++++++++++-------- .../test/ClangScanDeps/modules-include-tree.c | 2 +- 4 files changed, 81 insertions(+), 34 deletions(-) diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 977ff57bbcb5a..07d4123da7058 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -483,4 +483,26 @@ createIncludeTreeFileSystem(IncludeTreeRoot &Root); } // namespace cas } // namespace clang +namespace llvm { +template <> struct DenseMapInfo { + using FileEntry = clang::cas::IncludeTree::FileList::FileEntry; + + static FileEntry getEmptyKey() { + return {cas::ObjectRef::getDenseMapEmptyKey(), 0}; + } + + static FileEntry getTombstoneKey() { + return {cas::ObjectRef::getDenseMapTombstoneKey(), 0}; + } + + static unsigned getHashValue(FileEntry F) { + return F.FileRef.getDenseMapHash(); + } + + static bool isEqual(FileEntry LHS, FileEntry RHS) { + return LHS.FileRef == RHS.FileRef && LHS.Size == RHS.Size; + } +}; +} // namespace llvm + #endif diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index ae6deffe2052d..bae4ae5fcb041 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -460,9 +460,8 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { Directories{Alloc}; llvm::ErrorOr status(const Twine &Path) override { - SmallString<128> FilenameBuffer; - StringRef Filename = Path.toStringRef(FilenameBuffer); - + SmallString<128> Filename; + getPath(Path, Filename); auto FileEntry = Files.find(Filename); if (FileEntry != Files.end()) { return makeStatus(Filename, FileEntry->second.Size, @@ -482,8 +481,8 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { llvm::ErrorOr> openFileForRead(const Twine &Path) override { - SmallString<128> FilenameBuffer; - StringRef Filename = Path.toStringRef(FilenameBuffer); + SmallString<128> Filename; + getPath(Path, Filename); auto MaterializedFile = materialize(Filename); if (!MaterializedFile) return llvm::errorToErrorCode(MaterializedFile.takeError()); @@ -506,6 +505,15 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { return MaterializedFile{ContentsBlob->getData(), Entry->second}; } + /// Produce a filename compatible with our StringMaps. See comment in + /// \c createIncludeTreeFileSystem. + void getPath(const Twine &Path, SmallVectorImpl &Out) { + Path.toVector(Out); + // Strip dots, but do not eliminate a path consisting only of '.' + if (Out.size() != 1) + llvm::sys::path::remove_dots(Out); + } + static llvm::vfs::Status makeStatus(StringRef Filename, uint64_t Size, llvm::sys::fs::UniqueID UniqueID, llvm::sys::fs::file_type Type) { @@ -551,7 +559,11 @@ cas::createIncludeTreeFileSystem(IncludeTreeRoot &Root) { auto FilenameBlob = File.getFilename(); if (!FilenameBlob) return FilenameBlob.takeError(); - StringRef Filename = FilenameBlob->getData(); + SmallString<128> Filename(FilenameBlob->getData()); + // Strip './' in the filename to match the behaviour of ASTWriter; we + // also strip './' in IncludeTreeFileSystem::getPath. + assert(Filename != "."); + llvm::sys::path::remove_dots(Filename); StringRef DirName = llvm::sys::path::parent_path(Filename); if (DirName.empty()) diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index ced5e93d70cec..7c6e352979e6f 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -99,6 +99,7 @@ class IncludeTreeBuilder { llvm::SmallBitVector HasIncludeChecks; }; + Error addModuleInputs(ASTReader &Reader); Expected getObjectForFile(Preprocessor &PP, FileID FID); Expected getObjectForFileNonCached(FileManager &FM, const SrcMgr::FileInfo &FI); @@ -128,7 +129,7 @@ class IncludeTreeBuilder { // are recorded in the PCH, ordered by \p FileEntry::UID index. SmallVector PreIncludedFileNames; llvm::BitVector SeenIncludeFiles; - SmallVector IncludedFiles; + llvm::SetVector IncludedFiles; Optional PredefinesBufferRef; Optional ModuleIncludesBufferRef; Optional ModuleMapFileRef; @@ -494,30 +495,8 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, return Error::success(); // no need for additional work. // Go through all the recorded input files. - SmallVector NotSeenIncludes; - for (serialization::ModuleFile &MF : Reader->getModuleManager()) { - if (hasErrorOccurred()) - break; - Reader->visitInputFiles( - MF, /*IncludeSystem=*/true, /*Complain=*/false, - [&](const serialization::InputFile &IF, bool isSystem) { - Optional FE = IF.getFile(); - assert(FE); - if (FE->getUID() >= SeenIncludeFiles.size() || - !SeenIncludeFiles[FE->getUID()]) - NotSeenIncludes.push_back(*FE); - }); - } - // Sort so we can visit the files in deterministic order. - llvm::sort(NotSeenIncludes, [](const FileEntry *LHS, const FileEntry *RHS) { - return LHS->getUID() < RHS->getUID(); - }); - - for (const FileEntry *FE : NotSeenIncludes) { - auto FileNode = addToFileList(FM, FE); - if (!FileNode) - return FileNode.takeError(); - } + if (Error E = addModuleInputs(*Reader)) + return E; PreprocessorOptions &PPOpts = NewInvocation.getPreprocessorOpts(); if (PPOpts.ImplicitPCHInclude.empty()) @@ -543,7 +522,8 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, getCASTreeForFileIncludes(IncludeStack.pop_back_val()); if (!MainIncludeTree) return MainIncludeTree.takeError(); - auto FileList = cas::IncludeTree::FileList::create(DB, IncludedFiles); + auto FileList = + cas::IncludeTree::FileList::create(DB, IncludedFiles.getArrayRef()); if (!FileList) return FileList.takeError(); @@ -552,6 +532,39 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, ModuleMapFileRef); } +Error IncludeTreeBuilder::addModuleInputs(ASTReader &Reader) { + for (serialization::ModuleFile &MF : Reader.getModuleManager()) { + // Only add direct imports to avoid duplication. Each include tree is a + // superset of its imported modules' include trees. + if (!MF.isDirectlyImported()) + continue; + + assert(!MF.IncludeTreeID.empty() && "missing include-tree for import"); + + Optional ID; + if (Error E = DB.parseID(MF.IncludeTreeID).moveInto(ID)) + return E; + Optional Ref = DB.getReference(*ID); + if (!Ref) + return DB.createUnknownObjectError(*ID); + Optional Root; + if (Error E = cas::IncludeTreeRoot::get(DB, *Ref).moveInto(Root)) + return E; + Optional Files; + if (Error E = Root->getFileList().moveInto(Files)) + return E; + + Error E = Files->forEachFile([&](auto IF, auto Size) -> Error { + IncludedFiles.insert({IF.getRef(), Size}); + return Error::success(); + }); + if (E) + return E; + } + + return Error::success(); +} + Expected IncludeTreeBuilder::getObjectForFile(Preprocessor &PP, FileID FID) { SourceManager &SM = PP.getSourceManager(); @@ -627,7 +640,7 @@ IncludeTreeBuilder::addToFileList(FileManager &FM, const FileEntry *FE) { auto FileNode = createIncludeFile(Filename, **CASContents); if (!FileNode) return FileNode.takeError(); - IncludedFiles.push_back( + IncludedFiles.insert( {FileNode->getRef(), static_cast(FE->getSize())}); return FileNode->getRef(); diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c index 8ab36ff66ab02..e09c564231db5 100644 --- a/clang/test/ClangScanDeps/modules-include-tree.c +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -77,8 +77,8 @@ // fmodule-file, and fmodule-file-cache-key options. // CHECK: Files: -// CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} From 5d75294264c070148b23cd21079de3bbddba3d81 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 22 Mar 2023 16:59:04 -0700 Subject: [PATCH 12/38] [clang][cas] Create explicit IncludeTree::ModuleMap Instead of putting the modulemap file in the include-tree filesystem and parsing it at build time, create a data structure that represents just the parts of the modulemap we need when building the module: * Flags (explicit, system, framework, etc.) * Exports (export *, export Foo) * LinkLibraries (implicit framework autolink, link Foo) Additionally, we add modular headers lazily by inserting known headers when we encounter an include-tree header that is part of a submodule (this is needed for missing #include diagnostics). This removes the possibility of mismatches between header paths seen during modulemap parsing from the paths we embed due to #includes, for exmaple due to VFS virtual vs external paths. (cherry picked from commit a237e688354853ad782594d6158c2d28e26f8d01) Conflicts: clang/include/clang/Basic/Module.h clang/lib/Serialization/ASTWriter.cpp (cherry picked from commit c6b9298dfaa4a9a39ac63346c75e7d6123f3082f) --- .../clang/Basic/DiagnosticFrontendKinds.td | 2 + clang/include/clang/Basic/Module.h | 8 +- clang/include/clang/CAS/IncludeTree.h | 262 ++++++++++++- .../include/clang/Frontend/FrontendOptions.h | 4 + clang/lib/AST/Decl.cpp | 1 + clang/lib/CAS/IncludeTree.cpp | 343 ++++++++++++++++-- clang/lib/Frontend/CompilerInvocation.cpp | 51 +-- clang/lib/Frontend/FrontendAction.cpp | 148 +++++++- clang/lib/Frontend/FrontendActions.cpp | 2 + clang/lib/Frontend/IncludeTreePPActions.cpp | 6 + clang/lib/Sema/SemaModule.cpp | 1 + clang/lib/Serialization/ASTReaderDecl.cpp | 3 +- .../IncludeTreeActionController.cpp | 109 +++++- .../DependencyScanning/ScanAndUpdateArgs.cpp | 1 + .../modules-include-tree-implementation.c | 5 +- .../modules-include-tree-with-pch.c | 2 +- .../test/ClangScanDeps/modules-include-tree.c | 30 +- 17 files changed, 876 insertions(+), 102 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index 6f76ef2b42a90..81877ea252a7b 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -229,6 +229,8 @@ def err_missing_module_name : Error< DefaultFatal; def err_missing_module : Error< "no module named '%0' declared in module map file '%1'">, DefaultFatal; +def err_missing_module_include_tree : Error< + "no module named '%0' declared in include-tree module map '%1'">, DefaultFatal; def err_no_submodule : Error<"no submodule named %0 in module '%1'">; def err_no_submodule_suggest : Error< "no submodule named %0 in module '%1'; did you mean '%2'?">; diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index e216117194974..99f4f71a7b449 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -108,6 +108,10 @@ class alignas(8) Module { /// of header files. ModuleMapModule, + /// This is a module that was defined by a module map and built out + /// of header files as part of an \c IncludeTree. + IncludeTreeModuleMap, + /// This is a C++20 module interface unit. ModuleInterfaceUnit, @@ -175,7 +179,9 @@ class alignas(8) Module { bool isPrivateModule() const { return Kind == PrivateModuleFragment; } - bool isModuleMapModule() const { return Kind == ModuleMapModule; } + bool isModuleMapModule() const { + return Kind == ModuleMapModule || Kind == IncludeTreeModuleMap; + } private: /// The submodules of this module, indexed by name. diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 07d4123da7058..9c9723fd3bc90 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -54,7 +54,9 @@ class IncludeTree : public IncludeTreeBase { class File; class FileList; class Node; + class Module; class ModuleImport; + class ModuleMap; Expected getBaseFile(); @@ -381,6 +383,262 @@ class IncludeTree::Node { NodeKind K; }; +/// Module or submodule declaration as part of the \c IncludeTreeRoot module +/// map structure. +class IncludeTree::Module : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "Modu"; } + + class ExportList; + class LinkLibraryList; + + struct ModuleFlags { + bool IsFramework : 1; + bool IsExplicit : 1; + bool IsExternC : 1; + bool IsSystem : 1; + ModuleFlags() + : IsFramework(false), IsExplicit(false), IsExternC(false), + IsSystem(false) {} + }; + + ModuleFlags getFlags() const; + + /// The name of the current (sub)module. + StringRef getName() const { return dataAfterFlags(); } + + size_t getNumSubmodules() const; + + Expected getSubmodule(size_t I) { + assert(I < getNumSubmodules()); + auto Node = getCAS().getProxy(getReference(I)); + if (!Node) + return Node.takeError(); + return Module(*Node); + } + + llvm::Error forEachSubmodule(llvm::function_ref CB); + + std::optional getExportsRef() const { + if (std::optional Index = getExportsIndex()) + return getReference(*Index); + return std::nullopt; + } + std::optional getLinkLibrariesRef() const { + if (std::optional Index = getLinkLibrariesIndex()) + return getReference(*Index); + return std::nullopt; + } + + /// The list of modules that this submodule re-exports. + Expected> getExports(); + + /// The list of modules that this submodule re-exports. + Expected> getLinkLibraries(); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static Expected create(ObjectStore &DB, StringRef ModuleName, + ModuleFlags Flags, + ArrayRef Submodules, + std::optional ExportList, + std::optional LinkLibraries); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + return Base.getData().size() > 1; + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + char rawFlags() const { return getData()[0]; } + StringRef dataAfterFlags() const { return getData().drop_front(); } + bool hasExports() const; + bool hasLinkLibraries() const; + std::optional getExportsIndex() const; + std::optional getLinkLibrariesIndex() const; + + explicit Module(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } + + friend class IncludeTreeBase; + friend class IncludeTreeRoot; + friend class ModuleMap; +}; + +/// The set of modules re-exported by another module. +class IncludeTree::Module::ExportList : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "ExpL"; } + + /// An explicit export. + struct Export { + StringRef ModuleName; + bool Wildcard; + }; + + /// Whether this module exports all imported modules (`export *`). + bool hasGlobalWildcard() const; + + size_t getNumExplicitExports() const { return getNumReferences(); } + + /// Whether the explicit export at \p I has a wildcard (`export MyModule.*`). + bool exportHasWildcard(size_t I) const; + + Expected getExplicitExport(size_t I); + + /// Calls \p CB for each explicit export declaration. + llvm::Error forEachExplicitExport(llvm::function_ref CB); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static Expected create(ObjectStore &DB, ArrayRef Exports, + bool GlobalWildcard); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + size_t ExpectedBitSize = Base.getNumReferences() + 1; + size_t ExpectedSize = + ExpectedBitSize / 8 + (ExpectedBitSize % 8 == 0 ? 0 : 1); + return Base.getData().size() == ExpectedSize; + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + explicit ExportList(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } + + friend class IncludeTreeBase; + friend class Module; +}; + +/// The set of libraries to link against when using a module. +class IncludeTree::Module::LinkLibraryList + : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "LnkL"; } + + struct LinkLibrary { + StringRef Library; + bool IsFramework; + }; + + size_t getNumLibraries() const { return getNumReferences(); } + + /// Whether the library at \p I is a framework. + bool isFramework(size_t I) const; + + ObjectRef getLibraryNameRef(size_t I) const { return getReference(I); } + + Expected getLinkLibrary(size_t I) { + auto Name = getCAS().getProxy(getLibraryNameRef(I)); + if (!Name) + return Name.takeError(); + return LinkLibrary{Name->getData(), isFramework(I)}; + } + + /// Calls \p CB for each link libary. + llvm::Error + forEachLinkLibrary(llvm::function_ref CB); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static Expected create(ObjectStore &DB, + ArrayRef Exports); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + size_t ExpectedBitSize = Base.getNumReferences(); + size_t ExpectedSize = + ExpectedBitSize / 8 + (ExpectedBitSize % 8 == 0 ? 0 : 1); + return Base.getData().size() == ExpectedSize; + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + explicit LinkLibraryList(ObjectProxy Node) + : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } + + friend class IncludeTreeBase; + friend class Module; +}; + +class IncludeTree::ModuleMap : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "ModM"; } + + size_t getNumModules() const { return getNumReferences(); } + + Expected getModule(size_t I) { + auto Node = getCAS().getProxy(getReference(I)); + if (!Node) + return Node.takeError(); + return Module(*Node); + } + + /// Calls \p CB for each module declaration. + llvm::Error forEachModule(llvm::function_ref CB); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static Expected create(ObjectStore &DB, + ArrayRef Modules); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + return Base.getData().empty(); + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + explicit ModuleMap(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } + + friend class IncludeTreeBase; + friend class IncludeTreeRoot; +}; + /// Represents the include-tree result for a translation unit. class IncludeTreeRoot : public IncludeTreeBase { public: @@ -426,12 +684,12 @@ class IncludeTreeRoot : public IncludeTreeBase { return None; } - Expected> getModuleMapFile() { + Expected> getModuleMap() { if (Optional Ref = getModuleMapRef()) { auto Node = getCAS().getProxy(*Ref); if (!Node) return Node.takeError(); - return IncludeTree::File(*Node); + return IncludeTree::ModuleMap(*Node); } return std::nullopt; } diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h index 5554c2992ba42..7ab469a797e18 100644 --- a/clang/include/clang/Frontend/FrontendOptions.h +++ b/clang/include/clang/Frontend/FrontendOptions.h @@ -248,6 +248,10 @@ class FrontendInputFile { bool IsSystem = false) : File(File.str()), IncludeTree(std::move(Tree)), Kind(Kind), IsSystem(IsSystem) {} + FrontendInputFile(cas::ObjectRef Tree, llvm::MemoryBufferRef Buffer, + InputKind Kind, bool IsSystem = false) + : Buffer(Buffer), IncludeTree(std::move(Tree)), Kind(Kind), + IsSystem(IsSystem) {} InputKind getKind() const { return Kind; } bool isSystem() const { return IsSystem; } diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp index 151826f0d9c2b..4584111df0256 100644 --- a/clang/lib/AST/Decl.cpp +++ b/clang/lib/AST/Decl.cpp @@ -1565,6 +1565,7 @@ Module *Decl::getOwningModuleForLinkage(bool IgnoreLinkage) const { switch (M->Kind) { case Module::ModuleMapModule: + case Module::IncludeTreeModuleMap: // Module map modules have no special linkage semantics. return nullptr; diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index bae4ae5fcb041..dae8c92e52901 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -77,6 +77,33 @@ llvm::Error IncludeTree::forEachInclude( }); } +/// Write the bitset \p Bits to \p Writer, filling the final byte with zeros for +/// any unused values. Note: this does not store the size of the bitset. +static void writeBitSet(llvm::support::endian::Writer &Writer, + const llvm::SmallBitVector &Bits) { + uintptr_t Store; + ArrayRef BitWords = Bits.getData(Store); + size_t RemainingBitsCount = Bits.size(); + while (RemainingBitsCount > 0) { + if (BitWords.size() > 1) { + Writer.write(BitWords.front()); + BitWords = BitWords.drop_front(); + RemainingBitsCount -= sizeof(uintptr_t) * CHAR_BIT; + continue; + } + assert(RemainingBitsCount <= sizeof(uintptr_t) * CHAR_BIT); + uintptr_t LastWord = BitWords.front(); + unsigned BytesNum = RemainingBitsCount / CHAR_BIT; + if (RemainingBitsCount % CHAR_BIT != 0) + ++BytesNum; + while (BytesNum--) { + Writer.write(static_cast(LastWord & 0xFF)); + LastWord >>= CHAR_BIT; + } + break; + } +} + Expected IncludeTree::create( ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, ObjectRef BaseFile, ArrayRef Includes, @@ -118,27 +145,7 @@ Expected IncludeTree::create( if (SubmoduleName) Refs.push_back(*SubmoduleName); - uintptr_t Store; - ArrayRef BitWords = Checks.getData(Store); - size_t RemainingBitsCount = Checks.size(); - while (RemainingBitsCount > 0) { - if (BitWords.size() > 1) { - Writer.write(BitWords.front()); - BitWords = BitWords.drop_front(); - RemainingBitsCount -= sizeof(uintptr_t) * CHAR_BIT; - continue; - } - assert(RemainingBitsCount <= sizeof(uintptr_t) * CHAR_BIT); - uintptr_t LastWord = BitWords.front(); - unsigned BytesNum = RemainingBitsCount / CHAR_BIT; - if (RemainingBitsCount % CHAR_BIT != 0) - ++BytesNum; - while (BytesNum--) { - Buffer.push_back(LastWord & 0xFF); - LastWord >>= CHAR_BIT; - } - break; - } + writeBitSet(Writer, Checks); return IncludeTreeBase::create(DB, Refs, Buffer); } @@ -268,6 +275,236 @@ bool IncludeTree::FileList::isValid(const ObjectProxy &Node) { Base.getData().size() == NumFiles * sizeof(FileSizeTy); } +static constexpr char ModuleFlagFramework = 1 << 0; +static constexpr char ModuleFlagExplicit = 1 << 1; +static constexpr char ModuleFlagExternC = 1 << 2; +static constexpr char ModuleFlagSystem = 1 << 3; +static constexpr char ModuleFlagHasExports = 1 << 4; +static constexpr char ModuleFlagHasLinkLibraries = 1 << 5; + +IncludeTree::Module::ModuleFlags IncludeTree::Module::getFlags() const { + char Raw = rawFlags(); + ModuleFlags Flags; + Flags.IsFramework = Raw & ModuleFlagFramework; + Flags.IsExplicit = Raw & ModuleFlagExplicit; + Flags.IsExternC = Raw & ModuleFlagExternC; + Flags.IsSystem = Raw & ModuleFlagSystem; + return Flags; +} + +size_t IncludeTree::Module::getNumSubmodules() const { + size_t Count = getNumReferences(); + if (hasExports()) + Count -= 1; + if (hasLinkLibraries()) + Count -= 1; + return Count; +} + +llvm::Error IncludeTree::Module::forEachSubmodule( + llvm::function_ref CB) { + size_t Count = getNumSubmodules(); + return forEachReference([&](ObjectRef Ref) -> llvm::Error { + if (Count == 0) + return llvm::Error::success(); + Count -= 1; + auto Node = getCAS().getProxy(Ref); + if (!Node) + return Node.takeError(); + return CB(Module(*Node)); + }); +} + +Expected +IncludeTree::Module::create(ObjectStore &DB, StringRef ModuleName, + ModuleFlags Flags, ArrayRef Submodules, + std::optional ExportList, + std::optional LinkLibraries) { + // Data: + // - 1 byte for Flags + // - ModuleName (String) + // Refs: + // - Submodules (IncludeTreeModule) + // - (optional) ExportList + // - (optional) LinkLibaryList + + char RawFlags = 0; + if (Flags.IsFramework) + RawFlags |= ModuleFlagFramework; + if (Flags.IsExplicit) + RawFlags |= ModuleFlagExplicit; + if (Flags.IsExternC) + RawFlags |= ModuleFlagExternC; + if (Flags.IsSystem) + RawFlags |= ModuleFlagSystem; + if (ExportList) + RawFlags |= ModuleFlagHasExports; + if (LinkLibraries) + RawFlags |= ModuleFlagHasLinkLibraries; + + SmallString<64> Buffer; + Buffer.push_back(RawFlags); + Buffer.append(ModuleName); + + SmallVector Refs(Submodules); + if (ExportList) + Refs.push_back(*ExportList); + if (LinkLibraries) + Refs.push_back(*LinkLibraries); + + return IncludeTreeBase::create(DB, Refs, Buffer); +} + +bool IncludeTree::Module::hasExports() const { + return rawFlags() & ModuleFlagHasExports; +} +bool IncludeTree::Module::hasLinkLibraries() const { + return rawFlags() & ModuleFlagHasLinkLibraries; +} + +std::optional IncludeTree::Module::getExportsIndex() const { + if (hasExports()) + return getNumReferences() - (hasLinkLibraries() ? 2 : 1); + return std::nullopt; +} +std::optional IncludeTree::Module::getLinkLibrariesIndex() const { + if (hasLinkLibraries()) + return getNumReferences() - 1; + return std::nullopt; +} + +Expected> +IncludeTree::Module::getExports() { + if (auto Ref = getExportsRef()) { + auto N = getCAS().getProxy(*Ref); + if (!N) + return N.takeError(); + return ExportList(std::move(*N)); + } + return std::nullopt; +} + +/// The list of modules that this submodule re-exports. +Expected> +IncludeTree::Module::getLinkLibraries() { + if (auto Ref = getLinkLibrariesRef()) { + auto N = getCAS().getProxy(*Ref); + if (!N) + return N.takeError(); + return LinkLibraryList(std::move(*N)); + } + return std::nullopt; +} + +bool IncludeTree::Module::ExportList::hasGlobalWildcard() const { + // The bit after explicit exports is global. + return exportHasWildcard(getNumExplicitExports()); +} +bool IncludeTree::Module::ExportList::exportHasWildcard(size_t I) const { + assert(I < getNumExplicitExports() + 1); + unsigned ByteIndex = I / CHAR_BIT; + size_t RemainingIndex = I % CHAR_BIT; + uint8_t Bits = getData()[ByteIndex]; + return Bits & (1 << RemainingIndex); +} +Expected +IncludeTree::Module::ExportList::getExplicitExport(size_t I) { + Expected Name = getCAS().getProxy(getReference(I)); + if (!Name) + return Name.takeError(); + return Export{Name->getData(), exportHasWildcard(I)}; +} +llvm::Error IncludeTree::Module::ExportList::forEachExplicitExport( + llvm::function_ref CB) { + size_t ExportI = 0; + return forEachReference([&](ObjectRef Ref) { + Expected Name = getCAS().getProxy(Ref); + if (!Name) + return Name.takeError(); + return CB(Export{Name->getData(), exportHasWildcard(ExportI)}); + }); +} +Expected +IncludeTree::Module::ExportList::create(ObjectStore &DB, + ArrayRef Exports, + bool GlobalWildcard) { + // Data: + // - 1 bit per explicit export for wildcard + // - 1 bit for global wildcard + // Refs: export names + SmallString<64> Buffer; + llvm::raw_svector_ostream BufOS(Buffer); + llvm::support::endian::Writer Writer(BufOS, llvm::support::little); + SmallVector Refs; + llvm::SmallBitVector WildcardBits; + for (Export E : Exports) { + auto Ref = DB.storeFromString({}, E.ModuleName); + if (!Ref) + return Ref.takeError(); + Refs.push_back(*Ref); + WildcardBits.push_back(E.Wildcard); + } + WildcardBits.push_back(GlobalWildcard); + writeBitSet(Writer, WildcardBits); + + return IncludeTreeBase::create(DB, Refs, Buffer); +} + +bool IncludeTree::Module::LinkLibraryList::isFramework(size_t I) const { + assert(I < getNumLibraries()); + unsigned ByteIndex = I / CHAR_BIT; + size_t RemainingIndex = I % CHAR_BIT; + uint8_t Bits = getData()[ByteIndex]; + return Bits & (1 << RemainingIndex); +} +llvm::Error IncludeTree::Module::LinkLibraryList::forEachLinkLibrary( + llvm::function_ref CB) { + size_t I = 0; + return forEachReference([&](ObjectRef Ref) { + auto Name = getCAS().getProxy(getLibraryNameRef(I)); + if (!Name) + return Name.takeError(); + return CB({Name->getData(), isFramework(I++)}); + }); +} +Expected +IncludeTree::Module::LinkLibraryList::create(ObjectStore &DB, + ArrayRef Libraries) { + // Data: + // - 1 bit per library for IsFramework + // Refs: library names + SmallString<64> Buffer; + llvm::raw_svector_ostream BufOS(Buffer); + llvm::support::endian::Writer Writer(BufOS, llvm::support::little); + SmallVector Refs; + llvm::SmallBitVector FrameworkBits; + for (LinkLibrary L : Libraries) { + auto Ref = DB.storeFromString({}, L.Library); + if (!Ref) + return Ref.takeError(); + Refs.push_back(*Ref); + FrameworkBits.push_back(L.IsFramework); + } + writeBitSet(Writer, FrameworkBits); + + return IncludeTreeBase::create(DB, Refs, Buffer); +} + +Expected +IncludeTree::ModuleMap::create(ObjectStore &DB, ArrayRef Modules) { + return IncludeTreeBase::create(DB, Modules, {}); +} + +llvm::Error IncludeTree::ModuleMap::forEachModule( + llvm::function_ref CB) { + return forEachReference([&](ObjectRef Ref) { + auto N = getCAS().getProxy(Ref); + if (!N) + return N.takeError(); + return CB(Module(std::move(*N))); + }); +} + static constexpr char HasPCH = 0x01; static constexpr char HasModuleMap = 0x02; @@ -277,7 +514,7 @@ IncludeTreeRoot::create(ObjectStore &DB, ObjectRef MainFileTree, Optional ModuleMapRef) { assert(IncludeTree::isValid(DB, MainFileTree)); assert(IncludeTree::FileList::isValid(DB, FileList)); - assert(!ModuleMapRef || IncludeTree::File::isValid(DB, *ModuleMapRef)); + assert(!ModuleMapRef || IncludeTree::ModuleMap::isValid(DB, *ModuleMapRef)); std::array Data = {0}; if (PCHRef) @@ -376,6 +613,62 @@ llvm::Error IncludeTree::Node::print(llvm::raw_ostream &OS, unsigned Indent) { } } +llvm::Error IncludeTree::Module::print(llvm::raw_ostream &OS, unsigned Indent) { + OS.indent(Indent) << getName(); + ModuleFlags Flags = getFlags(); + if (Flags.IsFramework) + OS << " (framework)"; + if (Flags.IsExplicit) + OS << " (explicit)"; + if (Flags.IsExternC) + OS << " (extern_c)"; + if (Flags.IsSystem) + OS << " (system)"; + OS << '\n'; + auto ExportList = getExports(); + if (!ExportList) + return ExportList.takeError(); + if (*ExportList) + if (llvm::Error E = (*ExportList)->print(OS, Indent + 2)) + return E; + auto LinkLibraries = getLinkLibraries(); + if (!LinkLibraries) + return LinkLibraries.takeError(); + if (*LinkLibraries) + if (llvm::Error E = (*LinkLibraries)->print(OS, Indent + 2)) + return E; + return forEachSubmodule( + [&](Module Sub) { return Sub.print(OS, Indent + 2); }); +} +llvm::Error IncludeTree::Module::ExportList::print(llvm::raw_ostream &OS, + unsigned Indent) { + if (hasGlobalWildcard()) + OS.indent(Indent) << "export *\n"; + return forEachExplicitExport([&](Export E) { + OS.indent(Indent) << "export " << E.ModuleName; + if (E.Wildcard) + OS << ".*"; + OS << '\n'; + return llvm::Error::success(); + }); +} + +llvm::Error IncludeTree::Module::LinkLibraryList::print(llvm::raw_ostream &OS, + unsigned Indent) { + return forEachLinkLibrary([&](LinkLibrary E) { + OS.indent(Indent) << "link " << E.Library; + if (E.IsFramework) + OS << " (framework)"; + OS << '\n'; + return llvm::Error::success(); + }); +} + +llvm::Error IncludeTree::ModuleMap::print(llvm::raw_ostream &OS, + unsigned Indent) { + return forEachModule([&](Module M) { return M.print(OS, Indent); }); +} + llvm::Error IncludeTreeRoot::print(llvm::raw_ostream &OS, unsigned Indent) { if (Optional PCHRef = getPCHRef()) { OS.indent(Indent) << "(PCH) "; @@ -387,11 +680,11 @@ llvm::Error IncludeTreeRoot::print(llvm::raw_ostream &OS, unsigned Indent) { return E; if (llvm::Error E = MainTree->print(OS.indent(Indent), Indent)) return E; - Optional ModuleMap; - if (llvm::Error E = getModuleMapFile().moveInto(ModuleMap)) + Optional ModuleMap; + if (llvm::Error E = getModuleMap().moveInto(ModuleMap)) return E; if (ModuleMap) { - OS.indent(Indent) << "Module Map: "; + OS.indent(Indent) << "Module Map:\n"; if (llvm::Error E = ModuleMap->print(OS, Indent)) return E; } diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index ecf1848f86738..6ff91921007fa 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -3019,7 +3019,8 @@ static void GenerateFrontendArgs(const FrontendOptions &Opts, static void determineInputFromIncludeTree( StringRef IncludeTreeID, CASOptions &CASOpts, DiagnosticsEngine &Diags, - Optional &IncludeTree, StringRef &InputFilename) { + Optional &IncludeTree, + Optional &Buffer, StringRef &InputFilename) { assert(!IncludeTreeID.empty()); auto reportError = [&](llvm::Error &&E) { Diags.Report(diag::err_fe_unable_to_load_include_tree) @@ -3037,28 +3038,27 @@ static void determineInputFromIncludeTree( auto Root = cas::IncludeTreeRoot::get(*CAS, *Object); if (!Root) return reportError(Root.takeError()); - - Optional BaseFile; - - auto MaybeModuleMap = Root->getModuleMapFile(); - if (!MaybeModuleMap) - return reportError(MaybeModuleMap.takeError()); - if (*MaybeModuleMap) { - // Building a module from a modulemap, the modulemap is the primary input. - BaseFile = *MaybeModuleMap; - } else { - auto MainTree = Root->getMainFileTree(); - if (!MainTree) - return reportError(MainTree.takeError()); - if (llvm::Error E = MainTree->getBaseFile().moveInto(BaseFile)) - return reportError(std::move(E)); - } - + auto MainTree = Root->getMainFileTree(); + if (!MainTree) + return reportError(MainTree.takeError()); + auto BaseFile = MainTree->getBaseFile(); + if (!BaseFile) + return reportError(BaseFile.takeError()); auto FilenameBlob = BaseFile->getFilename(); if (!FilenameBlob) return reportError(FilenameBlob.takeError()); + InputFilename = FilenameBlob->getData(); IncludeTree = *Root; + + if (InputFilename != Module::getModuleInputBufferName()) + return; + + // Handle buffer + auto Contents = BaseFile->getContents(); + if (!Contents) + return reportError(Contents.takeError()); + Buffer = llvm::MemoryBufferRef(Contents->getData(), InputFilename); } static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, @@ -3286,13 +3286,14 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, Opts.Inputs.clear(); Optional Tree; + Optional TreeInputBuffer; if (!Opts.CASIncludeTreeID.empty()) { if (!Inputs.empty()) { Diags.Report(diag::err_drv_inputs_and_include_tree); } StringRef InputFilename; determineInputFromIncludeTree(Opts.CASIncludeTreeID, CASOpts, Diags, Tree, - InputFilename); + TreeInputBuffer, InputFilename); if (!InputFilename.empty()) Inputs.push_back(InputFilename.str()); } @@ -3331,8 +3332,16 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, if (Tree) { FrontendInputFile &InputFile = Opts.Inputs.back(); - InputFile = FrontendInputFile(Tree->getRef(), InputFile.getFile(), - InputFile.getKind(), InputFile.isSystem()); + if (TreeInputBuffer) { + // This is automatically set to modulemap when building a module; revert + // to a source file for the module includes buffer. + auto Kind = InputFile.getKind().withFormat(InputKind::Source); + InputFile = FrontendInputFile(Tree->getRef(), *TreeInputBuffer, Kind, + InputFile.isSystem()); + } else { + InputFile = FrontendInputFile(Tree->getRef(), InputFile.getFile(), + InputFile.getKind(), InputFile.isSystem()); + } } Opts.DashX = DashX; diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index b0e8a724310bb..f0259f591aa5b 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -546,6 +546,116 @@ static Module *prepareToBuildModule(CompilerInstance &CI, return M; } +static Expected makeIncludeTreeModule(CompilerInstance &CI, + cas::IncludeTree::Module Mod, + Module *Parent) { + ModuleMap &MMap = CI.getPreprocessor().getHeaderSearchInfo().getModuleMap(); + auto Flags = Mod.getFlags(); + Module *M = nullptr; + bool NewModule = false; + std::tie(M, NewModule) = MMap.findOrCreateModule( + Mod.getName(), Parent, Flags.IsFramework, Flags.IsExplicit); + assert(NewModule); + M->Kind = Module::IncludeTreeModuleMap; + M->IsExternC = Flags.IsExternC; + M->IsSystem = Flags.IsSystem; + + auto ExportList = Mod.getExports(); + if (!ExportList) + return ExportList.takeError(); + if (*ExportList) { + if ((*ExportList)->hasGlobalWildcard()) + M->Exports.push_back(Module::ExportDecl(nullptr, true)); + + llvm::Error Err = (*ExportList)->forEachExplicitExport([&](auto Export) { + Module::UnresolvedExportDecl UED; + SmallVector ModuleComponents; + Export.ModuleName.split(ModuleComponents, '.'); + for (StringRef Name : ModuleComponents) + UED.Id.push_back({std::string(Name), SourceLocation()}); + UED.Wildcard = Export.Wildcard; + M->UnresolvedExports.push_back(std::move(UED)); + return llvm::Error::success(); + }); + if (Err) + return std::move(Err); + } + + auto LinkLibs = Mod.getLinkLibraries(); + if (!LinkLibs) + return LinkLibs.takeError(); + if (*LinkLibs) { + llvm::Error Err = (*LinkLibs)->forEachLinkLibrary([&](auto LL) { + M->LinkLibraries.emplace_back(std::string(LL.Library), LL.IsFramework); + return llvm::Error::success(); + }); + if (Err) + return std::move(Err); + } + + llvm::Error Err = Mod.forEachSubmodule([&](cas::IncludeTree::Module Sub) { + return makeIncludeTreeModule(CI, Sub, M).takeError(); + }); + if (Err) + return std::move(Err); + return M; +} + +/// Loads the include tree modulemap \p MM. +/// \returns true if there was an error. +static bool loadIncludeTreeModuleMap(CompilerInstance &CI, + cas::IncludeTree::ModuleMap MM) { + llvm::Error Err = MM.forEachModule([&](auto M) -> llvm::Error { + return makeIncludeTreeModule(CI, M, /*Parent=*/nullptr).takeError(); + }); + if (Err) { + CI.getDiagnostics().Report(diag::err_fe_unable_to_load_include_tree) + << CI.getFrontendOpts().CASIncludeTreeID << std::move(Err); + return true; + } + return false; +} + +static Module *prepareToBuildModule(CompilerInstance &CI, + cas::IncludeTree::ModuleMap MM) { + if (CI.getLangOpts().CurrentModule.empty()) { + CI.getDiagnostics().Report(diag::err_missing_module_name); + + // FIXME: Eventually, we could consider asking whether there was just + // a single module described in the module map, and use that as a + // default. Then it would be fairly trivial to just "compile" a module + // map with a single module (the common case). + return nullptr; + } + + if (loadIncludeTreeModuleMap(CI, MM)) + return nullptr; + + // Dig out the module definition. + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + Module *M = HS.lookupModule(CI.getLangOpts().CurrentModule, SourceLocation(), + /*AllowSearch=*/false); + if (!M) { + CI.getDiagnostics().Report(diag::err_missing_module_include_tree) + << CI.getLangOpts().CurrentModule + << CI.getFrontendOpts().CASIncludeTreeID; + + return nullptr; + } + + if (auto CacheKey = CI.getCompileJobCacheKey()) + M->setModuleCacheKey(CacheKey->toString()); + + // If we're being run from the command-line, the module build stack will not + // have been filled in yet, so complete it now in order to allow us to detect + // module cycles. + SourceManager &SourceMgr = CI.getSourceManager(); + if (SourceMgr.getModuleBuildStack().empty()) + SourceMgr.pushModuleBuildStack(CI.getLangOpts().CurrentModule, + FullSourceLoc(SourceLocation(), SourceMgr)); + return M; +} + /// Compute the input buffer that should be used to build the specified module. static std::unique_ptr getInputBufferForModule(CompilerInstance &CI, Module *M) { @@ -882,6 +992,21 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, if (llvm::Error E = IncludeTreeRoot->getPCHBuffer().moveInto(IncludeTreePCHBuffer)) return reportError(std::move(E)); + + auto ModMap = IncludeTreeRoot->getModuleMap(); + if (!ModMap) + return reportError(ModMap.takeError()); + if (*ModMap) { + if (CI.getFrontendOpts().ProgramAction == frontend::GenerateModule) { + auto *CurrentModule = prepareToBuildModule(CI, **ModMap); + if (!CurrentModule) + return false; + CI.getLangOpts().setCompilingModule(LangOptions::CMK_ModuleMap); + } else { + if (loadIncludeTreeModuleMap(CI, **ModMap)) + return false; + } + } } if (!CI.InitializeSourceManager(Input)) @@ -924,25 +1049,10 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, // If the module contents are in the same file, skip to them. CI.getPreprocessor().setSkipMainFilePreamble(OffsetToContents, true); else { - std::unique_ptr Buffer; - if (Input.isIncludeTree()) { - // Get the existing module include buffer from the include-tree. - assert(IncludeTreeRoot); - auto MainTree = IncludeTreeRoot->getMainFileTree(); - if (!MainTree) - return reportError(MainTree.takeError()); - auto BaseFile = MainTree->getBaseFile(); - if (!BaseFile) - return reportError(BaseFile.takeError()); - if (auto E = BaseFile->getMemoryBuffer().moveInto(Buffer)) - return reportError(std::move(E)); - assert(Buffer); - } else { - // Otherwise, convert the module description to a suitable input buffer. - Buffer = getInputBufferForModule(CI, CurrentModule); - if (!Buffer) - return false; - } + // Otherwise, convert the module description to a suitable input buffer. + auto Buffer = getInputBufferForModule(CI, CurrentModule); + if (!Buffer) + return false; // Reinitialize the main file entry to refer to the new input. auto Kind = CurrentModule->IsSystem ? SrcMgr::C_System : SrcMgr::C_User; diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index e86c683c204b4..aa1c048e94ee6 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -831,6 +831,8 @@ static StringRef ModuleKindName(Module::ModuleKind MK) { switch (MK) { case Module::ModuleMapModule: return "Module Map Module"; + case Module::IncludeTreeModuleMap: + return "Include Tree Module"; case Module::ModuleInterfaceUnit: return "Interface Unit"; case Module::ModulePartitionInterface: diff --git a/clang/lib/Frontend/IncludeTreePPActions.cpp b/clang/lib/Frontend/IncludeTreePPActions.cpp index 3531f94c6d737..2d94a858c961a 100644 --- a/clang/lib/Frontend/IncludeTreePPActions.cpp +++ b/clang/lib/Frontend/IncludeTreePPActions.cpp @@ -167,6 +167,12 @@ class IncludeTreePPActions final : public PPCachedActions { return reportErrorTwine( llvm::Twine("failed to find or infer submodule '") + Sub + "'"); } + + // Add to known headers for the module. + ModuleMap &MMap = PP.getHeaderSearchInfo().getModuleMap(); + Module::Header H; + H.Entry = *FE; + MMap.addHeader(M, std::move(H), ModuleMap::NormalHeader); } return IncludeFile{FID, M}; diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp index 4b01f109fc881..6284ab346ee64 100644 --- a/clang/lib/Sema/SemaModule.cpp +++ b/clang/lib/Sema/SemaModule.cpp @@ -366,6 +366,7 @@ Sema::ActOnPrivateModuleFragmentDecl(SourceLocation ModuleLoc, switch (ModuleScopes.empty() ? Module::GlobalModuleFragment : ModuleScopes.back().Module->Kind) { case Module::ModuleMapModule: + case Module::IncludeTreeModuleMap: case Module::GlobalModuleFragment: case Module::ModulePartitionImplementation: case Module::ModulePartitionInterface: diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp index f6334988071e5..f45c49220bcca 100644 --- a/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/clang/lib/Serialization/ASTReaderDecl.cpp @@ -2980,8 +2980,7 @@ static bool isConsumerInterestedIn(ASTContext &Ctx, Decl *D, bool HasBody) { // emitted when we import the relevant module. if (isPartOfPerModuleInitializer(D)) { auto *M = D->getImportedOwningModule(); - if (M && M->Kind == Module::ModuleMapModule && - Ctx.DeclMustBeEmitted(D)) + if (M && M->isModuleMapModule() && Ctx.DeclMustBeEmitted(D)) return false; } diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 7c6e352979e6f..caea6662392ac 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -132,7 +132,7 @@ class IncludeTreeBuilder { llvm::SetVector IncludedFiles; Optional PredefinesBufferRef; Optional ModuleIncludesBufferRef; - Optional ModuleMapFileRef; + Optional ModuleMapRef; /// When the builder is created from an existing tree, the main include tree. Optional MainIncludeTreeRef; SmallVector IncludeStack; @@ -426,6 +426,59 @@ void IncludeTreeBuilder::exitedSubmodule(Preprocessor &PP, Module *M, // Submodule exit is handled automatically when leaving a modular file. } +static Expected +getIncludeTreeModule(cas::ObjectStore &DB, Module *M) { + using ITModule = cas::IncludeTree::Module; + SmallVector Submodules; + for (Module *Sub : M->submodules()) { + Expected SubTree = getIncludeTreeModule(DB, Sub); + if (!SubTree) + return SubTree.takeError(); + Submodules.push_back(SubTree->getRef()); + } + + ITModule::ModuleFlags Flags; + Flags.IsFramework = M->IsFramework; + Flags.IsExplicit = M->IsExplicit; + Flags.IsExternC = M->IsExternC; + Flags.IsSystem = M->IsSystem; + + bool GlobalWildcardExport = false; + SmallVector Exports; + llvm::BumpPtrAllocator Alloc; + llvm::StringSaver Saver(Alloc); + for (Module::ExportDecl &Export : M->Exports) { + if (Export.getPointer() == nullptr && Export.getInt()) { + GlobalWildcardExport = true; + } else if (Export.getPointer()) { + StringRef Name = Saver.save(Export.getPointer()->getFullModuleName()); + Exports.push_back({Name, Export.getInt()}); + } + } + std::optional ExportList; + if (GlobalWildcardExport || !Exports.empty()) { + auto EL = ITModule::ExportList::create(DB, Exports, GlobalWildcardExport); + if (!EL) + return EL.takeError(); + ExportList = EL->getRef(); + } + + SmallVector Libraries; + for (Module::LinkLibrary &LL : M->LinkLibraries) { + Libraries.push_back({LL.Library, LL.IsFramework}); + } + std::optional LinkLibraries; + if (!Libraries.empty()) { + auto LL = ITModule::LinkLibraryList::create(DB, Libraries); + if (!LL) + return LL.takeError(); + LinkLibraries = LL->getRef(); + } + + return ITModule::create(DB, M->Name, Flags, Submodules, ExportList, + LinkLibraries); +} + Expected IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { @@ -474,21 +527,6 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, return std::move(E); } - auto &FrontendOpts = NewInvocation.getFrontendOpts(); - if (!FrontendOpts.Inputs.empty() && - FrontendOpts.Inputs[0].getKind().getFormat() == InputKind::ModuleMap) { - // FIXME: handle inferred module maps - Expected FE = FM.getFileRef(FrontendOpts.Inputs[0].getFile()); - if (!FE) - return FE.takeError(); - if (Error E = addToFileList(FM, *FE).moveInto(ModuleMapFileRef)) - return std::move(E); - } - - for (StringRef ModuleMap : FrontendOpts.ModuleMapFiles) - if (Error E = addFile(ModuleMap)) - return std::move(E); - auto FinishIncludeTree = [&]() -> Error { IntrusiveRefCntPtr Reader = ScanInstance.getASTReader(); if (!Reader) @@ -522,14 +560,49 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, getCASTreeForFileIncludes(IncludeStack.pop_back_val()); if (!MainIncludeTree) return MainIncludeTree.takeError(); + + if (!ScanInstance.getLangOpts().CurrentModule.empty()) { + SmallVector Modules; + auto AddModule = [&](Module *M) -> llvm::Error { + Expected Mod = getIncludeTreeModule(DB, M); + if (!Mod) + return Mod.takeError(); + Modules.push_back(Mod->getRef()); + return Error::success(); + }; + if (Module *M = ScanInstance.getPreprocessor().getCurrentModule()) { + if (Error E = AddModule(M)) + return std::move(E); + } else { + // When building a TU or PCH, we can have headers files that are part of + // both the public and private modules that are included textually. In + // that case we need both of those modules. + ModuleMap &MMap = + ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap(); + M = MMap.findModule(ScanInstance.getLangOpts().CurrentModule); + assert(M && "missing current module?"); + if (Error E = AddModule(M)) + return std::move(E); + Module *PM = + MMap.findModule(ScanInstance.getLangOpts().ModuleName + "_Private"); + if (PM) + if (Error E = AddModule(M)) + return std::move(E); + } + + auto ModMap = cas::IncludeTree::ModuleMap::create(DB, Modules); + if (!ModMap) + return ModMap.takeError(); + ModuleMapRef = ModMap->getRef(); + } + auto FileList = cas::IncludeTree::FileList::create(DB, IncludedFiles.getArrayRef()); if (!FileList) return FileList.takeError(); return cas::IncludeTreeRoot::create(DB, MainIncludeTree->getRef(), - FileList->getRef(), PCHRef, - ModuleMapFileRef); + FileList->getRef(), PCHRef, ModuleMapRef); } Error IncludeTreeBuilder::addModuleInputs(ASTReader &Reader) { diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index af851c979f89c..7b3bd52de3f58 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -35,6 +35,7 @@ void tooling::dependencies::configureInvocationForCaching( if (ProduceIncludeTree) { FrontendOpts.CASIncludeTreeID = std::move(RootID); FrontendOpts.Inputs.clear(); + FrontendOpts.ModuleMapFiles.clear(); HeaderSearchOptions &HSOpts = CI.getHeaderSearchOpts(); HeaderSearchOptions OriginalHSOpts; std::swap(HSOpts, OriginalHSOpts); diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation.c b/clang/test/ClangScanDeps/modules-include-tree-implementation.c index 8d879d21236ed..40b1d51281acb 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-implementation.c +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation.c @@ -31,10 +31,9 @@ // CHECK: Files: // CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap // CHECK: [[PREFIX]]/Mod.h llvmcas://{{[[:xdigit:]]+}} - -// Despite not importing the module, we need its modulemap for submodule info. -// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap //--- cdb.json.template [{ diff --git a/clang/test/ClangScanDeps/modules-include-tree-with-pch.c b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c index 97168db219320..60b2ba3daf851 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-with-pch.c +++ b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c @@ -84,7 +84,7 @@ // CHECK-NEXT: "-cc1" // CHECK: "-fcas-path" // CHECK-NEXT: "[[PREFIX]]/cas" -// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK-NOT: -fmodule-map-file= // CHECK: "-disable-free" // CHECK: "-fcas-include-tree" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c index e09c564231db5..cb42ded1fc728 100644 --- a/clang/test/ClangScanDeps/modules-include-tree.c +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -39,20 +39,26 @@ // CHECK: llvmcas://{{[[:xdigit:]]+}} // CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} // CHECK: 2:1 [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} -// CHECK: Module Map: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: Module Map: +// CHECK: Top +// CHECK: export * // CHECK: Files: +// CHECK-NOT: [[PREFIX]]/module.modulemap // CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} -// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap // CHECK-LABEL: MODULE Left // CHECK: llvmcas://{{[[:xdigit:]]+}} // CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} // CHECK: 2:1 [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} // CHECK: 2:1 (Module) Top -// CHECK: Module Map: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: Module Map: +// CHECK: Left +// CHECK: export * // CHECK: Files: +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} -// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} // CHECK-LABEL: MODULE Right @@ -60,10 +66,13 @@ // CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} // CHECK: 2:1 [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} // CHECK: 2:1 (Module) Top -// CHECK: Module Map: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: Module Map: +// CHECK: Right +// CHECK: export * // CHECK: Files: +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} -// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} // CHECK-LABEL: TRANSLATION UNIT @@ -76,8 +85,9 @@ // recorded in the include-tree; it's handled entirely by fmodule-map-file, // fmodule-file, and fmodule-file-cache-key options. +// CHECK-NOT: Module Map // CHECK: Files: -// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} @@ -96,7 +106,7 @@ // CHECK-NEXT: "-cc1" // CHECK: "-fcas-path" // CHECK-NEXT: "[[PREFIX]]/cas" -// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK-NOT: -fmodule-map-file // CHECK: "-o" // CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" // CHECK: "-disable-free" @@ -131,7 +141,7 @@ // CHECK-NEXT: "-cc1" // CHECK: "-fcas-path" // CHECK-NEXT: "[[PREFIX]]/cas" -// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK-NOT: -fmodule-map-file // CHECK: "-o" // CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" // CHECK: "-disable-free" @@ -256,7 +266,7 @@ // CHECK-NEXT: "-cc1" // CHECK: "-fcas-path" // CHECK-NEXT: "[[PREFIX]]/cas" -// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK-NOT: -fmodule-map-file // CHECK: "-disable-free" // CHECK: "-fcas-include-tree" // CHECK-NEXT: "[[TU_TREE]]" From 165859332e8df37a92035824d019d7d028d686d0 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 23 Mar 2023 14:16:35 -0700 Subject: [PATCH 13/38] [clang][cas] Add tests for include-tree module map * Private modules with fmodule-name * Inferred framework modules * Framework autolink * VFS (cherry picked from commit 95659504cfb37ad5b9525933a505e2bc260a21aa) (cherry picked from commit 871b0e71a8cad1585b05d3a685999553de91d302) --- .../IncludeTreeActionController.cpp | 2 +- .../modules-include-tree-exports.c | 128 ++++++++++++++++++ ...ules-include-tree-implementation-private.c | 65 +++++++++ .../modules-include-tree-inferred.m | 48 +++++++ .../modules-include-tree-vfsoverlay.c | 99 ++++++++++++++ 5 files changed, 341 insertions(+), 1 deletion(-) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-exports.c create mode 100644 clang/test/ClangScanDeps/modules-include-tree-implementation-private.c create mode 100644 clang/test/ClangScanDeps/modules-include-tree-inferred.m create mode 100644 clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index caea6662392ac..93d8be92eece3 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -586,7 +586,7 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, Module *PM = MMap.findModule(ScanInstance.getLangOpts().ModuleName + "_Private"); if (PM) - if (Error E = AddModule(M)) + if (Error E = AddModule(PM)) return std::move(E); } diff --git a/clang/test/ClangScanDeps/modules-include-tree-exports.c b/clang/test/ClangScanDeps/modules-include-tree-exports.c new file mode 100644 index 0000000000000..0884e2852320d --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-exports.c @@ -0,0 +1,128 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// Extract the include-tree commands +// RUN: %deps-to-rsp %t/deps.json --module-name TwoSubs > %t/TwoSubs.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ExportExplicit > %t/ExportExplicit.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ExportWildcard > %t/ExportWildcard.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ExportGlobalWildcard > %t/ExportGlobalWildcard.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name NoExports > %t/NoExports.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu_export_explicit.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 1 > %t/tu_export_wildcard.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 2 > %t/tu_export_global_wildcard.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 3 > %t/tu_export_none.rsp + +// Build +// RUN: %clang @%t/TwoSubs.rsp +// RUN: %clang @%t/ExportExplicit.rsp +// RUN: %clang @%t/ExportWildcard.rsp +// RUN: %clang @%t/ExportGlobalWildcard.rsp +// RUN: %clang @%t/NoExports.rsp +// RUN: not %clang @%t/tu_export_explicit.rsp 2>&1 | FileCheck %s -check-prefix=tu_export_explicit +// RUN: %clang @%t/tu_export_wildcard.rsp 2>&1 | FileCheck %s -check-prefix=tu_export_wildcard -allow-empty +// RUN: %clang @%t/tu_export_global_wildcard.rsp 2>&1 | FileCheck %s -check-prefix=tu_export_global_wildcard -allow-empty +// RUN: not %clang @%t/tu_export_none.rsp 2>&1 | FileCheck %s -check-prefix=tu_export_none + +//--- cdb.json.template +[ +{ + "file": "DIR/tu_export_explicit.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_export_explicit.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +{ + "file": "DIR/tu_export_wildcard.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_export_wildcard.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +{ + "file": "DIR/tu_export_global_wildcard.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_export_global_wildcard.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +{ + "file": "DIR/tu_export_none.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_export_none.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +] + +//--- module.modulemap +module TwoSubs { + module Sub1 { header "Sub1.h" } + module Sub2 { header "Sub2.h" } +} + +module ExportExplicit { + header "Import.h" + export TwoSubs.Sub2 +} + +module ExportWildcard { + header "Import.h" + export TwoSubs.* +} + +module ExportGlobalWildcard { + header "Import.h" + export * +} + +module NoExports { + header "Import.h" +} + +//--- Sub1.h +void sub1(void); + +//--- Sub2.h +void sub2(void); + +//--- Import.h +#include "Sub1.h" +#include "Sub2.h" + +//--- tu_export_explicit.c +#pragma clang module import ExportExplicit +void tu1(void) { + sub2(); + // tu_export_explicit-NOT: error + sub1(); + // tu_export_explicit: error: call to undeclared function 'sub1' + // tu_export_explicit: error: missing '#include "Sub1.h"' +} + +//--- tu_export_wildcard.c +#pragma clang module import ExportWildcard +void tu1(void) { + sub1(); + sub2(); + // tu_export_wildcard-NOT: error +} + +//--- tu_export_global_wildcard.c +#pragma clang module import ExportGlobalWildcard +void tu1(void) { + sub1(); + sub2(); + // tu_export_global_wildcard-NOT: error +} + +//--- tu_export_none.c +#pragma clang module import NoExports +void tu1(void) { + sub1(); + // tu_export_none: error: call to undeclared function 'sub1' + // tu_export_none: error: missing '#include "Sub1.h"' + sub2(); + // tu_export_none: error: call to undeclared function 'sub2' + // tu_export_none: error: missing '#include "Sub2.h"' +} diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c new file mode 100644 index 0000000000000..9d21b042a1beb --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c @@ -0,0 +1,65 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: FileCheck %s -input-file %t/deps.json -check-prefix=NO_MODULES +// NO_MODULES: "modules": [] + +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid | FileCheck %s -DPREFIX=%/t +// RUN: %clang @%t/tu.rsp + +// CHECK: [[PREFIX]]/tu.m llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// +// CHECK: Submodule: Mod +// CHECK: 3:1 [[PREFIX]]/Mod.framework/PrivateHeaders/Priv.h llvmcas:// +// CHECK: Submodule: Mod_Private +// CHECK: Module Map: +// CHECK: Mod (framework) +// CHECK: link Mod (framework) +// CHECK: Mod_Private (framework) +// CHECK: link Mod_Private (framework) + +// CHECK: Files: +// CHECK: [[PREFIX]]/tu.m llvmcas:// +// CHECK-NOT: [[PREFIX]]/module.modulemap +// CHECK: [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// +// CHECK-NOT: [[PREFIX]]/module.modulemap +// CHECK: [[PREFIX]]/Mod.framework/PrivateHeaders/Priv.h llvmcas:// +// CHECK-NOT: [[PREFIX]]/module.modulemap + +//--- cdb.json.template +[{ + "file": "DIR/tu.m", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -F DIR -fmodule-name=Mod -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- Mod.framework/Modules/module.modulemap +framework module Mod { header "Mod.h" } + +//--- Mod.framework/Modules/module.private.modulemap +framework module Mod_Private { header "Priv.h" } + +//--- Mod.framework/Headers/Mod.h +void pub(void); + +//--- Mod.framework/PrivateHeaders/Priv.h +void priv(void); + +//--- tu.m +#import +#import +void tu(void) { + pub(); + priv(); +} diff --git a/clang/test/ClangScanDeps/modules-include-tree-inferred.m b/clang/test/ClangScanDeps/modules-include-tree-inferred.m new file mode 100644 index 0000000000000..df0668d54e8bd --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-inferred.m @@ -0,0 +1,48 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: %deps-to-rsp %t/deps.json --module-name Mod > %t/Mod.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: cat %t/Mod.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Mod.casid +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Mod.casid | FileCheck %s -DPREFIX=%/t +// RUN: %clang @%t/Mod.rsp +// RUN: %clang @%t/tu.rsp + +// CHECK: llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// +// CHECK: Submodule: Mod +// CHECK: Module Map: +// CHECK: Mod (framework) +// CHECK: link Mod (framework) +// CHECK: Files: +// CHECK-NOT: [[PREFIX]]/module.modulemap +// CHECK: [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// +// CHECK-NOT: [[PREFIX]]/module.modulemap + +//--- cdb.json.template +[{ + "file": "DIR/tu.m", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- module.modulemap +framework module * {} + +//--- Mod.framework/Headers/Mod.h +void pub(void); + +//--- tu.m +#import +void tu(void) { + pub(); +} diff --git a/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c b/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c new file mode 100644 index 0000000000000..1f5e102916775 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c @@ -0,0 +1,99 @@ +// Check include-tree-based caching works with vfsoverlay files. + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed -e "s|DIR|%/t|g" %t/vfs.yaml.template > %t/vfs.yaml + +// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: -cas-path %t/cas > %t/deps.json + +// RUN: %deps-to-rsp %t/deps.json --module-name=A > %t/A.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/A.rsp +// RUN: %clang @%t/tu.rsp + +// Extract include-tree casids +// RUN: cat %t/A.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/A.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE A" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/A.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// CHECK-LABEL: MODULE A +// CHECK: llvmcas:// +// CHECK: 2:1 [[PREFIX]]/elsewhere2/A.h llvmcas:// +// CHECK: Submodule: A +// CHECK: Module Map: +// CHECK: A (framework) +// CHECK: link A (framework) +// CHECK: Files: +// CHECK-NOT: modulemap +// CHECK: [[PREFIX]]/elsewhere2/A.h llvmcas:// +// CHECK-NOT: modulemap + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: Files: +// CHECK-NOT: .modulemap +// CHECK-NOT: .yaml +// CHECK: [[PREFIX]]/elsewhere2/A.h llvmcas:// +// CHECK-NOT: .modulemap +// CHECK-NOT: .yaml + +//--- cdb.json.template +[{ + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -ivfsoverlay DIR/vfs.yaml -F DIR", + "file": "DIR/tu.c" +}] + +//--- vfs.yaml.template +{ + "version": 0, + "case-sensitive": "false", + "roots": [ + { + "name": "DIR/A.framework", + "type": "directory" + "contents": [ + { + "name": "Modules", + "type": "directory" + "contents": [ + { + "external-contents": "DIR/elsewhere1/A.modulemap", + "name": "module.modulemap", + "type": "file" + } + ] + }, + { + "name": "Headers", + "type": "directory" + "contents": [ + { + "external-contents": "DIR/elsewhere2/A.h", + "name": "A.h", + "type": "file" + } + ] + } + ] + } + ] +} + +//--- elsewhere1/A.modulemap +framework module A { header "A.h" } + +//--- elsewhere2/A.h +typedef int A_t; + +//--- tu.c +#include "A/A.h" +A_t a = 0; From 6d400c9ce8c1da910f7d7ed648d684a81fbaff7d Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 23 Mar 2023 15:07:12 -0700 Subject: [PATCH 14/38] [clang][cas] Correctly split out modules that are only imported for visibility Modules imported for visibility only can happen when -fmodule-name matches a modular header that is imported more than once so that we trigger the correct re-exports but without trying to import a module. (cherry picked from commit d844c16c5b19e28db5736ae3da13e9ca05f3d866) (cherry picked from commit 02b64b7278e7c6165cba5f84f2d2f0b74f515793) --- .../include/clang/Basic/DiagnosticLexKinds.td | 3 ++ clang/include/clang/CAS/IncludeTree.h | 9 +++-- clang/include/clang/Lex/PPCachedActions.h | 2 + clang/lib/CAS/IncludeTree.cpp | 15 ++++++-- clang/lib/Frontend/IncludeTreePPActions.cpp | 2 +- clang/lib/Lex/PPDirectives.cpp | 37 ++++++++++++++----- .../DependencyScanningWorker.cpp | 4 ++ .../IncludeTreeActionController.cpp | 5 ++- ...ules-include-tree-implementation-private.c | 4 ++ .../modules-include-tree-implementation.c | 2 +- 10 files changed, 63 insertions(+), 20 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td index 12aed1386cdd8..3cba656af179d 100644 --- a/clang/include/clang/Basic/DiagnosticLexKinds.td +++ b/clang/include/clang/Basic/DiagnosticLexKinds.td @@ -918,6 +918,9 @@ def warn_defined_in_function_type_macro : Extension< "macro expansion producing 'defined' has undefined behavior">, InGroup; +def err_pp_missing_module_include_tree : Error< + "no module named '%0' declared in include-tree module map">, DefaultFatal; + let CategoryName = "Nullability Issue" in { def err_pp_assume_nonnull_syntax : Error<"expected 'begin' or 'end'">; diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 9c9723fd3bc90..77cbd3fcc47b6 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -331,9 +331,12 @@ class IncludeTree::ModuleImport : public IncludeTreeBase { public: static constexpr StringRef getNodeKind() { return "ModI"; } - static Expected create(ObjectStore &DB, StringRef ModuleName); + static Expected create(ObjectStore &DB, StringRef ModuleName, + bool VisibilityOnly); - StringRef getModuleName() { return getData(); } + StringRef getModuleName() const { return getData().drop_front(); } + /// Whether this module should only be "marked visible" rather than imported. + bool visibilityOnly() const { return (bool)getData()[0]; } llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); @@ -341,7 +344,7 @@ class IncludeTree::ModuleImport : public IncludeTreeBase { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); - return Base.getNumReferences() == 0 && !Base.getData().empty(); + return Base.getNumReferences() == 0 && Base.getData().size() > 1; } static bool isValid(ObjectStore &DB, ObjectRef Ref) { auto Node = DB.getProxy(Ref); diff --git a/clang/include/clang/Lex/PPCachedActions.h b/clang/include/clang/Lex/PPCachedActions.h index dde2c6f30ed93..c741fc022a5c4 100644 --- a/clang/include/clang/Lex/PPCachedActions.h +++ b/clang/include/clang/Lex/PPCachedActions.h @@ -39,6 +39,8 @@ class PPCachedActions { /// The module that is imported by an \c #include directive or \c @import. struct IncludeModule { SmallVector, 2> ImportPath; + // Whether this module should only be "marked visible" rather than imported. + bool VisibilityOnly; }; virtual ~PPCachedActions() = default; diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index dae8c92e52901..68bae3b5bf1a5 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -212,9 +212,12 @@ bool IncludeTree::isValid(const ObjectProxy &Node) { } Expected -IncludeTree::ModuleImport::create(ObjectStore &DB, StringRef ModuleName) { - return IncludeTreeBase::create(DB, {}, - llvm::arrayRefFromStringRef(ModuleName)); +IncludeTree::ModuleImport::create(ObjectStore &DB, StringRef ModuleName, + bool VisibilityOnly) { + SmallString<64> Buffer; + Buffer.push_back((char)VisibilityOnly); + Buffer.append(ModuleName); + return IncludeTreeBase::create(DB, {}, Buffer); } IncludeTree::FileList::FileSizeTy @@ -600,7 +603,11 @@ llvm::Error IncludeTree::FileList::print(llvm::raw_ostream &OS, llvm::Error IncludeTree::ModuleImport::print(llvm::raw_ostream &OS, unsigned Indent) { - OS << "(Module) " << getModuleName() << '\n'; + if (visibilityOnly()) + OS << "(Module for visibility only) "; + else + OS << "(Module) "; + OS << getModuleName() << '\n'; return llvm::Error::success(); } diff --git a/clang/lib/Frontend/IncludeTreePPActions.cpp b/clang/lib/Frontend/IncludeTreePPActions.cpp index 2d94a858c961a..87db33b85d52a 100644 --- a/clang/lib/Frontend/IncludeTreePPActions.cpp +++ b/clang/lib/Frontend/IncludeTreePPActions.cpp @@ -123,7 +123,7 @@ class IncludeTreePPActions final : public PPCachedActions { Import.getModuleName().split(ModuleComponents, '.'); for (StringRef Component : ModuleComponents) Path.emplace_back(PP.getIdentifierInfo(Component), IncludeLoc); - return IncludeModule{std::move(Path)}; + return IncludeModule{std::move(Path), Import.visibilityOnly()}; } assert(Node->getKind() == cas::IncludeTree::NodeKind::Tree); diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index 92b68ddf10a9c..c2278ba0b5380 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -2011,17 +2011,36 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, return; } if (auto *Import = std::get_if(&Include)) { - ModuleLoadResult Imported = TheModuleLoader.loadModule( - IncludeTok.getLocation(), Import->ImportPath, Module::Hidden, - /*IsIncludeDirective=*/true); - if (!Imported) { - assert(hadModuleLoaderFatalFailure() && "unexpected failure kind"); - if (hadModuleLoaderFatalFailure()) { - IncludeTok.setKind(tok::eof); - CurLexer->cutOffLexing(); + ModuleLoadResult Imported; + if (Import->VisibilityOnly) { + ModuleMap &MMap = getHeaderSearchInfo().getModuleMap(); + Module *M = nullptr; + for (auto &NameLoc : Import->ImportPath) { + M = MMap.lookupModuleQualified(NameLoc.first->getName(), M); + if (!M) + break; + } + if (!M) { + Diags->Report(diag::err_pp_missing_module_include_tree) + << getLangOpts().CurrentModule; + + return; + } + Imported = M; + } else { + Imported = TheModuleLoader.loadModule( + IncludeTok.getLocation(), Import->ImportPath, Module::Hidden, + /*IsIncludeDirective=*/true); + if (!Imported) { + assert(hadModuleLoaderFatalFailure() && "unexpected failure kind"); + if (hadModuleLoaderFatalFailure()) { + IncludeTok.setKind(tok::eof); + CurLexer->cutOffLexing(); + } + return; } - return; } + makeModuleVisible(Imported, EndLoc); if (IncludeTok.getIdentifierInfo()->getPPKeywordID() != tok::pp___include_macros) diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 5d5cd6c8326f5..fcefb3e7d4ffc 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -458,6 +458,10 @@ class DependencyScanningAction : public tooling::ToolAction { else Action = std::make_unique(); + // Normally this would be handled by GeneratePCHAction + if (ScanInstance.getFrontendOpts().ProgramAction == frontend::GeneratePCH) + ScanInstance.getLangOpts().CompilingPCH = true; + if (Error E = Controller.initialize(ScanInstance, OriginalInvocation)) return reportError(std::move(E)); diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 93d8be92eece3..9ea4bfa1763e2 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -397,8 +397,9 @@ void IncludeTreeBuilder::handleHasIncludeCheck(Preprocessor &PP, bool Result) { void IncludeTreeBuilder::moduleImport(Preprocessor &PP, const Module *M, SourceLocation EndLoc) { - auto Import = - check(cas::IncludeTree::ModuleImport::create(DB, M->getFullModuleName())); + bool VisibilityOnly = M->isForBuilding(PP.getLangOpts()); + auto Import = check(cas::IncludeTree::ModuleImport::create( + DB, M->getFullModuleName(), VisibilityOnly)); if (!Import) return; diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c index 9d21b042a1beb..912a72da3a174 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c @@ -23,6 +23,8 @@ // CHECK: Submodule: Mod // CHECK: 3:1 [[PREFIX]]/Mod.framework/PrivateHeaders/Priv.h llvmcas:// // CHECK: Submodule: Mod_Private +// CHECK: 4:1 (Module for visibility only) Mod +// CHECK: 5:1 (Module for visibility only) Mod_Private // CHECK: Module Map: // CHECK: Mod (framework) // CHECK: link Mod (framework) @@ -59,6 +61,8 @@ void priv(void); //--- tu.m #import #import +#import +#import void tu(void) { pub(); priv(); diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation.c b/clang/test/ClangScanDeps/modules-include-tree-implementation.c index 40b1d51281acb..b0ccbec601856 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-implementation.c +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation.c @@ -27,7 +27,7 @@ // CHECK: 2:1 [[PREFIX]]/Mod.h llvmcas://{{[[:xdigit:]]+}} // CHECK: Submodule: Mod -// CHECK: 3:1 (Module) Mod +// CHECK: 3:1 (Module for visibility only) Mod // CHECK: Files: // CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} From 0f206d4cfc97054d5b12ef6191afac058ba0bd92 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 23 Mar 2023 16:42:04 -0700 Subject: [PATCH 15/38] [clang][cas] Fix include-tree with pch importing public/private modules We were dropping the modular import of a private module if its public module was imported via PCH, because we skipped parsing the private modulemap during scanning. (cherry picked from commit 6034ccd1ea9d2fb5c2fea793c66d95dceedba8ac) (cherry picked from commit ab658ceefcece986379e7fbf6a0fcf0cab3a0808) --- .../IncludeTreeActionController.cpp | 56 ++++++++++++++++ .../modules-include-tree-pch-with-private.c | 65 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 9ea4bfa1763e2..60a2eb3a7b48f 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -153,6 +153,19 @@ struct PPCallbacksDependencyCollector : public DependencyCollector { PP.addPPCallbacks(std::move(CB)); } }; +/// A utility for adding \c ASTReaderListener to a compiler instance at the +/// appropriate time. +struct ASTReaderListenerDependencyCollector : public DependencyCollector { + using MakeL = + llvm::unique_function(ASTReader &R)>; + MakeL Create; + ASTReaderListenerDependencyCollector(MakeL Create) : Create(std::move(Create)) {} + void attachToASTReader(ASTReader &R) final { + std::unique_ptr L = Create(R); + assert(L); + R.addListener(std::move(L)); + } +}; struct IncludeTreePPCallbacks : public PPCallbacks { IncludeTreeBuilder &Builder; @@ -217,6 +230,40 @@ struct IncludeTreePPCallbacks : public PPCallbacks { Builder.exitedSubmodule(PP, M, ImportLoc, ForPragma); } }; + +/// Utility to trigger module lookup in header search for modules loaded via +/// PCH. This causes dependency scanning via PCH to parse modulemap files at +/// roughly the same point they would with modulemap files embedded in the pcms, +/// which is disabled with include-tree modules. Without this, we can fail to +/// find modules that are in the same directory as a named import, since +/// it may be skipped during search (see \c loadFrameworkModule). +/// +/// The specific lookup we do matches what happens in ASTReader for the +/// MODULE_DIRECTORY record, and ignores the result. +class LookupPCHModulesListener : public ASTReaderListener { +public: + LookupPCHModulesListener(Preprocessor &PP) : PP(PP) {} + +private: + void ReadModuleName(StringRef ModuleName) final { + if (PCHFinished) + return; + // Match MODULE_DIRECTORY: allow full search and ignore failure to find + // the module. + (void)PP.getHeaderSearchInfo().lookupModule( + ModuleName, SourceLocation(), /*AllowSearch=*/true, + /*AllowExtraModuleMapSearch=*/true); + } + void visitModuleFile(StringRef Filename, + serialization::ModuleKind Kind) final { + if (Kind == serialization::MK_PCH) + PCHFinished = true; + } + +private: + Preprocessor &PP; + bool PCHFinished = false; +}; } // namespace /// The PCH recorded file paths with canonical paths, create a VFS that @@ -274,6 +321,15 @@ Error IncludeTreeActionController::initialize( }); ScanInstance.addDependencyCollector(std::move(DC)); + // Attach callback for modules loaded via PCH. + if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty()) { + auto DC = std::make_shared( + [&](ASTReader &R) { + return std::make_unique(R.getPreprocessor()); + }); + ScanInstance.addDependencyCollector(std::move(DC)); + } + // Enable caching in the resulting commands. ScanInstance.getFrontendOpts().CacheCompileJob = true; CASOpts = ScanInstance.getCASOpts(); diff --git a/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c b/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c new file mode 100644 index 0000000000000..99910c5bb62f0 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c @@ -0,0 +1,65 @@ +// Test importing a private module whose public module was previously imported +// via a PCH. + +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json + +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch.json + +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Mod > %t/Mod.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/Mod.rsp +// RUN: %clang @%t/pch.rsp + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: %deps-to-rsp %t/deps.json --module-name Mod_Private > %t/Mod_Private.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/Mod_Private.rsp +// RUN: %clang @%t/tu.rsp + +//--- cdb.json.template +[{ + "file": "DIR/tu.m", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -include prefix.h -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- cdb_pch.json.template +[{ + "file": "DIR/prefix.h", + "directory": "DIR", + "command": "clang -x objective-c-header DIR/prefix.h -o DIR/prefix.h.pch -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- Mod.framework/Modules/module.modulemap +framework module Mod { header "Mod.h" } + +//--- Mod.framework/Modules/module.private.modulemap +framework module Mod_Private { header "Priv.h" } + +//--- Mod.framework/Headers/Mod.h +void pub(void); + +//--- Mod.framework/PrivateHeaders/Priv.h +void priv(void); + +//--- prefix.h +#import + +//--- tu.m +#import +void tu(void) { + pub(); + priv(); +} From 531d49bb953c08c7cd08c32c706ce49979cb1473 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Fri, 24 Mar 2023 15:48:09 -0700 Subject: [PATCH 16/38] [clang][cas] Make missing include-tree id an error This was review feedback on a previous PR that somehow went missing. Bring it back. (cherry picked from commit f1ccf9d17cd58d5c013e05fde249d008dd72d4dd) (cherry picked from commit 0b727cd37892e11441a9beac4b8ac8f4c415fd36) --- .../IncludeTreeActionController.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 60a2eb3a7b48f..29b72c4221efd 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -392,11 +392,14 @@ Error IncludeTreeActionController::finalizeModuleBuild( Error IncludeTreeActionController::finalizeModuleInvocation( CompilerInvocation &CI, const ModuleDeps &MD) { - if (auto ID = MD.IncludeTreeID) { - configureInvocationForCaching(CI, CASOpts, std::move(*ID), - /*CASFSWorkingDir=*/"", - /*ProduceIncludeTree=*/true); - } + if (!MD.IncludeTreeID) + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "missing include-tree for module '%s'", + MD.ID.ModuleName.c_str()); + + configureInvocationForCaching(CI, CASOpts, *MD.IncludeTreeID, + /*CASFSWorkingDir=*/"", + /*ProduceIncludeTree=*/true); DepscanPrefixMapping::remapInvocationPaths(CI, PrefixMapper); return Error::success(); From 8c8da77c677d0ea8b0af1e77f7ef9e8b0bdae395 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Sun, 26 Mar 2023 14:37:29 -0700 Subject: [PATCH 17/38] [clang][cas] Allow non-existent fmodule-name Missing the module named by fmodule-name is not an error for a TU or PCH. (cherry picked from commit fcd06ba0187401237b71018f8d9e509c0ea87e28) (cherry picked from commit 3e11b20d2ed0b175a215ca07e2561f65df9e1840) --- .../IncludeTreeActionController.cpp | 12 +++++------- .../modules-include-tree-implementation.c | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 29b72c4221efd..e218ae9c734dd 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -639,13 +639,11 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, // that case we need both of those modules. ModuleMap &MMap = ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap(); - M = MMap.findModule(ScanInstance.getLangOpts().CurrentModule); - assert(M && "missing current module?"); - if (Error E = AddModule(M)) - return std::move(E); - Module *PM = - MMap.findModule(ScanInstance.getLangOpts().ModuleName + "_Private"); - if (PM) + if (Module *M = MMap.findModule(ScanInstance.getLangOpts().CurrentModule)) + if (Error E = AddModule(M)) + return std::move(E); + if (Module *PM = + MMap.findModule(ScanInstance.getLangOpts().ModuleName + "_Private")) if (Error E = AddModule(PM)) return std::move(E); } diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation.c b/clang/test/ClangScanDeps/modules-include-tree-implementation.c index b0ccbec601856..282742ad803f1 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-implementation.c +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation.c @@ -4,7 +4,7 @@ // RUN: split-file %s %t // RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json -// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \ // RUN: -cas-path %t/cas -module-files-dir %t/outputs \ // RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ // RUN: > %t/deps.json @@ -35,12 +35,22 @@ // CHECK: [[PREFIX]]/Mod.h llvmcas://{{[[:xdigit:]]+}} // CHECK-NOT: [[PREFIX]]/module.modulemap +// RUN: %deps-to-rsp %t/deps.json --tu-index 1 > %t/tu_missing_module.rsp +// RUN: %clang @%t/tu_missing_module.rsp + //--- cdb.json.template -[{ +[ +{ "file": "DIR/tu.c", "directory": "DIR", "command": "clang -fsyntax-only DIR/tu.c -I DIR -fmodule-name=Mod -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" -}] +}, +{ + "file": "DIR/tu_missing_module.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_missing_module.c -I DIR -fmodule-name=NonExistent -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +} +] //--- module.modulemap module Mod { header "Mod.h" } @@ -55,3 +65,6 @@ void top(void); void tu(void) { top(); } + +//--- tu_missing_module.c + From d52f5d4b802c0e2517cbc6c6adde714f016c9579 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Mon, 27 Mar 2023 10:28:47 -0700 Subject: [PATCH 18/38] [clang][cas] Switch to chained representation of IncludeTree::FileList Instead of flattening file lists when importing PCH/modules, just create a chained reference to the list and flatten it only in FileList::forEachFile. This reduces the storage cost and performance overhead of merging file lists. (cherry picked from commit ee7ddda1e8e2fc16648ca18b12be168858751d4a) (cherry picked from commit 2fbfa490d0dab86eb042e9008969024ce1633863) --- clang/include/clang/CAS/IncludeTree.h | 44 ++------- clang/lib/CAS/IncludeTree.cpp | 78 +++++++++++++--- .../IncludeTreeActionController.cpp | 33 ++++--- clang/unittests/CAS/IncludeTreeTest.cpp | 92 ++++++++++++++----- 4 files changed, 163 insertions(+), 84 deletions(-) diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 77cbd3fcc47b6..b478a0614bb56 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -264,25 +264,15 @@ class IncludeTree::File : public IncludeTreeBase { } }; -/// A flat list of \c File entries. This is used along with a simple -/// implementation of a \p vfs::FileSystem produced via -/// \p createIncludeTreeFileSystem(). +/// A list of \c File entries. Multiple \c FileList can be combined without +/// copying their contents. This is used along with a simple implementation of a +/// \p vfs::FileSystem produced via \p createIncludeTreeFileSystem(). class IncludeTree::FileList : public IncludeTreeBase { public: static constexpr StringRef getNodeKind() { return "List"; } using FileSizeTy = uint32_t; - size_t getNumFiles() const { return getNumReferences(); } - - ObjectRef getFileRef(size_t I) const { - assert(I < getNumFiles()); - return getReference(I); - } - - Expected getFile(size_t I) { return getFile(getFileRef(I)); } - FileSizeTy getFileSize(size_t I) const; - /// \returns each \c File entry along with its file size. llvm::Error forEachFile(llvm::function_ref Callback); @@ -294,7 +284,8 @@ class IncludeTree::FileList : public IncludeTreeBase { ObjectRef FileRef; FileSizeTy Size; }; - static Expected create(ObjectStore &DB, ArrayRef Files); + static Expected create(ObjectStore &DB, ArrayRef Files, + ArrayRef FileLists); static Expected get(ObjectStore &CAS, ObjectRef Ref); @@ -308,6 +299,9 @@ class IncludeTree::FileList : public IncludeTreeBase { assert(isValid(*this)); } + size_t getNumFilesCurrentList() const; + FileSizeTy getFileSize(size_t I) const; + Expected getFile(ObjectRef Ref) { auto Node = getCAS().getProxy(Ref); if (!Node) @@ -744,26 +738,4 @@ createIncludeTreeFileSystem(IncludeTreeRoot &Root); } // namespace cas } // namespace clang -namespace llvm { -template <> struct DenseMapInfo { - using FileEntry = clang::cas::IncludeTree::FileList::FileEntry; - - static FileEntry getEmptyKey() { - return {cas::ObjectRef::getDenseMapEmptyKey(), 0}; - } - - static FileEntry getTombstoneKey() { - return {cas::ObjectRef::getDenseMapTombstoneKey(), 0}; - } - - static unsigned getHashValue(FileEntry F) { - return F.FileRef.getDenseMapHash(); - } - - static bool isEqual(FileEntry LHS, FileEntry RHS) { - return LHS.FileRef == RHS.FileRef && LHS.Size == RHS.Size; - } -}; -} // namespace llvm - #endif diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index 68bae3b5bf1a5..af850b68b0233 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -7,8 +7,11 @@ //===----------------------------------------------------------------------===// #include "clang/CAS/IncludeTree.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallBitVector.h" #include "llvm/Support/EndianStream.h" +#include "llvm/Support/Error.h" +#include using namespace clang; using namespace clang::cas; @@ -220,10 +223,15 @@ IncludeTree::ModuleImport::create(ObjectStore &DB, StringRef ModuleName, return IncludeTreeBase::create(DB, {}, Buffer); } +size_t IncludeTree::FileList::getNumFilesCurrentList() const { + return llvm::support::endian::read( + getData().data()); +} + IncludeTree::FileList::FileSizeTy IncludeTree::FileList::getFileSize(size_t I) const { - assert(I < getNumFiles()); - StringRef Data = getData(); + assert(I < getNumFilesCurrentList()); + StringRef Data = getData().drop_front(sizeof(uint32_t)); assert(Data.size() >= (I + 1) * sizeof(FileSizeTy)); return llvm::support::endian::read( Data.data() + I * sizeof(FileSizeTy)); @@ -232,29 +240,44 @@ IncludeTree::FileList::getFileSize(size_t I) const { llvm::Error IncludeTree::FileList::forEachFile( llvm::function_ref Callback) { size_t I = 0; + size_t FileCount = getNumFilesCurrentList(); return forEachReference([&](ObjectRef Ref) -> llvm::Error { - auto Include = getFile(Ref); - if (!Include) - return Include.takeError(); - return Callback(std::move(*Include), getFileSize(I++)); + if (I < FileCount) { + auto Include = getFile(Ref); + if (!Include) + return Include.takeError(); + return Callback(std::move(*Include), getFileSize(I++)); + } + // Otherwise, it's a chained FileList. + ++I; + auto Proxy = getCAS().getProxy(Ref); + if (!Proxy) + return Proxy.takeError(); + FileList FL(std::move(*Proxy)); + return FL.forEachFile(Callback); }); } Expected -IncludeTree::FileList::create(ObjectStore &DB, ArrayRef Files) { +IncludeTree::FileList::create(ObjectStore &DB, ArrayRef Files, + ArrayRef FileLists) { SmallVector Refs; - Refs.reserve(Files.size()); + Refs.reserve(Files.size() + FileLists.size()); SmallString<256> Buffer; - Buffer.reserve(Files.size() * sizeof(FileSizeTy)); + Buffer.reserve(sizeof(uint32_t) + Files.size() * sizeof(FileSizeTy)); llvm::raw_svector_ostream BufOS(Buffer); llvm::support::endian::Writer Writer(BufOS, llvm::support::little); + Writer.write(static_cast(Files.size())); for (const FileEntry &Entry : Files) { assert(File::isValid(DB, Entry.FileRef)); Refs.push_back(Entry.FileRef); Writer.write(Entry.Size); } + + Refs.append(FileLists.begin(), FileLists.end()); + return IncludeTreeBase::create(DB, Refs, Buffer); } @@ -273,9 +296,13 @@ bool IncludeTree::FileList::isValid(const ObjectProxy &Node) { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); - unsigned NumFiles = Base.getNumReferences(); - return NumFiles != 0 && - Base.getData().size() == NumFiles * sizeof(FileSizeTy); + StringRef Data = Base.getData(); + if (Data.size() < sizeof(uint32_t)) + return false; + unsigned NumFiles = + llvm::support::endian::read(Data.data()); + return NumFiles != 0 && NumFiles <= Base.getNumReferences() && + Data.size() == sizeof(uint32_t) + NumFiles * sizeof(FileSizeTy); } static constexpr char ModuleFlagFramework = 1 << 0; @@ -845,20 +872,47 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { }; } // namespace +static llvm::Error diagnoseFileChange(IncludeTree::File F, ObjectRef Content) { + auto FilenameBlob = F.getFilename(); + if (!FilenameBlob) + return FilenameBlob.takeError(); + cas::ObjectStore &DB = F.getCAS(); + std::string Filename(FilenameBlob->getData()); + std::string OldID = DB.getID(Content).toString(); + std::string NewID = DB.getID(F.getContentsRef()).toString(); + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "file '%s' changed during build; include-tree " + "contents changed from %s to %s", + Filename.c_str(), OldID.c_str(), + NewID.c_str()); +} + Expected> cas::createIncludeTreeFileSystem(IncludeTreeRoot &Root) { auto FileList = Root.getFileList(); if (!FileList) return FileList.takeError(); + // Map from FilenameRef to ContentsRef. + llvm::DenseMap SeenContents; + IntrusiveRefCntPtr IncludeTreeFS = new IncludeTreeFileSystem(Root.getCAS()); llvm::Error E = FileList->forEachFile( [&](IncludeTree::File File, IncludeTree::FileList::FileSizeTy Size) -> llvm::Error { + auto InsertPair = SeenContents.insert( + std::make_pair(File.getFilenameRef(), File.getContentsRef())); + if (!InsertPair.second) { + if (InsertPair.first->second != File.getContentsRef()) + return diagnoseFileChange(File, InsertPair.first->second); + return llvm::Error::success(); + } + auto FilenameBlob = File.getFilename(); if (!FilenameBlob) return FilenameBlob.takeError(); + SmallString<128> Filename(FilenameBlob->getData()); // Strip './' in the filename to match the behaviour of ASTWriter; we // also strip './' in IncludeTreeFileSystem::getPath. diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index e218ae9c734dd..10df1417740d4 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -129,7 +129,8 @@ class IncludeTreeBuilder { // are recorded in the PCH, ordered by \p FileEntry::UID index. SmallVector PreIncludedFileNames; llvm::BitVector SeenIncludeFiles; - llvm::SetVector IncludedFiles; + SmallVector IncludedFiles; + SmallVector IncludedFileLists; Optional PredefinesBufferRef; Optional ModuleIncludesBufferRef; Optional ModuleMapRef; @@ -454,9 +455,23 @@ void IncludeTreeBuilder::handleHasIncludeCheck(Preprocessor &PP, bool Result) { IncludeStack.back().HasIncludeChecks.push_back(Result); } +// FIXME: duplicates code in PPDirectives +static bool isForModuleBuilding(const Module *M, StringRef CurrentModule, + StringRef ModuleName) { + StringRef TopLevelName = M->getTopLevelModuleName(); + + // When building framework Foo, we wanna make sure that Foo *and* Foo_Private + // are textually included and no modules are built for both. + if (M->getTopLevelModule()->IsFramework && CurrentModule == ModuleName && + !CurrentModule.endswith("_Private") && TopLevelName.endswith("_Private")) + TopLevelName = TopLevelName.drop_back(8); + + return TopLevelName == CurrentModule; +} + void IncludeTreeBuilder::moduleImport(Preprocessor &PP, const Module *M, SourceLocation EndLoc) { - bool VisibilityOnly = M->isForBuilding(PP.getLangOpts()); + bool VisibilityOnly = isForModuleBuilding(M, PP.getLangOpts().CurrentModule, PP.getLangOpts().ModuleName); auto Import = check(cas::IncludeTree::ModuleImport::create( DB, M->getFullModuleName(), VisibilityOnly)); if (!Import) @@ -655,7 +670,7 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, } auto FileList = - cas::IncludeTree::FileList::create(DB, IncludedFiles.getArrayRef()); + cas::IncludeTree::FileList::create(DB, IncludedFiles, IncludedFileLists); if (!FileList) return FileList.takeError(); @@ -681,16 +696,8 @@ Error IncludeTreeBuilder::addModuleInputs(ASTReader &Reader) { Optional Root; if (Error E = cas::IncludeTreeRoot::get(DB, *Ref).moveInto(Root)) return E; - Optional Files; - if (Error E = Root->getFileList().moveInto(Files)) - return E; - Error E = Files->forEachFile([&](auto IF, auto Size) -> Error { - IncludedFiles.insert({IF.getRef(), Size}); - return Error::success(); - }); - if (E) - return E; + IncludedFileLists.push_back(Root->getFileListRef()); } return Error::success(); @@ -771,7 +778,7 @@ IncludeTreeBuilder::addToFileList(FileManager &FM, const FileEntry *FE) { auto FileNode = createIncludeFile(Filename, **CASContents); if (!FileNode) return FileNode.takeError(); - IncludedFiles.insert( + IncludedFiles.push_back( {FileNode->getRef(), static_cast(FE->getSize())}); return FileNode->getRef(); diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index c7739611bfce3..c85b4f767d625 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -4,6 +4,7 @@ #include "llvm/CAS/CASProvidingFileSystem.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/CAS/ObjectStore.h" +#include "llvm/Support/Error.h" #include "llvm/Testing/Support/Error.h" #include "gtest/gtest.h" @@ -135,29 +136,74 @@ TEST(IncludeTree, IncludeTreeScan) { Optional FileList; ASSERT_THAT_ERROR(Root->getFileList().moveInto(FileList), llvm::Succeeded()); - ASSERT_EQ(FileList->getNumFiles(), size_t(4)); - { - Optional File; - ASSERT_THAT_ERROR(FileList->getFile(0).moveInto(File), llvm::Succeeded()); - EXPECT_EQ(File->getRef(), MainFile->getRef()); - EXPECT_EQ(FileList->getFileSize(0), MainContents.size()); - } - { - Optional File; - ASSERT_THAT_ERROR(FileList->getFile(1).moveInto(File), llvm::Succeeded()); - EXPECT_EQ(File->getRef(), A1File->getRef()); - EXPECT_EQ(FileList->getFileSize(1), A1Contents.size()); - } - { - Optional File; - ASSERT_THAT_ERROR(FileList->getFile(2).moveInto(File), llvm::Succeeded()); - EXPECT_EQ(File->getRef(), B1File->getRef()); - EXPECT_EQ(FileList->getFileSize(2), IncludeTree::FileList::FileSizeTy(0)); - } - { + + SmallVector> + Files; + ASSERT_THAT_ERROR(FileList->forEachFile([&](auto F, auto S) -> llvm::Error { + Files.push_back({F, S}); + return llvm::Error::success(); + }), + llvm::Succeeded()); + + ASSERT_EQ(Files.size(), size_t(4)); + EXPECT_EQ(Files[0].first.getRef(), MainFile->getRef()); + EXPECT_EQ(Files[0].second, MainContents.size()); + EXPECT_EQ(Files[1].first.getRef(), A1File->getRef()); + EXPECT_EQ(Files[1].second, A1Contents.size()); + EXPECT_EQ(Files[2].first.getRef(), B1File->getRef()); + EXPECT_EQ(Files[2].second, IncludeTree::FileList::FileSizeTy(0)); + EXPECT_EQ(Files[3].first.getRef(), SysFile->getRef()); + EXPECT_EQ(Files[3].second, IncludeTree::FileList::FileSizeTy(0)); +} + +TEST(IncludeTree, IncludeTreeFileList) { + std::shared_ptr DB = llvm::cas::createInMemoryCAS(); + SmallVector Files; + for (unsigned I = 0; I < 10; ++I) { Optional File; - ASSERT_THAT_ERROR(FileList->getFile(3).moveInto(File), llvm::Succeeded()); - EXPECT_EQ(File->getRef(), SysFile->getRef()); - EXPECT_EQ(FileList->getFileSize(3), IncludeTree::FileList::FileSizeTy(0)); + std::string Path = "/file" + std::to_string(I); + static constexpr StringRef Bytes = "123456789"; + Optional Content; + ASSERT_THAT_ERROR( + DB->storeFromString({}, Bytes.substr(0, I)).moveInto(Content), + llvm::Succeeded()); + ASSERT_THAT_ERROR( + IncludeTree::File::create(*DB, Path, *Content).moveInto(File), + llvm::Succeeded()); + Files.push_back(std::move(*File)); } + + auto MakeFileList = [&](unsigned Begin, unsigned End, + ArrayRef Lists) { + SmallVector Entries; + for (; Begin != End; ++Begin) + Entries.push_back({Files[Begin].getRef(), Begin}); + return IncludeTree::FileList::create(*DB, Entries, Lists); + }; + + Optional L89, L7, L29, L; + ASSERT_THAT_ERROR(MakeFileList(8, 10, {}).moveInto(L89), llvm::Succeeded()); + EXPECT_EQ(L89->getNumReferences(), 2u); + ASSERT_THAT_ERROR(MakeFileList(7, 8, {}).moveInto(L7), llvm::Succeeded()); + EXPECT_EQ(L7->getNumReferences(), 1u); + ASSERT_THAT_ERROR( + MakeFileList(2, 7, {L7->getRef(), L89->getRef()}).moveInto(L29), + llvm::Succeeded()); + EXPECT_EQ(L29->getNumReferences(), 7u); // 2,3,4,5,6, {7}, {8, 9} + ASSERT_THAT_ERROR(MakeFileList(0, 2, {L29->getRef()}).moveInto(L), + llvm::Succeeded()); + EXPECT_EQ(L->getNumReferences(), 3u); // 0, 1, {2, ...} + + size_t I = 0; + ASSERT_THAT_ERROR( + L->forEachFile([&](IncludeTree::File F, auto Size) -> llvm::Error { + EXPECT_EQ(F.getFilenameRef(), Files[I].getFilenameRef()) + << "filename mismatch at " << I; + EXPECT_EQ(F.getContentsRef(), Files[I].getContentsRef()) + << "contents mismatch at " << I; + EXPECT_EQ(Size, I) << "size mismatch at " << I; + I += 1; + return llvm::Error::success(); + }), + llvm::Succeeded()); } From fe61f9d96fd2d87fa6e742d439fe5353f1ed2bf5 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Sun, 26 Mar 2023 14:37:25 -0700 Subject: [PATCH 19/38] [clang][cas] Add simple prefix-mapping test for include-tree modules (cherry picked from commit f796c8fd6ed20acd97fe1b686a2bd07d09e96507) (cherry picked from commit c2817eacc92fce41ce6427ccf93709affa2b5d72) --- .../modules-include-tree-prefix-map.c | 305 ++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-prefix-map.c diff --git a/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c b/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c new file mode 100644 index 0000000000000..d96f6b21233b8 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c @@ -0,0 +1,305 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t/dir1 +// RUN: cp -r %t/dir1 %t/dir2 +// RUN: sed "s|DIR|%/t/dir1|g" %t/dir1/cdb.json.template > %t/cdb1.json +// RUN: sed "s|DIR|%/t/dir2|g" %t/dir1/cdb.json.template > %t/cdb2.json + +// RUN: clang-scan-deps -compilation-database %t/cdb1.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/dir1/outputs \ +// RUN: -prefix-map=%t/dir1/outputs=/^modules -prefix-map=%t/dir1=/^src -prefix-map-sdk=/^sdk -prefix-map-toolchain=/^tc \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// Extract the include-tree commands +// RUN: %deps-to-rsp %t/deps.json --module-name Top > %t/Top.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Left > %t/Left.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp + +// Extract include-tree casids +// RUN: cat %t/Top.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Top.casid +// RUN: cat %t/Left.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Left.casid +// RUN: cat %t/Right.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Right.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE Top" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Top.casid >> %t/result.txt +// RUN: echo "MODULE Left" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Left.casid >> %t/result.txt +// RUN: echo "MODULE Right" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Right.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t -check-prefix=NO_PATHS +// NO_PATHS-NOT: [[PREFIX]] + +// RUN: cat %t/deps.json >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// CHECK-LABEL: MODULE Top +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 /^src/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Module Map: +// CHECK: Top +// CHECK: export * +// CHECK: Files: +// CHECK-NOT: module.modulemap +// CHECK: /^src/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: module.modulemap + +// CHECK-LABEL: MODULE Left +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 /^src/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Top +// CHECK: Module Map: +// CHECK: Left +// CHECK: export * +// CHECK: Files: +// CHECK-NOT: module.modulemap +// CHECK: /^src/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: module.modulemap +// CHECK: /^src/Top.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: MODULE Right +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 /^src/Right.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Top +// CHECK: Module Map: +// CHECK: Right +// CHECK: export * +// CHECK: Files: +// CHECK-NOT: module.modulemap +// CHECK: /^src/Right.h llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: module.modulemap +// CHECK: /^src/Top.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: /^src/tu.m llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Left +// CHECK: 3:1 (Module) Right + +// Note: the modules with explicit imports are imported via parser and are not +// recorded in the include-tree; it's handled entirely by fmodule-map-file, +// fmodule-file, and fmodule-file-cache-key options. + +// CHECK-NOT: Module Map +// CHECK: Files: +// CHECK-NOT: module.modulemap +// CHECK: /^src/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: /^src/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: /^src/Right.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK: { +// CHECK-NEXT "modules": [ +// CHECK-NEXT { +// CHECK: "cas-include-tree-id": "[[LEFT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK: { +// CHECK: "module-name": "Top" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/dir1/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/dir1/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[LEFT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file=Top=/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Left" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/dir1/Left.h" +// CHECK-NEXT: "[[PREFIX]]/dir1/module.modulemap" +// CHECK: ] +// CHECK: "name": "Left" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Top" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/dir1/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/dir1/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[RIGHT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file-cache-key +// CHECK-NEXT: "/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file=Top=/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Right" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/dir1/Right.h" +// CHECK-NEXT: "[[PREFIX]]/dir1/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Right" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[TOP_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/dir1/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/dir1/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[TOP_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Top" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/dir1/Top.h" +// CHECK-NEXT: "[[PREFIX]]/dir1/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Top" +// CHECK: } +// CHECK: ] +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "commands": [ +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[TU_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Left" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "module-name": "Right" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file +// CHECK: "-disable-free" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[TU_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-fsyntax-only" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "/^modules/{{.*}}/Left-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "/^modules/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file=Left=/^modules/{{.*}}/Left-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=/^modules/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/dir1/tu.m" +// CHECK-NEXT: ] +// CHECK: "input-file": "[[PREFIX]]/dir1/tu.m" +// CHECK: } +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } + +// Build the include-tree commands +// RUN: %clang @%t/Top.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: %clang @%t/Left.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: %clang @%t/Right.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: %clang @%t/tu.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS + +// Scan in a different directory +// RUN: clang-scan-deps -compilation-database %t/cdb2.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/dir2/outputs \ +// RUN: -prefix-map=%t/dir2/outputs=/^modules -prefix-map=%t/dir2=/^src -prefix-map-sdk=/^sdk -prefix-map-toolchain=/^tc \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps2.json + +// RUN: %deps-to-rsp %t/deps2.json --module-name Top > %t/Top2.rsp +// RUN: %deps-to-rsp %t/deps2.json --module-name Left > %t/Left2.rsp +// RUN: %deps-to-rsp %t/deps2.json --module-name Right > %t/Right2.rsp +// RUN: %deps-to-rsp %t/deps2.json --tu-index 0 > %t/tu2.rsp + +// Check cache hits +// RUN: %clang @%t/Top2.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/Left2.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/Right2.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/tu2.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT + +// CACHE_MISS: compile job cache miss +// CACHE_HIT: compile job cache hit + +//--- cdb.json.template +[{ + "file": "DIR/tu.m", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" +}] + +//--- module.modulemap +module Top { header "Top.h" export *} +module Left { header "Left.h" export *} +module Right { header "Right.h" export *} + +//--- Top.h +#pragma once +struct Top { + int x; +}; +void top(void); + +//--- Left.h +#include "Top.h" +void left(void); + +//--- Right.h +#include "Top.h" +void right(void); + +//--- tu.m +#import "Left.h" +#import + +void tu(void) { + top(); + left(); + right(); +} From 0d7dd5ec0f082cc9144c558ebba871724159e928 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Mon, 27 Mar 2023 14:12:20 -0700 Subject: [PATCH 20/38] [clang][cas] Skip duplicates in forEachFile If we see the same File or FileList more than once, skip visiting it again. This improves performance if the same module is reached through multiple dependency chains, or the same file is in multiple modules/PCH/TU file lists explicitly. (cherry picked from commit 591fa525181ff2c17033f56f160a43f977386da1) (cherry picked from commit 653e01be976e31a5c804a6e8e1b01a04434fc2b8) --- clang/include/clang/CAS/IncludeTree.h | 4 ++ clang/lib/CAS/IncludeTree.cpp | 23 ++++++++--- clang/unittests/CAS/IncludeTreeTest.cpp | 53 +++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index b478a0614bb56..826bb85397f6b 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -309,6 +309,10 @@ class IncludeTree::FileList : public IncludeTreeBase { return File(std::move(*Node)); } + llvm::Error + forEachFileImpl(llvm::DenseSet &Seen, + llvm::function_ref Callback); + static bool isValid(const ObjectProxy &Node); static bool isValid(ObjectStore &CAS, ObjectRef Ref) { auto Node = CAS.getProxy(Ref); diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index af850b68b0233..54fa397340ecc 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -237,27 +237,38 @@ IncludeTree::FileList::getFileSize(size_t I) const { Data.data() + I * sizeof(FileSizeTy)); } -llvm::Error IncludeTree::FileList::forEachFile( +llvm::Error IncludeTree::FileList::forEachFileImpl( + llvm::DenseSet &Seen, llvm::function_ref Callback) { - size_t I = 0; + size_t Next = 0; size_t FileCount = getNumFilesCurrentList(); return forEachReference([&](ObjectRef Ref) -> llvm::Error { - if (I < FileCount) { + size_t Index = Next++; + if (!Seen.insert(Ref).second) + return llvm::Error::success(); + + if (Index < FileCount) { auto Include = getFile(Ref); if (!Include) return Include.takeError(); - return Callback(std::move(*Include), getFileSize(I++)); + return Callback(std::move(*Include), getFileSize(Index)); } + // Otherwise, it's a chained FileList. - ++I; auto Proxy = getCAS().getProxy(Ref); if (!Proxy) return Proxy.takeError(); FileList FL(std::move(*Proxy)); - return FL.forEachFile(Callback); + return FL.forEachFileImpl(Seen, Callback); }); } +llvm::Error IncludeTree::FileList::forEachFile( + llvm::function_ref Callback) { + llvm::DenseSet Seen; + return forEachFileImpl(Seen, Callback); +} + Expected IncludeTree::FileList::create(ObjectStore &DB, ArrayRef Files, ArrayRef FileLists) { diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index c85b4f767d625..1514cba3f89c5 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -197,6 +197,8 @@ TEST(IncludeTree, IncludeTreeFileList) { size_t I = 0; ASSERT_THAT_ERROR( L->forEachFile([&](IncludeTree::File F, auto Size) -> llvm::Error { + if (I >= Files.size()) + return llvm::Error::success(); // diagnosed later. EXPECT_EQ(F.getFilenameRef(), Files[I].getFilenameRef()) << "filename mismatch at " << I; EXPECT_EQ(F.getContentsRef(), Files[I].getContentsRef()) @@ -206,4 +208,55 @@ TEST(IncludeTree, IncludeTreeFileList) { return llvm::Error::success(); }), llvm::Succeeded()); + EXPECT_EQ(I, Files.size()); +} + +TEST(IncludeTree, IncludeTreeFileListDuplicates) { + std::shared_ptr DB = llvm::cas::createInMemoryCAS(); + SmallVector Files; + for (unsigned I = 0; I < 10; ++I) { + std::optional File; + std::string Path = "/file" + std::to_string(I); + static constexpr StringRef Bytes = "123456789"; + std::optional Content; + ASSERT_THAT_ERROR( + DB->storeFromString({}, Bytes.substr(0, I)).moveInto(Content), + llvm::Succeeded()); + ASSERT_THAT_ERROR( + IncludeTree::File::create(*DB, Path, *Content).moveInto(File), + llvm::Succeeded()); + Files.push_back(std::move(*File)); + } + + auto MakeFileList = [&](unsigned Begin, unsigned End, + ArrayRef Lists) { + SmallVector Entries; + for (; Begin != End; ++Begin) + Entries.push_back({Files[Begin].getRef(), Begin}); + return IncludeTree::FileList::create(*DB, Entries, Lists); + }; + + std::optional L89, L; + ASSERT_THAT_ERROR(MakeFileList(8, 10, {}).moveInto(L89), llvm::Succeeded()); + EXPECT_EQ(L89->getNumReferences(), 2u); + ASSERT_THAT_ERROR( + MakeFileList(0, 9, {L89->getRef(), L89->getRef()}).moveInto(L), + llvm::Succeeded()); + EXPECT_EQ(L->getNumReferences(), 11u); // 0, 1, ..., 8, {8, 9}, {8, 9} + + size_t I = 0; + ASSERT_THAT_ERROR( + L->forEachFile([&](IncludeTree::File F, auto Size) -> llvm::Error { + if (I >= Files.size()) + return llvm::Error::success(); // diagnosed later. + EXPECT_EQ(F.getFilenameRef(), Files[I].getFilenameRef()) + << "filename mismatch at " << I; + EXPECT_EQ(F.getContentsRef(), Files[I].getContentsRef()) + << "contents mismatch at " << I; + EXPECT_EQ(Size, I) << "size mismatch at " << I; + I += 1; + return llvm::Error::success(); + }), + llvm::Succeeded()); + EXPECT_EQ(I, Files.size()); } From 91b39eaf4e2610d5ac5883893770a8b63c1deb41 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Mon, 27 Mar 2023 21:23:42 -0700 Subject: [PATCH 21/38] [clang][cas] Update tests for stable/20221013 branch * Framework autolink is not automatic unless the binary exists * Inputs are sorted by name in dep scan --- clang/test/ClangScanDeps/modules-include-tree-exports.c | 6 +++--- .../modules-include-tree-implementation-private.c | 2 -- clang/test/ClangScanDeps/modules-include-tree-inferred.m | 1 - clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c | 1 - 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/clang/test/ClangScanDeps/modules-include-tree-exports.c b/clang/test/ClangScanDeps/modules-include-tree-exports.c index 0884e2852320d..381854d4b94f3 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-exports.c +++ b/clang/test/ClangScanDeps/modules-include-tree-exports.c @@ -16,9 +16,9 @@ // RUN: %deps-to-rsp %t/deps.json --module-name ExportGlobalWildcard > %t/ExportGlobalWildcard.rsp // RUN: %deps-to-rsp %t/deps.json --module-name NoExports > %t/NoExports.rsp // RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu_export_explicit.rsp -// RUN: %deps-to-rsp %t/deps.json --tu-index 1 > %t/tu_export_wildcard.rsp -// RUN: %deps-to-rsp %t/deps.json --tu-index 2 > %t/tu_export_global_wildcard.rsp -// RUN: %deps-to-rsp %t/deps.json --tu-index 3 > %t/tu_export_none.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 1 > %t/tu_export_global_wildcard.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 2 > %t/tu_export_none.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 3 > %t/tu_export_wildcard.rsp // Build // RUN: %clang @%t/TwoSubs.rsp diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c index 912a72da3a174..279a88fcf3d06 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c @@ -27,9 +27,7 @@ // CHECK: 5:1 (Module for visibility only) Mod_Private // CHECK: Module Map: // CHECK: Mod (framework) -// CHECK: link Mod (framework) // CHECK: Mod_Private (framework) -// CHECK: link Mod_Private (framework) // CHECK: Files: // CHECK: [[PREFIX]]/tu.m llvmcas:// diff --git a/clang/test/ClangScanDeps/modules-include-tree-inferred.m b/clang/test/ClangScanDeps/modules-include-tree-inferred.m index df0668d54e8bd..f42847ef69004 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-inferred.m +++ b/clang/test/ClangScanDeps/modules-include-tree-inferred.m @@ -22,7 +22,6 @@ // CHECK: Submodule: Mod // CHECK: Module Map: // CHECK: Mod (framework) -// CHECK: link Mod (framework) // CHECK: Files: // CHECK-NOT: [[PREFIX]]/module.modulemap // CHECK: [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// diff --git a/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c b/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c index 1f5e102916775..59589ed535a58 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c +++ b/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c @@ -31,7 +31,6 @@ // CHECK: Submodule: A // CHECK: Module Map: // CHECK: A (framework) -// CHECK: link A (framework) // CHECK: Files: // CHECK-NOT: modulemap // CHECK: [[PREFIX]]/elsewhere2/A.h llvmcas:// From 7a736484abe99044efdf79b2ec94c877aeb1aee3 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Tue, 28 Mar 2023 13:47:39 -0700 Subject: [PATCH 22/38] [clang][cas] Fall back to textual include for missing inferred submodules When an inferred submodule is missing, because the umbrella header does not actually include it, we need to fall back to textual include. This was not working when included via PCH, because we were not serializing the inference flag(s) in the include-tree. rdar://107281193 (cherry picked from commit ecf79468fe5bff8acaa1435c722a838ef005652c) --- clang/include/clang/CAS/IncludeTree.h | 12 ++- clang/lib/CAS/IncludeTree.cpp | 49 ++++++++-- clang/lib/Frontend/FrontendAction.cpp | 3 + .../IncludeTreeActionController.cpp | 3 + .../modules-include-tree-missing-submodule.c | 92 +++++++++++++++++++ 5 files changed, 145 insertions(+), 14 deletions(-) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-missing-submodule.c diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 826bb85397f6b..cf36c8c90277b 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -398,9 +398,13 @@ class IncludeTree::Module : public IncludeTreeBase { bool IsExplicit : 1; bool IsExternC : 1; bool IsSystem : 1; + bool InferSubmodules : 1; + bool InferExplicitSubmodules : 1; + bool InferExportWildcard : 1; ModuleFlags() : IsFramework(false), IsExplicit(false), IsExternC(false), - IsSystem(false) {} + IsSystem(false), InferSubmodules(false), + InferExplicitSubmodules(false), InferExportWildcard(false) {} }; ModuleFlags getFlags() const; @@ -449,7 +453,7 @@ class IncludeTree::Module : public IncludeTreeBase { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); - return Base.getData().size() > 1; + return Base.getData().size() > 2; } static bool isValid(ObjectStore &DB, ObjectRef Ref) { auto Node = DB.getProxy(Ref); @@ -461,8 +465,8 @@ class IncludeTree::Module : public IncludeTreeBase { } private: - char rawFlags() const { return getData()[0]; } - StringRef dataAfterFlags() const { return getData().drop_front(); } + uint16_t rawFlags() const; + StringRef dataAfterFlags() const { return getData().drop_front(2); } bool hasExports() const; bool hasLinkLibraries() const; std::optional getExportsIndex() const; diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index 54fa397340ecc..0cd6869339db1 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -316,20 +316,26 @@ bool IncludeTree::FileList::isValid(const ObjectProxy &Node) { Data.size() == sizeof(uint32_t) + NumFiles * sizeof(FileSizeTy); } -static constexpr char ModuleFlagFramework = 1 << 0; -static constexpr char ModuleFlagExplicit = 1 << 1; -static constexpr char ModuleFlagExternC = 1 << 2; -static constexpr char ModuleFlagSystem = 1 << 3; -static constexpr char ModuleFlagHasExports = 1 << 4; -static constexpr char ModuleFlagHasLinkLibraries = 1 << 5; +static constexpr uint16_t ModuleFlagFramework = 1 << 0; +static constexpr uint16_t ModuleFlagExplicit = 1 << 1; +static constexpr uint16_t ModuleFlagExternC = 1 << 2; +static constexpr uint16_t ModuleFlagSystem = 1 << 3; +static constexpr uint16_t ModuleFlagInferSubmodules = 1 << 4; +static constexpr uint16_t ModuleFlagInferExplicitSubmodules = 1 << 5; +static constexpr uint16_t ModuleFlagInferInferExportWildcard = 1 << 6; +static constexpr uint16_t ModuleFlagHasExports = 1 << 7; +static constexpr uint16_t ModuleFlagHasLinkLibraries = 1 << 8; IncludeTree::Module::ModuleFlags IncludeTree::Module::getFlags() const { - char Raw = rawFlags(); + uint16_t Raw = rawFlags(); ModuleFlags Flags; Flags.IsFramework = Raw & ModuleFlagFramework; Flags.IsExplicit = Raw & ModuleFlagExplicit; Flags.IsExternC = Raw & ModuleFlagExternC; Flags.IsSystem = Raw & ModuleFlagSystem; + Flags.InferSubmodules = Raw & ModuleFlagInferSubmodules; + Flags.InferExplicitSubmodules = Raw & ModuleFlagInferExplicitSubmodules; + Flags.InferExportWildcard = Raw & ModuleFlagInferInferExportWildcard; return Flags; } @@ -362,14 +368,14 @@ IncludeTree::Module::create(ObjectStore &DB, StringRef ModuleName, std::optional ExportList, std::optional LinkLibraries) { // Data: - // - 1 byte for Flags + // - 2 bytes for Flags // - ModuleName (String) // Refs: // - Submodules (IncludeTreeModule) // - (optional) ExportList // - (optional) LinkLibaryList - char RawFlags = 0; + uint16_t RawFlags = 0; if (Flags.IsFramework) RawFlags |= ModuleFlagFramework; if (Flags.IsExplicit) @@ -378,13 +384,22 @@ IncludeTree::Module::create(ObjectStore &DB, StringRef ModuleName, RawFlags |= ModuleFlagExternC; if (Flags.IsSystem) RawFlags |= ModuleFlagSystem; + if (Flags.InferSubmodules) + RawFlags |= ModuleFlagInferSubmodules; + if (Flags.InferExplicitSubmodules) + RawFlags |= ModuleFlagInferExplicitSubmodules; + if (Flags.InferExportWildcard) + RawFlags |= ModuleFlagInferInferExportWildcard; if (ExportList) RawFlags |= ModuleFlagHasExports; if (LinkLibraries) RawFlags |= ModuleFlagHasLinkLibraries; SmallString<64> Buffer; - Buffer.push_back(RawFlags); + llvm::raw_svector_ostream BufOS(Buffer); + llvm::support::endian::Writer Writer(BufOS, llvm::support::little); + Writer.write(RawFlags); + Buffer.append(ModuleName); SmallVector Refs(Submodules); @@ -396,6 +411,11 @@ IncludeTree::Module::create(ObjectStore &DB, StringRef ModuleName, return IncludeTreeBase::create(DB, Refs, Buffer); } +uint16_t IncludeTree::Module::rawFlags() const { + return llvm::support::endian::read( + getData().data()); +} + bool IncludeTree::Module::hasExports() const { return rawFlags() & ModuleFlagHasExports; } @@ -670,6 +690,15 @@ llvm::Error IncludeTree::Module::print(llvm::raw_ostream &OS, unsigned Indent) { if (Flags.IsSystem) OS << " (system)"; OS << '\n'; + if (Flags.InferSubmodules) { + if (Flags.InferExplicitSubmodules) + OS << " explicit module *"; + else + OS << " module *"; + if (Flags.InferExportWildcard) + OS << " { export * }"; + OS << '\n'; + } auto ExportList = getExports(); if (!ExportList) return ExportList.takeError(); diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index f0259f591aa5b..f91fc30e78242 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -559,6 +559,9 @@ static Expected makeIncludeTreeModule(CompilerInstance &CI, M->Kind = Module::IncludeTreeModuleMap; M->IsExternC = Flags.IsExternC; M->IsSystem = Flags.IsSystem; + M->InferSubmodules = Flags.InferSubmodules; + M->InferExplicitSubmodules = Flags.InferExplicitSubmodules; + M->InferExportWildcard = Flags.InferExportWildcard; auto ExportList = Mod.getExports(); if (!ExportList) diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 10df1417740d4..95cb66b067f5e 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -517,6 +517,9 @@ getIncludeTreeModule(cas::ObjectStore &DB, Module *M) { Flags.IsExplicit = M->IsExplicit; Flags.IsExternC = M->IsExternC; Flags.IsSystem = M->IsSystem; + Flags.InferSubmodules = M->InferSubmodules; + Flags.InferExplicitSubmodules = M->InferExplicitSubmodules; + Flags.InferExportWildcard = M->InferExportWildcard; bool GlobalWildcardExport = false; SmallVector Exports; diff --git a/clang/test/ClangScanDeps/modules-include-tree-missing-submodule.c b/clang/test/ClangScanDeps/modules-include-tree-missing-submodule.c new file mode 100644 index 0000000000000..f599b6bb94687 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-missing-submodule.c @@ -0,0 +1,92 @@ +// Ensure we fallback to textual inclusion for headers in incomplete umbrellas. + +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json + +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch.json + +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Foo > %t/Foo.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/Foo.rsp +// RUN: %clang @%t/pch.rsp + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp + +// Extract include-tree casids +// RUN: cat %t/Foo.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Foo.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE Foo" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Foo.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// CHECK-LABEL: MODULE Foo +// CHECK: llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/Foo.framework/Headers/Foo.h llvmcas:// +// CHECK: Submodule: Foo +// CHECK-NOT: Bar +// CHECK: Module Map: +// CHECK: Foo (framework) +// CHECK-NOT: Bar +// CHECK: module * +// CHECK-NOT: Bar + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: (PCH) llvmcas:// +// CHECK: [[PREFIX]]/tu.c llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/Foo.framework/Headers/Bar.h llvmcas:// + +// RUN: %clang @%t/tu.rsp + +//--- cdb_pch.json.template +[{ + "file": "DIR/prefix.h", + "directory": "DIR", + "command": "clang -x c-header DIR/prefix.h -o DIR/prefix.h.pch -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" +}] + +//--- cdb.json.template +[{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -include prefix.h -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" +}] + +//--- Foo.framework/Modules/module.modulemap +framework module Foo { + umbrella header "Foo.h" + module * { export * } +} + +//--- Foo.framework/Headers/Foo.h +// Do not import Bar.h +void foo(void); + +//--- Foo.framework/Headers/Bar.h +void bar(void); + +//--- prefix.h +#include + +//--- tu.c +#include +// FIXME: -Wincomplete-umbrella warning +void tu(void) { + bar(); +} From 5fa22a83c4ce24e44203fe9f8a020c8685542f91 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Fri, 31 Mar 2023 16:01:23 -0700 Subject: [PATCH 23/38] [clang][cas] Extend fix for private modules to module builds In 6034ccd1ea9 we fixed an issue where if a public module is imported via PCH we cannot import the private module in a TU because the modulemap is not parsed. This is also an issue when building a module that imports the private module and the TU used a PCH, because even though we don't import the PCH itself, we still use the prebuilt/explicit module found via the PCH instead of building an implicit scanner module. This commit generalizes the original fix to handle all prebuilt/explicit modules. rdar://107446573 (cherry picked from commit 159fe25a5162f09a64add08b39ba6fc5f88043f7) --- .../IncludeTreeActionController.cpp | 92 ++++++++++--------- .../modules-include-tree-pch-with-private.c | 57 ++++++++++++ 2 files changed, 104 insertions(+), 45 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 95cb66b067f5e..563afd75c68d4 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -141,30 +141,33 @@ class IncludeTreeBuilder { Optional ErrorToReport; }; -/// A utility for adding \c PPCallbacks to a compiler instance at the -/// appropriate time. -struct PPCallbacksDependencyCollector : public DependencyCollector { - using MakeCB = +/// A utility for adding \c PPCallbacks and/or \cASTReaderListener to a compiler +/// instance at the appropriate time. +struct AttachOnlyDependencyCollector : public DependencyCollector { + using MakePPCB = llvm::unique_function(Preprocessor &)>; - MakeCB Create; - PPCallbacksDependencyCollector(MakeCB Create) : Create(std::move(Create)) {} + using MakeASTReaderL = + llvm::unique_function(ASTReader &R)>; + MakePPCB CreatePPCB; + MakeASTReaderL CreateASTReaderL; + AttachOnlyDependencyCollector(MakePPCB CreatePPCB, MakeASTReaderL CreateL) + : CreatePPCB(std::move(CreatePPCB)), + CreateASTReaderL(std::move(CreateL)) {} + void attachToPreprocessor(Preprocessor &PP) final { - std::unique_ptr CB = Create(PP); - assert(CB); - PP.addPPCallbacks(std::move(CB)); + if (CreatePPCB) { + std::unique_ptr CB = CreatePPCB(PP); + assert(CB); + PP.addPPCallbacks(std::move(CB)); + } } -}; -/// A utility for adding \c ASTReaderListener to a compiler instance at the -/// appropriate time. -struct ASTReaderListenerDependencyCollector : public DependencyCollector { - using MakeL = - llvm::unique_function(ASTReader &R)>; - MakeL Create; - ASTReaderListenerDependencyCollector(MakeL Create) : Create(std::move(Create)) {} + void attachToASTReader(ASTReader &R) final { - std::unique_ptr L = Create(R); - assert(L); - R.addListener(std::move(L)); + if (CreateASTReaderL) { + std::unique_ptr L = CreateASTReaderL(R); + assert(L); + R.addListener(std::move(L)); + } } }; @@ -243,27 +246,29 @@ struct IncludeTreePPCallbacks : public PPCallbacks { /// MODULE_DIRECTORY record, and ignores the result. class LookupPCHModulesListener : public ASTReaderListener { public: - LookupPCHModulesListener(Preprocessor &PP) : PP(PP) {} + LookupPCHModulesListener(ASTReader &R) : Reader(R) {} private: - void ReadModuleName(StringRef ModuleName) final { - if (PCHFinished) - return; - // Match MODULE_DIRECTORY: allow full search and ignore failure to find - // the module. - (void)PP.getHeaderSearchInfo().lookupModule( - ModuleName, SourceLocation(), /*AllowSearch=*/true, - /*AllowExtraModuleMapSearch=*/true); - } void visitModuleFile(StringRef Filename, serialization::ModuleKind Kind) final { - if (Kind == serialization::MK_PCH) - PCHFinished = true; + // Any prebuilt or explicit modules seen during scanning are "full" modules + // rather than implicitly built scanner modules. + if (Kind == serialization::MK_PrebuiltModule || + Kind == serialization::MK_ExplicitModule) { + serialization::ModuleManager &Manager = Reader.getModuleManager(); + serialization::ModuleFile *MF = Manager.lookupByFileName(Filename); + assert(MF && "module file missing in visitModuleFile"); + // Match MODULE_DIRECTORY: allow full search and ignore failure to find + // the module. + HeaderSearch &HS = Reader.getPreprocessor().getHeaderSearchInfo(); + (void)HS.lookupModule(MF->ModuleName, SourceLocation(), + /*AllowSearch=*/true, + /*AllowExtraModuleMapSearch=*/true); + } } private: - Preprocessor &PP; - bool PCHFinished = false; + ASTReader &Reader; }; } // namespace @@ -316,21 +321,15 @@ Error IncludeTreeActionController::initialize( // Attach callbacks for the IncludeTree of the TU. The preprocessor // does not exist yet, so we need to indirect this via DependencyCollector. - auto DC = std::make_shared( + auto DC = std::make_shared( [&Builder = current()](Preprocessor &PP) { return std::make_unique(Builder, PP); + }, + [](ASTReader &R) { + return std::make_unique(R); }); ScanInstance.addDependencyCollector(std::move(DC)); - // Attach callback for modules loaded via PCH. - if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty()) { - auto DC = std::make_shared( - [&](ASTReader &R) { - return std::make_unique(R.getPreprocessor()); - }); - ScanInstance.addDependencyCollector(std::move(DC)); - } - // Enable caching in the resulting commands. ScanInstance.getFrontendOpts().CacheCompileJob = true; CASOpts = ScanInstance.getCASOpts(); @@ -366,9 +365,12 @@ Error IncludeTreeActionController::initializeModuleBuild( // Attach callbacks for the IncludeTree of the module. The preprocessor // does not exist yet, so we need to indirect this via DependencyCollector. - auto DC = std::make_shared( + auto DC = std::make_shared( [&Builder = current()](Preprocessor &PP) { return std::make_unique(Builder, PP); + }, + [](ASTReader &R) { + return std::make_unique(R); }); ModuleScanInstance.addDependencyCollector(std::move(DC)); diff --git a/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c b/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c index 99910c5bb62f0..1f24620484ace 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c +++ b/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c @@ -14,8 +14,10 @@ // RUN: > %t/deps_pch.json // RUN: %deps-to-rsp %t/deps_pch.json --module-name Mod > %t/Mod.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Indirect1 > %t/Indirect1.rsp // RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp // RUN: %clang @%t/Mod.rsp +// RUN: %clang @%t/Indirect1.rsp // RUN: %clang @%t/pch.rsp // RUN: clang-scan-deps -compilation-database %t/cdb.json \ @@ -24,10 +26,39 @@ // RUN: > %t/deps.json // RUN: %deps-to-rsp %t/deps.json --module-name Mod_Private > %t/Mod_Private.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Indirect2 > %t/Indirect2.rsp // RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp // RUN: %clang @%t/Mod_Private.rsp +// RUN: %clang @%t/Indirect2.rsp // RUN: %clang @%t/tu.rsp +// Extract include-tree casids +// RUN: cat %t/Indirect2.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Indirect.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE Indirect2" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Indirect.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt + +// Explicitly check that Mod_Private is imported as a module and not a header. +// RUN: FileCheck %s -DPREFIX=%/t -input-file %t/result.txt + +// CHECK-LABEL: MODULE Indirect2 +// CHECK: llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/indirect2.h llvmcas:// +// CHECK: Submodule: Indirect2 +// CHECK: 2:1 (Module) Indirect1 +// CHECK: 3:1 (Module) Mod_Private + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: (PCH) llvmcas:// +// CHECK: [[PREFIX]]/tu.m llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 (Module) Mod_Private +// CHECK: 3:1 (Module) Indirect2 + //--- cdb.json.template [{ "file": "DIR/tu.m", @@ -54,12 +85,38 @@ void pub(void); //--- Mod.framework/PrivateHeaders/Priv.h void priv(void); +//--- module.modulemap +module Indirect1 { + header "indirect1.h" + export * +} +module Indirect2 { + header "indirect2.h" + export * +} + +//--- indirect1.h +#import + +//--- indirect2.h +#import "indirect1.h" +#import + +static inline void indirect(void) { + pub(); + priv(); +} + //--- prefix.h #import +#import "indirect1.h" //--- tu.m #import +#import "indirect2.h" + void tu(void) { pub(); priv(); + indirect(); } From 20d4eb71d7f95c4b11fb0327d87e12aa84e79687 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 30 Mar 2023 15:47:53 -0700 Subject: [PATCH 24/38] [clang][deps] Remove -coverage-data-file and -coverage-notes-file from modules When not performing codegen, we can strip the coverage-data-file and coverage-notes-file options to improve canonicalization. rdar://107443796 Differential Revision: https://reviews.llvm.org/D147282 (cherry picked from commit 758bca6483853a743297b68bd88a5dba9d5247f2) --- clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp | 2 ++ .../test/ClangScanDeps/Inputs/removed-args/cdb.json.template | 2 +- clang/test/ClangScanDeps/removed-args.c | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index d157350603a91..960a3c7d7bfd9 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -106,6 +106,8 @@ ModuleDepCollector::makeInvocationForModuleBuildWithoutOutputs( if (!CI.getLangOpts()->ModulesCodegen) { CI.getCodeGenOpts().DebugCompilationDir.clear(); CI.getCodeGenOpts().CoverageCompilationDir.clear(); + CI.getCodeGenOpts().CoverageDataFile.clear(); + CI.getCodeGenOpts().CoverageNotesFile.clear(); } // Map output paths that affect behaviour to "-" so their existence is in the diff --git a/clang/test/ClangScanDeps/Inputs/removed-args/cdb.json.template b/clang/test/ClangScanDeps/Inputs/removed-args/cdb.json.template index 2bdc340a3fccd..7ae3c88aedd8d 100644 --- a/clang/test/ClangScanDeps/Inputs/removed-args/cdb.json.template +++ b/clang/test/ClangScanDeps/Inputs/removed-args/cdb.json.template @@ -1,7 +1,7 @@ [ { "directory": "DIR", - "command": "clang -fsyntax-only DIR/tu.c -fmodules -fimplicit-module-maps -fmodules-validate-once-per-build-session -fbuild-session-file=DIR/build-session -fmodules-prune-interval=123 -fmodules-prune-after=123 -fmodules-cache-path=DIR/cache -include DIR/header.h -grecord-command-line -fdebug-compilation-dir=DIR/debug -fcoverage-compilation-dir=DIR/coverage -o DIR/tu.o -serialize-diagnostics DIR/tu.diag -MT tu -MD -MF DIR/tu.d", + "command": "clang -fsyntax-only DIR/tu.c -fmodules -fimplicit-module-maps -fmodules-validate-once-per-build-session -fbuild-session-file=DIR/build-session -fmodules-prune-interval=123 -fmodules-prune-after=123 -fmodules-cache-path=DIR/cache -include DIR/header.h -grecord-command-line -fdebug-compilation-dir=DIR/debug -fcoverage-compilation-dir=DIR/coverage -ftest-coverage -o DIR/tu.o -serialize-diagnostics DIR/tu.diag -MT tu -MD -MF DIR/tu.d", "file": "DIR/tu.c" } ] diff --git a/clang/test/ClangScanDeps/removed-args.c b/clang/test/ClangScanDeps/removed-args.c index 02fbb8cacff2c..9a4ef25838e46 100644 --- a/clang/test/ClangScanDeps/removed-args.c +++ b/clang/test/ClangScanDeps/removed-args.c @@ -23,6 +23,8 @@ // CHECK-NEXT: "-cc1" // CHECK-NOT: "-fdebug-compilation-dir=" // CHECK-NOT: "-fcoverage-compilation-dir=" +// CHECK-NOT: "-coverage-notes-file +// CHECK-NOT: "-coverage-data-file // CHECK-NOT: "-dwarf-debug-flags" // CHECK-NOT: "-main-file-name" // CHECK-NOT: "-include" @@ -46,6 +48,8 @@ // CHECK-NEXT: "-cc1" // CHECK-NOT: "-fdebug-compilation-dir= // CHECK-NOT: "-fcoverage-compilation-dir= +// CHECK-NOT: "-coverage-notes-file +// CHECK-NOT: "-coverage-data-file // CHECK-NOT: "-dwarf-debug-flags" // CHECK-NOT: "-main-file-name" // CHECK-NOT: "-include" From 379b7b42f9d24be9c7b8e3ed23ebfc68a1d87876 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 30 Mar 2023 16:15:43 -0700 Subject: [PATCH 25/38] [clang][cas] Strip -coverage-data-file and -coverage-notes-file from PCH Companion to https://reviews.llvm.org/D147282; when caching a PCH, remove the -coverage-data-file and -coverage-notes-file options. These paths cannot currently be prefix-mapped, because they are emitted into the binary for the coverage runtime to use. However, they have no effect on PCH, so remove them. rdar://107443796 (cherry picked from commit 0d086a5125e09158dbab9985702c6a704a786daf) (cherry picked from commit eb9236e51cd666588e88a4c3102a58b4aa8de42a) --- .../lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp | 6 ++++++ clang/test/CAS/depscan-include-tree.c | 6 +++++- clang/test/CAS/fcas-include-tree-with-pch.c | 8 ++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index 7b3bd52de3f58..7076c3cdea57a 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -29,6 +29,12 @@ void tooling::dependencies::configureInvocationForCaching( // Clear this otherwise it defeats the purpose of making the compilation key // independent of certain arguments. CI.getCodeGenOpts().DwarfDebugFlags.clear(); + if (FrontendOpts.ProgramAction == frontend::GeneratePCH) { + // Clear paths that are emitted into binaries; they do not affect PCH. + // For modules this is handled in ModuleDepCollector. + CI.getCodeGenOpts().CoverageDataFile.clear(); + CI.getCodeGenOpts().CoverageNotesFile.clear(); + } // "Fix" the CAS options. auto &FileSystemOpts = CI.getFileSystemOpts(); diff --git a/clang/test/CAS/depscan-include-tree.c b/clang/test/CAS/depscan-include-tree.c index f78a027fc2944..ae3f79ea43428 100644 --- a/clang/test/CAS/depscan-include-tree.c +++ b/clang/test/CAS/depscan-include-tree.c @@ -2,7 +2,8 @@ // RUN: split-file %s %t // RUN: %clang -cc1depscan -o %t/inline.rsp -fdepscan=inline -fdepscan-include-tree -cc1-args -cc1 -triple x86_64-apple-macos11.0 \ -// RUN: -emit-obj %t/t.c -o %t/t.o -dwarf-ext-refs -fmodule-format=obj \ +// RUN: -emit-obj %t/t.c -o %t/t.o -dwarf-ext-refs -fmodule-format=obj \ +// RUN: -ftest-coverage -coverage-notes-file %t/t.gcno -coverage-data-file %t/t.gcda \ // RUN: -I %t/includes -isysroot %S/Inputs/SDK -fcas-path %t/cas -DSOME_MACRO -dependency-file %t/inline.d -MT deps // RUN: FileCheck %s -input-file %t/inline.rsp -DPREFIX=%t @@ -13,6 +14,9 @@ // CHECK: "-fcas-path" "[[PREFIX]]/cas" // CHECK: "-fcas-include-tree" // CHECK: "-isysroot" +// CHECK: "-ftest-coverage" +// CHECK: "-coverage-data-file" "[[PREFIX]]/t.gcda" +// CHECK: "-coverage-notes-file" "[[PREFIX]]/t.gcno" // SHOULD-NOT: "-fcas-fs" // SHOULD-NOT: "-fcas-fs-working-directory" // SHOULD-NOT: "-I" diff --git a/clang/test/CAS/fcas-include-tree-with-pch.c b/clang/test/CAS/fcas-include-tree-with-pch.c index 8438e07977f2e..026e4fe785031 100644 --- a/clang/test/CAS/fcas-include-tree-with-pch.c +++ b/clang/test/CAS/fcas-include-tree-with-pch.c @@ -37,6 +37,14 @@ // RUN: %clang @tu2.rsp -emit-llvm -o tree-rel.ll // RUN: diff -u source-rel.ll tree-rel.ll +// Check that -coverage-notes-file and -coverage-data-file are stripped +// RUN: %clang -cc1depscan -o pch3.rsp -fdepscan=inline -fdepscan-include-tree -cc1-args \ +// RUN: -cc1 -emit-pch -x c-header prefix.h -I %t/inc -DCMD_MACRO=1 -fcas-path %t/cas \ +// RUN: -ftest-coverage -coverage-notes-file %t/pch.gcno -coverage-data-file %t/pch.gcda +// RUN: FileCheck %s -check-prefix=COVERAGE -input-file %t/pch3.rsp +// COVERAGE-NOT: -coverage-data-file +// COVERAGE-NOT: -coverage-notes-file + //--- t1.c #if S2_MACRO #include "s2-link.h" From a707b128b4639e5ebb1133270174c0b1cc30a064 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 29 Mar 2023 11:22:30 -0700 Subject: [PATCH 26/38] [clang][cas] Move -fdepscan-prefix-map-* options to clang driver Centralize the handling of prefix mapping options so that the driver is responsible for lowering `-fdepscan-prefix-map-sdk` and `-fdepscan-prefix-map-toolchain` to use `-fdepscan-prefix-map`, and all the tools take their prefix mappings from `FrontendOptions::PathPrefixMappings` instead of each having their own handling. This makes it easier for new code paths to use prefix mapping, since they get configuration "for free". cc1depscan[d] no longer needs special handling of these options at all, and in tests we just move to the driver or cc1 options. For clang-scan-deps it's move convenient to keep the command-line options, but implement them in the argument adjuster by adding the right driver options. rdar://106307646 (cherry picked from commit 5f5cf44b6f7d2d2ee829576088bebba283b7f23e) --- .../DependencyScanningTool.h | 22 ++--- .../DependencyScanningWorker.h | 7 -- .../DependencyScanning/ScanAndUpdateArgs.h | 14 ++-- clang/lib/Driver/ToolChains/Clang.cpp | 84 +++++++++++++++---- clang/lib/Driver/ToolChains/Clang.h | 8 +- .../CASFSActionController.cpp | 20 ++--- .../DependencyScanning/CachingActions.h | 6 +- .../DependencyScanningTool.cpp | 49 ++++------- .../DependencyScanningWorker.cpp | 13 +-- .../IncludeTreeActionController.cpp | 19 +---- .../DependencyScanning/ScanAndUpdateArgs.cpp | 81 +++++------------- clang/test/CAS/cached-diagnostics.c | 8 +- clang/test/CAS/depscan-prefix-map.c | 60 ++++--------- clang/test/CAS/driver-cache-launcher.c | 5 +- .../CAS/fcas-include-tree-prefix-mapping.c | 24 ++++-- clang/test/CAS/pgo-profile.c | 8 +- clang/test/Driver/fdepscan-prefix-map-sdk.c | 8 ++ .../Driver/fdepscan-prefix-map-toolchain.c | 15 ++++ clang/test/Driver/fdepscan-prefix-map.c | 8 ++ clang/tools/clang-scan-deps/ClangScanDeps.cpp | 45 +++++----- clang/tools/driver/cc1depscanProtocol.cpp | 64 ++------------ clang/tools/driver/cc1depscanProtocol.h | 22 +---- clang/tools/driver/cc1depscan_main.cpp | 56 +++---------- clang/unittests/CAS/IncludeTreeTest.cpp | 4 +- 24 files changed, 261 insertions(+), 389 deletions(-) create mode 100644 clang/test/Driver/fdepscan-prefix-map-sdk.c create mode 100644 clang/test/Driver/fdepscan-prefix-map-toolchain.c create mode 100644 clang/test/Driver/fdepscan-prefix-map.c diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h index 87d56443a9e33..52cb75423d911 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -106,8 +106,7 @@ class DependencyScanningTool { /// Collect dependency tree. llvm::Expected - getDependencyTree(const std::vector &CommandLine, StringRef CWD, - DepscanPrefixMapping PrefixMapping = {}); + getDependencyTree(const std::vector &CommandLine, StringRef CWD); /// If \p DiagGenerationAsCompilation is true it will generate error /// diagnostics same way as the normal compilation, with "N errors generated" @@ -117,14 +116,12 @@ class DependencyScanningTool { getDependencyTreeFromCompilerInvocation( std::shared_ptr Invocation, StringRef CWD, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, - bool DiagGenerationAsCompilation, - DepscanPrefixMapping PrefixMapping = {}); + bool DiagGenerationAsCompilation); Expected getIncludeTree(cas::ObjectStore &DB, const std::vector &CommandLine, StringRef CWD, - LookupModuleOutputCallback LookupModuleOutput, - const DepscanPrefixMapping &PrefixMapping); + LookupModuleOutputCallback LookupModuleOutput); /// If \p DiagGenerationAsCompilation is true it will generate error /// diagnostics same way as the normal compilation, with "N errors generated" @@ -133,7 +130,6 @@ class DependencyScanningTool { Expected getIncludeTreeFromCompilerInvocation( cas::ObjectStore &DB, std::shared_ptr Invocation, StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, - const DepscanPrefixMapping &PrefixMapping, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, bool DiagGenerationAsCompilation); @@ -155,8 +151,7 @@ class DependencyScanningTool { getTranslationUnitDependencies(const std::vector &CommandLine, StringRef CWD, const llvm::StringSet<> &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping); + LookupModuleOutputCallback LookupModuleOutput); /// Given a compilation context specified via the Clang driver command-line, /// gather modular dependencies of module with the given name, and return the @@ -165,8 +160,7 @@ class DependencyScanningTool { getModuleDependencies(StringRef ModuleName, const std::vector &CommandLine, StringRef CWD, const llvm::StringSet<> &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping); + LookupModuleOutputCallback LookupModuleOutput); ScanningOutputFormat getScanningFormat() const { return Worker.getScanningFormat(); @@ -187,13 +181,11 @@ class DependencyScanningTool { static std::unique_ptr createActionController(DependencyScanningWorker &Worker, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping); + LookupModuleOutputCallback LookupModuleOutput); private: std::unique_ptr - createActionController(LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping); + createActionController(LookupModuleOutputCallback LookupModuleOutput); private: DependencyScanningWorker Worker; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h index a0df3970696c7..3ddfc2f761492 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -31,7 +31,6 @@ using CachingOnDiskFileSystemPtr = llvm::IntrusiveRefCntPtr; class DependencyScanningWorkerFilesystem; -struct DepscanPrefixMapping; /// A command-line tool invocation that is part of building a TU. /// @@ -96,12 +95,6 @@ class DependencyActionController { const ModuleDeps &MD) { return llvm::Error::success(); } - - /// FIXME: This is temporary until we eliminate the split between consumers in - /// \p DependencyScanningTool and collectors in \p DependencyScanningWorker - /// and have them both in the same file. see FIXME in \p - /// DependencyScanningAction::runInvocation. - virtual const DepscanPrefixMapping *getPrefixMapping() { return nullptr; } }; /// An individual dependency scanning worker that is able to run on its own diff --git a/clang/include/clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h b/clang/include/clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h index 42f0db456e190..792bee4ef0861 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h +++ b/clang/include/clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h @@ -41,14 +41,13 @@ void configureInvocationForCaching(CompilerInvocation &CI, CASOptions CASOpts, bool ProduceIncludeTree); struct DepscanPrefixMapping { - Optional NewSDKPath; - Optional NewToolchainPath; - SmallVector PrefixMap; + /// Add path mappings to the \p Mapper. + static void configurePrefixMapper(const CompilerInvocation &Invocation, + llvm::PrefixMapper &Mapper); - /// Add path mappings from the current path in \p Invocation to the new path - /// from \c DepscanPrefixMapping to the \p Mapper. - llvm::Error configurePrefixMapper(const CompilerInvocation &Invocation, - llvm::PrefixMapper &Mapper) const; + /// Add path mappings to the \p Mapper. + static void configurePrefixMapper(ArrayRef PathPrefixMappings, + llvm::PrefixMapper &Mapper); /// Apply the mappings from \p Mapper to \p Invocation. static void remapInvocationPaths(CompilerInvocation &Invocation, @@ -61,7 +60,6 @@ Expected scanAndUpdateCC1InlineWithTool( tooling::dependencies::DependencyScanningTool &Tool, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, CompilerInvocation &Invocation, StringRef WorkingDirectory, - const tooling::dependencies::DepscanPrefixMapping &PrefixMapping, llvm::cas::ObjectStore &DB); } // end namespace clang diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index ec21cbfcfcb13..1008c1b376bc4 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -1190,7 +1190,8 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA, const Driver &D, const ArgList &Args, ArgStringList &CmdArgs, const InputInfo &Output, - const InputInfoList &Inputs) const { + const InputInfoList &Inputs, + std::optional &Sysroot) const { const bool IsIAMCU = getToolChain().getTriple().isOSIAMCU(); CheckPreprocessingOptions(D, Args); @@ -1428,10 +1429,15 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA, // -isysroot to the CC1 invocation. StringRef sysroot = C.getSysRoot(); if (sysroot != "") { - if (!Args.hasArg(options::OPT_isysroot)) { + if (Arg *A = Args.getLastArg(options::OPT_isysroot)) { + Sysroot = A->getValue(); + } else { CmdArgs.push_back("-isysroot"); CmdArgs.push_back(C.getArgs().MakeArgString(sysroot)); + Sysroot = sysroot; } + } else if (Arg *A = Args.getLastArg(options::OPT_isysroot)) { + Sysroot = A->getValue(); } // Parse additional include paths from environment variables. @@ -4524,6 +4530,52 @@ static void ProcessVSRuntimeLibrary(const ArgList &Args, } } +void Clang::AddPrefixMappingOptions(const ArgList &Args, ArgStringList &CmdArgs, + const Driver &D, + std::optional Sysroot) const { + auto IsPathApplicableAsPrefix = [](std::optional Path) { + return Path && llvm::sys::path::is_absolute(*Path) && + *Path != llvm::sys::path::root_path(*Path); + }; + + if (Arg *A = Args.getLastArg(options::OPT_fdepscan_prefix_map_sdk_EQ)) { + if (IsPathApplicableAsPrefix(Sysroot)) { + CmdArgs.push_back(Args.MakeArgString(Twine("-fdepscan-prefix-map=") + + *Sysroot + "=" + A->getValue())); + } else { + // FIXME: warning if we cannot infer sdk + } + } + + if (Arg *A = Args.getLastArg(options::OPT_fdepscan_prefix_map_toolchain_EQ)) { + StringRef ResourceDir = D.ResourceDir; + StringRef Guess = llvm::sys::path::parent_path(ResourceDir); + for (StringRef Dir : {"clang", "lib", "usr"}) { + if (llvm::sys::path::filename(Guess) != Dir) + break; + Guess = llvm::sys::path::parent_path(Guess); + } + if (IsPathApplicableAsPrefix(Guess)) { + CmdArgs.push_back(Args.MakeArgString(Twine("-fdepscan-prefix-map=") + + Guess + "=" + A->getValue())); + } else { + // FIXME: warning if we cannot infer toolchain + } + } + + for (const Arg *A : Args.filtered(options::OPT_fdepscan_prefix_map_EQ)) { + A->claim(); + StringRef Map = A->getValue(); + StringRef Prefix = Map.split('=').first; + if (Prefix.size() == Map.size() || !IsPathApplicableAsPrefix(Prefix)) { + D.Diag(diag::err_drv_invalid_argument_to_option) + << A->getValue() << A->getOption().getName(); + } else { + A->render(Args, CmdArgs); + } + } +} + void Clang::ConstructJob(Compilation &C, const JobAction &Job, const InputInfo &Output, const InputInfoList &Inputs, const ArgList &Args, const char *LinkingOutput) const { @@ -4640,19 +4692,15 @@ void Clang::ConstructJob(Compilation &C, const JobAction &Job, D.Diag(diag::err_drv_clang_unsupported) << "C++ for IAMCU"; { - const OptSpecifier DepScanOpts[] = { - options::OPT_fdepscan_EQ, - options::OPT_fdepscan_include_tree, - options::OPT_fdepscan_share_EQ, - options::OPT_fdepscan_share_identifier, - options::OPT_fdepscan_share_parent, - options::OPT_fdepscan_share_parent_EQ, - options::OPT_fno_depscan_share, - options::OPT_fdepscan_share_stop_EQ, - options::OPT_fdepscan_daemon_EQ, - options::OPT_fdepscan_prefix_map_EQ, - options::OPT_fdepscan_prefix_map_sdk_EQ, - options::OPT_fdepscan_prefix_map_toolchain_EQ}; + const OptSpecifier DepScanOpts[] = {options::OPT_fdepscan_EQ, + options::OPT_fdepscan_include_tree, + options::OPT_fdepscan_share_EQ, + options::OPT_fdepscan_share_identifier, + options::OPT_fdepscan_share_parent, + options::OPT_fdepscan_share_parent_EQ, + options::OPT_fno_depscan_share, + options::OPT_fdepscan_share_stop_EQ, + options::OPT_fdepscan_daemon_EQ}; // Handle depscan. if (Job.getKind() == Action::DepscanJobClass) { @@ -5890,8 +5938,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &Job, // preprocessor. // // FIXME: Support -fpreprocessed + std::optional Sysroot; if (types::getPreprocessedType(InputType) != types::TY_INVALID) - AddPreprocessingOptions(C, JA, D, Args, CmdArgs, Output, Inputs); + AddPreprocessingOptions(C, JA, D, Args, CmdArgs, Output, Inputs, Sysroot); + + // Handle -fdepscan-prefix-map-* options + AddPrefixMappingOptions(Args, CmdArgs, D, Sysroot); // Don't warn about "clang -c -DPIC -fPIC test.i" because libtool.m4 assumes // that "The compiler can only warn and ignore the option if not recognized". diff --git a/clang/lib/Driver/ToolChains/Clang.h b/clang/lib/Driver/ToolChains/Clang.h index d7550665539cc..e5940b45f6620 100644 --- a/clang/lib/Driver/ToolChains/Clang.h +++ b/clang/lib/Driver/ToolChains/Clang.h @@ -43,7 +43,8 @@ class LLVM_LIBRARY_VISIBILITY Clang : public Tool { const Driver &D, const llvm::opt::ArgList &Args, llvm::opt::ArgStringList &CmdArgs, const InputInfo &Output, - const InputInfoList &Inputs) const; + const InputInfoList &Inputs, + std::optional &Sysroot) const; void RenderTargetOptions(const llvm::Triple &EffectiveTriple, const llvm::opt::ArgList &Args, bool KernelOrKext, @@ -104,6 +105,11 @@ class LLVM_LIBRARY_VISIBILITY Clang : public Tool { StringRef Dir, Compilation &C, StringRef Target, const InputInfo &Output, const InputInfo &Input, const llvm::opt::ArgList &Args) const; + void AddPrefixMappingOptions(const llvm::opt::ArgList &Args, + llvm::opt::ArgStringList &CmdArgs, + const Driver &D, + std::optional Sysroot) const; + public: Clang(const ToolChain &TC, bool HasIntegratedBackend = true); ~Clang() override; diff --git a/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp b/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp index e1a6167923be3..7f940c6336a2b 100644 --- a/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp @@ -22,8 +22,7 @@ namespace { class CASFSActionController : public CallbackActionController { public: CASFSActionController(LookupModuleOutputCallback LookupModuleOutput, - llvm::cas::CachingOnDiskFileSystem &CacheFS, - DepscanPrefixMapping PrefixMapping); + llvm::cas::CachingOnDiskFileSystem &CacheFS); llvm::Error initialize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) override; @@ -38,7 +37,6 @@ class CASFSActionController : public CallbackActionController { private: llvm::cas::CachingOnDiskFileSystem &CacheFS; - DepscanPrefixMapping PrefixMapping; std::optional Mapper; CASOptions CASOpts; }; @@ -46,17 +44,15 @@ class CASFSActionController : public CallbackActionController { CASFSActionController::CASFSActionController( LookupModuleOutputCallback LookupModuleOutput, - llvm::cas::CachingOnDiskFileSystem &CacheFS, - DepscanPrefixMapping PrefixMapping) - : CallbackActionController(std::move(LookupModuleOutput)), CacheFS(CacheFS), - PrefixMapping(std::move(PrefixMapping)) {} + llvm::cas::CachingOnDiskFileSystem &CacheFS) + : CallbackActionController(std::move(LookupModuleOutput)), + CacheFS(CacheFS) {} Error CASFSActionController::initialize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { // Setup prefix mapping. Mapper.emplace(&CacheFS); - if (Error E = PrefixMapping.configurePrefixMapper(NewInvocation, *Mapper)) - return E; + DepscanPrefixMapping::configurePrefixMapper(NewInvocation, *Mapper); const PreprocessorOptions &PPOpts = ScanInstance.getPreprocessorOpts(); if (!PPOpts.Includes.empty() || !PPOpts.ImplicitPCHInclude.empty()) @@ -197,8 +193,6 @@ Error CASFSActionController::finalizeModuleInvocation(CompilerInvocation &CI, std::unique_ptr dependencies::createCASFSActionController( LookupModuleOutputCallback LookupModuleOutput, - llvm::cas::CachingOnDiskFileSystem &CacheFS, - DepscanPrefixMapping PrefixMapping) { - return std::make_unique(LookupModuleOutput, CacheFS, - std::move(PrefixMapping)); + llvm::cas::CachingOnDiskFileSystem &CacheFS) { + return std::make_unique(LookupModuleOutput, CacheFS); } diff --git a/clang/lib/Tooling/DependencyScanning/CachingActions.h b/clang/lib/Tooling/DependencyScanning/CachingActions.h index 8dc3e33b61553..8cb39dc649641 100644 --- a/clang/lib/Tooling/DependencyScanning/CachingActions.h +++ b/clang/lib/Tooling/DependencyScanning/CachingActions.h @@ -20,13 +20,11 @@ namespace clang::tooling::dependencies { std::unique_ptr createIncludeTreeActionController(LookupModuleOutputCallback LookupModuleOutput, - cas::ObjectStore &DB, - DepscanPrefixMapping PrefixMapping); + cas::ObjectStore &DB); std::unique_ptr createCASFSActionController(LookupModuleOutputCallback LookupModuleOutput, - llvm::cas::CachingOnDiskFileSystem &CacheFS, - DepscanPrefixMapping PrefixMapping); + llvm::cas::CachingOnDiskFileSystem &CacheFS); /// The PCH recorded file paths with canonical paths, create a VFS that /// allows remapping back to the non-canonical source paths so that they are diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp index 90b92d3790cc2..2097a16bbd350 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -162,11 +162,9 @@ class GetIncludeTree : public EmptyDependencyConsumer { llvm::Expected DependencyScanningTool::getDependencyTree( - const std::vector &CommandLine, StringRef CWD, - DepscanPrefixMapping PrefixMapping) { + const std::vector &CommandLine, StringRef CWD) { GetDependencyTree Consumer(*Worker.getCASFS()); - auto Controller = createCASFSActionController(nullptr, *Worker.getCASFS(), - std::move(PrefixMapping)); + auto Controller = createCASFSActionController(nullptr, *Worker.getCASFS()); auto Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller); if (Result) @@ -178,10 +176,9 @@ llvm::Expected DependencyScanningTool::getDependencyTreeFromCompilerInvocation( std::shared_ptr Invocation, StringRef CWD, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, - bool DiagGenerationAsCompilation, DepscanPrefixMapping PrefixMapping) { + bool DiagGenerationAsCompilation) { GetDependencyTree Consumer(*Worker.getCASFS()); - auto Controller = createCASFSActionController(nullptr, *Worker.getCASFS(), - std::move(PrefixMapping)); + auto Controller = createCASFSActionController(nullptr, *Worker.getCASFS()); Worker.computeDependenciesFromCompilerInvocation( std::move(Invocation), CWD, Consumer, *Controller, DiagsConsumer, VerboseOS, DiagGenerationAsCompilation); @@ -190,11 +187,9 @@ DependencyScanningTool::getDependencyTreeFromCompilerInvocation( Expected DependencyScanningTool::getIncludeTree( cas::ObjectStore &DB, const std::vector &CommandLine, - StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, - const DepscanPrefixMapping &PrefixMapping) { + StringRef CWD, LookupModuleOutputCallback LookupModuleOutput) { GetIncludeTree Consumer(DB); - auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB, - std::move(PrefixMapping)); + auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB); llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller); if (Result) @@ -206,12 +201,10 @@ Expected DependencyScanningTool::getIncludeTreeFromCompilerInvocation( cas::ObjectStore &DB, std::shared_ptr Invocation, StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, - const DepscanPrefixMapping &PrefixMapping, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, bool DiagGenerationAsCompilation) { GetIncludeTree Consumer(DB); - auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB, - std::move(PrefixMapping)); + auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB); Worker.computeDependenciesFromCompilerInvocation( std::move(Invocation), CWD, Consumer, *Controller, DiagsConsumer, VerboseOS, DiagGenerationAsCompilation); @@ -222,11 +215,9 @@ llvm::Expected DependencyScanningTool::getTranslationUnitDependencies( const std::vector &CommandLine, StringRef CWD, const llvm::StringSet<> &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping) { + LookupModuleOutputCallback LookupModuleOutput) { FullDependencyConsumer Consumer(AlreadySeen); - auto Controller = - createActionController(LookupModuleOutput, std::move(PrefixMapping)); + auto Controller = createActionController(LookupModuleOutput); llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller); if (Result) @@ -237,11 +228,9 @@ DependencyScanningTool::getTranslationUnitDependencies( llvm::Expected DependencyScanningTool::getModuleDependencies( StringRef ModuleName, const std::vector &CommandLine, StringRef CWD, const llvm::StringSet<> &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping) { + LookupModuleOutputCallback LookupModuleOutput) { FullDependencyConsumer Consumer(AlreadySeen); - auto Controller = - createActionController(LookupModuleOutput, std::move(PrefixMapping)); + auto Controller = createActionController(LookupModuleOutput); llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller, ModuleName); if (Result) @@ -293,21 +282,17 @@ CallbackActionController::~CallbackActionController() {} std::unique_ptr DependencyScanningTool::createActionController( DependencyScanningWorker &Worker, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping) { + LookupModuleOutputCallback LookupModuleOutput) { if (Worker.getScanningFormat() == ScanningOutputFormat::FullIncludeTree) - return createIncludeTreeActionController( - LookupModuleOutput, *Worker.getCAS(), std::move(PrefixMapping)); + return createIncludeTreeActionController(LookupModuleOutput, + *Worker.getCAS()); if (auto CacheFS = Worker.getCASFS()) - return createCASFSActionController(LookupModuleOutput, *CacheFS, - std::move(PrefixMapping)); + return createCASFSActionController(LookupModuleOutput, *CacheFS); return std::make_unique(LookupModuleOutput); } std::unique_ptr DependencyScanningTool::createActionController( - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping) { - return createActionController(Worker, std::move(LookupModuleOutput), - std::move(PrefixMapping)); + LookupModuleOutputCallback LookupModuleOutput) { + return createActionController(Worker, std::move(LookupModuleOutput)); } diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index fcefb3e7d4ffc..d547429955506 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -176,17 +176,14 @@ class ReversePrefixMappingDependencyFileGenerator const DependencyOutputOptions &Opts) : DependencyFileGenerator(Opts) {} - Error initialize(const CompilerInvocation &CI, - const DepscanPrefixMapping &PrefixMapping) { + void initialize(const CompilerInvocation &CI) { llvm::PrefixMapper Mapper; - if (Error E = PrefixMapping.configurePrefixMapper(CI, Mapper)) - return E; + DepscanPrefixMapping::configurePrefixMapper(CI, Mapper); if (Mapper.empty()) - return Error::success(); + return; ReverseMapper.addInverseRange(Mapper.getMappings()); ReverseMapper.sort(); - return Error::success(); } void maybeAddDependency(StringRef Filename, bool FromModule, bool IsSystem, @@ -425,9 +422,7 @@ class DependencyScanningAction : public tooling::ToolAction { auto DFG = std::make_shared( *Opts); - if (auto *Mapping = Controller.getPrefixMapping()) - if (auto E = DFG->initialize(ScanInstance.getInvocation(), *Mapping)) - return reportError(std::move(E)); + DFG->initialize(ScanInstance.getInvocation()); ScanInstance.addDependencyCollector(std::move(DFG)); } diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 563afd75c68d4..1331e19be742e 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -25,10 +25,8 @@ class IncludeTreeBuilder; class IncludeTreeActionController : public CallbackActionController { public: IncludeTreeActionController(cas::ObjectStore &DB, - DepscanPrefixMapping PrefixMapping, LookupModuleOutputCallback LookupOutput) - : CallbackActionController(LookupOutput), DB(DB), - PrefixMapping(std::move(PrefixMapping)) {} + : CallbackActionController(LookupOutput), DB(DB) {} Expected getIncludeTree(); @@ -43,10 +41,6 @@ class IncludeTreeActionController : public CallbackActionController { Error finalizeModuleInvocation(CompilerInvocation &CI, const ModuleDeps &MD) override; - const DepscanPrefixMapping *getPrefixMapping() override { - return &PrefixMapping; - } - private: IncludeTreeBuilder ¤t() { assert(!BuilderStack.empty()); @@ -56,7 +50,6 @@ class IncludeTreeActionController : public CallbackActionController { private: cas::ObjectStore &DB; CASOptions CASOpts; - DepscanPrefixMapping PrefixMapping; llvm::PrefixMapper PrefixMapper; // IncludeTreePPCallbacks keeps a pointer to the current builder, so use a // pointer so the builder cannot move when resizing. @@ -296,9 +289,7 @@ Expected IncludeTreeActionController::getIncludeTree() { Error IncludeTreeActionController::initialize( CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { - if (Error E = - PrefixMapping.configurePrefixMapper(NewInvocation, PrefixMapper)) - return E; + DepscanPrefixMapping::configurePrefixMapper(NewInvocation, PrefixMapper); auto ensurePathRemapping = [&]() { if (PrefixMapper.empty()) @@ -822,8 +813,6 @@ IncludeTreeBuilder::createIncludeFile(StringRef Filename, std::unique_ptr dependencies::createIncludeTreeActionController( - LookupModuleOutputCallback LookupModuleOutput, cas::ObjectStore &DB, - DepscanPrefixMapping PrefixMapping) { - return std::make_unique( - DB, std::move(PrefixMapping), LookupModuleOutput); + LookupModuleOutputCallback LookupModuleOutput, cas::ObjectStore &DB) { + return std::make_unique(DB, LookupModuleOutput); } diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index 7076c3cdea57a..ff02143402c7d 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -179,66 +179,28 @@ void DepscanPrefixMapping::remapInvocationPaths(CompilerInvocation &Invocation, Mapper.mapInPlace(CodeGenOpts.ProfileRemappingFile); } -Error DepscanPrefixMapping::configurePrefixMapper( - const CompilerInvocation &Invocation, llvm::PrefixMapper &Mapper) const { - auto isPathApplicableAsPrefix = [](StringRef Path) -> bool { - if (Path.empty()) - return false; - if (llvm::sys::path::is_relative(Path)) - return false; - if (Path == llvm::sys::path::root_path(Path)) - return false; - return true; - }; +void DepscanPrefixMapping::configurePrefixMapper(const CompilerInvocation &CI, + llvm::PrefixMapper &Mapper) { + return configurePrefixMapper(CI.getFrontendOpts().PathPrefixMappings, Mapper); +} - const HeaderSearchOptions &HSOpts = Invocation.getHeaderSearchOpts(); +void DepscanPrefixMapping::configurePrefixMapper( + ArrayRef PathPrefixMappings, llvm::PrefixMapper &Mapper) { + if (PathPrefixMappings.empty()) + return; - if (NewSDKPath) { - StringRef SDK = HSOpts.Sysroot; - if (isPathApplicableAsPrefix(SDK)) - // Need a new copy of the string since the invocation will be modified. - Mapper.add({SDK, *NewSDKPath}); - } - if (NewToolchainPath) { - // Look up for the toolchain, assuming resources are at - // /usr/lib/clang/. Return a shallower guess if the - // directories do not match. - // - // FIXME: Should this append ".." instead of calling parent_path? - StringRef ResourceDir = HSOpts.ResourceDir; - StringRef Guess = llvm::sys::path::parent_path(ResourceDir); - for (StringRef Dir : {"clang", "lib", "usr"}) { - if (llvm::sys::path::filename(Guess) != Dir) - break; - Guess = llvm::sys::path::parent_path(Guess); - } - if (isPathApplicableAsPrefix(Guess)) - // Need a new copy of the string since the invocation will be modified. - Mapper.add({Guess, *NewToolchainPath}); - } - if (!PrefixMap.empty()) { - llvm::SmallVector Split; - llvm::MappedPrefix::transformJoinedIfValid(PrefixMap, Split); - for (auto &MappedPrefix : Split) { - if (isPathApplicableAsPrefix(MappedPrefix.Old)) { - Mapper.add(MappedPrefix); - } else { - return createStringError(llvm::errc::invalid_argument, - "invalid prefix map: '" + MappedPrefix.Old + - "=" + MappedPrefix.New + "'"); - } - } - } + llvm::SmallVector Split; + llvm::MappedPrefix::transformJoinedIfValid(PathPrefixMappings, Split); + for (auto &MappedPrefix : Split) + Mapper.add(MappedPrefix); Mapper.sort(); - return Error::success(); } Expected clang::scanAndUpdateCC1InlineWithTool( DependencyScanningTool &Tool, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, CompilerInvocation &Invocation, - StringRef WorkingDirectory, const DepscanPrefixMapping &PrefixMapping, - llvm::cas::ObjectStore &DB) { + StringRef WorkingDirectory, llvm::cas::ObjectStore &DB) { // Override the CASOptions. They may match (the caller having sniffed them // out of InputArgs) but if they have been overridden we want the new ones. Invocation.getCASOpts() = Tool.getCASOpts(); @@ -255,8 +217,7 @@ Expected clang::scanAndUpdateCC1InlineWithTool( Tool.getCachingFileSystem()); } llvm::PrefixMapper &Mapper = *MapperPtr; - if (Error E = PrefixMapping.configurePrefixMapper(Invocation, Mapper)) - return std::move(E); + DepscanPrefixMapping::configurePrefixMapper(Invocation, Mapper); auto ScanInvocation = std::make_shared(Invocation); // An error during dep-scanning is treated as if the main compilation has @@ -265,18 +226,18 @@ Expected clang::scanAndUpdateCC1InlineWithTool( Optional Root; if (ProduceIncludeTree) { - if (Error E = Tool.getIncludeTreeFromCompilerInvocation( - DB, std::move(ScanInvocation), WorkingDirectory, - /*LookupModuleOutput=*/nullptr, PrefixMapping, - DiagsConsumer, VerboseOS, - /*DiagGenerationAsCompilation*/ true) - .moveInto(Root)) + if (Error E = + Tool.getIncludeTreeFromCompilerInvocation( + DB, std::move(ScanInvocation), WorkingDirectory, + /*LookupModuleOutput=*/nullptr, DiagsConsumer, VerboseOS, + /*DiagGenerationAsCompilation*/ true) + .moveInto(Root)) return std::move(E); } else { if (Error E = Tool.getDependencyTreeFromCompilerInvocation( std::move(ScanInvocation), WorkingDirectory, DiagsConsumer, VerboseOS, - /*DiagGenerationAsCompilation*/ true, PrefixMapping) + /*DiagGenerationAsCompilation*/ true) .moveInto(Root)) return std::move(E); } diff --git a/clang/test/CAS/cached-diagnostics.c b/clang/test/CAS/cached-diagnostics.c index 509417333d382..6f11adf48f88a 100644 --- a/clang/test/CAS/cached-diagnostics.c +++ b/clang/test/CAS/cached-diagnostics.c @@ -4,8 +4,8 @@ // RUN: %clang_cc1 -triple x86_64-apple-macos12 -fsyntax-only %t/src/main.c -I %t/src/inc -Wunknown-pragmas 2> %t/regular-diags1.txt -// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -fdepscan-prefix-map=%t/src=/^src -o %t/t1.rsp -cc1-args \ -// RUN: -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas \ +// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t/t1.rsp -cc1-args \ +// RUN: -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas -fdepscan-prefix-map=%t/src=/^src \ // RUN: -emit-obj %t/src/main.c -o %t/out/output.o -I %t/src/inc -Wunknown-pragmas // Compare diagnostics after a miss. @@ -41,8 +41,8 @@ // RUN: %clang_cc1 -triple x86_64-apple-macos12 -fsyntax-only %t/src2/main.c -I %t/src2/inc -Wunknown-pragmas 2> %t/regular-diags2.txt -// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -fdepscan-prefix-map=%t/src2=/^src -o %t/t2.rsp -cc1-args \ -// RUN: -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas \ +// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t/t2.rsp -cc1-args \ +// RUN: -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas -fdepscan-prefix-map=%t/src2=/^src \ // RUN: -emit-obj %t/src2/main.c -o %t/out2/output.o -I %t/src2/inc -Wunknown-pragmas // RUN: %clang @%t/t2.rsp -Rcompile-job-cache 2> %t/diags-hit2.txt diff --git a/clang/test/CAS/depscan-prefix-map.c b/clang/test/CAS/depscan-prefix-map.c index 062d74b1f0742..1b5f6b5e1cdb0 100644 --- a/clang/test/CAS/depscan-prefix-map.c +++ b/clang/test/CAS/depscan-prefix-map.c @@ -5,46 +5,46 @@ // RUN: rm -rf %t.d // RUN: mkdir %t.d // RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=inline \ -// RUN: -fdepscan-prefix-map=%S=/^source \ -// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ -// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ -// RUN: -fdepscan-prefix-map-toolchain=/^toolchain \ -// RUN: -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ // RUN: -isysroot %S/Inputs/SDK \ // RUN: -resource-dir %S/Inputs/toolchain_dir/usr/lib/clang/1000 \ // RUN: -internal-isystem %S/Inputs/toolchain_dir/usr/lib/clang/1000/include \ // RUN: -working-directory %t.d \ // RUN: -fcas-path %t.d/cas \ +// RUN: -fdepscan-prefix-map=%S=/^source \ +// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ +// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: | FileCheck %s -DPREFIX=%t.d // RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=inline \ -// RUN: -fdepscan-prefix-map=%S=/^source \ -// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ -// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ -// RUN: -fdepscan-prefix-map-toolchain=/^toolchain \ -// RUN: -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ // RUN: -isysroot %S/Inputs/SDK \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 \ // RUN: -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -working-directory %t.d \ // RUN: -fcas-path %t.d/cas \ +// RUN: -fdepscan-prefix-map=%S=/^source \ +// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ +// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: | FileCheck %s -DPREFIX=%t.d // RUN: %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ // RUN: -cas-args -fcas-path %t.d/cas -- \ // RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=daemon \ // RUN: -fdepscan-daemon=%{clang-daemon-dir}/%basename_t \ -// RUN: -fdepscan-prefix-map=%S=/^source \ -// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ -// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ -// RUN: -fdepscan-prefix-map-toolchain=/^toolchain \ -// RUN: -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ // RUN: -isysroot %S/Inputs/SDK \ // RUN: -resource-dir %S/Inputs/toolchain_dir/usr/lib/clang/1000 \ // RUN: -internal-isystem %S/Inputs/toolchain_dir/usr/lib/clang/1000/include \ // RUN: -working-directory %t.d \ // RUN: -fcas-path %t.d/cas \ +// RUN: -fdepscan-prefix-map=%S=/^source \ +// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ +// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: | FileCheck %s -DPREFIX=%t.d // // CHECK: "-fcas-path" "[[PREFIX]]/cas" @@ -64,35 +64,5 @@ // CHECK-ROOT-NEXT: file {{.*}} /^source/depscan-prefix-map.c{{$}} // CHECK-ROOT-NEXT: file {{.*}} /^toolchain/usr/lib/clang/1000/include/stdarg.h{{$}} -// RUN: not %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ -// RUN: -cas-args -fcas-path %t.d/cas -- \ -// RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=daemon \ -// RUN: -fdepscan-daemon=%{clang-daemon-dir}/%basename_t \ -// RUN: -fdepscan-prefix-map=/=/^foo \ -// RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ -// RUN: -fcas-path %t.d/cas \ -// RUN: 2>&1 | FileCheck %s -DPREFIX=%t.d -check-prefix=ERROR_ROOT -// ERROR_ROOT: invalid prefix map: '/=/^foo' - -// RUN: not %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ -// RUN: -cas-args -fcas-path %t.d/cas -- \ -// RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=daemon \ -// RUN: -fdepscan-daemon=%{clang-daemon-dir}/%basename_t \ -// RUN: -fdepscan-prefix-map==/^foo \ -// RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ -// RUN: -fcas-path %t.d/cas \ -// RUN: 2>&1 | FileCheck %s -DPREFIX=%t.d -check-prefix=ERROR_EMPTY -// ERROR_EMPTY: invalid prefix map: '=/^foo' - -// RUN: not %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ -// RUN: -cas-args -fcas-path %t.d/cas -- \ -// RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=daemon \ -// RUN: -fdepscan-daemon=%{clang-daemon-dir}/%basename_t \ -// RUN: -fdepscan-prefix-map=relative=/^foo \ -// RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ -// RUN: -fcas-path %t.d/cas \ -// RUN: 2>&1 | FileCheck %s -DPREFIX=%t.d -check-prefix=ERROR_RELATIVE -// ERROR_RELATIVE: invalid prefix map: 'relative=/^foo' - #include int test() { return 0; } diff --git a/clang/test/CAS/driver-cache-launcher.c b/clang/test/CAS/driver-cache-launcher.c index e12f622830a5d..195e62c86d3ab 100644 --- a/clang/test/CAS/driver-cache-launcher.c +++ b/clang/test/CAS/driver-cache-launcher.c @@ -58,14 +58,15 @@ // RUN: cp -R %S/Inputs/cmake-build %t/cmake-build // RUN: pushd %t/cmake-build // RUN: cache-build-session -prefix-map-cmake -v echo 2>&1 | FileCheck %s -check-prefix=SESSION-CMAKE-PREFIX -// RUN: env LLVM_CACHE_CAS_PATH=%t/cas cache-build-session -prefix-map-cmake %clang-cache %clang -c %s -o %t.o -### 2>&1 | FileCheck %s -check-prefix=CLANG-CMAKE-PREFIX -DPREFIX=%t +// RUN: env LLVM_CACHE_CAS_PATH=%t/cas cache-build-session -prefix-map-cmake %clang-cache %clang -c %s -o %t.o -isysroot %S/Inputs/SDK -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -### 2>&1 | FileCheck %s -check-prefix=CLANG-CMAKE-PREFIX -DPREFIX=%t -DINPUTS=%S/Inputs // RUN: popd // SESSION-CMAKE-PREFIX: note: setting LLVM_CACHE_PREFIX_MAPS=/llvm/build=/^build;/llvm/llvm-project/llvm=/^src;/llvm/llvm-project/clang=/^src-clang;/llvm/llvm-project/clang-tools-extra=/^src-clang-tools-extra;/llvm/llvm-project/third-party/benchmark=/^src-benchmark;/llvm/llvm-project/other/benchmark=/^src-benchmark-1;/llvm/llvm-project/another/benchmark=/^src-benchmark-2{{$}} // SESSION-CMAKE-PREFIX: note: setting LLVM_CACHE_BUILD_SESSION_ID= // CLANG-CMAKE-PREFIX: "-cc1depscan" "-fdepscan=daemon" "-fdepscan-share-identifier" -// CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map-sdk=/^sdk" "-fdepscan-prefix-map-toolchain=/^toolchain" +// CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=[[INPUTS]]/SDK=/^sdk" +// CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=[[INPUTS]]/toolchain_dir=/^toolchain" // CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=/llvm/build=/^build" // CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=/llvm/llvm-project/llvm=/^src" // CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=/llvm/llvm-project/clang=/^src-clang" diff --git a/clang/test/CAS/fcas-include-tree-prefix-mapping.c b/clang/test/CAS/fcas-include-tree-prefix-mapping.c index d6de19be71139..088b8af61480d 100644 --- a/clang/test/CAS/fcas-include-tree-prefix-mapping.c +++ b/clang/test/CAS/fcas-include-tree-prefix-mapping.c @@ -3,11 +3,13 @@ // RUN: mkdir %t/out // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/t1.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out \ // RUN: -emit-llvm %t/src/main.c -o %t/out/output.ll -include %t/src/prefix.h -I %t/src/inc \ // RUN: -MT deps -dependency-file %t/t1.d @@ -41,11 +43,13 @@ // RUN: mkdir %t/out2 // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/t2.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out2 \ // RUN: -emit-llvm %t/src2/main.c -o %t/out2/output.ll -include %t/src2/prefix.h -I %t/src2/inc \ // RUN: -MT deps -dependency-file %t/t2.d @@ -77,22 +81,26 @@ // Check with PCH. // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/pch1.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out \ // RUN: -emit-pch -x c-header %t/src/prefix.h -o %t/out/prefix.h.pch -include %t/src/prefix.h -I %t/src/inc // RUN: %clang @%t/pch1.rsp // With different cas path to avoid cache hit. // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/pch2.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas2 -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out2 \ // RUN: -emit-pch -x c-header %t/src2/prefix.h -o %t/out2/prefix.h.pch -include %t/src2/prefix.h -I %t/src2/inc // RUN: %clang @%t/pch2.rsp @@ -100,11 +108,13 @@ // RUN: diff %t/out/prefix.h.pch %t/out2/prefix.h.pch // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/t3.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out \ // RUN: -emit-obj %t/src/main.c -o %t/out/main.o -include-pch %t/out/prefix.h.pch -I %t/src/inc \ // RUN: -MT deps -dependency-file %t/t1.pch.d @@ -115,11 +125,13 @@ // RUN: -e "s/' .*$//" > %t/cache-key3 // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/t4.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out2 \ // RUN: -emit-obj %t/src2/main.c -o %t/out2/main.o -include-pch %t/out2/prefix.h.pch -I %t/src2/inc \ // RUN: -MT deps -dependency-file %t/t2.pch.d diff --git a/clang/test/CAS/pgo-profile.c b/clang/test/CAS/pgo-profile.c index b64716c34cf98..387b3e8fa7f70 100644 --- a/clang/test/CAS/pgo-profile.c +++ b/clang/test/CAS/pgo-profile.c @@ -37,9 +37,9 @@ // RUN: mkdir -p %t.dir/a && mkdir -p %t.dir/b // RUN: cp %t.profdata %t.dir/a/a.profdata // RUN: cp %t.profdata %t.dir/b/a.profdata -// RUN: %clang -cc1depscan -fdepscan=inline -o %t4.rsp -fdepscan-prefix-map=%t.dir/a=/^testdir -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache \ +// RUN: %clang -cc1depscan -fdepscan=inline -o %t4.rsp -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache -fdepscan-prefix-map=%t.dir/a=/^testdir \ // RUN: -x c %s -o %t.o -fcas-path %t.dir/cas -fprofile-instrument-use-path=%t.dir/a/a.profdata -// RUN: %clang -cc1depscan -fdepscan=inline -o %t5.rsp -fdepscan-prefix-map=%t.dir/b=/^testdir -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache \ +// RUN: %clang -cc1depscan -fdepscan=inline -o %t5.rsp -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache -fdepscan-prefix-map=%t.dir/b=/^testdir \ // RUN: -x c %s -o %t.o -fcas-path %t.dir/cas -fprofile-instrument-use-path=%t.dir/b/a.profdata // RUN: cat %t4.rsp | FileCheck %s --check-prefix=REMAP // RUN: %clang @%t4.rsp 2>&1 | FileCheck %s --check-prefix=CACHE-MISS @@ -54,9 +54,9 @@ // RUN: grep llvmcas %t.dir/cache-key1 // RUN: diff -u %t.dir/cache-key1 %t.dir/cache-key2 -// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t4.inc.rsp -fdepscan-prefix-map=%t.dir/a=/^testdir -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache \ +// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t4.inc.rsp -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache -fdepscan-prefix-map=%t.dir/a=/^testdir \ // RUN: -x c %s -o %t.o -fcas-path %t.dir/cas -fprofile-instrument-use-path=%t.dir/a/a.profdata -// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t5.inc.rsp -fdepscan-prefix-map=%t.dir/b=/^testdir -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache \ +// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t5.inc.rsp -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache -fdepscan-prefix-map=%t.dir/b=/^testdir \ // RUN: -x c %s -o %t.o -fcas-path %t.dir/cas -fprofile-instrument-use-path=%t.dir/b/a.profdata // RUN: cat %t4.inc.rsp | FileCheck %s --check-prefix=REMAP // RUN: %clang @%t4.inc.rsp 2>&1 | FileCheck %s --check-prefix=CACHE-MISS diff --git a/clang/test/Driver/fdepscan-prefix-map-sdk.c b/clang/test/Driver/fdepscan-prefix-map-sdk.c new file mode 100644 index 0000000000000..421148c1a1a6d --- /dev/null +++ b/clang/test/Driver/fdepscan-prefix-map-sdk.c @@ -0,0 +1,8 @@ +// RUN: %clang -fdepscan-prefix-map-sdk=/^sdk -### %s 2>&1 | FileCheck %s -check-prefix=NONE +// RUN: %clang -fdepscan-prefix-map-sdk=/^sdk -isysroot relative -### %s 2>&1 | FileCheck %s -check-prefix=NONE + +// NONE-NOT: -fdepscan-prefix-map + +// RUN: %clang -fdepscan-prefix-map-sdk=/^sdk -isysroot /sys/path -### %s 2>&1 | FileCheck %s +// RUN: %clang -fdepscan-prefix-map-sdk=/^sdk --sysroot /sys/path -### %s 2>&1 | FileCheck %s +// CHECK: -fdepscan-prefix-map=/sys/path=/^sdk diff --git a/clang/test/Driver/fdepscan-prefix-map-toolchain.c b/clang/test/Driver/fdepscan-prefix-map-toolchain.c new file mode 100644 index 0000000000000..1bdbdfe1ccd78 --- /dev/null +++ b/clang/test/Driver/fdepscan-prefix-map-toolchain.c @@ -0,0 +1,15 @@ +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir '' -### %s 2>&1 | FileCheck %s -check-prefix=NONE +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir relative -### %s 2>&1 | FileCheck %s -check-prefix=NONE +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir /lib/clang/10 -### %s 2>&1 | FileCheck %s -check-prefix=NONE + +// NONE-NOT: -fdepscan-prefix-map + +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir /tc/10 -### %s 2>&1 | FileCheck %s +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir /tc/lib/clang/10 -### %s 2>&1 | FileCheck %s +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir /tc/usr/lib/clang/10 -### %s 2>&1 | FileCheck %s + +// CHECK: -fdepscan-prefix-map=/tc=/^tc + +// Implicit resource-dir +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -### %s 2>&1 | FileCheck %s -check-prefix=CHECK_IMPLICIT +// CHECK_IMPLICIT: -fdepscan-prefix-map={{.*}}=/^tc diff --git a/clang/test/Driver/fdepscan-prefix-map.c b/clang/test/Driver/fdepscan-prefix-map.c new file mode 100644 index 0000000000000..4c5c34bb98bfd --- /dev/null +++ b/clang/test/Driver/fdepscan-prefix-map.c @@ -0,0 +1,8 @@ +// RUN: %clang -fdepscan-prefix-map=/^bad -### %s 2>&1 | FileCheck %s -check-prefix=INVALID +// RUN: %clang -fdepscan-prefix-map==/^bad -### %s 2>&1 | FileCheck %s -check-prefix=INVALID +// RUN: %clang -fdepscan-prefix-map=relative=/^bad -### %s 2>&1 | FileCheck %s -check-prefix=INVALID +// RUN: %clang -fdepscan-prefix-map=/=/^bad -### %s 2>&1 | FileCheck %s -check-prefix=INVALID +// INVALID: error: invalid argument '{{.*}}/^bad' to -fdepscan-prefix-map= + +// RUN: %clang -fdepscan-prefix-map=/good=/^good -### %s 2>&1 | FileCheck %s +// CHECK: "-fdepscan-prefix-map=/good=/^good" diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index ecb3d04f80adf..761e7e5f35083 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -265,10 +265,8 @@ llvm::cl::opt Verbose("v", llvm::cl::Optional, static bool emitCompilationDBWithCASTreeArguments( std::shared_ptr DB, std::vector Inputs, - DiagnosticConsumer &DiagsConsumer, - const DepscanPrefixMapping &PrefixMapping, - DependencyScanningService &Service, llvm::ThreadPool &Pool, - llvm::raw_ostream &OS) { + DiagnosticConsumer &DiagsConsumer, DependencyScanningService &Service, + llvm::ThreadPool &Pool, llvm::raw_ostream &OS) { // Follow `-cc1depscan` and also ignore diagnostics. // FIXME: Seems not a good idea to do this.. @@ -329,7 +327,6 @@ static bool emitCompilationDBWithCASTreeArguments( tooling::dependencies::DependencyScanningTool &WorkerTool; DiagnosticConsumer &DiagsConsumer; StringRef CWD; - const DepscanPrefixMapping &PrefixMapping; SmallVectorImpl &OutputArgs; llvm::StringSaver &Saver; @@ -338,12 +335,10 @@ static bool emitCompilationDBWithCASTreeArguments( llvm::cas::ObjectStore &DB, tooling::dependencies::DependencyScanningTool &WorkerTool, DiagnosticConsumer &DiagsConsumer, StringRef CWD, - const DepscanPrefixMapping &PrefixMapping, SmallVectorImpl &OutputArgs, llvm::StringSaver &Saver) : DB(DB), WorkerTool(WorkerTool), DiagsConsumer(DiagsConsumer), - CWD(CWD), PrefixMapping(PrefixMapping), OutputArgs(OutputArgs), - Saver(Saver) {} + CWD(CWD), OutputArgs(OutputArgs), Saver(Saver) {} bool runInvocation(std::shared_ptr Invocation, @@ -352,7 +347,7 @@ static bool emitCompilationDBWithCASTreeArguments( DiagnosticConsumer *DiagConsumer) override { Expected Root = scanAndUpdateCC1InlineWithTool( WorkerTool, DiagsConsumer, /*VerboseOS*/ nullptr, *Invocation, - CWD, PrefixMapping, DB); + CWD, DB); if (!Root) { llvm::consumeError(Root.takeError()); return false; @@ -369,7 +364,7 @@ static bool emitCompilationDBWithCASTreeArguments( llvm::StringSaver &Saver = PerThreadStates[I]->Saver; OutputArgs.push_back(Saver.save(Input->CommandLine.front()).data()); ScanForCC1Action Action(*DB, WorkerTool, *IgnoringDiagsConsumer, CWD, - PrefixMapping, OutputArgs, Saver); + OutputArgs, Saver); llvm::IntrusiveRefCntPtr FileMgr = WorkerTool.getOrCreateFileManager(); @@ -867,6 +862,14 @@ int main(int argc, const char **argv) { } } AdjustedArgs.insert(AdjustedArgs.end(), FlagsEnd, Args.end()); + if (!PrefixMapToolchain.empty()) + AdjustedArgs.push_back("-fdepscan-prefix-map-toolchain=" + + PrefixMapToolchain); + if (!PrefixMapSDK.empty()) + AdjustedArgs.push_back("-fdepscan-prefix-map-sdk=" + PrefixMapSDK); + for (StringRef Map : PrefixMaps) { + AdjustedArgs.push_back("-fdepscan-prefix-map=" + std::string(Map)); + } return AdjustedArgs; }); @@ -898,13 +901,6 @@ int main(int argc, const char **argv) { FS = llvm::cantFail(llvm::cas::createCachingOnDiskFileSystem(*CAS)); } - DepscanPrefixMapping PrefixMapping; - if (!PrefixMapToolchain.empty()) - PrefixMapping.NewToolchainPath = PrefixMapToolchain; - if (!PrefixMapSDK.empty()) - PrefixMapping.NewSDKPath = PrefixMapSDK; - PrefixMapping.PrefixMap.append(PrefixMaps.begin(), PrefixMaps.end()); - DependencyScanningService Service(ScanMode, Format, CASOpts, CAS, Cache, FS, OptimizeArgs, EagerLoadModules); llvm::ThreadPool Pool(llvm::hardware_concurrency(NumThreads)); @@ -916,7 +912,7 @@ int main(int argc, const char **argv) { } return emitCompilationDBWithCASTreeArguments( CAS, AdjustingCompilations->getAllCompileCommands(), *DiagsConsumer, - PrefixMapping, Service, Pool, llvm::outs()); + Service, Pool, llvm::outs()); } std::vector> WorkerTools; @@ -959,7 +955,7 @@ int main(int argc, const char **argv) { << " files using " << Pool.getThreadCount() << " workers\n"; } for (unsigned I = 0; I < Pool.getThreadCount(); ++I) { - Pool.async([I, &CAS, &PrefixMapping, &Lock, &Index, &Inputs, &TreeResults, + Pool.async([I, &CAS, &Lock, &Index, &Inputs, &TreeResults, &HadErrors, &FD, &WorkerTools, &DependencyOS, &Errs]() { llvm::StringSet<> AlreadySeenModules; while (true) { @@ -996,28 +992,27 @@ int main(int argc, const char **argv) { Errs)) HadErrors = true; } else if (Format == ScanningOutputFormat::Tree) { - auto MaybeTree = WorkerTools[I]->getDependencyTree( - Input->CommandLine, CWD, PrefixMapping); + auto MaybeTree = + WorkerTools[I]->getDependencyTree(Input->CommandLine, CWD); std::unique_lock LockGuard(Lock); TreeResults.emplace_back(LocalIndex, std::move(Filename), std::move(MaybeTree)); } else if (Format == ScanningOutputFormat::IncludeTree) { auto MaybeTree = WorkerTools[I]->getIncludeTree( - *CAS, Input->CommandLine, CWD, LookupOutput, PrefixMapping); + *CAS, Input->CommandLine, CWD, LookupOutput); std::unique_lock LockGuard(Lock); TreeResults.emplace_back(LocalIndex, std::move(Filename), std::move(MaybeTree)); } else if (MaybeModuleName) { auto MaybeFullDeps = WorkerTools[I]->getModuleDependencies( *MaybeModuleName, Input->CommandLine, CWD, AlreadySeenModules, - LookupOutput, PrefixMapping); + LookupOutput); if (handleModuleResult(Filename, MaybeFullDeps, FD, LocalIndex, DependencyOS, Errs)) HadErrors = true; } else { auto MaybeTUDeps = WorkerTools[I]->getTranslationUnitDependencies( - Input->CommandLine, CWD, AlreadySeenModules, LookupOutput, - PrefixMapping); + Input->CommandLine, CWD, AlreadySeenModules, LookupOutput); if (handleTranslationUnitResult(Filename, MaybeTUDeps, FD, LocalIndex, DependencyOS, Errs)) HadErrors = true; diff --git a/clang/tools/driver/cc1depscanProtocol.cpp b/clang/tools/driver/cc1depscanProtocol.cpp index b65d77682c9c9..7e77f4b483f4f 100644 --- a/clang/tools/driver/cc1depscanProtocol.cpp +++ b/clang/tools/driver/cc1depscanProtocol.cpp @@ -280,73 +280,23 @@ llvm::Error CC1DepScanDProtocol::getArgs(llvm::StringSaver &Saver, return Error::success(); } -llvm::Error -CC1DepScanDProtocol::putCommand(StringRef WorkingDirectory, - ArrayRef Args, - const DepscanPrefixMapping &Mapping) { +llvm::Error CC1DepScanDProtocol::putCommand(StringRef WorkingDirectory, + ArrayRef Args) { if (llvm::Error E = putString(WorkingDirectory)) return E; if (llvm::Error E = putArgs(Args)) return E; - return putDepscanPrefixMapping(Mapping); + return llvm::Error::success(); } -llvm::Error CC1DepScanDProtocol::getCommand(llvm::StringSaver &Saver, - StringRef &WorkingDirectory, - SmallVectorImpl &Args, - DepscanPrefixMapping &Mapping) { +llvm::Error +CC1DepScanDProtocol::getCommand(llvm::StringSaver &Saver, + StringRef &WorkingDirectory, + SmallVectorImpl &Args) { if (llvm::Error E = getString(Saver, WorkingDirectory)) return E; if (llvm::Error E = getArgs(Saver, Args)) return E; - return getDepscanPrefixMapping(Saver, Mapping); -} - -llvm::Error CC1DepScanDProtocol::putDepscanPrefixMapping( - const DepscanPrefixMapping &Mapping) { - // Construct the message. - SmallString<256> FullMapping; - if (Mapping.NewSDKPath) - FullMapping.append(*Mapping.NewSDKPath); - FullMapping.push_back(0); - if (Mapping.NewToolchainPath) - FullMapping.append(*Mapping.NewToolchainPath); - FullMapping.push_back(0); - for (StringRef Map : Mapping.PrefixMap) { - FullMapping.append(Map); - FullMapping.push_back(0); - } - return putString(FullMapping); -} - -llvm::Error -CC1DepScanDProtocol::getDepscanPrefixMapping(llvm::StringSaver &Saver, - DepscanPrefixMapping &Mapping) { - StringRef FullMapping; - if (llvm::Error E = getString(Saver, FullMapping)) - return E; - - // Parse the mapping. - size_t Count = 0; - for (auto I = FullMapping.begin(), B = I, E = FullMapping.end(); I != E; - ++I) { - if (I != B && I[-1]) - continue; // Wait for null-terminator. - StringRef Map = I; - switch (Count++) { - case 0: - if (!Map.empty()) - Mapping.NewSDKPath = std::string(Map); - break; - case 1: - if (!Map.empty()) - Mapping.NewToolchainPath = std::string(Map); - break; - default: - Mapping.PrefixMap.push_back(std::string(Map)); - break; - } - } return llvm::Error::success(); } diff --git a/clang/tools/driver/cc1depscanProtocol.h b/clang/tools/driver/cc1depscanProtocol.h index 575ee4f6f36b6..2600306d391c9 100644 --- a/clang/tools/driver/cc1depscanProtocol.h +++ b/clang/tools/driver/cc1depscanProtocol.h @@ -21,14 +21,7 @@ #include // FIXME: Unix-only. Not portable. #include // FIXME: Unix-only. Not portable. -namespace clang { -namespace tooling::dependencies { -struct DepscanPrefixMapping; -} - -namespace cc1depscand { -using tooling::dependencies::DepscanPrefixMapping; - +namespace clang::cc1depscand { struct DepscanSharing { bool OnlyShareParent = false; bool ShareViaIdentifier = false; @@ -137,20 +130,14 @@ class CC1DepScanDProtocol { return putMessage(String.size(), String.begin()); } - llvm::Error putDepscanPrefixMapping(const DepscanPrefixMapping &Mapping); - llvm::Error getDepscanPrefixMapping(llvm::StringSaver &Saver, - DepscanPrefixMapping &Mapping); - llvm::Error putArgs(ArrayRef Args); llvm::Error getArgs(llvm::StringSaver &Saver, SmallVectorImpl &Args); llvm::Error putCommand(StringRef WorkingDirectory, - ArrayRef Args, - const DepscanPrefixMapping &Mapping); + ArrayRef Args); llvm::Error getCommand(llvm::StringSaver &Saver, StringRef &WorkingDirectory, - SmallVectorImpl &Args, - DepscanPrefixMapping &Mapping); + SmallVectorImpl &Args); llvm::Error putScanResultFailed(StringRef Reason); llvm::Error putScanResultSuccess(StringRef RootID, @@ -240,8 +227,7 @@ class ScanDaemon : public OpenSocket { explicit ScanDaemon(OpenSocket S) : OpenSocket(std::move(S)) {} }; -} // namespace cc1depscand -} // namespace clang +} // namespace clang::cc1depscand #endif /* LLVM_ON_UNIX */ #endif // LLVM_CLANG_TOOLS_DRIVER_CC1DEPSCANPROTOCOL_H diff --git a/clang/tools/driver/cc1depscan_main.cpp b/clang/tools/driver/cc1depscan_main.cpp index 030fb270fa02f..85abbe55e31dd 100644 --- a/clang/tools/driver/cc1depscan_main.cpp +++ b/clang/tools/driver/cc1depscan_main.cpp @@ -64,7 +64,6 @@ using namespace clang; using namespace llvm::opt; using cc1depscand::DepscanSharing; -using clang::tooling::dependencies::DepscanPrefixMapping; using llvm::Error; #define DEBUG_TYPE "cc1depscand" @@ -357,7 +356,6 @@ static Expected scanAndUpdateCC1Inline( const char *Exec, ArrayRef InputArgs, StringRef WorkingDirectory, SmallVectorImpl &OutputArgs, bool ProduceIncludeTree, bool &DiagnosticErrorOccurred, - const DepscanPrefixMapping &PrefixMapping, llvm::function_ref SaveArg, const CASOptions &CASOpts, std::shared_ptr DB, std::shared_ptr Cache); @@ -366,15 +364,14 @@ static Expected scanAndUpdateCC1InlineWithTool( tooling::dependencies::DependencyScanningTool &Tool, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, const char *Exec, ArrayRef InputArgs, StringRef WorkingDirectory, - SmallVectorImpl &OutputArgs, - const DepscanPrefixMapping &PrefixMapping, llvm::cas::ObjectStore &DB, + SmallVectorImpl &OutputArgs, llvm::cas::ObjectStore &DB, llvm::function_ref SaveArg); static llvm::Expected scanAndUpdateCC1UsingDaemon( const char *Exec, ArrayRef OldArgs, StringRef WorkingDirectory, SmallVectorImpl &NewArgs, - bool &DiagnosticErrorOccurred, const DepscanPrefixMapping &Mapping, - StringRef Path, const DepscanSharing &Sharing, + bool &DiagnosticErrorOccurred, StringRef Path, + const DepscanSharing &Sharing, llvm::function_ref SaveArg, llvm::cas::ObjectStore &CAS) { using namespace clang::cc1depscand; @@ -391,7 +388,7 @@ static llvm::Expected scanAndUpdateCC1UsingDaemon( CC1DepScanDProtocol Comms(*Daemon); // llvm::dbgs() << "sending request...\n"; - if (auto E = Comms.putCommand(WorkingDirectory, OldArgs, Mapping)) + if (auto E = Comms.putCommand(WorkingDirectory, OldArgs)) return std::move(E); llvm::BumpPtrAllocator Alloc; @@ -441,29 +438,6 @@ static void writeResponseFile(raw_ostream &OS, OS << "\n"; } -static DepscanPrefixMapping -parseCASFSAutoPrefixMappings(DiagnosticsEngine &Diag, const ArgList &Args) { - using namespace clang::driver; - DepscanPrefixMapping Mapping; - for (const Arg *A : Args.filtered(options::OPT_fdepscan_prefix_map_EQ)) { - StringRef Map = A->getValue(); - size_t Equals = Map.find('='); - if (Equals == StringRef::npos) - Diag.Report(diag::err_drv_invalid_argument_to_option) - << Map << A->getOption().getName(); - else - Mapping.PrefixMap.push_back(std::string(Map)); - A->claim(); - } - if (const Arg *A = Args.getLastArg(options::OPT_fdepscan_prefix_map_sdk_EQ)) - Mapping.NewSDKPath = A->getValue(); - if (const Arg *A = - Args.getLastArg(options::OPT_fdepscan_prefix_map_toolchain_EQ)) - Mapping.NewToolchainPath = A->getValue(); - - return Mapping; -} - static int scanAndUpdateCC1(const char *Exec, ArrayRef OldArgs, SmallVectorImpl &NewArgs, DiagnosticsEngine &Diag, @@ -531,8 +505,6 @@ static int scanAndUpdateCC1(const char *Exec, ArrayRef OldArgs, bool ProduceIncludeTree = Args.hasArg(options::OPT_fdepscan_include_tree); - DepscanPrefixMapping PrefixMapping = parseCASFSAutoPrefixMappings(Diag, Args); - auto SaveArg = [&Args](const Twine &T) { return Args.MakeArgString(T); }; CompilerInvocation::GenerateCASArgs(CASOpts, Sharing.CASArgs, SaveArg); if (ProduceIncludeTree) @@ -543,10 +515,10 @@ static int scanAndUpdateCC1(const char *Exec, ArrayRef OldArgs, if (Optional DaemonPath = makeDepscanDaemonPath(Mode, Sharing)) return scanAndUpdateCC1UsingDaemon( Exec, OldArgs, WorkingDirectory, NewArgs, DiagnosticErrorOccurred, - PrefixMapping, *DaemonPath, Sharing, SaveArg, *DB); + *DaemonPath, Sharing, SaveArg, *DB); return scanAndUpdateCC1Inline(Exec, OldArgs, WorkingDirectory, NewArgs, ProduceIncludeTree, DiagnosticErrorOccurred, - PrefixMapping, SaveArg, CASOpts, DB, Cache); + SaveArg, CASOpts, DB, Cache); }; if (llvm::Error E = ScanAndUpdate().moveInto(RootID)) { Diag.Report(diag::err_cas_depscan_failed) << std::move(E); @@ -943,9 +915,7 @@ int ScanServer::listen() { llvm::StringSaver Saver(Alloc); StringRef WorkingDirectory; SmallVector Args; - DepscanPrefixMapping PrefixMapping; - if (llvm::Error E = - Comms.getCommand(Saver, WorkingDirectory, Args, PrefixMapping)) { + if (llvm::Error E = Comms.getCommand(Saver, WorkingDirectory, Args)) { SharedOS.applyLocked([&](raw_ostream &OS) { OS << I << ": failed to get command\n"; logAllUnhandledErrors(std::move(E), OS); @@ -988,8 +958,7 @@ int ScanServer::listen() { SmallVector NewArgs; auto RootID = scanAndUpdateCC1InlineWithTool( *Tool, *DiagsConsumer, &DiagsOS, Argv0, Args, WorkingDirectory, - NewArgs, PrefixMapping, *CAS, - [&](const Twine &T) { return Saver.save(T).data(); }); + NewArgs, *CAS, [&](const Twine &T) { return Saver.save(T).data(); }); if (!RootID) { consumeError(Comms.putScanResultFailed(toString(RootID.takeError()))); SharedOS.applyLocked([&](raw_ostream &OS) { @@ -1071,8 +1040,7 @@ static Expected scanAndUpdateCC1InlineWithTool( tooling::dependencies::DependencyScanningTool &Tool, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, const char *Exec, ArrayRef InputArgs, StringRef WorkingDirectory, - SmallVectorImpl &OutputArgs, - const DepscanPrefixMapping &PrefixMapping, llvm::cas::ObjectStore &DB, + SmallVectorImpl &OutputArgs, llvm::cas::ObjectStore &DB, llvm::function_ref SaveArg) { DiagnosticsEngine Diags(new DiagnosticIDs(), new DiagnosticOptions()); Diags.setClient(&DiagsConsumer, /*ShouldOwnClient=*/false); @@ -1082,8 +1050,7 @@ static Expected scanAndUpdateCC1InlineWithTool( "failed to create compiler invocation"); Expected Root = scanAndUpdateCC1InlineWithTool( - Tool, DiagsConsumer, VerboseOS, *Invocation, WorkingDirectory, - PrefixMapping, DB); + Tool, DiagsConsumer, VerboseOS, *Invocation, WorkingDirectory, DB); if (!Root) return Root; @@ -1097,7 +1064,6 @@ static Expected scanAndUpdateCC1Inline( const char *Exec, ArrayRef InputArgs, StringRef WorkingDirectory, SmallVectorImpl &OutputArgs, bool ProduceIncludeTree, bool &DiagnosticErrorOccurred, - const DepscanPrefixMapping &PrefixMapping, llvm::function_ref SaveArg, const CASOptions &CASOpts, std::shared_ptr DB, std::shared_ptr Cache) { @@ -1128,7 +1094,7 @@ static Expected scanAndUpdateCC1Inline( auto Result = scanAndUpdateCC1InlineWithTool( Tool, *DiagsConsumer, /*VerboseOS*/ nullptr, Exec, InputArgs, - WorkingDirectory, OutputArgs, PrefixMapping, *DB, SaveArg); + WorkingDirectory, OutputArgs, *DB, SaveArg); DiagnosticErrorOccurred = DiagsConsumer->getNumErrors() != 0; return Result; } diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index 1514cba3f89c5..46121711aedd8 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -54,10 +54,8 @@ TEST(IncludeTree, IncludeTreeScan) { "-o" "t.cpp.o"}; Optional Root; - DepscanPrefixMapping PrefixMapping; ASSERT_THAT_ERROR( - ScanTool - .getIncludeTree(*DB, CommandLine, /*CWD*/ "", nullptr, PrefixMapping) + ScanTool.getIncludeTree(*DB, CommandLine, /*CWD*/ "", nullptr) .moveInto(Root), llvm::Succeeded()); From 3c0fb9fdba897867bbde804ef9d3e2cba595f434 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Tue, 4 Apr 2023 09:15:56 -0700 Subject: [PATCH 27/38] [libclang][cas] Update for prefix mapping API change --- clang/tools/libclang/CDependencies.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 4e202d2153282..6c96106e30a0f 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -216,7 +216,7 @@ static CXErrorCode getFullDependencies(DependencyScanningWorker *Worker, llvm::StringSet<> AlreadySeen; FullDependencyConsumer DepConsumer(AlreadySeen); auto Controller = DependencyScanningTool::createActionController( - *Worker, std::move(LookupOutput), /*PrefixMapping=*/{}); + *Worker, std::move(LookupOutput)); bool HasDiagConsumer = DiagConsumer; bool HasError = Error; From a8fedd5b670b492fefde1f5eae5d25670942ffcb Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Tue, 4 Apr 2023 09:17:24 -0700 Subject: [PATCH 28/38] [clang][cas] Add a depscan-prefix-map feature flag For rdar://106307646 --- clang/tools/driver/features.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clang/tools/driver/features.json b/clang/tools/driver/features.json index 770b73c613387..3989988e1bff0 100644 --- a/clang/tools/driver/features.json +++ b/clang/tools/driver/features.json @@ -15,6 +15,9 @@ { "name": "extract-api-ignores" }, + { + "name": "depscan-prefix-map" + }, { "name": "deployment-target-environment-variables", "value": [ From 4f89fbea180ac7654244d198856d7bcdd7cd4a06 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 30 Mar 2023 16:28:21 -0700 Subject: [PATCH 29/38] [clang][cas] Prefix map -fdepfile-entry For rdar://106307646 --- .../lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp | 8 ++++++++ clang/test/CAS/depscan-prefix-map.c | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index ff02143402c7d..88457d2f0a8e3 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -177,6 +177,14 @@ void DepscanPrefixMapping::remapInvocationPaths(CompilerInvocation &Invocation, Mapper.mapInPlace(CodeGenOpts.ProfileInstrumentUsePath); Mapper.mapInPlace(CodeGenOpts.SampleProfileFile); Mapper.mapInPlace(CodeGenOpts.ProfileRemappingFile); + + // Dependency output options. + // Note: these are not in the cache key, but they are in the module context + // hash, which indirectly impacts the cache key when importing a module. + // In the future we may change how -fmodule-file-cache-key works when + // remapping to avoid needing this. + for (auto &ExtraDep : Invocation.getDependencyOutputOpts().ExtraDeps) + Mapper.mapInPlace(ExtraDep.first); } void DepscanPrefixMapping::configurePrefixMapper(const CompilerInvocation &CI, diff --git a/clang/test/CAS/depscan-prefix-map.c b/clang/test/CAS/depscan-prefix-map.c index 1b5f6b5e1cdb0..25d1fa177b85b 100644 --- a/clang/test/CAS/depscan-prefix-map.c +++ b/clang/test/CAS/depscan-prefix-map.c @@ -16,6 +16,7 @@ // RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ // RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ // RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ +// RUN: -fdepfile-entry=%t.d/extra \ // RUN: | FileCheck %s -DPREFIX=%t.d // RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=inline \ // RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ @@ -29,6 +30,7 @@ // RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ // RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ // RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ +// RUN: -fdepfile-entry=%t.d/extra \ // RUN: | FileCheck %s -DPREFIX=%t.d // RUN: %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ // RUN: -cas-args -fcas-path %t.d/cas -- \ @@ -45,12 +47,14 @@ // RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ // RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ // RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ +// RUN: -fdepfile-entry=%t.d/extra \ // RUN: | FileCheck %s -DPREFIX=%t.d // // CHECK: "-fcas-path" "[[PREFIX]]/cas" // CHECK-SAME: "-working-directory" "/^testdir" // CHECK-SAME: "-x" "c" "/^source/depscan-prefix-map.c" // CHECK-SAME: "-isysroot" "/^sdk" +// CHECK-SAME: "-fdepfile-entry=/^testdir/extra" // RUN: llvm-cas --cas %t.d/cas --ls-tree-recursive @%t.root \ // RUN: | FileCheck %s -check-prefix=CHECK-ROOT From 5cff5c02fab8541f420e4d16e33e8a30fc333510 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Mon, 3 Apr 2023 15:22:20 -0700 Subject: [PATCH 30/38] [clang][modules] Handle explicit modules when checking for .Private -> _Private While we eventually want to remove the mapping from .Private to _Private modules, until we do, ensure that it behaves the same for explicit modules. rdar://107449872 Differential Revision: https://reviews.llvm.org/D147477 (cherry picked from commit 8ec36e6956cb03d80f3fee8e593808c43a8a1ec3) --- clang/lib/Frontend/CompilerInstance.cpp | 6 +++- ...implicit-private-with-submodule-explicit.m | 31 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 clang/test/Modules/implicit-private-with-submodule-explicit.m diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index f53c34d9bff87..96da881f2b0ea 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -2081,8 +2081,12 @@ CompilerInstance::loadModule(SourceLocation ImportLoc, PrivateModule, PP->getIdentifierInfo(Module->Name)->getTokenID()); PrivPath.push_back(std::make_pair(&II, Path[0].second)); + std::string FileName; + // If there is a modulemap module or prebuilt module, load it. if (PP->getHeaderSearchInfo().lookupModule(PrivateModule, ImportLoc, true, - !IsInclusionDirective)) + !IsInclusionDirective) || + selectModuleSource(nullptr, PrivateModule, FileName, BuiltModules, + PP->getHeaderSearchInfo()) != MS_ModuleNotFound) Sub = loadModule(ImportLoc, PrivPath, Visibility, IsInclusionDirective); if (Sub) { MapPrivateSubModToTopLevel = true; diff --git a/clang/test/Modules/implicit-private-with-submodule-explicit.m b/clang/test/Modules/implicit-private-with-submodule-explicit.m new file mode 100644 index 0000000000000..a2e9950ec3181 --- /dev/null +++ b/clang/test/Modules/implicit-private-with-submodule-explicit.m @@ -0,0 +1,31 @@ +// Checks that the use of .Private to refer to _Private modules works with an +// explicit module. + +// RUN: rm -rf %t +// RUN: split-file %s %t + +// RUN: %clang_cc1 -x objective-c -fmodules -fno-implicit-modules -emit-module -fmodule-name=A %t/module.modulemap -o %t/A.pcm +// RUN: %clang_cc1 -x objective-c -fmodules -fno-implicit-modules -emit-module -fmodule-name=A_Private %t/module.modulemap -o %t/A_Private.pcm + +// Check lazily-loaded module +// RUN: %clang_cc1 -x objective-c -verify -fmodules -fno-implicit-modules -fmodule-file=A=%t/A.pcm -fmodule-file=A_Private=%t/A_Private.pcm -fsyntax-only %t/tu.m + +// Check eagerly-loaded module +// RUN: %clang_cc1 -x objective-c -verify -fmodules -fno-implicit-modules -fmodule-file=%t/A.pcm -fmodule-file=%t/A_Private.pcm -fsyntax-only %t/tu.m + +//--- module.modulemap +module A { header "a.h" } +module A_Private { header "priv.h" } + +//--- a.h + +//--- priv.h +void priv(void); + +//--- tu.m +@import A.Private; // expected-warning{{no submodule named 'Private' in module 'A'; using top level 'A_Private'}} +// expected-note@*:* {{defined here}} + +void tu(void) { + priv(); +} \ No newline at end of file From 3409d7d91dade6b40dc9fcbc3e01926ba1f1aeec Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Tue, 21 Mar 2023 16:33:28 -0700 Subject: [PATCH 31/38] [libclang] Add APIs to report dependency scanning info as opaque objects This is useful so we can evolve the reported scanning information without breaking ABI. (cherry picked from commit 55779305c69335faba3e020faea98a56a1e05b62) --- clang/include/clang-c/CXString.h | 9 + clang/include/clang-c/Dependencies.h | 245 +++++++++++++++++++++++++ clang/tools/c-index-test/core_main.cpp | 135 ++++++++------ clang/tools/libclang/CDependencies.cpp | 235 +++++++++++++++++++++++- clang/tools/libclang/libclang.map | 22 +++ 5 files changed, 589 insertions(+), 57 deletions(-) diff --git a/clang/include/clang-c/CXString.h b/clang/include/clang-c/CXString.h index f117010c71a46..f132a2191a79e 100644 --- a/clang/include/clang-c/CXString.h +++ b/clang/include/clang-c/CXString.h @@ -16,6 +16,7 @@ #include "clang-c/ExternC.h" #include "clang-c/Platform.h" +#include LLVM_CLANG_C_EXTERN_C_BEGIN @@ -44,6 +45,14 @@ typedef struct { unsigned Count; } CXStringSet; +/** + * An array of C strings. + */ +typedef struct { + const char **Strings; + size_t Count; +} CXCStringArray; + /** * Retrieve the character data associated with the given string. */ diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index 2daf9f38b0fc5..ec04090365eec 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -365,6 +365,251 @@ clang_experimental_DependencyScannerWorker_getFileDependencies_v5( CXModuleLookupOutputCallback *MLO, unsigned Options, CXFileDependenciesList **Out, CXDiagnosticSet *OutDiags); +/** + * Output of \c clang_experimental_DependencyScannerWorker_getDepGraph. + */ +typedef struct CXOpaqueDepGraph *CXDepGraph; + +/** + * An individual module dependency that is part of an overall compilation + * \c CXDepGraph. + */ +typedef struct CXOpaqueDepGraphModule *CXDepGraphModule; + +/** + * An individual command-line invocation that is part of an overall compilation + * \c CXDepGraph. + */ +typedef struct CXOpaqueDepGraphTUCommand *CXDepGraphTUCommand; + +/** + * Settings to use for the + * \c clang_experimental_DependencyScannerWorker_getDepGraph action. + */ +typedef struct CXOpaqueDependencyScannerWorkerScanSettings + *CXDependencyScannerWorkerScanSettings; + +/** + * Creates a set of settings for + * \c clang_experimental_DependencyScannerWorker_getDepGraph action. + * Must be disposed with + * \c clang_experimental_DependencyScannerWorkerScanSettings_dispose. + * Memory for settings is not copied. Any provided pointers must be valid until + * the call to \c clang_experimental_DependencyScannerWorker_getDepGraph. + * + * \param argc the number of compiler invocation arguments (including argv[0]). + * \param argv the compiler driver invocation arguments (including argv[0]). + * \param ModuleName If non-null, the dependencies of the named module are + * returned. Otherwise, the dependencies of the whole + * translation unit are returned. + * \param WorkingDirectory the directory in which the invocation runs. + * \param MLOContext the context that will be passed to \c MLO each time it is + * called. + * \param MLO a callback that is called to determine the paths of output files + * for each module dependency. This may receive the same module on + * different workers. This should be NULL if + * \c clang_experimental_DependencyScannerService_create_v1 was + * called with \c CXDependencyMode_Flat. This callback will be called + * on the same thread that called \c + * clang_experimental_DependencyScannerWorker_getDepGraph. + */ +CINDEX_LINKAGE CXDependencyScannerWorkerScanSettings +clang_experimental_DependencyScannerWorkerScanSettings_create( + int argc, const char *const *argv, const char *ModuleName, + const char *WorkingDirectory, void *MLOContext, + CXModuleLookupOutputCallback *MLO); + +/** + * Dispose of a \c CXDependencyScannerWorkerScanSettings object. + */ +CINDEX_LINKAGE void + clang_experimental_DependencyScannerWorkerScanSettings_dispose( + CXDependencyScannerWorkerScanSettings); + +/** + * Produces the dependency graph for a particular compiler invocation. + * + * \param Settings object created via + * \c clang_experimental_DependencyScannerWorkerScanSettings_create. + * \param [out] Out A non-NULL pointer to store the resulting dependencies. The + * output must be freed by calling + * \c clang_experimental_DepGraph_dispose. + * + * \returns \c CXError_Success on success; otherwise a non-zero \c CXErrorCode + * indicating the kind of error. When returning \c CXError_Failure there will + * be a \c CXDepGraph object on \p Out that can be used to get diagnostics via + * \c clang_experimental_DepGraph_getDiagnostics. + */ +CINDEX_LINKAGE enum CXErrorCode +clang_experimental_DependencyScannerWorker_getDepGraph( + CXDependencyScannerWorker, CXDependencyScannerWorkerScanSettings Settings, + CXDepGraph *Out); + +/** + * Dispose of a \c CXDepGraph object. + */ +CINDEX_LINKAGE void clang_experimental_DepGraph_dispose(CXDepGraph); + +/** + * \returns the number of \c CXDepGraphModule objects in the graph. + */ +CINDEX_LINKAGE size_t clang_experimental_DepGraph_getNumModules(CXDepGraph); + +/** + * \returns the \c CXDepGraphModule object at the given \p Index. + * + * The \c CXDepGraphModule object is only valid to use while \c CXDepGraph is + * valid. Must be disposed with \c clang_experimental_DepGraphModule_dispose. + */ +CINDEX_LINKAGE CXDepGraphModule +clang_experimental_DepGraph_getModule(CXDepGraph, size_t Index); + +CINDEX_LINKAGE void clang_experimental_DepGraphModule_dispose(CXDepGraphModule); + +/** + * \returns the name of the module. This may include `:` for C++20 module + * partitions, or a header-name for C++20 header units. + * + * The string is only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE +const char *clang_experimental_DepGraphModule_getName(CXDepGraphModule); + +/** + * \returns the context hash of a module represents the set of compiler options + * that may make one version of a module incompatible from another. This + * includes things like language mode, predefined macros, header search paths, + * etc... + * + * Modules with the same name but a different \c ContextHash should be treated + * as separate modules for the purpose of a build. + * + * The string is only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE +const char *clang_experimental_DepGraphModule_getContextHash(CXDepGraphModule); + +/** + * \returns the path to the modulemap file which defines this module. If there's + * no modulemap (e.g. for a C++ module) returns \c NULL. + * + * This can be used to explicitly build this module. This file will + * additionally appear in \c FileDeps as a dependency. + * + * The string is only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE const char * + clang_experimental_DepGraphModule_getModuleMapPath(CXDepGraphModule); + +/** + * \returns the list of files which this module directly depends on. + * + * If any of these change then the module needs to be rebuilt. + * + * The strings are only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE CXCStringArray + clang_experimental_DepGraphModule_getFileDeps(CXDepGraphModule); + +/** + * \returns the list of modules which this module direct depends on. + * + * This does include the context hash. The format is + * `:` + * + * The strings are only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE CXCStringArray + clang_experimental_DepGraphModule_getModuleDeps(CXDepGraphModule); + +/** + * \returns the canonical command line to build this module. + * + * The strings are only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE CXCStringArray + clang_experimental_DepGraphModule_getBuildArguments(CXDepGraphModule); + +/** + * \returns the number \c CXDepGraphTUCommand objects in the graph. + */ +CINDEX_LINKAGE size_t clang_experimental_DepGraph_getNumTUCommands(CXDepGraph); + +/** + * \returns the \c CXDepGraphTUCommand object at the given \p Index. + * + * The \c CXDepGraphTUCommand object is only valid to use while \c CXDepGraph is + * valid. Must be disposed with \c clang_experimental_DepGraphTUCommand_dispose. + */ +CINDEX_LINKAGE CXDepGraphTUCommand +clang_experimental_DepGraph_getTUCommand(CXDepGraph, size_t Index); + +/** + * Dispose of a \c CXDepGraphTUCommand object. + */ +CINDEX_LINKAGE void + clang_experimental_DepGraphTUCommand_dispose(CXDepGraphTUCommand); + +/** + * \returns the executable name for the command. + * + * The string is only valid to use while the \c CXDepGraphTUCommand object is + * valid. + */ +CINDEX_LINKAGE const char * + clang_experimental_DepGraphTUCommand_getExecutable(CXDepGraphTUCommand); + +/** + * \returns the canonical command line to build this translation unit. + * + * The strings are only valid to use while the \c CXDepGraphTUCommand object is + * valid. + */ +CINDEX_LINKAGE CXCStringArray + clang_experimental_DepGraphTUCommand_getBuildArguments(CXDepGraphTUCommand); + +/** + * \returns the list of files which this translation unit directly depends on. + * + * The strings are only valid to use while the \c CXDepGraph object is valid. + */ +CINDEX_LINKAGE +CXCStringArray clang_experimental_DepGraph_getTUFileDeps(CXDepGraph); + +/** + * \returns the list of modules which this translation unit direct depends on. + * + * This does include the context hash. The format is + * `:` + * + * The strings are only valid to use while the \c CXDepGraph object is valid. + */ +CINDEX_LINKAGE +CXCStringArray clang_experimental_DepGraph_getTUModuleDeps(CXDepGraph); + +/** + * \returns the context hash of the C++20 module this translation unit exports. + * + * If the translation unit is not a module then this is empty. + * + * The string is only valid to use while the \c CXDepGraph object is valid. + */ +CINDEX_LINKAGE +const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph); + +/** + * \returns The diagnostics emitted during scanning. These must be always freed + * by calling \c clang_disposeDiagnosticSet. + */ +CINDEX_LINKAGE +CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph); + /** * @} */ diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index 2fa66355c8daf..e5512c9d33678 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -719,35 +719,6 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, clang_experimental_DependencyScannerService_dispose_v0(Service); }); - auto Callback = [&](CXModuleDependencySet *MDS) { - llvm::outs() << "modules:\n"; - for (const auto &M : llvm::makeArrayRef(MDS->Modules, MDS->Count)) { - llvm::outs() << " module:\n" - << " name: " << clang_getCString(M.Name) << "\n" - << " context-hash: " << clang_getCString(M.ContextHash) - << "\n" - << " module-map-path: " - << clang_getCString(M.ModuleMapPath) << "\n" - << " module-deps:\n"; - for (const auto &ModuleName : - llvm::makeArrayRef(M.ModuleDeps->Strings, M.ModuleDeps->Count)) - llvm::outs() << " " << clang_getCString(ModuleName) << "\n"; - llvm::outs() << " file-deps:\n"; - for (const auto &FileName : - llvm::makeArrayRef(M.FileDeps->Strings, M.FileDeps->Count)) - llvm::outs() << " " << clang_getCString(FileName) << "\n"; - llvm::outs() << " build-args:"; - for (const auto &Arg : llvm::makeArrayRef(M.BuildArguments->Strings, - M.BuildArguments->Count)) - llvm::outs() << " " << clang_getCString(Arg); - llvm::outs() << "\n"; - } - clang_experimental_ModuleDependencySet_dispose(MDS); - }; - - auto CB = - functionObjectToCCallbackRef(Callback); - auto LookupOutput = [&](const char *ModuleName, const char *ContextHash, CXOutputKind Kind, char *Output, size_t MaxLen) { std::string Out = OutputPath + "/" + ModuleName + "_" + ContextHash; @@ -781,46 +752,98 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, char *Output, size_t MaxLen)>(LookupOutput); unsigned CommandIndex = 0; - auto HandleCommand = [&](CXString ContextHash, CXStringSet *ModuleDeps, - CXStringSet *FileDeps, CXStringSet *Args) { + auto HandleCommand = [&](const char *ContextHash, CXCStringArray ModuleDeps, + CXCStringArray FileDeps, CXCStringArray Args) { llvm::outs() << " command " << CommandIndex++ << ":\n"; - llvm::outs() << " context-hash: " << clang_getCString(ContextHash) - << "\n" + llvm::outs() << " context-hash: " << ContextHash << "\n" << " module-deps:\n"; for (const auto &ModuleName : - llvm::makeArrayRef(ModuleDeps->Strings, ModuleDeps->Count)) - llvm::outs() << " " << clang_getCString(ModuleName) << "\n"; + llvm::makeArrayRef(ModuleDeps.Strings, ModuleDeps.Count)) + llvm::outs() << " " << ModuleName << "\n"; llvm::outs() << " file-deps:\n"; - for (const auto &FileName : - llvm::makeArrayRef(FileDeps->Strings, FileDeps->Count)) - llvm::outs() << " " << clang_getCString(FileName) << "\n"; + for (const auto &FileName : ArrayRef(FileDeps.Strings, FileDeps.Count)) + llvm::outs() << " " << FileName << "\n"; llvm::outs() << " build-args:"; - for (const auto &Arg : llvm::makeArrayRef(Args->Strings, Args->Count)) - llvm::outs() << " " << clang_getCString(Arg); + for (const auto &Arg : ArrayRef(Args.Strings, Args.Count)) + llvm::outs() << " " << Arg; llvm::outs() << "\n"; }; - CXFileDependenciesList *Result = nullptr; - CXDiagnosticSet Diags; - auto DisposeDiagnosticSet = - llvm::make_scope_exit([&]() { clang_disposeDiagnosticSet(Diags); }); - CXErrorCode Err = - clang_experimental_DependencyScannerWorker_getFileDependencies_v5( - Worker, Args.size(), Args.data(), - ModuleName ? ModuleName->c_str() : nullptr, WorkingDirectory.c_str(), - CB.Context, CB.Callback, LookupOutputCB.Context, - LookupOutputCB.Callback, - /*Options=*/0, &Result, &Diags); + CXDependencyScannerWorkerScanSettings ScanSettings = + clang_experimental_DependencyScannerWorkerScanSettings_create( + Args.size(), Args.data(), ModuleName ? ModuleName->c_str() : nullptr, + WorkingDirectory.c_str(), LookupOutputCB.Context, + LookupOutputCB.Callback); + auto DisposeScanSettings = llvm::make_scope_exit([&]() { + clang_experimental_DependencyScannerWorkerScanSettings_dispose( + ScanSettings); + }); + CXDepGraph Graph = nullptr; + auto DisposeDepGraph = llvm::make_scope_exit( + [&]() { clang_experimental_DepGraph_dispose(Graph); }); + CXErrorCode Err = clang_experimental_DependencyScannerWorker_getDepGraph( + Worker, ScanSettings, &Graph); + if (Err == CXError_Success) { + llvm::outs() << "modules:\n"; + for (size_t I = 0, E = clang_experimental_DepGraph_getNumModules(Graph); + I < E; ++I) { + CXDepGraphModule Mod = clang_experimental_DepGraph_getModule(Graph, I); + const char *Name = clang_experimental_DepGraphModule_getName(Mod); + const char *ContextHash = + clang_experimental_DepGraphModule_getContextHash(Mod); + const char *ModuleMapPath = + clang_experimental_DepGraphModule_getModuleMapPath(Mod); + CXCStringArray ModuleDeps = + clang_experimental_DepGraphModule_getModuleDeps(Mod); + CXCStringArray FileDeps = + clang_experimental_DepGraphModule_getFileDeps(Mod); + CXCStringArray BuildArguments = + clang_experimental_DepGraphModule_getBuildArguments(Mod); + auto Dispose = llvm::make_scope_exit( + [&]() { clang_experimental_DepGraphModule_dispose(Mod); }); + llvm::outs() << " module:\n" + << " name: " << Name << "\n" + << " context-hash: " << ContextHash << "\n" + << " module-map-path: " + << (ModuleMapPath ? ModuleMapPath : "") << "\n" + << " module-deps:\n"; + for (const auto &ModuleName : + ArrayRef(ModuleDeps.Strings, ModuleDeps.Count)) + llvm::outs() << " " << ModuleName << "\n"; + llvm::outs() << " file-deps:\n"; + for (const auto &FileName : ArrayRef(FileDeps.Strings, FileDeps.Count)) + llvm::outs() << " " << FileName << "\n"; + llvm::outs() << " build-args:"; + for (const auto &Arg : + ArrayRef(BuildArguments.Strings, BuildArguments.Count)) + llvm::outs() << " " << Arg; + llvm::outs() << "\n"; + } + llvm::outs() << "dependencies:\n"; - for (size_t I = 0; I < Result->NumCommands; ++I) - HandleCommand( - Result->Commands[I].ContextHash, Result->Commands[I].ModuleDeps, - Result->Commands[I].FileDeps, Result->Commands[I].BuildArguments); - clang_experimental_FileDependenciesList_dispose(Result); + const char *TUContextHash = + clang_experimental_DepGraph_getTUContextHash(Graph); + CXCStringArray TUModuleDeps = + clang_experimental_DepGraph_getTUModuleDeps(Graph); + CXCStringArray TUFileDeps = + clang_experimental_DepGraph_getTUFileDeps(Graph); + for (size_t I = 0, E = clang_experimental_DepGraph_getNumTUCommands(Graph); + I < E; ++I) { + CXDepGraphTUCommand Cmd = + clang_experimental_DepGraph_getTUCommand(Graph, I); + CXCStringArray Args = + clang_experimental_DepGraphTUCommand_getBuildArguments(Cmd); + auto Dispose = llvm::make_scope_exit( + [&]() { clang_experimental_DepGraphTUCommand_dispose(Cmd); }); + HandleCommand(TUContextHash, TUModuleDeps, TUFileDeps, Args); + } return 0; } llvm::errs() << "error: failed to get dependencies\n"; + CXDiagnosticSet Diags = clang_experimental_DepGraph_getDiagnostics(Graph); + auto DisposeDiagnosticSet = + llvm::make_scope_exit([&]() { clang_disposeDiagnosticSet(Diags); }); for (unsigned I = 0, N = clang_getNumDiagnosticsInSet(Diags); I < N; ++I) { CXDiagnostic Diag = clang_getDiagnosticInSet(Diags, I); CXString Spelling = clang_getDiagnosticSpelling(Diag); diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 6c96106e30a0f..81a3ca943f129 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -239,7 +239,7 @@ static CXErrorCode getFullDependencies(DependencyScanningWorker *Worker, TranslationUnitDeps TU = DepConsumer.takeTranslationUnitDeps(); - if (!TU.ModuleGraph.empty()) { + if (MDC && !TU.ModuleGraph.empty()) { CXModuleDependencySet *MDS = new CXModuleDependencySet; MDS->Count = TU.ModuleGraph.size(); MDS->Modules = new CXModuleDependency[MDS->Count]; @@ -385,6 +385,239 @@ CXErrorCode clang_experimental_DependencyScannerWorker_getFileDependencies_v5( return Result; } +namespace { + +struct DependencyScannerWorkerScanSettings { + int argc; + const char *const *argv; + const char *ModuleName; + const char *WorkingDirectory; + void *MLOContext; + CXModuleLookupOutputCallback *MLO; +}; + +struct CStringsManager { + SmallVector>> OwnedCStr; + SmallVector>> OwnedStdStr; + + /// Doesn't own the string contents. + CXCStringArray createCStringsRef(ArrayRef Strings) { + OwnedCStr.push_back(std::make_unique>()); + std::vector &CStrings = *OwnedCStr.back(); + CStrings.reserve(Strings.size()); + for (const auto &String : Strings) + CStrings.push_back(String.c_str()); + return {CStrings.data(), CStrings.size()}; + } + + /// Doesn't own the string contents. + CXCStringArray createCStringsRef(const llvm::StringSet<> &StringsUnordered) { + std::vector Strings; + + for (auto SI = StringsUnordered.begin(), SE = StringsUnordered.end(); + SI != SE; ++SI) + Strings.push_back(SI->getKey()); + + llvm::sort(Strings); + + OwnedCStr.push_back(std::make_unique>()); + std::vector &CStrings = *OwnedCStr.back(); + CStrings.reserve(Strings.size()); + for (const auto &String : Strings) + CStrings.push_back(String.data()); + return {CStrings.data(), CStrings.size()}; + } + + /// Gets ownership of string contents. + CXCStringArray createCStringsOwned(std::vector &&Strings) { + OwnedStdStr.push_back( + std::make_unique>(std::move(Strings))); + return createCStringsRef(*OwnedStdStr.back()); + } +}; + +struct DependencyGraph { + TranslationUnitDeps TUDeps; + CXDiagnosticSetDiagnosticConsumer DiagConsumer; + CStringsManager StrMgr{}; +}; + +struct DependencyGraphModule { + ModuleDeps *ModDeps; + CStringsManager StrMgr{}; +}; + +struct DependencyGraphTUCommand { + Command *TUCmd; + CStringsManager StrMgr{}; +}; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerWorkerScanSettings, + CXDependencyScannerWorkerScanSettings) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyGraph, CXDepGraph) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyGraphModule, CXDepGraphModule) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyGraphTUCommand, + CXDepGraphTUCommand) + +} // end anonymous namespace + +CXDependencyScannerWorkerScanSettings +clang_experimental_DependencyScannerWorkerScanSettings_create( + int argc, const char *const *argv, const char *ModuleName, + const char *WorkingDirectory, void *MLOContext, + CXModuleLookupOutputCallback *MLO) { + return wrap(new DependencyScannerWorkerScanSettings{ + argc, argv, ModuleName, WorkingDirectory, MLOContext, MLO}); +} + +void clang_experimental_DependencyScannerWorkerScanSettings_dispose( + CXDependencyScannerWorkerScanSettings Settings) { + delete unwrap(Settings); +} + +enum CXErrorCode clang_experimental_DependencyScannerWorker_getDepGraph( + CXDependencyScannerWorker W, + CXDependencyScannerWorkerScanSettings CXSettings, CXDepGraph *Out) { + DependencyScannerWorkerScanSettings &Settings = *unwrap(CXSettings); + int argc = Settings.argc; + const char *const *argv = Settings.argv; + const char *ModuleName = Settings.ModuleName; + const char *WorkingDirectory = Settings.WorkingDirectory; + void *MLOContext = Settings.MLOContext; + CXModuleLookupOutputCallback *MLO = Settings.MLO; + + OutputLookup OL(MLOContext, MLO); + auto LookupOutputs = [&](const ModuleID &ID, ModuleOutputKind MOK) { + return OL.lookupModuleOutput(ID, MOK); + }; + + if (!Out) + return CXError_InvalidArguments; + *Out = nullptr; + + DependencyGraph *DepGraph = new DependencyGraph(); + + CXErrorCode Result = getFileDependencies( + W, argc, argv, WorkingDirectory, /*MDC=*/nullptr, /*MDCContext=*/nullptr, + /*Error=*/nullptr, &DepGraph->DiagConsumer, LookupOutputs, + ModuleName ? Optional(ModuleName) : std::nullopt, + [&](TranslationUnitDeps TU) { DepGraph->TUDeps = std::move(TU); }); + + *Out = wrap(DepGraph); + return Result; +} + +void clang_experimental_DepGraph_dispose(CXDepGraph Graph) { + delete unwrap(Graph); +} + +size_t clang_experimental_DepGraph_getNumModules(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return TUDeps.ModuleGraph.size(); +} + +CXDepGraphModule clang_experimental_DepGraph_getModule(CXDepGraph Graph, + size_t Index) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return wrap(new DependencyGraphModule{&TUDeps.ModuleGraph[Index]}); +} + +void clang_experimental_DepGraphModule_dispose(CXDepGraphModule CXDepMod) { + delete unwrap(CXDepMod); +} + +const char * +clang_experimental_DepGraphModule_getName(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + return ModDeps.ID.ModuleName.c_str(); +} + +const char * +clang_experimental_DepGraphModule_getContextHash(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + return ModDeps.ID.ContextHash.c_str(); +} + +const char * +clang_experimental_DepGraphModule_getModuleMapPath(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + if (ModDeps.ClangModuleMapFile.empty()) + return nullptr; + return ModDeps.ClangModuleMapFile.c_str(); +} + +CXCStringArray +clang_experimental_DepGraphModule_getFileDeps(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + return unwrap(CXDepMod)->StrMgr.createCStringsRef(ModDeps.FileDeps); +} + +CXCStringArray +clang_experimental_DepGraphModule_getModuleDeps(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + std::vector Modules; + Modules.reserve(ModDeps.ClangModuleDeps.size()); + for (const ModuleID &MID : ModDeps.ClangModuleDeps) + Modules.push_back(MID.ModuleName + ":" + MID.ContextHash); + return unwrap(CXDepMod)->StrMgr.createCStringsOwned(std::move(Modules)); +} + +CXCStringArray +clang_experimental_DepGraphModule_getBuildArguments(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + return unwrap(CXDepMod)->StrMgr.createCStringsRef(ModDeps.BuildArguments); +} + +size_t clang_experimental_DepGraph_getNumTUCommands(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return TUDeps.Commands.size(); +} + +CXDepGraphTUCommand clang_experimental_DepGraph_getTUCommand(CXDepGraph Graph, + size_t Index) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return wrap(new DependencyGraphTUCommand{&TUDeps.Commands[Index]}); +} + +void clang_experimental_DepGraphTUCommand_dispose(CXDepGraphTUCommand CXCmd) { + delete unwrap(CXCmd); +} + +const char * +clang_experimental_DepGraphTUCommand_getExecutable(CXDepGraphTUCommand CXCmd) { + Command &TUCmd = *unwrap(CXCmd)->TUCmd; + return TUCmd.Executable.c_str(); +} + +CXCStringArray clang_experimental_DepGraphTUCommand_getBuildArguments( + CXDepGraphTUCommand CXCmd) { + Command &TUCmd = *unwrap(CXCmd)->TUCmd; + return unwrap(CXCmd)->StrMgr.createCStringsRef(TUCmd.Arguments); +} + +CXCStringArray clang_experimental_DepGraph_getTUFileDeps(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return unwrap(Graph)->StrMgr.createCStringsRef(TUDeps.FileDeps); +} + +CXCStringArray clang_experimental_DepGraph_getTUModuleDeps(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + std::vector Modules; + Modules.reserve(TUDeps.ClangModuleDeps.size()); + for (const ModuleID &MID : TUDeps.ClangModuleDeps) + Modules.push_back(MID.ModuleName + ":" + MID.ContextHash); + return unwrap(Graph)->StrMgr.createCStringsOwned(std::move(Modules)); +} + +const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return TUDeps.ID.ContextHash.c_str(); +} + +CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph Graph) { + return unwrap(Graph)->DiagConsumer.getDiagnosticSet(); +} + static std::string lookupModuleOutput(const ModuleID &ID, ModuleOutputKind MOK, void *MLOContext, CXModuleLookupOutputCallback *MLO) { diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index 6ba53c57c1f15..e9b5ffbc610e4 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -489,7 +489,29 @@ LLVM_16 { clang_experimental_DependencyScannerServiceOptions_setCASDatabases; clang_experimental_DependencyScannerServiceOptions_setDependencyMode; clang_experimental_DependencyScannerServiceOptions_setObjectStore; + clang_experimental_DependencyScannerWorker_getDepGraph; clang_experimental_DependencyScannerWorker_getFileDependencies_v5; + clang_experimental_DependencyScannerWorkerScanSettings_create; + clang_experimental_DependencyScannerWorkerScanSettings_dispose; + clang_experimental_DepGraph_dispose; + clang_experimental_DepGraph_getDiagnostics; + clang_experimental_DepGraph_getModule; + clang_experimental_DepGraph_getNumModules; + clang_experimental_DepGraph_getNumTUCommands; + clang_experimental_DepGraph_getTUCommand; + clang_experimental_DepGraph_getTUContextHash; + clang_experimental_DepGraph_getTUFileDeps; + clang_experimental_DepGraph_getTUModuleDeps; + clang_experimental_DepGraphModule_dispose; + clang_experimental_DepGraphModule_getBuildArguments; + clang_experimental_DepGraphModule_getContextHash; + clang_experimental_DepGraphModule_getFileDeps; + clang_experimental_DepGraphModule_getModuleDeps; + clang_experimental_DepGraphModule_getModuleMapPath; + clang_experimental_DepGraphModule_getName; + clang_experimental_DepGraphTUCommand_dispose; + clang_experimental_DepGraphTUCommand_getBuildArguments; + clang_experimental_DepGraphTUCommand_getExecutable; clang_getUnqualifiedType; clang_getNonReferenceType; clang_CXXMethod_isDeleted; From 0e9192b0647bafd23ab4ce2a5319b1b78d4476b1 Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Thu, 30 Mar 2023 17:54:57 -0700 Subject: [PATCH 32/38] [libclang/depscan] Fix use-after-free issue when using diagnostics of `clang_experimental_DepGraph_getDiagnostics` The `StoredDiagnostic`s captured from the `getFileDependencies()` call reference a `SourceManager` object that gets destroyed and is invalid to use for getting diagnostic location info. To address this, capture diagnostics as a serialized diagnostics buffer and "materialize" it for the `clang_experimental_DepGraph_getDiagnostics` call, by re-using existing libclang machinery for loading serialized diagnostics. Also delete `clang_experimental_DependencyScannerWorker_getFileDependencies_v5` since it's unsafe to use to get diagnostics and its functionality is superseeded by `clang_experimental_DependencyScannerWorker_getDepGraph`. rdar://105978877 (cherry picked from commit 397a30d9d20ed76388700605076da468d02dbf3b) --- clang/include/clang-c/Dependencies.h | 21 ++---- .../Frontend/SerializedDiagnosticReader.h | 3 + .../Frontend/SerializedDiagnosticPrinter.cpp | 3 + .../Frontend/SerializedDiagnosticReader.cpp | 7 +- clang/test/Index/Core/scan-deps-with-diags.m | 6 ++ clang/tools/c-index-test/core_main.cpp | 3 +- clang/tools/libclang/CDependencies.cpp | 70 +++++++------------ clang/tools/libclang/CXLoadedDiagnostic.cpp | 52 ++++++++++---- clang/tools/libclang/CXLoadedDiagnostic.h | 13 +++- clang/tools/libclang/libclang.map | 1 - 10 files changed, 99 insertions(+), 80 deletions(-) create mode 100644 clang/test/Index/Core/scan-deps-with-diags.m diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index ec04090365eec..955217fa3301a 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -310,18 +310,8 @@ typedef size_t CXModuleLookupOutputCallback(void *Context, char *Output, size_t MaxLen); /** - * See \c clang_experimental_DependencyScannerWorker_getFileDependencies_v5. - * Returns diagnostics in an unstructured CXString instead of CXDiagnosticSet. - */ -CINDEX_LINKAGE enum CXErrorCode -clang_experimental_DependencyScannerWorker_getFileDependencies_v4( - CXDependencyScannerWorker Worker, int argc, const char *const *argv, - const char *ModuleName, const char *WorkingDirectory, void *MDCContext, - CXModuleDiscoveredCallback *MDC, void *MLOContext, - CXModuleLookupOutputCallback *MLO, unsigned Options, - CXFileDependenciesList **Out, CXString *error); - -/** + * Deprecated, use \c clang_experimental_DependencyScannerWorker_getDepGraph. + * * Calculates the list of file dependencies for a particular compiler * invocation. * @@ -351,19 +341,18 @@ clang_experimental_DependencyScannerWorker_getFileDependencies_v4( * \param [out] Out A non-NULL pointer to store the resulting dependencies. The * output must be freed by calling * \c clang_experimental_FileDependenciesList_dispose. - * \param [out] OutDiags The diagnostics emitted during scanning. These must be - * always freed by calling \c clang_disposeDiagnosticSet. + * \param [out] error the error string to pass back to client (if any). * * \returns \c CXError_Success on success; otherwise a non-zero \c CXErrorCode * indicating the kind of error. */ CINDEX_LINKAGE enum CXErrorCode -clang_experimental_DependencyScannerWorker_getFileDependencies_v5( +clang_experimental_DependencyScannerWorker_getFileDependencies_v4( CXDependencyScannerWorker Worker, int argc, const char *const *argv, const char *ModuleName, const char *WorkingDirectory, void *MDCContext, CXModuleDiscoveredCallback *MDC, void *MLOContext, CXModuleLookupOutputCallback *MLO, unsigned Options, - CXFileDependenciesList **Out, CXDiagnosticSet *OutDiags); + CXFileDependenciesList **Out, CXString *error); /** * Output of \c clang_experimental_DependencyScannerWorker_getDepGraph. diff --git a/clang/include/clang/Frontend/SerializedDiagnosticReader.h b/clang/include/clang/Frontend/SerializedDiagnosticReader.h index be7d3a10a1705..1d95fd23c774d 100644 --- a/clang/include/clang/Frontend/SerializedDiagnosticReader.h +++ b/clang/include/clang/Frontend/SerializedDiagnosticReader.h @@ -65,6 +65,9 @@ class SerializedDiagnosticReader { /// Read the diagnostics in \c File std::error_code readDiagnostics(StringRef File); + /// Read the diagnostics in \c Buffer. + std::error_code readDiagnostics(llvm::MemoryBufferRef Buffer); + private: enum class Cursor; diff --git a/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp b/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp index 8d5039935aa7f..a75ad2bbe066d 100644 --- a/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp +++ b/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp @@ -608,6 +608,9 @@ void SDiagsWriter::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel, return; } + // Call base class to update diagnostic counts. + DiagnosticConsumer::HandleDiagnostic(DiagLevel, Info); + // Enter the block for a non-note diagnostic immediately, rather than waiting // for beginDiagnostic, in case associated notes are emitted before we get // there. diff --git a/clang/lib/Frontend/SerializedDiagnosticReader.cpp b/clang/lib/Frontend/SerializedDiagnosticReader.cpp index aca3f77c2de3e..984d039fe7820 100644 --- a/clang/lib/Frontend/SerializedDiagnosticReader.cpp +++ b/clang/lib/Frontend/SerializedDiagnosticReader.cpp @@ -34,7 +34,12 @@ std::error_code SerializedDiagnosticReader::readDiagnostics(StringRef File) { if (!Buffer) return SDError::CouldNotLoad; - llvm::BitstreamCursor Stream(**Buffer); + return readDiagnostics(**Buffer); +} + +std::error_code +SerializedDiagnosticReader::readDiagnostics(llvm::MemoryBufferRef Buffer) { + llvm::BitstreamCursor Stream(Buffer); Optional BlockInfo; if (Stream.AtEndOfStream()) diff --git a/clang/test/Index/Core/scan-deps-with-diags.m b/clang/test/Index/Core/scan-deps-with-diags.m new file mode 100644 index 0000000000000..a47d3646385ff --- /dev/null +++ b/clang/test/Index/Core/scan-deps-with-diags.m @@ -0,0 +1,6 @@ +// RUN: not c-index-test core --scan-deps %S -output-dir=%t -- \ +// RUN: %clang -c %s -o %t/t.o 2> %t.err.txt +// RUN: FileCheck -input-file=%t.err.txt %s + +// CHECK: [[@LINE+1]]:10: fatal error: 'not-existent.h' file not found +#include "not-existent.h" diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index e5512c9d33678..a7cade1af6ec2 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -846,7 +846,8 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, llvm::make_scope_exit([&]() { clang_disposeDiagnosticSet(Diags); }); for (unsigned I = 0, N = clang_getNumDiagnosticsInSet(Diags); I < N; ++I) { CXDiagnostic Diag = clang_getDiagnosticInSet(Diags, I); - CXString Spelling = clang_getDiagnosticSpelling(Diag); + CXString Spelling = + clang_formatDiagnostic(Diag, clang_defaultDiagnosticDisplayOptions()); llvm::errs() << clang_getCString(Spelling) << "\n"; clang_disposeString(Spelling); clang_disposeDiagnostic(Diag); diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 81a3ca943f129..222e878834634 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -13,11 +13,13 @@ #include "CASUtils.h" #include "CXDiagnosticSetDiagnosticConsumer.h" +#include "CXLoadedDiagnostic.h" #include "CXString.h" #include "clang-c/Dependencies.h" #include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/SerializedDiagnosticPrinter.h" #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" @@ -342,49 +344,6 @@ CXErrorCode clang_experimental_DependencyScannerWorker_getFileDependencies_v4( return Result; } -CXErrorCode clang_experimental_DependencyScannerWorker_getFileDependencies_v5( - CXDependencyScannerWorker W, int argc, const char *const *argv, - const char *ModuleName, const char *WorkingDirectory, void *MDCContext, - CXModuleDiscoveredCallback *MDC, void *MLOContext, - CXModuleLookupOutputCallback *MLO, unsigned, CXFileDependenciesList **Out, - CXDiagnosticSet *OutDiags) { - OutputLookup OL(MLOContext, MLO); - auto LookupOutputs = [&](const ModuleID &ID, ModuleOutputKind MOK) { - return OL.lookupModuleOutput(ID, MOK); - }; - - if (!Out) - return CXError_InvalidArguments; - *Out = nullptr; - - CXDiagnosticSetDiagnosticConsumer DiagConsumer; - - CXErrorCode Result = getFileDependencies( - W, argc, argv, WorkingDirectory, MDC, MDCContext, nullptr, &DiagConsumer, - LookupOutputs, ModuleName ? Optional(ModuleName) : None, - [&](TranslationUnitDeps TU) { - assert(TU.DriverCommandLine.empty()); - std::vector Modules; - for (const ModuleID &MID : TU.ClangModuleDeps) - Modules.push_back(MID.ModuleName + ":" + MID.ContextHash); - auto *Commands = new CXTranslationUnitCommand[TU.Commands.size()]; - for (size_t I = 0, E = TU.Commands.size(); I < E; ++I) { - Commands[I].ContextHash = cxstring::createDup(TU.ID.ContextHash); - Commands[I].FileDeps = cxstring::createSet(TU.FileDeps); - Commands[I].ModuleDeps = cxstring::createSet(Modules); - Commands[I].Executable = - cxstring::createDup(TU.Commands[I].Executable); - Commands[I].BuildArguments = - cxstring::createSet(TU.Commands[I].Arguments); - } - *Out = new CXFileDependenciesList{TU.Commands.size(), Commands}; - }); - - *OutDiags = DiagConsumer.getDiagnosticSet(); - - return Result; -} - namespace { struct DependencyScannerWorkerScanSettings { @@ -438,8 +397,18 @@ struct CStringsManager { struct DependencyGraph { TranslationUnitDeps TUDeps; - CXDiagnosticSetDiagnosticConsumer DiagConsumer; + SmallString<256> SerialDiagBuf; CStringsManager StrMgr{}; + + CXDiagnosticSet getDiagnosticSet() const { + CXLoadDiag_Error Error; + CXString ErrorString; + CXDiagnosticSet DiagSet = loadCXDiagnosticsFromBuffer( + llvm::MemoryBufferRef(SerialDiagBuf, ""), &Error, &ErrorString); + assert(Error == CXLoadDiag_None); + clang_disposeString(ErrorString); + return DiagSet; + } }; struct DependencyGraphModule { @@ -497,9 +466,18 @@ enum CXErrorCode clang_experimental_DependencyScannerWorker_getDepGraph( DependencyGraph *DepGraph = new DependencyGraph(); + // We capture diagnostics as a serialized diagnostics buffer, so that we don't + // need to keep a valid SourceManager in order to access diagnostic locations. + auto DiagOpts = llvm::makeIntrusiveRefCnt(); + auto DiagOS = + std::make_unique(DepGraph->SerialDiagBuf); + std::unique_ptr SerialDiagConsumer = + serialized_diags::create("", DiagOpts.get(), + /*MergeChildRecords=*/false, std::move(DiagOS)); + CXErrorCode Result = getFileDependencies( W, argc, argv, WorkingDirectory, /*MDC=*/nullptr, /*MDCContext=*/nullptr, - /*Error=*/nullptr, &DepGraph->DiagConsumer, LookupOutputs, + /*Error=*/nullptr, SerialDiagConsumer.get(), LookupOutputs, ModuleName ? Optional(ModuleName) : std::nullopt, [&](TranslationUnitDeps TU) { DepGraph->TUDeps = std::move(TU); }); @@ -615,7 +593,7 @@ const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph Graph) { } CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph Graph) { - return unwrap(Graph)->DiagConsumer.getDiagnosticSet(); + return unwrap(Graph)->getDiagnosticSet(); } static std::string lookupModuleOutput(const ModuleID &ID, ModuleOutputKind MOK, diff --git a/clang/tools/libclang/CXLoadedDiagnostic.cpp b/clang/tools/libclang/CXLoadedDiagnostic.cpp index 3d2b3047f47b5..6481109a17824 100644 --- a/clang/tools/libclang/CXLoadedDiagnostic.cpp +++ b/clang/tools/libclang/CXLoadedDiagnostic.cpp @@ -249,6 +249,9 @@ class DiagLoader : serialized_diags::SerializedDiagnosticReader { } CXDiagnosticSet load(const char *file); + CXDiagnosticSet load(llvm::MemoryBufferRef Buffer); + + CXDiagnosticSet reportError(std::error_code EC); }; } // end anonymous namespace @@ -256,24 +259,38 @@ CXDiagnosticSet DiagLoader::load(const char *file) { TopDiags = std::make_unique(); std::error_code EC = readDiagnostics(file); - if (EC) { - switch (EC.value()) { - case static_cast(serialized_diags::SDError::HandlerFailed): - // We've already reported the problem. - break; - case static_cast(serialized_diags::SDError::CouldNotLoad): - reportBad(CXLoadDiag_CannotLoad, EC.message()); - break; - default: - reportInvalidFile(EC.message()); - break; - } - return nullptr; - } + if (EC) + return reportError(EC); + + return (CXDiagnosticSet)TopDiags.release(); +} + +CXDiagnosticSet DiagLoader::load(llvm::MemoryBufferRef Buffer) { + TopDiags = std::make_unique(); + + std::error_code EC = readDiagnostics(Buffer); + if (EC) + return reportError(EC); return (CXDiagnosticSet)TopDiags.release(); } +CXDiagnosticSet DiagLoader::reportError(std::error_code EC) { + assert(EC); + switch (EC.value()) { + case static_cast(serialized_diags::SDError::HandlerFailed): + // We've already reported the problem. + break; + case static_cast(serialized_diags::SDError::CouldNotLoad): + reportBad(CXLoadDiag_CannotLoad, EC.message()); + break; + default: + reportInvalidFile(EC.message()); + break; + } + return nullptr; +} + std::error_code DiagLoader::readLocation(const serialized_diags::Location &SDLoc, CXLoadedDiagnostic::Location &LoadedLoc) { @@ -420,3 +437,10 @@ CXDiagnosticSet clang_loadDiagnostics(const char *file, DiagLoader L(error, errorString); return L.load(file); } + +CXDiagnosticSet clang::loadCXDiagnosticsFromBuffer(llvm::MemoryBufferRef buffer, + enum CXLoadDiag_Error *error, + CXString *errorString) { + DiagLoader L(error, errorString); + return L.load(buffer); +} diff --git a/clang/tools/libclang/CXLoadedDiagnostic.h b/clang/tools/libclang/CXLoadedDiagnostic.h index 93995d7bb798c..eb76dfbfe11b5 100644 --- a/clang/tools/libclang/CXLoadedDiagnostic.h +++ b/clang/tools/libclang/CXLoadedDiagnostic.h @@ -19,6 +19,10 @@ #include "clang/Basic/LLVM.h" #include +namespace llvm { +class MemoryBufferRef; +} + namespace clang { class CXLoadedDiagnostic : public CXDiagnosticImpl { public: @@ -88,6 +92,13 @@ class CXLoadedDiagnostic : public CXDiagnosticImpl { unsigned severity; unsigned category; }; -} + +/// Read a serialized diagnostics \p buffer and create a \c CXDiagnosticSet +/// object for the loaded diagnostics. +CXDiagnosticSet loadCXDiagnosticsFromBuffer(llvm::MemoryBufferRef buffer, + enum CXLoadDiag_Error *error, + CXString *errorString); + +} // namespace clang #endif diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index e9b5ffbc610e4..00ded6d77e700 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -490,7 +490,6 @@ LLVM_16 { clang_experimental_DependencyScannerServiceOptions_setDependencyMode; clang_experimental_DependencyScannerServiceOptions_setObjectStore; clang_experimental_DependencyScannerWorker_getDepGraph; - clang_experimental_DependencyScannerWorker_getFileDependencies_v5; clang_experimental_DependencyScannerWorkerScanSettings_create; clang_experimental_DependencyScannerWorkerScanSettings_dispose; clang_experimental_DepGraph_dispose; From 6a5c8b794773dec530822f4b12be0b231532b86b Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Mon, 20 Mar 2023 17:10:51 -0700 Subject: [PATCH 33/38] [clang/cas/dep-scan] Include the translation unit cache key in the scanning output Also add translation unit and module cache keys in the json output of `clang-scan-deps` for testing purposes. (cherry picked from commit c1784bd8f6ecc21eb41e820b77e65deafaded82a) --- .../DependencyScanningWorker.h | 3 ++ .../DependencyScanningWorker.cpp | 17 +++++++++-- clang/test/ClangScanDeps/cas-trees.c | 28 ++++++++++++++++--- clang/test/ClangScanDeps/include-tree.c | 13 ++++++++- .../modules-cas-full-by-mod-name.c | 13 +++++++-- clang/test/ClangScanDeps/modules-cas-trees.c | 10 +++++-- clang/tools/clang-scan-deps/ClangScanDeps.cpp | 4 +++ 7 files changed, 76 insertions(+), 12 deletions(-) diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h index 3ddfc2f761492..eb944a1046213 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -38,6 +38,9 @@ class DependencyScanningWorkerFilesystem; struct Command { std::string Executable; std::vector Arguments; + + /// The \c ActionCache key for this translation unit, if any. + std::optional TUCacheKey; }; class DependencyConsumer { diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index d547429955506..fe63145171747 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -15,6 +15,7 @@ #include "clang/Driver/Driver.h" #include "clang/Driver/Job.h" #include "clang/Driver/Tool.h" +#include "clang/Frontend/CompileJobCacheKey.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/FrontendActions.h" @@ -479,6 +480,13 @@ class DependencyScanningAction : public tooling::ToolAction { LastCC1Arguments = OriginalInvocation.getCC1CommandLine(); + if (ScanInstance.getFrontendOpts().CacheCompileJob) { + auto &CAS = ScanInstance.getOrCreateObjectStore(); + if (auto Key = createCompileJobCacheKey( + CAS, ScanInstance.getDiagnostics(), OriginalInvocation)) + TUCacheKey = Key->toString(); + } + return true; } @@ -493,6 +501,8 @@ class DependencyScanningAction : public tooling::ToolAction { return Result; } + const std::optional &getTUCacheKey() const { return TUCacheKey; } + IntrusiveRefCntPtr getDepScanFS() { if (DepFS) { assert(!DepCASFS && "CAS DepFS should not be set"); @@ -523,6 +533,7 @@ class DependencyScanningAction : public tooling::ToolAction { Optional ScanInstanceStorage; std::shared_ptr MDC; std::vector LastCC1Arguments; + std::optional TUCacheKey; bool Scanned = false; raw_ostream *VerboseOS; }; @@ -701,7 +712,8 @@ bool DependencyScanningWorker::computeDependencies( // consumer. Consumer.handleBuildCommand( {Cmd.getExecutable(), - {Cmd.getArguments().begin(), Cmd.getArguments().end()}}); + {Cmd.getArguments().begin(), Cmd.getArguments().end()}, + /*TUCacheKey=*/std::nullopt}); return true; } @@ -722,7 +734,8 @@ bool DependencyScanningWorker::computeDependencies( return false; std::vector Args = Action.takeLastCC1Arguments(); - Consumer.handleBuildCommand({Cmd.getExecutable(), std::move(Args)}); + Consumer.handleBuildCommand( + {Cmd.getExecutable(), std::move(Args), Action.getTUCacheKey()}); return true; }); diff --git a/clang/test/ClangScanDeps/cas-trees.c b/clang/test/ClangScanDeps/cas-trees.c index c54f9a276e8de..47dbac6c47e44 100644 --- a/clang/test/ClangScanDeps/cas-trees.c +++ b/clang/test/ClangScanDeps/cas-trees.c @@ -18,7 +18,8 @@ // FULL-TREE-NEXT: "modules": [], // FULL-TREE-NEXT: "translation-units": [ // FULL-TREE-NEXT: { -// FULL-TREE: "casfs-root-id": "[[T1_ROOT_ID:llvmcas://[[:xdigit:]]+]]" +// FULL-TREE: "cache-key": "[[T1_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// FULL-TREE-NEXT: "casfs-root-id": "[[T1_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // FULL-TREE-NEXT: "clang-context-hash": "{{[A-Z0-9]+}}", // FULL-TREE-NEXT: "clang-module-deps": [], // FULL-TREE-NEXT: "command-line": [ @@ -36,7 +37,8 @@ // FULL-TREE-NEXT: "input-file": "[[PREFIX]]/t1.c" // FULL-TREE-NEXT: } // FULL-TREE: { -// FULL-TREE: "casfs-root-id": "[[T2_ROOT_ID:llvmcas://[[:xdigit:]]+]]" +// FULL-TREE: "cache-key": "[[T2_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// FULL-TREE-NEXT: "casfs-root-id": "[[T2_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // FULL-TREE-NEXT: "clang-context-hash": "{{[A-Z0-9]+}}", // FULL-TREE-NEXT: "clang-module-deps": [], // FULL-TREE-NEXT: "command-line": [ @@ -56,13 +58,31 @@ // Build with caching // RUN: %deps-to-rsp %t/full_result.json --tu-index 0 > %t/t1.cc1.rsp // RUN: %deps-to-rsp %t/full_result.json --tu-index 1 > %t/t2.cc1.rsp -// RUN: %clang @%t/t1.cc1.rsp -Rcompile-job-cache 2>&1 | FileCheck %s -check-prefix=CACHE-MISS +// RUN: %clang @%t/t1.cc1.rsp -Rcompile-job-cache 2> %t/t1-miss.err +// RUN: FileCheck %s -input-file=%t/t1-miss.err -check-prefix=CACHE-MISS // RUN: %clang @%t/t1.cc1.rsp -Rcompile-job-cache 2>&1 | FileCheck %s -check-prefix=CACHE-HIT -// RUN: %clang @%t/t2.cc1.rsp -Rcompile-job-cache 2>&1 | FileCheck %s -check-prefix=CACHE-MISS +// RUN: %clang @%t/t2.cc1.rsp -Rcompile-job-cache 2> %t/t2-miss.err +// RUN: FileCheck %s -input-file=%t/t2-miss.err -check-prefix=CACHE-MISS // RUN: %clang @%t/t2.cc1.rsp -Rcompile-job-cache 2>&1 | FileCheck %s -check-prefix=CACHE-HIT // CACHE-HIT: remark: compile job cache hit // CACHE-MISS: remark: compile job cache miss +// Check cache keys. +// RUN: cp %t/full_result.json %t/combined.txt +// RUN: cat %t/t1-miss.err >> %t/combined.txt +// RUN: cat %t/t2-miss.err >> %t/combined.txt +// RUN: FileCheck %s -input-file=%t/combined.txt -check-prefix=COMBINED + +// COMBINED: "commands": [ +// COMBINED-NEXT: { +// COMBINED-NEXT: "cache-key": "[[T1_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// COMBINED: } +// COMBINED: "commands": [ +// COMBINED: { +// COMBINED-NEXT: "cache-key": "[[T2_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// COMBINED: remark: compile job cache miss for '[[T1_CACHE_KEY]]' +// COMBINED-NEXT: remark: compile job cache miss for '[[T2_CACHE_KEY]]' + // RUN: clang-scan-deps -compilation-database %t/cdb.json -cas-path %t/cas -format experimental-tree -emit-cas-compdb | FileCheck %s -DPREFIX=%/t -check-prefix=COMPDB // COMPDB: [ // COMPDB: { diff --git a/clang/test/ClangScanDeps/include-tree.c b/clang/test/ClangScanDeps/include-tree.c index eed9b3c784015..d21c408650156 100644 --- a/clang/test/ClangScanDeps/include-tree.c +++ b/clang/test/ClangScanDeps/include-tree.c @@ -48,6 +48,7 @@ // FULL-NEXT: { // FULL-NEXT: "commands": [ // FULL-NEXT: { +// FULL-NEXT: "cache-key": "[[TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // FULL: "clang-module-deps": [] // FULL: "command-line": [ // FULL-NEXT: "-cc1" @@ -84,7 +85,17 @@ // Build the include-tree command // RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp -// RUN: %clang @%t/tu.rsp +// RUN: %clang @%t/tu.rsp -Rcompile-job-cache 2> %t/t.err + +// Check cache key. +// RUN: cp %t/full.txt %t/combined.txt +// RUN: cat %t/t.err >> %t/combined.txt +// RUN: FileCheck %s -input-file=%t/combined.txt -check-prefix=COMBINED + +// COMBINED: "commands": [ +// COMBINED-NEXT: { +// COMBINED-NEXT: "cache-key": "[[TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// COMBINED: remark: compile job cache miss for '[[TU_CACHE_KEY]]' //--- cdb.json.template [ diff --git a/clang/test/ClangScanDeps/modules-cas-full-by-mod-name.c b/clang/test/ClangScanDeps/modules-cas-full-by-mod-name.c index 11a32b40cb1b0..c6ad2068c6c97 100644 --- a/clang/test/ClangScanDeps/modules-cas-full-by-mod-name.c +++ b/clang/test/ClangScanDeps/modules-cas-full-by-mod-name.c @@ -35,6 +35,7 @@ module transitive { header "transitive.h" } // CHECK: { // CHECK-NEXT: "modules": [ // CHECK-NEXT: { +// CHECK-NEXT: "cache-key": "[[DIRECT_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK-NEXT: "casfs-root-id": "[[LEFT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK-NEXT: "clang-module-deps": [ // CHECK-NEXT: { @@ -44,6 +45,9 @@ module transitive { header "transitive.h" } // CHECK-NEXT: ], // CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", // CHECK-NEXT: "command-line": [ +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "{{.*transitive-.*\.pcm}}" +// CHECK-NEXT: "[[TRANSITIVE_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: ], // CHECK-NEXT: "context-hash": "{{.*}}", // CHECK-NEXT: "file-deps": [ @@ -53,7 +57,8 @@ module transitive { header "transitive.h" } // CHECK-NEXT: "name": "direct" // CHECK-NEXT: }, // CHECK-NEXT: { -// CHECK-NEXT: "casfs-root-id": "[[LEFT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" +// CHECK-NEXT: "cache-key": "[[ROOT_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// CHECK-NEXT: "casfs-root-id": "[[ROOT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK-NEXT: "clang-module-deps": [ // CHECK-NEXT: { // CHECK-NEXT: "context-hash": "{{.*}}", @@ -62,6 +67,9 @@ module transitive { header "transitive.h" } // CHECK-NEXT: ], // CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", // CHECK-NEXT: "command-line": [ +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "{{.*direct-.*\.pcm}}" +// CHECK-NEXT: "[[DIRECT_CACHE_KEY]]" // CHECK: ], // CHECK-NEXT: "context-hash": "{{.*}}", // CHECK-NEXT: "file-deps": [ @@ -72,7 +80,8 @@ module transitive { header "transitive.h" } // CHECK-NEXT: "name": "root" // CHECK-NEXT: }, // CHECK-NEXT: { -// CHECK-NEXT: "casfs-root-id": "[[LEFT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" +// CHECK-NEXT: "cache-key": "[[TRANSITIVE_CACHE_KEY]]" +// CHECK-NEXT: "casfs-root-id": "[[LEFT_ROOT_ID]]" // CHECK-NEXT: "clang-module-deps": [], // CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", // CHECK-NEXT: "command-line": [ diff --git a/clang/test/ClangScanDeps/modules-cas-trees.c b/clang/test/ClangScanDeps/modules-cas-trees.c index f3d6cf578e01c..9aad19541aaef 100644 --- a/clang/test/ClangScanDeps/modules-cas-trees.c +++ b/clang/test/ClangScanDeps/modules-cas-trees.c @@ -64,6 +64,7 @@ // CHECK: { // CHECK-NEXT: "modules": [ // CHECK-NEXT: { +// CHECK: "cache-key": "[[LEFT_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: "casfs-root-id": "[[LEFT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK: "clang-module-deps": [ // CHECK: { @@ -92,6 +93,7 @@ // CHECK: "name": "Left" // CHECK: } // CHECK-NEXT: { +// CHECK: "cache-key": "[[RIGHT_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: "casfs-root-id": "[[RIGHT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK: "clang-module-deps": [ // CHECK: { @@ -110,7 +112,7 @@ // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" // CHECK: "[[TOP_PCM]]" -// CHECK: "[[TOP_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// CHECK: "[[TOP_CACHE_KEY]]" // CHECK: "-fmodule-file={{(Top=)?}}[[TOP_PCM]]" // CHECK: ] // CHECK: "file-deps": [ @@ -120,6 +122,7 @@ // CHECK: "name": "Right" // CHECK: } // CHECK-NEXT: { +// CHECK: "cache-key": "[[TOP_CACHE_KEY]]" // CHECK: "casfs-root-id": "[[TOP_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK: "clang-module-deps": [] // CHECK: "command-line": [ @@ -144,6 +147,7 @@ // CHECK: { // CHECK: "commands": [ // CHECK: { +// CHECK: "cache-key": "[[TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: "casfs-root-id": "[[TU_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK: "clang-module-deps": [ // CHECK: { @@ -162,10 +166,10 @@ // CHECK: "-fcache-compile-job" // CHECK: "-fmodule-file-cache-key" // CHECK: "[[LEFT_PCM]]" -// CHECK: "{{llvmcas://[[:xdigit:]]+}}" +// CHECK: "[[LEFT_CACHE_KEY]]" // CHECK: "-fmodule-file-cache-key" // CHECK: "[[RIGHT_PCM]]" -// CHECK: "{{llvmcas://[[:xdigit:]]+}}" +// CHECK: "[[RIGHT_CACHE_KEY]]" // CHECK: "-fmodule-file={{(Left=)?}}[[LEFT_PCM]]" // CHECK: "-fmodule-file={{(Right=)?}}[[RIGHT_PCM]]" // CHECK: ] diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index 761e7e5f35083..9858575bbbb20 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -618,6 +618,8 @@ class FullDeps { {"clang-modulemap-file", MD.ClangModuleMapFile}, {"command-line", MD.BuildArguments}, }; + if (MD.ModuleCacheKey) + O.try_emplace("cache-key", MD.ModuleCacheKey); if (MD.CASFileSystemRootID) O.try_emplace("casfs-root-id", MD.CASFileSystemRootID->toString()); if (MD.IncludeTreeID) @@ -638,6 +640,8 @@ class FullDeps { {"executable", Cmd.Executable}, {"command-line", Cmd.Arguments}, }; + if (Cmd.TUCacheKey) + O.try_emplace("cache-key", *Cmd.TUCacheKey); if (I.CASFileSystemRootID) O.try_emplace("casfs-root-id", I.CASFileSystemRootID); if (I.IncludeTreeID) From fa6ed105ca3ba864165260d0993655453a5c83fa Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Wed, 22 Mar 2023 16:09:30 -0700 Subject: [PATCH 34/38] [libclang][cas] Expose the module and TU cache keys from the libclang dep-scan API (cherry picked from commit b5ca7144d571408eb402d6017256955d442eea4b) --- clang/include/clang-c/Dependencies.h | 12 ++++++++++++ clang/test/Index/Core/scan-deps-cas.m | 3 +++ clang/tools/c-index-test/core_main.cpp | 21 +++++++++++++++------ clang/tools/libclang/CDependencies.cpp | 16 ++++++++++++++++ clang/tools/libclang/libclang.map | 2 ++ 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index 955217fa3301a..2334495605fe6 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -525,6 +525,12 @@ CINDEX_LINKAGE CXCStringArray CINDEX_LINKAGE CXCStringArray clang_experimental_DepGraphModule_getBuildArguments(CXDepGraphModule); +/** + * \returns the \c ActionCache key for this module, if any. + */ +CINDEX_LINKAGE +const char *clang_experimental_DepGraphModule_getCacheKey(CXDepGraphModule); + /** * \returns the number \c CXDepGraphTUCommand objects in the graph. */ @@ -563,6 +569,12 @@ CINDEX_LINKAGE const char * CINDEX_LINKAGE CXCStringArray clang_experimental_DepGraphTUCommand_getBuildArguments(CXDepGraphTUCommand); +/** + * \returns the \c ActionCache key for this translation unit, if any. + */ +CINDEX_LINKAGE const char * + clang_experimental_DepGraphTUCommand_getCacheKey(CXDepGraphTUCommand); + /** * \returns the list of files which this translation unit directly depends on. * diff --git a/clang/test/Index/Core/scan-deps-cas.m b/clang/test/Index/Core/scan-deps-cas.m index 54b13da1e025f..e9909433e7547 100644 --- a/clang/test/Index/Core/scan-deps-cas.m +++ b/clang/test/Index/Core/scan-deps-cas.m @@ -33,6 +33,7 @@ // CHECK-NEXT: name: ModA // CHECK-NEXT: context-hash: [[HASH_MOD_A:[A-Z0-9]+]] // CHECK-NEXT: module-map-path: [[PREFIX]]/Inputs/module/module.modulemap +// CHECK-NEXT: cache-key: [[CASFS_MODA_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // CHECK-NEXT: module-deps: // CHECK-NEXT: file-deps: // CHECK-NEXT: [[PREFIX]]/Inputs/module/ModA.h @@ -51,6 +52,7 @@ // CHECK-NEXT: dependencies: // CHECK-NEXT: command 0: // CHECK-NEXT: context-hash: [[HASH_TU:[A-Z0-9]+]] +// CHECK-NEXT: cache-key: [[CASFS_TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // CHECK-NEXT: module-deps: // CHECK-NEXT: ModA:[[HASH_MOD_A]] // CHECK-NEXT: file-deps: @@ -66,6 +68,7 @@ // INCLUDE_TREE: dependencies: // INCLUDE_TREE-NEXT: command 0: // INCLUDE_TREE-NEXT: context-hash: [[HASH_TU:[A-Z0-9]+]] +// INCLUDE_TREE-NEXT: cache-key: [[INC_TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // INCLUDE_TREE-NEXT: module-deps: // INCLUDE_TREE-NEXT: file-deps: // INCLUDE_TREE-NEXT: [[PREFIX]]/scan-deps-cas.m diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index a7cade1af6ec2..d9aa02fce3228 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -753,10 +753,13 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, unsigned CommandIndex = 0; auto HandleCommand = [&](const char *ContextHash, CXCStringArray ModuleDeps, - CXCStringArray FileDeps, CXCStringArray Args) { + CXCStringArray FileDeps, CXCStringArray Args, + const char *CacheKey) { llvm::outs() << " command " << CommandIndex++ << ":\n"; - llvm::outs() << " context-hash: " << ContextHash << "\n" - << " module-deps:\n"; + llvm::outs() << " context-hash: " << ContextHash << "\n"; + if (CacheKey) + llvm::outs() << " cache-key: " << CacheKey << "\n"; + llvm::outs() << " module-deps:\n"; for (const auto &ModuleName : llvm::makeArrayRef(ModuleDeps.Strings, ModuleDeps.Count)) llvm::outs() << " " << ModuleName << "\n"; @@ -794,6 +797,8 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, clang_experimental_DepGraphModule_getContextHash(Mod); const char *ModuleMapPath = clang_experimental_DepGraphModule_getModuleMapPath(Mod); + const char *ModuleCacheKey = + clang_experimental_DepGraphModule_getCacheKey(Mod); CXCStringArray ModuleDeps = clang_experimental_DepGraphModule_getModuleDeps(Mod); CXCStringArray FileDeps = @@ -806,8 +811,10 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, << " name: " << Name << "\n" << " context-hash: " << ContextHash << "\n" << " module-map-path: " - << (ModuleMapPath ? ModuleMapPath : "") << "\n" - << " module-deps:\n"; + << (ModuleMapPath ? ModuleMapPath : "") << "\n"; + if (ModuleCacheKey) + llvm::outs() << " cache-key: " << ModuleCacheKey << "\n"; + llvm::outs() << " module-deps:\n"; for (const auto &ModuleName : ArrayRef(ModuleDeps.Strings, ModuleDeps.Count)) llvm::outs() << " " << ModuleName << "\n"; @@ -834,9 +841,11 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, clang_experimental_DepGraph_getTUCommand(Graph, I); CXCStringArray Args = clang_experimental_DepGraphTUCommand_getBuildArguments(Cmd); + const char *CacheKey = + clang_experimental_DepGraphTUCommand_getCacheKey(Cmd); auto Dispose = llvm::make_scope_exit( [&]() { clang_experimental_DepGraphTUCommand_dispose(Cmd); }); - HandleCommand(TUContextHash, TUModuleDeps, TUFileDeps, Args); + HandleCommand(TUContextHash, TUModuleDeps, TUFileDeps, Args, CacheKey); } return 0; } diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 222e878834634..ba192b1fe8b91 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -546,6 +546,14 @@ clang_experimental_DepGraphModule_getBuildArguments(CXDepGraphModule CXDepMod) { return unwrap(CXDepMod)->StrMgr.createCStringsRef(ModDeps.BuildArguments); } +const char * +clang_experimental_DepGraphModule_getCacheKey(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + if (ModDeps.ModuleCacheKey) + return ModDeps.ModuleCacheKey->c_str(); + return nullptr; +} + size_t clang_experimental_DepGraph_getNumTUCommands(CXDepGraph Graph) { TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; return TUDeps.Commands.size(); @@ -573,6 +581,14 @@ CXCStringArray clang_experimental_DepGraphTUCommand_getBuildArguments( return unwrap(CXCmd)->StrMgr.createCStringsRef(TUCmd.Arguments); } +const char * +clang_experimental_DepGraphTUCommand_getCacheKey(CXDepGraphTUCommand CXCmd) { + Command &TUCmd = *unwrap(CXCmd)->TUCmd; + if (TUCmd.TUCacheKey) + return TUCmd.TUCacheKey->c_str(); + return nullptr; +} + CXCStringArray clang_experimental_DepGraph_getTUFileDeps(CXDepGraph Graph) { TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; return unwrap(Graph)->StrMgr.createCStringsRef(TUDeps.FileDeps); diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index 00ded6d77e700..1b7035d732ee1 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -503,6 +503,7 @@ LLVM_16 { clang_experimental_DepGraph_getTUModuleDeps; clang_experimental_DepGraphModule_dispose; clang_experimental_DepGraphModule_getBuildArguments; + clang_experimental_DepGraphModule_getCacheKey; clang_experimental_DepGraphModule_getContextHash; clang_experimental_DepGraphModule_getFileDeps; clang_experimental_DepGraphModule_getModuleDeps; @@ -510,6 +511,7 @@ LLVM_16 { clang_experimental_DepGraphModule_getName; clang_experimental_DepGraphTUCommand_dispose; clang_experimental_DepGraphTUCommand_getBuildArguments; + clang_experimental_DepGraphTUCommand_getCacheKey; clang_experimental_DepGraphTUCommand_getExecutable; clang_getUnqualifiedType; clang_getNonReferenceType; From f560e4adf64b2e08bb7c7c219a099846be01c273 Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Mon, 3 Apr 2023 07:54:07 -0700 Subject: [PATCH 35/38] [libclang][cas] Avoid using `CachingOnDiskFileSystem` when include-tree output is requested It has overhead that is not needed for the include-tree mechanism. (cherry picked from commit 32bd17b4dcc20baf665fc94b9cff0a2a4c00e3c6) --- clang/tools/libclang/CDependencies.cpp | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index ba192b1fe8b91..d418caaf20b42 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -23,6 +23,7 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "llvm/CAS/CASProvidingFileSystem.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/Support/Process.h" @@ -145,7 +146,7 @@ ScanningOutputFormat DependencyScannerServiceOptions::getFormat() const { return ScanningOutputFormat::FullTree; // Note: default caching behaviour is currently cas-fs. - return ConfiguredFormat; + return ScanningOutputFormat::FullTree; } CXDependencyScannerService @@ -155,15 +156,18 @@ clang_experimental_DependencyScannerService_create_v1( std::shared_ptr CAS = unwrap(Opts)->CAS; std::shared_ptr Cache = unwrap(Opts)->Cache; IntrusiveRefCntPtr FS; - if (CAS && Cache) { + ScanningOutputFormat Format = unwrap(Opts)->getFormat(); + bool IsCASFSOutput = Format == ScanningOutputFormat::Tree || + Format == ScanningOutputFormat::FullTree; + if (CAS && Cache && IsCASFSOutput) { assert(unwrap(Opts)->CASOpts.getKind() != CASOptions::UnknownCAS && "CAS and ActionCache must match CASOptions"); FS = llvm::cantFail( llvm::cas::createCachingOnDiskFileSystem(CAS)); } return wrap(new DependencyScanningService( - ScanningMode::DependencyDirectivesScan, unwrap(Opts)->getFormat(), - unwrap(Opts)->CASOpts, std::move(CAS), std::move(Cache), std::move(FS), + ScanningMode::DependencyDirectivesScan, Format, unwrap(Opts)->CASOpts, + std::move(CAS), std::move(Cache), std::move(FS), /*ReuseFilemanager=*/false)); } @@ -195,8 +199,16 @@ void clang_experimental_FileDependenciesList_dispose( CXDependencyScannerWorker clang_experimental_DependencyScannerWorker_create_v0( CXDependencyScannerService Service) { - return wrap(new DependencyScanningWorker( - *unwrap(Service), llvm::vfs::createPhysicalFileSystem())); + ScanningOutputFormat Format = unwrap(Service)->getFormat(); + bool IsIncludeTreeOutput = Format == ScanningOutputFormat::IncludeTree || + Format == ScanningOutputFormat::FullIncludeTree; + llvm::IntrusiveRefCntPtr FS = + llvm::vfs::createPhysicalFileSystem(); + if (IsIncludeTreeOutput) + FS = llvm::cas::createCASProvidingFileSystem(unwrap(Service)->getCAS(), + std::move(FS)); + + return wrap(new DependencyScanningWorker(*unwrap(Service), FS)); } void clang_experimental_DependencyScannerWorker_dispose_v0( From fd562de508df4eb2141845d96b221f6842e3c992 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Mon, 3 Apr 2023 13:56:56 -0700 Subject: [PATCH 36/38] [clang][cas] Switch to include-tree by default As of 3a924f2d4d we have significantly fewer known isuses with include-tree caching than with cas-fs, so it's time to switch the default. rdar://107575958 (cherry picked from commit cfdc12edaef901b2858cd36bf59c83be0526231b) --- clang/test/Index/Core/scan-deps-cas.m | 35 ++++++++++++++++++++------ clang/tools/libclang/CDependencies.cpp | 4 +-- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/clang/test/Index/Core/scan-deps-cas.m b/clang/test/Index/Core/scan-deps-cas.m index e9909433e7547..a74c6431364a1 100644 --- a/clang/test/Index/Core/scan-deps-cas.m +++ b/clang/test/Index/Core/scan-deps-cas.m @@ -1,26 +1,26 @@ -// RUN: rm -rf %t.mcp %t +// RUN: rm -rf %t // RUN: c-index-test core --scan-deps %S -output-dir=%t -cas-path %t/cas \ // RUN: -- %clang -c -I %S/Inputs/module \ -// RUN: -fmodules -fmodules-cache-path=%t.mcp \ +// RUN: -fmodules -fmodules-cache-path=%t/mcpit \ // RUN: -o FoE.o -x objective-c %s > %t.result -// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t +// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t -check-prefix=INCLUDE_TREE // RUN: env CLANG_CACHE_USE_CASFS_DEPSCAN=1 c-index-test core --scan-deps %S -output-dir=%t -cas-path %t/cas \ // RUN: -- %clang -c -I %S/Inputs/module \ -// RUN: -fmodules -fmodules-cache-path=%t.mcp \ +// RUN: -fmodules -fmodules-cache-path=%t/mcp \ // RUN: -o FoE.o -x objective-c %s > %t.casfs.result // RUN: cat %t.casfs.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t -// FIXME: enable modules when supported. // RUN: env CLANG_CACHE_USE_INCLUDE_TREE=1 c-index-test core --scan-deps %S -output-dir=%t -cas-path %t/cas \ // RUN: -- %clang -c -I %S/Inputs/module \ +// RUN: -fmodules -fmodules-cache-path=%t/mcpit \ // RUN: -o FoE.o -x objective-c %s > %t.includetree.result // RUN: cat %t.includetree.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t -check-prefix=INCLUDE_TREE // RUN: c-index-test core --scan-deps %S -output-dir=%t \ // RUN: -- %clang -c -I %S/Inputs/module \ -// RUN: -fmodules -fmodules-cache-path=%t.mcp \ +// RUN: -fmodules -fmodules-cache-path=%t/mcp \ // RUN: -o FoE.o -x objective-c %s | FileCheck %s -check-prefix=NO_CAS // NO_CAS-NOT: fcas // NO_CAS-NOT: faction-cache @@ -65,16 +65,37 @@ // CHECK-SAME: -fmodule-file-cache-key [[PCM:.*ModA_.*pcm]] llvmcas://{{[[:xdigit:]]+}} // CHECK-SAME: -fmodule-file={{(ModA=)?}}[[PCM]] + +// INCLUDE_TREE: modules: +// INCLUDE_TREE-NEXT: module: +// INCLUDE_TREE-NEXT: name: ModA +// INCLUDE_TREE-NEXT: context-hash: [[HASH_MOD_A:[A-Z0-9]+]] +// INCLUDE_TREE-NEXT: module-map-path: /Users/blangmuir/src/cas/llvm-project/clang/test/Index/Core/Inputs/module/module.modulemap +// INCLUDE_TREE-NEXT: cache-key: [[ModA_CACHE_KEY:llvmcas://[[:xdigit:]]+]] +// INCLUDE_TREE-NEXT: module-deps: +// INCLUDE_TREE-NEXT: file-deps: +// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/ModA.h +// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/SubModA.h +// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/SubSubModA.h +// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/module.modulemap +// INCLUDE_TREE-NEXT: build-args: +// INCLUDE_TREE-SAME: -cc1 +// INCLUDE_TREE-SAME: -fcas-path +// INCLUDE_TREE-SAME: -fcas-include-tree llvmcas://{{[[:xdigit:]]+}} +// INCLUDE_TREE-SAME: -fcache-compile-job + // INCLUDE_TREE: dependencies: // INCLUDE_TREE-NEXT: command 0: // INCLUDE_TREE-NEXT: context-hash: [[HASH_TU:[A-Z0-9]+]] // INCLUDE_TREE-NEXT: cache-key: [[INC_TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // INCLUDE_TREE-NEXT: module-deps: +// INCLUDE_TREE-NEXT: ModA:[[HASH_MOD_A]] // INCLUDE_TREE-NEXT: file-deps: // INCLUDE_TREE-NEXT: [[PREFIX]]/scan-deps-cas.m -// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/ModA.h // INCLUDE_TREE-NEXT: build-args: // INCLUDE_TREE-SAME: -cc1 // INCLUDE_TREE-SAME: -fcas-path // INCLUDE_TREE-SAME: -fcas-include-tree llvmcas://{{[[:xdigit:]]+}} // INCLUDE_TREE-SAME: -fcache-compile-job +// INCLUDE_TREE-SAME: -fmodule-file-cache-key [[PCM:.*ModA_.*pcm]] [[ModA_CACHE_KEY]] +// INCLUDE_TREE-SAME: -fmodule-file={{(ModA=)?}}[[PCM]] \ No newline at end of file diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index d418caaf20b42..093b9cba6d952 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -145,8 +145,8 @@ ScanningOutputFormat DependencyScannerServiceOptions::getFormat() const { if (llvm::sys::Process::GetEnv("CLANG_CACHE_USE_CASFS_DEPSCAN")) return ScanningOutputFormat::FullTree; - // Note: default caching behaviour is currently cas-fs. - return ScanningOutputFormat::FullTree; + // Use include-tree by default. + return ScanningOutputFormat::FullIncludeTree; } CXDependencyScannerService From 55fdf8ac7e229b88108725825ebfa1379b7d2e0a Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Tue, 4 Apr 2023 16:17:33 -0700 Subject: [PATCH 37/38] [libclang][cas] Fix local path that showed up in test rdar://107634986 (cherry picked from commit 3c2b29ff2554ddbd4f20934b31863df7d2722528) --- clang/test/Index/Core/scan-deps-cas.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/test/Index/Core/scan-deps-cas.m b/clang/test/Index/Core/scan-deps-cas.m index a74c6431364a1..afeba129d9ecc 100644 --- a/clang/test/Index/Core/scan-deps-cas.m +++ b/clang/test/Index/Core/scan-deps-cas.m @@ -70,7 +70,7 @@ // INCLUDE_TREE-NEXT: module: // INCLUDE_TREE-NEXT: name: ModA // INCLUDE_TREE-NEXT: context-hash: [[HASH_MOD_A:[A-Z0-9]+]] -// INCLUDE_TREE-NEXT: module-map-path: /Users/blangmuir/src/cas/llvm-project/clang/test/Index/Core/Inputs/module/module.modulemap +// INCLUDE_TREE-NEXT: module-map-path: [[PREFIX]]/Inputs/module/module.modulemap // INCLUDE_TREE-NEXT: cache-key: [[ModA_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // INCLUDE_TREE-NEXT: module-deps: // INCLUDE_TREE-NEXT: file-deps: From a8b1c4ffcfb72b3163aec237bc0d4bea99b0c29d Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 5 Apr 2023 19:20:31 -0700 Subject: [PATCH 38/38] Fix doxygen comment Fix doxygen warning: '\c' command does not have a valid word argument [-Wdocumentation] --- clang/include/clang/Lex/PPCachedActions.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/include/clang/Lex/PPCachedActions.h b/clang/include/clang/Lex/PPCachedActions.h index c741fc022a5c4..a76a2ac1a5302 100644 --- a/clang/include/clang/Lex/PPCachedActions.h +++ b/clang/include/clang/Lex/PPCachedActions.h @@ -36,7 +36,7 @@ class PPCachedActions { FileID FID; Module *Submodule; }; - /// The module that is imported by an \c #include directive or \c @import. + /// The module that is imported by an \c #include directive or \c \@import. struct IncludeModule { SmallVector, 2> ImportPath; // Whether this module should only be "marked visible" rather than imported.