diff --git a/clang/include/clang-c/CXString.h b/clang/include/clang-c/CXString.h index f117010c71a46..f132a2191a79e 100644 --- a/clang/include/clang-c/CXString.h +++ b/clang/include/clang-c/CXString.h @@ -16,6 +16,7 @@ #include "clang-c/ExternC.h" #include "clang-c/Platform.h" +#include LLVM_CLANG_C_EXTERN_C_BEGIN @@ -44,6 +45,14 @@ typedef struct { unsigned Count; } CXStringSet; +/** + * An array of C strings. + */ +typedef struct { + const char **Strings; + size_t Count; +} CXCStringArray; + /** * Retrieve the character data associated with the given string. */ diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index 2daf9f38b0fc5..2334495605fe6 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -310,18 +310,8 @@ typedef size_t CXModuleLookupOutputCallback(void *Context, char *Output, size_t MaxLen); /** - * See \c clang_experimental_DependencyScannerWorker_getFileDependencies_v5. - * Returns diagnostics in an unstructured CXString instead of CXDiagnosticSet. - */ -CINDEX_LINKAGE enum CXErrorCode -clang_experimental_DependencyScannerWorker_getFileDependencies_v4( - CXDependencyScannerWorker Worker, int argc, const char *const *argv, - const char *ModuleName, const char *WorkingDirectory, void *MDCContext, - CXModuleDiscoveredCallback *MDC, void *MLOContext, - CXModuleLookupOutputCallback *MLO, unsigned Options, - CXFileDependenciesList **Out, CXString *error); - -/** + * Deprecated, use \c clang_experimental_DependencyScannerWorker_getDepGraph. + * * Calculates the list of file dependencies for a particular compiler * invocation. * @@ -351,19 +341,275 @@ clang_experimental_DependencyScannerWorker_getFileDependencies_v4( * \param [out] Out A non-NULL pointer to store the resulting dependencies. The * output must be freed by calling * \c clang_experimental_FileDependenciesList_dispose. - * \param [out] OutDiags The diagnostics emitted during scanning. These must be - * always freed by calling \c clang_disposeDiagnosticSet. + * \param [out] error the error string to pass back to client (if any). * * \returns \c CXError_Success on success; otherwise a non-zero \c CXErrorCode * indicating the kind of error. */ CINDEX_LINKAGE enum CXErrorCode -clang_experimental_DependencyScannerWorker_getFileDependencies_v5( +clang_experimental_DependencyScannerWorker_getFileDependencies_v4( CXDependencyScannerWorker Worker, int argc, const char *const *argv, const char *ModuleName, const char *WorkingDirectory, void *MDCContext, CXModuleDiscoveredCallback *MDC, void *MLOContext, CXModuleLookupOutputCallback *MLO, unsigned Options, - CXFileDependenciesList **Out, CXDiagnosticSet *OutDiags); + CXFileDependenciesList **Out, CXString *error); + +/** + * Output of \c clang_experimental_DependencyScannerWorker_getDepGraph. + */ +typedef struct CXOpaqueDepGraph *CXDepGraph; + +/** + * An individual module dependency that is part of an overall compilation + * \c CXDepGraph. + */ +typedef struct CXOpaqueDepGraphModule *CXDepGraphModule; + +/** + * An individual command-line invocation that is part of an overall compilation + * \c CXDepGraph. + */ +typedef struct CXOpaqueDepGraphTUCommand *CXDepGraphTUCommand; + +/** + * Settings to use for the + * \c clang_experimental_DependencyScannerWorker_getDepGraph action. + */ +typedef struct CXOpaqueDependencyScannerWorkerScanSettings + *CXDependencyScannerWorkerScanSettings; + +/** + * Creates a set of settings for + * \c clang_experimental_DependencyScannerWorker_getDepGraph action. + * Must be disposed with + * \c clang_experimental_DependencyScannerWorkerScanSettings_dispose. + * Memory for settings is not copied. Any provided pointers must be valid until + * the call to \c clang_experimental_DependencyScannerWorker_getDepGraph. + * + * \param argc the number of compiler invocation arguments (including argv[0]). + * \param argv the compiler driver invocation arguments (including argv[0]). + * \param ModuleName If non-null, the dependencies of the named module are + * returned. Otherwise, the dependencies of the whole + * translation unit are returned. + * \param WorkingDirectory the directory in which the invocation runs. + * \param MLOContext the context that will be passed to \c MLO each time it is + * called. + * \param MLO a callback that is called to determine the paths of output files + * for each module dependency. This may receive the same module on + * different workers. This should be NULL if + * \c clang_experimental_DependencyScannerService_create_v1 was + * called with \c CXDependencyMode_Flat. This callback will be called + * on the same thread that called \c + * clang_experimental_DependencyScannerWorker_getDepGraph. + */ +CINDEX_LINKAGE CXDependencyScannerWorkerScanSettings +clang_experimental_DependencyScannerWorkerScanSettings_create( + int argc, const char *const *argv, const char *ModuleName, + const char *WorkingDirectory, void *MLOContext, + CXModuleLookupOutputCallback *MLO); + +/** + * Dispose of a \c CXDependencyScannerWorkerScanSettings object. + */ +CINDEX_LINKAGE void + clang_experimental_DependencyScannerWorkerScanSettings_dispose( + CXDependencyScannerWorkerScanSettings); + +/** + * Produces the dependency graph for a particular compiler invocation. + * + * \param Settings object created via + * \c clang_experimental_DependencyScannerWorkerScanSettings_create. + * \param [out] Out A non-NULL pointer to store the resulting dependencies. The + * output must be freed by calling + * \c clang_experimental_DepGraph_dispose. + * + * \returns \c CXError_Success on success; otherwise a non-zero \c CXErrorCode + * indicating the kind of error. When returning \c CXError_Failure there will + * be a \c CXDepGraph object on \p Out that can be used to get diagnostics via + * \c clang_experimental_DepGraph_getDiagnostics. + */ +CINDEX_LINKAGE enum CXErrorCode +clang_experimental_DependencyScannerWorker_getDepGraph( + CXDependencyScannerWorker, CXDependencyScannerWorkerScanSettings Settings, + CXDepGraph *Out); + +/** + * Dispose of a \c CXDepGraph object. + */ +CINDEX_LINKAGE void clang_experimental_DepGraph_dispose(CXDepGraph); + +/** + * \returns the number of \c CXDepGraphModule objects in the graph. + */ +CINDEX_LINKAGE size_t clang_experimental_DepGraph_getNumModules(CXDepGraph); + +/** + * \returns the \c CXDepGraphModule object at the given \p Index. + * + * The \c CXDepGraphModule object is only valid to use while \c CXDepGraph is + * valid. Must be disposed with \c clang_experimental_DepGraphModule_dispose. + */ +CINDEX_LINKAGE CXDepGraphModule +clang_experimental_DepGraph_getModule(CXDepGraph, size_t Index); + +CINDEX_LINKAGE void clang_experimental_DepGraphModule_dispose(CXDepGraphModule); + +/** + * \returns the name of the module. This may include `:` for C++20 module + * partitions, or a header-name for C++20 header units. + * + * The string is only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE +const char *clang_experimental_DepGraphModule_getName(CXDepGraphModule); + +/** + * \returns the context hash of a module represents the set of compiler options + * that may make one version of a module incompatible from another. This + * includes things like language mode, predefined macros, header search paths, + * etc... + * + * Modules with the same name but a different \c ContextHash should be treated + * as separate modules for the purpose of a build. + * + * The string is only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE +const char *clang_experimental_DepGraphModule_getContextHash(CXDepGraphModule); + +/** + * \returns the path to the modulemap file which defines this module. If there's + * no modulemap (e.g. for a C++ module) returns \c NULL. + * + * This can be used to explicitly build this module. This file will + * additionally appear in \c FileDeps as a dependency. + * + * The string is only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE const char * + clang_experimental_DepGraphModule_getModuleMapPath(CXDepGraphModule); + +/** + * \returns the list of files which this module directly depends on. + * + * If any of these change then the module needs to be rebuilt. + * + * The strings are only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE CXCStringArray + clang_experimental_DepGraphModule_getFileDeps(CXDepGraphModule); + +/** + * \returns the list of modules which this module direct depends on. + * + * This does include the context hash. The format is + * `:` + * + * The strings are only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE CXCStringArray + clang_experimental_DepGraphModule_getModuleDeps(CXDepGraphModule); + +/** + * \returns the canonical command line to build this module. + * + * The strings are only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE CXCStringArray + clang_experimental_DepGraphModule_getBuildArguments(CXDepGraphModule); + +/** + * \returns the \c ActionCache key for this module, if any. + */ +CINDEX_LINKAGE +const char *clang_experimental_DepGraphModule_getCacheKey(CXDepGraphModule); + +/** + * \returns the number \c CXDepGraphTUCommand objects in the graph. + */ +CINDEX_LINKAGE size_t clang_experimental_DepGraph_getNumTUCommands(CXDepGraph); + +/** + * \returns the \c CXDepGraphTUCommand object at the given \p Index. + * + * The \c CXDepGraphTUCommand object is only valid to use while \c CXDepGraph is + * valid. Must be disposed with \c clang_experimental_DepGraphTUCommand_dispose. + */ +CINDEX_LINKAGE CXDepGraphTUCommand +clang_experimental_DepGraph_getTUCommand(CXDepGraph, size_t Index); + +/** + * Dispose of a \c CXDepGraphTUCommand object. + */ +CINDEX_LINKAGE void + clang_experimental_DepGraphTUCommand_dispose(CXDepGraphTUCommand); + +/** + * \returns the executable name for the command. + * + * The string is only valid to use while the \c CXDepGraphTUCommand object is + * valid. + */ +CINDEX_LINKAGE const char * + clang_experimental_DepGraphTUCommand_getExecutable(CXDepGraphTUCommand); + +/** + * \returns the canonical command line to build this translation unit. + * + * The strings are only valid to use while the \c CXDepGraphTUCommand object is + * valid. + */ +CINDEX_LINKAGE CXCStringArray + clang_experimental_DepGraphTUCommand_getBuildArguments(CXDepGraphTUCommand); + +/** + * \returns the \c ActionCache key for this translation unit, if any. + */ +CINDEX_LINKAGE const char * + clang_experimental_DepGraphTUCommand_getCacheKey(CXDepGraphTUCommand); + +/** + * \returns the list of files which this translation unit directly depends on. + * + * The strings are only valid to use while the \c CXDepGraph object is valid. + */ +CINDEX_LINKAGE +CXCStringArray clang_experimental_DepGraph_getTUFileDeps(CXDepGraph); + +/** + * \returns the list of modules which this translation unit direct depends on. + * + * This does include the context hash. The format is + * `:` + * + * The strings are only valid to use while the \c CXDepGraph object is valid. + */ +CINDEX_LINKAGE +CXCStringArray clang_experimental_DepGraph_getTUModuleDeps(CXDepGraph); + +/** + * \returns the context hash of the C++20 module this translation unit exports. + * + * If the translation unit is not a module then this is empty. + * + * The string is only valid to use while the \c CXDepGraph object is valid. + */ +CINDEX_LINKAGE +const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph); + +/** + * \returns The diagnostics emitted during scanning. These must be always freed + * by calling \c clang_disposeDiagnosticSet. + */ +CINDEX_LINKAGE +CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph); /** * @} diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h index e6f4fece8a4ad..4f62b11a05db4 100644 --- a/clang/include/clang/AST/ASTContext.h +++ b/clang/include/clang/AST/ASTContext.h @@ -477,6 +477,14 @@ class ASTContext : public RefCountedBase { /// For module code-gen cases, this is the top-level module we are building. Module *TopLevelModule = nullptr; + /// The include tree that is being built, if any. + /// See \c FrontendOptions::CASIncludeTreeID. + std::optional CASIncludeTreeID; + + /// The cas-fs tree that is being built, if any. + /// See \c FileSystemOptions::CASFileSystemRootID. + std::optional CASFileSystemRootID; + static constexpr unsigned ConstantArrayTypesLog2InitSize = 8; static constexpr unsigned GeneralTypesLog2InitSize = 9; static constexpr unsigned FunctionProtoTypesLog2InitSize = 12; @@ -1086,6 +1094,20 @@ class ASTContext : public RefCountedBase { /// Get module under construction, nullptr if this is not a C++20 module. Module *getModuleForCodeGen() const { return TopLevelModule; } + std::optional getCASIncludeTreeID() const { + return CASIncludeTreeID; + } + void setCASIncludeTreeID(std::string ID) { + CASIncludeTreeID = std::move(ID); + } + + std::optional getCASFileSystemRootID() const { + return CASFileSystemRootID; + } + void setCASFileSystemRootID(std::string ID) { + CASFileSystemRootID = std::move(ID); + } + TranslationUnitDecl *getTranslationUnitDecl() const { return TUDecl->getMostRecentDecl(); } diff --git a/clang/include/clang/Basic/DiagnosticCASKinds.td b/clang/include/clang/Basic/DiagnosticCASKinds.td index c173f434be6fe..d08b034b98d6f 100644 --- a/clang/include/clang/Basic/DiagnosticCASKinds.td +++ b/clang/include/clang/Basic/DiagnosticCASKinds.td @@ -36,6 +36,10 @@ def err_cas_missing_root_id : Error< "CAS missing expected root-id '%0'">, DefaultFatal; def err_cas_cannot_parse_root_id_for_module : Error< "CAS cannot parse root-id '%0' for module '%1'">, DefaultFatal; +def err_cas_cannot_parse_include_tree_id : Error< + "CAS cannot parse include-tree-id '%0'">, DefaultFatal; +def err_cas_missing_include_tree_id : Error< + "CAS missing expected include-tree '%0'">, DefaultFatal; def warn_clang_cache_disabled_caching: Warning< "caching disabled because %0">, diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index 6f76ef2b42a90..81877ea252a7b 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -229,6 +229,8 @@ def err_missing_module_name : Error< DefaultFatal; def err_missing_module : Error< "no module named '%0' declared in module map file '%1'">, DefaultFatal; +def err_missing_module_include_tree : Error< + "no module named '%0' declared in include-tree module map '%1'">, DefaultFatal; def err_no_submodule : Error<"no submodule named %0 in module '%1'">; def err_no_submodule_suggest : Error< "no submodule named %0 in module '%1'; did you mean '%2'?">; diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td index 12aed1386cdd8..3cba656af179d 100644 --- a/clang/include/clang/Basic/DiagnosticLexKinds.td +++ b/clang/include/clang/Basic/DiagnosticLexKinds.td @@ -918,6 +918,9 @@ def warn_defined_in_function_type_macro : Extension< "macro expansion producing 'defined' has undefined behavior">, InGroup; +def err_pp_missing_module_include_tree : Error< + "no module named '%0' declared in include-tree module map">, DefaultFatal; + let CategoryName = "Nullability Issue" in { def err_pp_assume_nonnull_syntax : Error<"expected 'begin' or 'end'">; diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index 3218e33bb88ff..99f4f71a7b449 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -108,6 +108,10 @@ class alignas(8) Module { /// of header files. ModuleMapModule, + /// This is a module that was defined by a module map and built out + /// of header files as part of an \c IncludeTree. + IncludeTreeModuleMap, + /// This is a C++20 module interface unit. ModuleInterfaceUnit, @@ -175,7 +179,9 @@ class alignas(8) Module { bool isPrivateModule() const { return Kind == PrivateModuleFragment; } - bool isModuleMapModule() const { return Kind == ModuleMapModule; } + bool isModuleMapModule() const { + return Kind == ModuleMapModule || Kind == IncludeTreeModuleMap; + } private: /// The submodules of this module, indexed by name. @@ -192,10 +198,6 @@ class alignas(8) Module { /// The \c ActionCache key for this module, if any. Optional ModuleCacheKey; - /// The CAS filesystem root ID for implicit modules built with the dependency - /// scanner, if any. - Optional CASFileSystemRootID; - /// The top-level headers associated with this module. llvm::SmallSetVector TopHeaders; @@ -636,14 +638,6 @@ class alignas(8) Module { getTopLevelModule()->ModuleCacheKey = std::move(Key); } - Optional getCASFileSystemRootID() const { - return getTopLevelModule()->CASFileSystemRootID; - } - - void setCASFileSystemRootID(std::string ID) { - assert(!getCASFileSystemRootID() || *getCASFileSystemRootID() == ID); - getTopLevelModule()->CASFileSystemRootID = std::move(ID); - } /// Retrieve the directory for which this module serves as the /// umbrella. diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 5519010160937..cf36c8c90277b 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -39,115 +39,84 @@ template class IncludeTreeBase : public ObjectProxy { return Node.getData().startswith(NodeT::getNodeKind()); } - friend class IncludeFile; - friend class IncludeFileList; - friend class IncludeTree; - friend class IncludeTreeRoot; + friend NodeT; }; -/// Represents a \p SourceManager file (or buffer in the case of preprocessor -/// predefines) that got included by the preprocessor. -class IncludeFile : public IncludeTreeBase { +/// Represents a DAG of included files by the preprocessor and module imports. +/// Each node in the DAG represents a particular inclusion of a file or module +/// that encompasses inclusions of other files as sub-trees, along with all the +/// \p __has_include() preprocessor checks that occurred during preprocessing +/// of that file. +class IncludeTree : public IncludeTreeBase { public: - static constexpr StringRef getNodeKind() { return "File"; } + static constexpr StringRef getNodeKind() { return "Tree"; } - ObjectRef getFilenameRef() const { return getReference(0); } - ObjectRef getContentsRef() const { return getReference(1); } + class File; + class FileList; + class Node; + class Module; + class ModuleImport; + class ModuleMap; - Expected getFilename() { - return getCAS().getProxy(getFilenameRef()); - } + Expected getBaseFile(); - Expected getContents() { - return getCAS().getProxy(getContentsRef()); - } + /// The include file that resulted in this include-tree. + ObjectRef getBaseFileRef() const { return getReference(0); } struct FileInfo { StringRef Filename; StringRef Contents; }; - Expected getFileInfo() { - auto Filename = getFilename(); - if (!Filename) - return Filename.takeError(); - auto Contents = getContents(); - if (!Contents) - return Contents.takeError(); - return FileInfo{Filename->getData(), Contents->getData()}; - } - - static Expected create(ObjectStore &DB, StringRef Filename, - ObjectRef Contents); - - llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + Expected getBaseFileInfo(); - static bool isValid(const ObjectProxy &Node) { - if (!IncludeTreeBase::isValid(Node)) - return false; - IncludeTreeBase Base(Node); - return Base.getNumReferences() == 2 && Base.getData().empty(); - } - static bool isValid(ObjectStore &DB, ObjectRef Ref) { - auto Node = DB.getProxy(Ref); - if (!Node) { - llvm::consumeError(Node.takeError()); - return false; - } - return isValid(*Node); + SrcMgr::CharacteristicKind getFileCharacteristic() const { + return (SrcMgr::CharacteristicKind)(getData().front() & ~IsSubmoduleBit); } -private: - friend class IncludeTreeBase; - friend class IncludeFileList; - friend class IncludeTree; - friend class IncludeTreeRoot; + bool isSubmodule() const { return (getData().front() & IsSubmoduleBit) != 0; } - explicit IncludeFile(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { - assert(isValid(*this)); + std::optional getSubmoduleNameRef() const { + if (isSubmodule()) + return getReference(getNumReferences() - 1); + return std::nullopt; } -}; - -/// Represents a DAG of included files by the preprocessor. -/// Each node in the DAG represents a particular inclusion of a file that -/// encompasses inclusions of other files as sub-trees, along with all the -/// \p __has_include() preprocessor checks that occurred during preprocessing -/// of that file. -class IncludeTree : public IncludeTreeBase { -public: - static constexpr StringRef getNodeKind() { return "Tree"; } - Expected getBaseFile() { - auto Node = getCAS().getProxy(getBaseFileRef()); + /// If \c getBaseFile() is a modular header, get its submodule name. + Expected> getSubmoduleName() { + auto Ref = getSubmoduleNameRef(); + if (!Ref) + return std::nullopt; + auto Node = getCAS().getProxy(*Ref); if (!Node) return Node.takeError(); - return IncludeFile(std::move(*Node)); - } - - /// The include file that resulted in this include-tree. - ObjectRef getBaseFileRef() const { return getReference(0); } - - Expected getBaseFileInfo() { - auto File = getBaseFile(); - if (!File) - return File.takeError(); - return File->getFileInfo(); + return Node->getData(); } - SrcMgr::CharacteristicKind getFileCharacteristic() const { - return (SrcMgr::CharacteristicKind)dataSkippingIncludes().front(); - } - - size_t getNumIncludes() const { return getNumReferences() - 1; } + size_t getNumIncludes() const { + return getNumReferences() - (isSubmodule() ? 2 : 1); + }; ObjectRef getIncludeRef(size_t I) const { assert(I < getNumIncludes()); return getReference(I + 1); } + enum class NodeKind : uint8_t { + Tree, + ModuleImport, + }; + + /// The kind of node included at the given index. + NodeKind getIncludeKind(size_t I) const; + + /// The sub-include-tree or module import for the given index. + Expected getIncludeNode(size_t I); + /// The sub-include-trees of included files, in the order that they occurred. - Expected getInclude(size_t I) { - return getInclude(getIncludeRef(I)); + Expected getIncludeTree(size_t I) { + assert(getIncludeKind(I) == NodeKind::Tree); + return getIncludeTree(getIncludeRef(I)); } /// The source byte offset for a particular include, pointing to the beginning @@ -172,15 +141,20 @@ class IncludeTree : public IncludeTreeBase { /// occur in the same order. bool getCheckResult(size_t I) const; - /// Passes pairs of (IncludeTree, include offset) to \p Callback. + /// Passes pairs of (Node, include offset) to \p Callback. llvm::Error forEachInclude( - llvm::function_ref)> - Callback); + llvm::function_ref)> Callback); + + struct IncludeInfo { + ObjectRef Ref; ///< IncludeTree or IncludeTreeModuleImport + uint32_t Offset; + NodeKind Kind; + }; static Expected create(ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, - ObjectRef BaseFile, ArrayRef> Includes, - llvm::SmallBitVector Checks); + ObjectRef BaseFile, ArrayRef Includes, + Optional SubmoduleName, llvm::SmallBitVector Checks); static Expected get(ObjectStore &DB, ObjectRef Ref); @@ -190,19 +164,25 @@ class IncludeTree : public IncludeTreeBase { friend class IncludeTreeBase; friend class IncludeTreeRoot; + static constexpr char IsSubmoduleBit = 0x80; + explicit IncludeTree(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { assert(isValid(*this)); } - Expected getInclude(ObjectRef Ref) { + Expected getIncludeTree(ObjectRef Ref) { auto Node = getCAS().getProxy(Ref); if (!Node) return Node.takeError(); return IncludeTree(std::move(*Node)); } + Expected getIncludeNode(ObjectRef Ref, NodeKind Kind); + + StringRef dataSkippingFlags() const { return getData().drop_front(); } StringRef dataSkippingIncludes() const { - return getData().drop_front(getNumIncludes() * sizeof(uint32_t)); + return dataSkippingFlags().drop_front(getNumIncludes() * + (sizeof(uint32_t) + 1)); } static bool isValid(const ObjectProxy &Node); @@ -216,28 +196,86 @@ class IncludeTree : public IncludeTreeBase { } }; -/// A flat list of \p IncludeFile entries. This is used along with a simple -/// implementation of a \p vfs::FileSystem produced via -/// \p createIncludeTreeFileSystem(). -class IncludeFileList : public IncludeTreeBase { +/// Represents a \p SourceManager file (or buffer in the case of preprocessor +/// predefines) that got included by the preprocessor. +class IncludeTree::File : public IncludeTreeBase { public: - static constexpr StringRef getNodeKind() { return "List"; } + static constexpr StringRef getNodeKind() { return "File"; } - using FileSizeTy = uint32_t; + ObjectRef getFilenameRef() const { return getReference(0); } + ObjectRef getContentsRef() const { return getReference(1); } + + Expected getFilename() { + return getCAS().getProxy(getFilenameRef()); + } - size_t getNumFiles() const { return getNumReferences(); } + Expected getContents() { + return getCAS().getProxy(getContentsRef()); + } - ObjectRef getFileRef(size_t I) const { - assert(I < getNumFiles()); - return getReference(I); + Expected getFileInfo() { + auto Filename = getFilename(); + if (!Filename) + return Filename.takeError(); + auto Contents = getContents(); + if (!Contents) + return Contents.takeError(); + return IncludeTree::FileInfo{Filename->getData(), Contents->getData()}; } - Expected getFile(size_t I) { return getFile(getFileRef(I)); } - FileSizeTy getFileSize(size_t I) const; + Expected> getMemoryBuffer() { + auto Filename = getFilename(); + if (!Filename) + return Filename.takeError(); + auto Contents = getContents(); + if (!Contents) + return Contents.takeError(); + return Contents->getMemoryBuffer(Filename->getData()); + } + + static Expected create(ObjectStore &DB, StringRef Filename, + ObjectRef Contents); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + return Base.getNumReferences() == 2 && Base.getData().empty(); + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + friend class IncludeTreeBase; + friend class FileList; + friend class IncludeTree; + friend class IncludeTreeRoot; + + explicit File(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } +}; + +/// A list of \c File entries. Multiple \c FileList can be combined without +/// copying their contents. This is used along with a simple implementation of a +/// \p vfs::FileSystem produced via \p createIncludeTreeFileSystem(). +class IncludeTree::FileList : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "List"; } - /// \returns each \p IncludeFile entry along with its file size. - llvm::Error forEachFile( - llvm::function_ref Callback); + using FileSizeTy = uint32_t; + + /// \returns each \c File entry along with its file size. + llvm::Error + forEachFile(llvm::function_ref Callback); /// We record the file size as well to avoid needing to materialize the /// underlying buffer for the \p IncludeTreeFileSystem::status() @@ -246,29 +284,35 @@ class IncludeFileList : public IncludeTreeBase { ObjectRef FileRef; FileSizeTy Size; }; - static Expected create(ObjectStore &DB, - ArrayRef Files); + static Expected create(ObjectStore &DB, ArrayRef Files, + ArrayRef FileLists); - static Expected get(ObjectStore &CAS, ObjectRef Ref); + static Expected get(ObjectStore &CAS, ObjectRef Ref); llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); private: - friend class IncludeTreeBase; + friend class IncludeTreeBase; friend class IncludeTreeRoot; - explicit IncludeFileList(ObjectProxy Node) - : IncludeTreeBase(std::move(Node)) { + explicit FileList(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { assert(isValid(*this)); } - Expected getFile(ObjectRef Ref) { + size_t getNumFilesCurrentList() const; + FileSizeTy getFileSize(size_t I) const; + + Expected getFile(ObjectRef Ref) { auto Node = getCAS().getProxy(Ref); if (!Node) return Node.takeError(); - return IncludeFile(std::move(*Node)); + return File(std::move(*Node)); } + llvm::Error + forEachFileImpl(llvm::DenseSet &Seen, + llvm::function_ref Callback); + static bool isValid(const ObjectProxy &Node); static bool isValid(ObjectStore &CAS, ObjectRef Ref) { auto Node = CAS.getProxy(Ref); @@ -280,6 +324,326 @@ class IncludeFileList : public IncludeTreeBase { } }; +/// Represents a module imported by an IncludeTree. +class IncludeTree::ModuleImport : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "ModI"; } + + static Expected create(ObjectStore &DB, StringRef ModuleName, + bool VisibilityOnly); + + StringRef getModuleName() const { return getData().drop_front(); } + /// Whether this module should only be "marked visible" rather than imported. + bool visibilityOnly() const { return (bool)getData()[0]; } + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + return Base.getNumReferences() == 0 && Base.getData().size() > 1; + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + friend class IncludeTreeBase; + friend class Node; + + explicit ModuleImport(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } +}; + +/// Represents an \c IncludeTree or \c ModuleImport. +class IncludeTree::Node { +public: + IncludeTree getIncludeTree() const { + assert(K == NodeKind::Tree); + return IncludeTree(N); + } + ModuleImport getModuleImport() const { + assert(K == NodeKind::ModuleImport); + return ModuleImport(N); + } + NodeKind getKind() const { return K; } + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + +private: + friend class IncludeTree; + Node(ObjectProxy N, NodeKind K) : N(std::move(N)), K(K) {} + ObjectProxy N; + NodeKind K; +}; + +/// Module or submodule declaration as part of the \c IncludeTreeRoot module +/// map structure. +class IncludeTree::Module : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "Modu"; } + + class ExportList; + class LinkLibraryList; + + struct ModuleFlags { + bool IsFramework : 1; + bool IsExplicit : 1; + bool IsExternC : 1; + bool IsSystem : 1; + bool InferSubmodules : 1; + bool InferExplicitSubmodules : 1; + bool InferExportWildcard : 1; + ModuleFlags() + : IsFramework(false), IsExplicit(false), IsExternC(false), + IsSystem(false), InferSubmodules(false), + InferExplicitSubmodules(false), InferExportWildcard(false) {} + }; + + ModuleFlags getFlags() const; + + /// The name of the current (sub)module. + StringRef getName() const { return dataAfterFlags(); } + + size_t getNumSubmodules() const; + + Expected getSubmodule(size_t I) { + assert(I < getNumSubmodules()); + auto Node = getCAS().getProxy(getReference(I)); + if (!Node) + return Node.takeError(); + return Module(*Node); + } + + llvm::Error forEachSubmodule(llvm::function_ref CB); + + std::optional getExportsRef() const { + if (std::optional Index = getExportsIndex()) + return getReference(*Index); + return std::nullopt; + } + std::optional getLinkLibrariesRef() const { + if (std::optional Index = getLinkLibrariesIndex()) + return getReference(*Index); + return std::nullopt; + } + + /// The list of modules that this submodule re-exports. + Expected> getExports(); + + /// The list of modules that this submodule re-exports. + Expected> getLinkLibraries(); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static Expected create(ObjectStore &DB, StringRef ModuleName, + ModuleFlags Flags, + ArrayRef Submodules, + std::optional ExportList, + std::optional LinkLibraries); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + return Base.getData().size() > 2; + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + uint16_t rawFlags() const; + StringRef dataAfterFlags() const { return getData().drop_front(2); } + bool hasExports() const; + bool hasLinkLibraries() const; + std::optional getExportsIndex() const; + std::optional getLinkLibrariesIndex() const; + + explicit Module(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } + + friend class IncludeTreeBase; + friend class IncludeTreeRoot; + friend class ModuleMap; +}; + +/// The set of modules re-exported by another module. +class IncludeTree::Module::ExportList : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "ExpL"; } + + /// An explicit export. + struct Export { + StringRef ModuleName; + bool Wildcard; + }; + + /// Whether this module exports all imported modules (`export *`). + bool hasGlobalWildcard() const; + + size_t getNumExplicitExports() const { return getNumReferences(); } + + /// Whether the explicit export at \p I has a wildcard (`export MyModule.*`). + bool exportHasWildcard(size_t I) const; + + Expected getExplicitExport(size_t I); + + /// Calls \p CB for each explicit export declaration. + llvm::Error forEachExplicitExport(llvm::function_ref CB); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static Expected create(ObjectStore &DB, ArrayRef Exports, + bool GlobalWildcard); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + size_t ExpectedBitSize = Base.getNumReferences() + 1; + size_t ExpectedSize = + ExpectedBitSize / 8 + (ExpectedBitSize % 8 == 0 ? 0 : 1); + return Base.getData().size() == ExpectedSize; + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + explicit ExportList(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } + + friend class IncludeTreeBase; + friend class Module; +}; + +/// The set of libraries to link against when using a module. +class IncludeTree::Module::LinkLibraryList + : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "LnkL"; } + + struct LinkLibrary { + StringRef Library; + bool IsFramework; + }; + + size_t getNumLibraries() const { return getNumReferences(); } + + /// Whether the library at \p I is a framework. + bool isFramework(size_t I) const; + + ObjectRef getLibraryNameRef(size_t I) const { return getReference(I); } + + Expected getLinkLibrary(size_t I) { + auto Name = getCAS().getProxy(getLibraryNameRef(I)); + if (!Name) + return Name.takeError(); + return LinkLibrary{Name->getData(), isFramework(I)}; + } + + /// Calls \p CB for each link libary. + llvm::Error + forEachLinkLibrary(llvm::function_ref CB); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static Expected create(ObjectStore &DB, + ArrayRef Exports); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + size_t ExpectedBitSize = Base.getNumReferences(); + size_t ExpectedSize = + ExpectedBitSize / 8 + (ExpectedBitSize % 8 == 0 ? 0 : 1); + return Base.getData().size() == ExpectedSize; + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + explicit LinkLibraryList(ObjectProxy Node) + : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } + + friend class IncludeTreeBase; + friend class Module; +}; + +class IncludeTree::ModuleMap : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "ModM"; } + + size_t getNumModules() const { return getNumReferences(); } + + Expected getModule(size_t I) { + auto Node = getCAS().getProxy(getReference(I)); + if (!Node) + return Node.takeError(); + return Module(*Node); + } + + /// Calls \p CB for each module declaration. + llvm::Error forEachModule(llvm::function_ref CB); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static Expected create(ObjectStore &DB, + ArrayRef Modules); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + return Base.getData().empty(); + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + explicit ModuleMap(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } + + friend class IncludeTreeBase; + friend class IncludeTreeRoot; +}; + /// Represents the include-tree result for a translation unit. class IncludeTreeRoot : public IncludeTreeBase { public: @@ -290,9 +654,15 @@ class IncludeTreeRoot : public IncludeTreeBase { ObjectRef getFileListRef() const { return getReference(1); } Optional getPCHRef() const { - if (getNumReferences() > 2) - return getReference(2); - return None; + if (auto Index = getPCHRefIndex()) + return getReference(*Index); + return std::nullopt; + } + + Optional getModuleMapRef() const { + if (auto Index = getModuleMapRefIndex()) + return getReference(*Index); + return std::nullopt; } Expected getMainFileTree() { @@ -302,11 +672,11 @@ class IncludeTreeRoot : public IncludeTreeBase { return IncludeTree(std::move(*Node)); } - Expected getFileList() { + Expected getFileList() { auto Node = getCAS().getProxy(getFileListRef()); if (!Node) return Node.takeError(); - return IncludeFileList(std::move(*Node)); + return IncludeTree::FileList(std::move(*Node)); } Expected> getPCHBuffer() { @@ -319,10 +689,20 @@ class IncludeTreeRoot : public IncludeTreeBase { return None; } - static Expected create(ObjectStore &DB, - ObjectRef MainFileTree, - ObjectRef FileList, - Optional PCHRef); + Expected> getModuleMap() { + if (Optional Ref = getModuleMapRef()) { + auto Node = getCAS().getProxy(*Ref); + if (!Node) + return Node.takeError(); + return IncludeTree::ModuleMap(*Node); + } + return std::nullopt; + } + + static Expected + create(ObjectStore &DB, ObjectRef MainFileTree, ObjectRef FileList, + Optional PCHRef, + Optional ModuleMapRef); static Expected get(ObjectStore &DB, ObjectRef Ref); @@ -332,8 +712,8 @@ class IncludeTreeRoot : public IncludeTreeBase { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); - return (Base.getNumReferences() == 2 || Base.getNumReferences() == 3) && - Base.getData().empty(); + return (Base.getNumReferences() >= 2 && Base.getNumReferences() <= 4) && + Base.getData().size() == 1; } static bool isValid(ObjectStore &DB, ObjectRef Ref) { auto Node = DB.getProxy(Ref); @@ -347,6 +727,9 @@ class IncludeTreeRoot : public IncludeTreeBase { private: friend class IncludeTreeBase; + std::optional getPCHRefIndex() const; + std::optional getModuleMapRefIndex() const; + explicit IncludeTreeRoot(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { assert(isValid(*this)); diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h index 5554c2992ba42..7ab469a797e18 100644 --- a/clang/include/clang/Frontend/FrontendOptions.h +++ b/clang/include/clang/Frontend/FrontendOptions.h @@ -248,6 +248,10 @@ class FrontendInputFile { bool IsSystem = false) : File(File.str()), IncludeTree(std::move(Tree)), Kind(Kind), IsSystem(IsSystem) {} + FrontendInputFile(cas::ObjectRef Tree, llvm::MemoryBufferRef Buffer, + InputKind Kind, bool IsSystem = false) + : Buffer(Buffer), IncludeTree(std::move(Tree)), Kind(Kind), + IsSystem(IsSystem) {} InputKind getKind() const { return Kind; } bool isSystem() const { return IsSystem; } diff --git a/clang/include/clang/Frontend/SerializedDiagnosticReader.h b/clang/include/clang/Frontend/SerializedDiagnosticReader.h index be7d3a10a1705..1d95fd23c774d 100644 --- a/clang/include/clang/Frontend/SerializedDiagnosticReader.h +++ b/clang/include/clang/Frontend/SerializedDiagnosticReader.h @@ -65,6 +65,9 @@ class SerializedDiagnosticReader { /// Read the diagnostics in \c File std::error_code readDiagnostics(StringRef File); + /// Read the diagnostics in \c Buffer. + std::error_code readDiagnostics(llvm::MemoryBufferRef Buffer); + private: enum class Cursor; diff --git a/clang/include/clang/Lex/PPCachedActions.h b/clang/include/clang/Lex/PPCachedActions.h index d9ae64030d621..a76a2ac1a5302 100644 --- a/clang/include/clang/Lex/PPCachedActions.h +++ b/clang/include/clang/Lex/PPCachedActions.h @@ -15,9 +15,12 @@ #define LLVM_CLANG_LEX_PPCACHEDACTIONS_H #include "clang/Basic/SourceLocation.h" +#include "llvm/ADT/SmallVector.h" namespace clang { +class IdentifierInfo; +class Module; class Preprocessor; /// This interface provides a way to override the actions of the preprocessor as @@ -28,6 +31,18 @@ class PPCachedActions { virtual void anchor(); public: + /// The file that is included by an \c #include directive. + struct IncludeFile { + FileID FID; + Module *Submodule; + }; + /// The module that is imported by an \c #include directive or \c \@import. + struct IncludeModule { + SmallVector, 2> ImportPath; + // Whether this module should only be "marked visible" rather than imported. + bool VisibilityOnly; + }; + virtual ~PPCachedActions() = default; /// \returns the \p FileID that should be used for predefines. @@ -37,9 +52,10 @@ class PPCachedActions { virtual bool evaluateHasInclude(Preprocessor &PP, SourceLocation Loc, bool IsIncludeNext) = 0; - /// \returns the \p FileID that should be entered for an include directive. - /// \p None indicates that the directive should be skipped. - virtual Optional + /// \returns the file that should be entered or module that should be imported + /// for an \c #include directive. \c {} indicates that the directive + /// should be skipped. + virtual std::variant handleIncludeDirective(Preprocessor &PP, SourceLocation IncludeLoc, SourceLocation AfterDirectiveLoc) = 0; diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h index 34792dae0e36e..1cfeaf5bc81c5 100644 --- a/clang/include/clang/Serialization/ASTBitCodes.h +++ b/clang/include/clang/Serialization/ASTBitCodes.h @@ -41,7 +41,7 @@ namespace serialization { /// Version 4 of AST files also requires that the version control branch and /// revision match exactly, since there is no backward compatibility of /// AST files at this time. -const unsigned VERSION_MAJOR = 25; +const unsigned VERSION_MAJOR = 26; /// AST file minor version number supported by this version of /// Clang. @@ -367,6 +367,10 @@ enum ControlRecordTypes { /// Record code for the (optional) CAS filesystem root ID for implicit modules /// built with the dependency scanner. CASFS_ROOT_ID, + + /// Record code for the (optional) include-tree ID for implicit modules + /// built with the dependency scanner. + CAS_INCLUDE_TREE_ID, }; /// Record types that occur within the options block inside diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index cad25a360c553..191372471c5ff 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -246,6 +246,11 @@ class ASTReaderListener { return false; } + /// Called for each CAS include-tree root ID. + /// + /// \returns true to indicate \p RootID is invalid, or false otherwise. + virtual bool readIncludeTreeID(StringRef ID, bool Complain) { return false; } + /// Called for each module cache key. /// /// \returns true to indicate the key cannot be loaded. @@ -300,6 +305,7 @@ class ChainedASTReaderListener : public ASTReaderListener { bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden, bool isExplicitModule) override; bool readCASFileSystemRootID(StringRef RootID, bool Complain) override; + bool readIncludeTreeID(StringRef ID, bool Complain) override; bool readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) override; void readModuleFileExtension( diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 2d0be293d93d3..c94b089bc7936 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -143,6 +143,10 @@ class ModuleFile { /// scanner, or empty. std::string CASFileSystemRootID; + /// The include-tree root ID for implicit modules built with the dependency + /// scanner, or empty. + std::string IncludeTreeID; + /// The name of the module. std::string ModuleName; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h index c9f96805f5a4e..52cb75423d911 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -71,6 +71,9 @@ struct TranslationUnitDeps { /// The CASID for input file dependency tree. llvm::Optional CASFileSystemRootID; + /// The include-tree for input file dependency tree. + llvm::Optional IncludeTreeID; + /// The sequence of commands required to build the translation unit. Commands /// should be executed in order. /// @@ -103,8 +106,7 @@ class DependencyScanningTool { /// Collect dependency tree. llvm::Expected - getDependencyTree(const std::vector &CommandLine, StringRef CWD, - DepscanPrefixMapping PrefixMapping = {}); + getDependencyTree(const std::vector &CommandLine, StringRef CWD); /// If \p DiagGenerationAsCompilation is true it will generate error /// diagnostics same way as the normal compilation, with "N errors generated" @@ -114,13 +116,12 @@ class DependencyScanningTool { getDependencyTreeFromCompilerInvocation( std::shared_ptr Invocation, StringRef CWD, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, - bool DiagGenerationAsCompilation, - DepscanPrefixMapping PrefixMapping = {}); + bool DiagGenerationAsCompilation); Expected getIncludeTree(cas::ObjectStore &DB, const std::vector &CommandLine, StringRef CWD, - const DepscanPrefixMapping &PrefixMapping); + LookupModuleOutputCallback LookupModuleOutput); /// If \p DiagGenerationAsCompilation is true it will generate error /// diagnostics same way as the normal compilation, with "N errors generated" @@ -128,7 +129,7 @@ class DependencyScanningTool { /// \p DiagOpts.DiagnosticSerializationFile setting is set for the invocation. Expected getIncludeTreeFromCompilerInvocation( cas::ObjectStore &DB, std::shared_ptr Invocation, - StringRef CWD, const DepscanPrefixMapping &PrefixMapping, + StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, bool DiagGenerationAsCompilation); @@ -150,8 +151,7 @@ class DependencyScanningTool { getTranslationUnitDependencies(const std::vector &CommandLine, StringRef CWD, const llvm::StringSet<> &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping); + LookupModuleOutputCallback LookupModuleOutput); /// Given a compilation context specified via the Clang driver command-line, /// gather modular dependencies of module with the given name, and return the @@ -160,8 +160,7 @@ class DependencyScanningTool { getModuleDependencies(StringRef ModuleName, const std::vector &CommandLine, StringRef CWD, const llvm::StringSet<> &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping); + LookupModuleOutputCallback LookupModuleOutput); ScanningOutputFormat getScanningFormat() const { return Worker.getScanningFormat(); @@ -182,13 +181,11 @@ class DependencyScanningTool { static std::unique_ptr createActionController(DependencyScanningWorker &Worker, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping); + LookupModuleOutputCallback LookupModuleOutput); private: std::unique_ptr - createActionController(LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping); + createActionController(LookupModuleOutputCallback LookupModuleOutput); private: DependencyScanningWorker Worker; @@ -225,6 +222,10 @@ class FullDependencyConsumer : public DependencyConsumer { CASFileSystemRootID = std::move(ID); } + void handleIncludeTreeID(std::string ID) override { + IncludeTreeID = std::move(ID); + } + TranslationUnitDeps takeTranslationUnitDeps(); ModuleDepsGraph takeModuleGraphDeps(); @@ -236,6 +237,7 @@ class FullDependencyConsumer : public DependencyConsumer { std::vector Commands; std::string ContextHash; Optional CASFileSystemRootID; + Optional IncludeTreeID; std::vector OutputPaths; const llvm::StringSet<> &AlreadySeen; }; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h index 1c2182811da40..eb944a1046213 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -31,7 +31,6 @@ using CachingOnDiskFileSystemPtr = llvm::IntrusiveRefCntPtr; class DependencyScanningWorkerFilesystem; -struct DepscanPrefixMapping; /// A command-line tool invocation that is part of building a TU. /// @@ -39,6 +38,9 @@ struct DepscanPrefixMapping; struct Command { std::string Executable; std::vector Arguments; + + /// The \c ActionCache key for this translation unit, if any. + std::optional TUCacheKey; }; class DependencyConsumer { @@ -96,19 +98,6 @@ class DependencyActionController { const ModuleDeps &MD) { return llvm::Error::success(); } - - virtual void enteredInclude(Preprocessor &PP, FileID FID) {} - - virtual void exitedInclude(Preprocessor &PP, FileID IncludedBy, - FileID Include, SourceLocation ExitLoc) {} - - virtual void handleHasIncludeCheck(Preprocessor &PP, bool Result) {} - - /// FIXME: This is temporary until we eliminate the split between consumers in - /// \p DependencyScanningTool and collectors in \p DependencyScanningWorker - /// and have them both in the same file. see FIXME in \p - /// DependencyScanningAction::runInvocation. - virtual const DepscanPrefixMapping *getPrefixMapping() { return nullptr; } }; /// An individual dependency scanning worker that is able to run on its own diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h index b4b8b1206098c..809abf722e5ec 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -121,6 +121,9 @@ struct ModuleDeps { /// The CASID for the module input dependency tree, if any. llvm::Optional CASFileSystemRootID; + /// The CASID for the module include-tree, if any. + llvm::Optional IncludeTreeID; + /// The \c ActionCache key for this module, if any. llvm::Optional ModuleCacheKey; diff --git a/clang/include/clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h b/clang/include/clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h index 42f0db456e190..792bee4ef0861 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h +++ b/clang/include/clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h @@ -41,14 +41,13 @@ void configureInvocationForCaching(CompilerInvocation &CI, CASOptions CASOpts, bool ProduceIncludeTree); struct DepscanPrefixMapping { - Optional NewSDKPath; - Optional NewToolchainPath; - SmallVector PrefixMap; + /// Add path mappings to the \p Mapper. + static void configurePrefixMapper(const CompilerInvocation &Invocation, + llvm::PrefixMapper &Mapper); - /// Add path mappings from the current path in \p Invocation to the new path - /// from \c DepscanPrefixMapping to the \p Mapper. - llvm::Error configurePrefixMapper(const CompilerInvocation &Invocation, - llvm::PrefixMapper &Mapper) const; + /// Add path mappings to the \p Mapper. + static void configurePrefixMapper(ArrayRef PathPrefixMappings, + llvm::PrefixMapper &Mapper); /// Apply the mappings from \p Mapper to \p Invocation. static void remapInvocationPaths(CompilerInvocation &Invocation, @@ -61,7 +60,6 @@ Expected scanAndUpdateCC1InlineWithTool( tooling::dependencies::DependencyScanningTool &Tool, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, CompilerInvocation &Invocation, StringRef WorkingDirectory, - const tooling::dependencies::DepscanPrefixMapping &PrefixMapping, llvm::cas::ObjectStore &DB); } // end namespace clang diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp index 151826f0d9c2b..4584111df0256 100644 --- a/clang/lib/AST/Decl.cpp +++ b/clang/lib/AST/Decl.cpp @@ -1565,6 +1565,7 @@ Module *Decl::getOwningModuleForLinkage(bool IgnoreLinkage) const { switch (M->Kind) { case Module::ModuleMapModule: + case Module::IncludeTreeModuleMap: // Module map modules have no special linkage semantics. return nullptr; diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index 435522869d7c8..0cd6869339db1 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -7,8 +7,11 @@ //===----------------------------------------------------------------------===// #include "clang/CAS/IncludeTree.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallBitVector.h" #include "llvm/Support/EndianStream.h" +#include "llvm/Support/Error.h" +#include using namespace clang; using namespace clang::cas; @@ -33,8 +36,9 @@ Expected IncludeTreeBase::create(ObjectStore &DB, return NodeT(*Proxy); } -Expected IncludeFile::create(ObjectStore &DB, StringRef Filename, - ObjectRef Contents) { +Expected IncludeTree::File::create(ObjectStore &DB, + StringRef Filename, + ObjectRef Contents) { auto PathRef = DB.storeFromString({}, Filename); if (!PathRef) return PathRef.takeError(); @@ -42,58 +46,47 @@ Expected IncludeFile::create(ObjectStore &DB, StringRef Filename, return IncludeTreeBase::create(DB, Refs, {}); } +Expected IncludeTree::getBaseFile() { + auto Node = getCAS().getProxy(getBaseFileRef()); + if (!Node) + return Node.takeError(); + return File(std::move(*Node)); +} + +Expected IncludeTree::getBaseFileInfo() { + auto File = getBaseFile(); + if (!File) + return File.takeError(); + return File->getFileInfo(); +} + llvm::Error IncludeTree::forEachInclude( - llvm::function_ref)> - Callback) { + llvm::function_ref)> Callback) { size_t RefI = 0; + const size_t IncludeEnd = getNumIncludes(); return forEachReference([&](ObjectRef Ref) -> llvm::Error { if (RefI == 0) { ++RefI; return llvm::Error::success(); } size_t IncludeI = RefI - 1; + if (IncludeI >= IncludeEnd) + return llvm::Error::success(); ++RefI; - auto Include = getInclude(Ref); + auto Include = getIncludeNode(Ref, getIncludeKind(IncludeI)); if (!Include) return Include.takeError(); return Callback({*Include, getIncludeOffset(IncludeI)}); }); } -Expected IncludeTree::create( - ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, - ObjectRef BaseFile, ArrayRef> Includes, - llvm::SmallBitVector Checks) { - // The data buffer is composed of - // 1. `uint32_t` offsets of includes - // 2. 1 byte for `CharacteristicKind` - // 3. variable number of bitset bytes for `Checks`. - - char Kind = FileCharacteristic; - assert(Kind == FileCharacteristic && "SrcMgr::CharacteristicKind too big!"); - assert(IncludeFile::isValid(DB, BaseFile)); - SmallVector Refs; - Refs.reserve(Includes.size() + 1); - Refs.push_back(BaseFile); - SmallString<64> Buffer; - Buffer.reserve(Includes.size() * sizeof(uint32_t) + 1); - - llvm::raw_svector_ostream BufOS(Buffer); - llvm::support::endian::Writer Writer(BufOS, llvm::support::little); - - for (const auto &Include : Includes) { - ObjectRef FileRef = Include.first; - uint32_t Offset = Include.second; - assert(IncludeTree::isValid(DB, FileRef)); - Refs.push_back(FileRef); - Writer.write(Offset); - } - - Buffer += Kind; - +/// Write the bitset \p Bits to \p Writer, filling the final byte with zeros for +/// any unused values. Note: this does not store the size of the bitset. +static void writeBitSet(llvm::support::endian::Writer &Writer, + const llvm::SmallBitVector &Bits) { uintptr_t Store; - ArrayRef BitWords = Checks.getData(Store); - size_t RemainingBitsCount = Checks.size(); + ArrayRef BitWords = Bits.getData(Store); + size_t RemainingBitsCount = Bits.size(); while (RemainingBitsCount > 0) { if (BitWords.size() > 1) { Writer.write(BitWords.front()); @@ -107,11 +100,55 @@ Expected IncludeTree::create( if (RemainingBitsCount % CHAR_BIT != 0) ++BytesNum; while (BytesNum--) { - Buffer.push_back(LastWord & 0xFF); + Writer.write(static_cast(LastWord & 0xFF)); LastWord >>= CHAR_BIT; } break; } +} + +Expected IncludeTree::create( + ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, + ObjectRef BaseFile, ArrayRef Includes, + Optional SubmoduleName, llvm::SmallBitVector Checks) { + // The data buffer is composed of + // 1. 1 byte for `CharacteristicKind` and IsSubmodule + // 2. `uint32_t` offset and `uint8_t` kind for each includes + // 3. variable number of bitset bytes for `Checks`. + + char Kind = FileCharacteristic; + assert(Kind == FileCharacteristic && (Kind & IsSubmoduleBit) == 0 && + "SrcMgr::CharacteristicKind too big!"); + if (SubmoduleName) + Kind |= IsSubmoduleBit; + + assert(File::isValid(DB, BaseFile)); + SmallVector Refs; + Refs.reserve(Includes.size() + 2); + Refs.push_back(BaseFile); + SmallString<64> Buffer; + Buffer.reserve(Includes.size() * sizeof(uint32_t) + 1); + + Buffer += Kind; + + llvm::raw_svector_ostream BufOS(Buffer); + llvm::support::endian::Writer Writer(BufOS, llvm::support::little); + + for (const auto &Include : Includes) { + assert((Include.Kind == NodeKind::Tree && + IncludeTree::isValid(DB, Include.Ref)) || + (Include.Kind == NodeKind::ModuleImport && + ModuleImport::isValid(DB, Include.Ref))); + Refs.push_back(Include.Ref); + Writer.write(Include.Offset); + static_assert(sizeof(uint8_t) == sizeof(Kind)); + Writer.write(static_cast(Include.Kind)); + } + + if (SubmoduleName) + Refs.push_back(*SubmoduleName); + + writeBitSet(Writer, Checks); return IncludeTreeBase::create(DB, Refs, Buffer); } @@ -126,102 +163,433 @@ Expected IncludeTree::get(ObjectStore &DB, ObjectRef Ref) { return IncludeTree(std::move(*Node)); } +IncludeTree::NodeKind IncludeTree::getIncludeKind(size_t I) const { + assert(I < getNumIncludes()); + StringRef Data = dataSkippingFlags(); + assert(Data.size() >= (I + 1) * (sizeof(uint32_t) + 1)); + uint8_t K = *(Data.data() + I * (sizeof(uint32_t) + 1) + sizeof(uint32_t)); + return NodeKind(K); +} + uint32_t IncludeTree::getIncludeOffset(size_t I) const { assert(I < getNumIncludes()); - StringRef Data = getData(); + StringRef Data = dataSkippingFlags(); assert(Data.size() >= (I + 1) * sizeof(uint32_t)); uint32_t Offset = llvm::support::endian::read( - Data.data() + I * sizeof(uint32_t)); + Data.data() + I * (sizeof(uint32_t) + 1)); return Offset; } bool IncludeTree::getCheckResult(size_t I) const { // Skip include offsets and CharacteristicKind. - StringRef Data = dataSkippingIncludes().drop_front(); + StringRef Data = dataSkippingIncludes(); unsigned ByteIndex = I / CHAR_BIT; size_t RemainingIndex = I % CHAR_BIT; uint8_t Bits = Data[ByteIndex]; return Bits & (1 << RemainingIndex); } +Expected IncludeTree::getIncludeNode(size_t I) { + return getIncludeNode(getIncludeRef(I), getIncludeKind(I)); +} + +Expected IncludeTree::getIncludeNode(ObjectRef Ref, + NodeKind K) { + auto N = getCAS().getProxy(Ref); + if (!N) + return N.takeError(); + return Node(std::move(*N), K); +} + bool IncludeTree::isValid(const ObjectProxy &Node) { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); - if (Base.getNumReferences() == 0) + if (Base.getNumReferences() == 0 || Base.getData().empty()) return false; unsigned NumIncludes = Base.getNumReferences() - 1; - return Base.getData().size() >= NumIncludes * sizeof(uint32_t) + 1; + if (Base.getData().front() & IsSubmoduleBit) + NumIncludes -= 1; + return Base.getData().size() >= NumIncludes * (sizeof(uint32_t) + 1) + 1; } -IncludeFileList::FileSizeTy IncludeFileList::getFileSize(size_t I) const { - assert(I < getNumFiles()); - StringRef Data = getData(); +Expected +IncludeTree::ModuleImport::create(ObjectStore &DB, StringRef ModuleName, + bool VisibilityOnly) { + SmallString<64> Buffer; + Buffer.push_back((char)VisibilityOnly); + Buffer.append(ModuleName); + return IncludeTreeBase::create(DB, {}, Buffer); +} + +size_t IncludeTree::FileList::getNumFilesCurrentList() const { + return llvm::support::endian::read( + getData().data()); +} + +IncludeTree::FileList::FileSizeTy +IncludeTree::FileList::getFileSize(size_t I) const { + assert(I < getNumFilesCurrentList()); + StringRef Data = getData().drop_front(sizeof(uint32_t)); assert(Data.size() >= (I + 1) * sizeof(FileSizeTy)); return llvm::support::endian::read( Data.data() + I * sizeof(FileSizeTy)); } -llvm::Error IncludeFileList::forEachFile( - llvm::function_ref Callback) { - size_t I = 0; +llvm::Error IncludeTree::FileList::forEachFileImpl( + llvm::DenseSet &Seen, + llvm::function_ref Callback) { + size_t Next = 0; + size_t FileCount = getNumFilesCurrentList(); return forEachReference([&](ObjectRef Ref) -> llvm::Error { - auto Include = getFile(Ref); - if (!Include) - return Include.takeError(); - return Callback(std::move(*Include), getFileSize(I++)); + size_t Index = Next++; + if (!Seen.insert(Ref).second) + return llvm::Error::success(); + + if (Index < FileCount) { + auto Include = getFile(Ref); + if (!Include) + return Include.takeError(); + return Callback(std::move(*Include), getFileSize(Index)); + } + + // Otherwise, it's a chained FileList. + auto Proxy = getCAS().getProxy(Ref); + if (!Proxy) + return Proxy.takeError(); + FileList FL(std::move(*Proxy)); + return FL.forEachFileImpl(Seen, Callback); }); } -Expected IncludeFileList::create(ObjectStore &DB, - ArrayRef Files) { +llvm::Error IncludeTree::FileList::forEachFile( + llvm::function_ref Callback) { + llvm::DenseSet Seen; + return forEachFileImpl(Seen, Callback); +} + +Expected +IncludeTree::FileList::create(ObjectStore &DB, ArrayRef Files, + ArrayRef FileLists) { SmallVector Refs; - Refs.reserve(Files.size()); + Refs.reserve(Files.size() + FileLists.size()); SmallString<256> Buffer; - Buffer.reserve(Files.size() * sizeof(FileSizeTy)); + Buffer.reserve(sizeof(uint32_t) + Files.size() * sizeof(FileSizeTy)); llvm::raw_svector_ostream BufOS(Buffer); llvm::support::endian::Writer Writer(BufOS, llvm::support::little); + Writer.write(static_cast(Files.size())); for (const FileEntry &Entry : Files) { - assert(IncludeFile::isValid(DB, Entry.FileRef)); + assert(File::isValid(DB, Entry.FileRef)); Refs.push_back(Entry.FileRef); Writer.write(Entry.Size); } + + Refs.append(FileLists.begin(), FileLists.end()); + return IncludeTreeBase::create(DB, Refs, Buffer); } -Expected IncludeFileList::get(ObjectStore &DB, ObjectRef Ref) { +Expected IncludeTree::FileList::get(ObjectStore &DB, + ObjectRef Ref) { auto Node = DB.getProxy(Ref); if (!Node) return Node.takeError(); if (!isValid(*Node)) return llvm::createStringError(llvm::inconvertibleErrorCode(), "not a IncludeFileList node kind"); - return IncludeFileList(std::move(*Node)); + return FileList(std::move(*Node)); } -bool IncludeFileList::isValid(const ObjectProxy &Node) { +bool IncludeTree::FileList::isValid(const ObjectProxy &Node) { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); - unsigned NumFiles = Base.getNumReferences(); - return NumFiles != 0 && - Base.getData().size() == NumFiles * sizeof(FileSizeTy); + StringRef Data = Base.getData(); + if (Data.size() < sizeof(uint32_t)) + return false; + unsigned NumFiles = + llvm::support::endian::read(Data.data()); + return NumFiles != 0 && NumFiles <= Base.getNumReferences() && + Data.size() == sizeof(uint32_t) + NumFiles * sizeof(FileSizeTy); } -Expected IncludeTreeRoot::create(ObjectStore &DB, - ObjectRef MainFileTree, - ObjectRef FileList, - Optional PCHRef) { - assert(IncludeTree::isValid(DB, MainFileTree)); - assert(IncludeFileList::isValid(DB, FileList)); - if (PCHRef) { - return IncludeTreeBase::create(DB, {MainFileTree, FileList, *PCHRef}, {}); - } else { - return IncludeTreeBase::create(DB, {MainFileTree, FileList}, {}); +static constexpr uint16_t ModuleFlagFramework = 1 << 0; +static constexpr uint16_t ModuleFlagExplicit = 1 << 1; +static constexpr uint16_t ModuleFlagExternC = 1 << 2; +static constexpr uint16_t ModuleFlagSystem = 1 << 3; +static constexpr uint16_t ModuleFlagInferSubmodules = 1 << 4; +static constexpr uint16_t ModuleFlagInferExplicitSubmodules = 1 << 5; +static constexpr uint16_t ModuleFlagInferInferExportWildcard = 1 << 6; +static constexpr uint16_t ModuleFlagHasExports = 1 << 7; +static constexpr uint16_t ModuleFlagHasLinkLibraries = 1 << 8; + +IncludeTree::Module::ModuleFlags IncludeTree::Module::getFlags() const { + uint16_t Raw = rawFlags(); + ModuleFlags Flags; + Flags.IsFramework = Raw & ModuleFlagFramework; + Flags.IsExplicit = Raw & ModuleFlagExplicit; + Flags.IsExternC = Raw & ModuleFlagExternC; + Flags.IsSystem = Raw & ModuleFlagSystem; + Flags.InferSubmodules = Raw & ModuleFlagInferSubmodules; + Flags.InferExplicitSubmodules = Raw & ModuleFlagInferExplicitSubmodules; + Flags.InferExportWildcard = Raw & ModuleFlagInferInferExportWildcard; + return Flags; +} + +size_t IncludeTree::Module::getNumSubmodules() const { + size_t Count = getNumReferences(); + if (hasExports()) + Count -= 1; + if (hasLinkLibraries()) + Count -= 1; + return Count; +} + +llvm::Error IncludeTree::Module::forEachSubmodule( + llvm::function_ref CB) { + size_t Count = getNumSubmodules(); + return forEachReference([&](ObjectRef Ref) -> llvm::Error { + if (Count == 0) + return llvm::Error::success(); + Count -= 1; + auto Node = getCAS().getProxy(Ref); + if (!Node) + return Node.takeError(); + return CB(Module(*Node)); + }); +} + +Expected +IncludeTree::Module::create(ObjectStore &DB, StringRef ModuleName, + ModuleFlags Flags, ArrayRef Submodules, + std::optional ExportList, + std::optional LinkLibraries) { + // Data: + // - 2 bytes for Flags + // - ModuleName (String) + // Refs: + // - Submodules (IncludeTreeModule) + // - (optional) ExportList + // - (optional) LinkLibaryList + + uint16_t RawFlags = 0; + if (Flags.IsFramework) + RawFlags |= ModuleFlagFramework; + if (Flags.IsExplicit) + RawFlags |= ModuleFlagExplicit; + if (Flags.IsExternC) + RawFlags |= ModuleFlagExternC; + if (Flags.IsSystem) + RawFlags |= ModuleFlagSystem; + if (Flags.InferSubmodules) + RawFlags |= ModuleFlagInferSubmodules; + if (Flags.InferExplicitSubmodules) + RawFlags |= ModuleFlagInferExplicitSubmodules; + if (Flags.InferExportWildcard) + RawFlags |= ModuleFlagInferInferExportWildcard; + if (ExportList) + RawFlags |= ModuleFlagHasExports; + if (LinkLibraries) + RawFlags |= ModuleFlagHasLinkLibraries; + + SmallString<64> Buffer; + llvm::raw_svector_ostream BufOS(Buffer); + llvm::support::endian::Writer Writer(BufOS, llvm::support::little); + Writer.write(RawFlags); + + Buffer.append(ModuleName); + + SmallVector Refs(Submodules); + if (ExportList) + Refs.push_back(*ExportList); + if (LinkLibraries) + Refs.push_back(*LinkLibraries); + + return IncludeTreeBase::create(DB, Refs, Buffer); +} + +uint16_t IncludeTree::Module::rawFlags() const { + return llvm::support::endian::read( + getData().data()); +} + +bool IncludeTree::Module::hasExports() const { + return rawFlags() & ModuleFlagHasExports; +} +bool IncludeTree::Module::hasLinkLibraries() const { + return rawFlags() & ModuleFlagHasLinkLibraries; +} + +std::optional IncludeTree::Module::getExportsIndex() const { + if (hasExports()) + return getNumReferences() - (hasLinkLibraries() ? 2 : 1); + return std::nullopt; +} +std::optional IncludeTree::Module::getLinkLibrariesIndex() const { + if (hasLinkLibraries()) + return getNumReferences() - 1; + return std::nullopt; +} + +Expected> +IncludeTree::Module::getExports() { + if (auto Ref = getExportsRef()) { + auto N = getCAS().getProxy(*Ref); + if (!N) + return N.takeError(); + return ExportList(std::move(*N)); } + return std::nullopt; +} + +/// The list of modules that this submodule re-exports. +Expected> +IncludeTree::Module::getLinkLibraries() { + if (auto Ref = getLinkLibrariesRef()) { + auto N = getCAS().getProxy(*Ref); + if (!N) + return N.takeError(); + return LinkLibraryList(std::move(*N)); + } + return std::nullopt; +} + +bool IncludeTree::Module::ExportList::hasGlobalWildcard() const { + // The bit after explicit exports is global. + return exportHasWildcard(getNumExplicitExports()); +} +bool IncludeTree::Module::ExportList::exportHasWildcard(size_t I) const { + assert(I < getNumExplicitExports() + 1); + unsigned ByteIndex = I / CHAR_BIT; + size_t RemainingIndex = I % CHAR_BIT; + uint8_t Bits = getData()[ByteIndex]; + return Bits & (1 << RemainingIndex); +} +Expected +IncludeTree::Module::ExportList::getExplicitExport(size_t I) { + Expected Name = getCAS().getProxy(getReference(I)); + if (!Name) + return Name.takeError(); + return Export{Name->getData(), exportHasWildcard(I)}; +} +llvm::Error IncludeTree::Module::ExportList::forEachExplicitExport( + llvm::function_ref CB) { + size_t ExportI = 0; + return forEachReference([&](ObjectRef Ref) { + Expected Name = getCAS().getProxy(Ref); + if (!Name) + return Name.takeError(); + return CB(Export{Name->getData(), exportHasWildcard(ExportI)}); + }); +} +Expected +IncludeTree::Module::ExportList::create(ObjectStore &DB, + ArrayRef Exports, + bool GlobalWildcard) { + // Data: + // - 1 bit per explicit export for wildcard + // - 1 bit for global wildcard + // Refs: export names + SmallString<64> Buffer; + llvm::raw_svector_ostream BufOS(Buffer); + llvm::support::endian::Writer Writer(BufOS, llvm::support::little); + SmallVector Refs; + llvm::SmallBitVector WildcardBits; + for (Export E : Exports) { + auto Ref = DB.storeFromString({}, E.ModuleName); + if (!Ref) + return Ref.takeError(); + Refs.push_back(*Ref); + WildcardBits.push_back(E.Wildcard); + } + WildcardBits.push_back(GlobalWildcard); + writeBitSet(Writer, WildcardBits); + + return IncludeTreeBase::create(DB, Refs, Buffer); +} + +bool IncludeTree::Module::LinkLibraryList::isFramework(size_t I) const { + assert(I < getNumLibraries()); + unsigned ByteIndex = I / CHAR_BIT; + size_t RemainingIndex = I % CHAR_BIT; + uint8_t Bits = getData()[ByteIndex]; + return Bits & (1 << RemainingIndex); +} +llvm::Error IncludeTree::Module::LinkLibraryList::forEachLinkLibrary( + llvm::function_ref CB) { + size_t I = 0; + return forEachReference([&](ObjectRef Ref) { + auto Name = getCAS().getProxy(getLibraryNameRef(I)); + if (!Name) + return Name.takeError(); + return CB({Name->getData(), isFramework(I++)}); + }); +} +Expected +IncludeTree::Module::LinkLibraryList::create(ObjectStore &DB, + ArrayRef Libraries) { + // Data: + // - 1 bit per library for IsFramework + // Refs: library names + SmallString<64> Buffer; + llvm::raw_svector_ostream BufOS(Buffer); + llvm::support::endian::Writer Writer(BufOS, llvm::support::little); + SmallVector Refs; + llvm::SmallBitVector FrameworkBits; + for (LinkLibrary L : Libraries) { + auto Ref = DB.storeFromString({}, L.Library); + if (!Ref) + return Ref.takeError(); + Refs.push_back(*Ref); + FrameworkBits.push_back(L.IsFramework); + } + writeBitSet(Writer, FrameworkBits); + + return IncludeTreeBase::create(DB, Refs, Buffer); +} + +Expected +IncludeTree::ModuleMap::create(ObjectStore &DB, ArrayRef Modules) { + return IncludeTreeBase::create(DB, Modules, {}); +} + +llvm::Error IncludeTree::ModuleMap::forEachModule( + llvm::function_ref CB) { + return forEachReference([&](ObjectRef Ref) { + auto N = getCAS().getProxy(Ref); + if (!N) + return N.takeError(); + return CB(Module(std::move(*N))); + }); +} + +static constexpr char HasPCH = 0x01; +static constexpr char HasModuleMap = 0x02; + +Expected +IncludeTreeRoot::create(ObjectStore &DB, ObjectRef MainFileTree, + ObjectRef FileList, Optional PCHRef, + Optional ModuleMapRef) { + assert(IncludeTree::isValid(DB, MainFileTree)); + assert(IncludeTree::FileList::isValid(DB, FileList)); + assert(!ModuleMapRef || IncludeTree::ModuleMap::isValid(DB, *ModuleMapRef)); + + std::array Data = {0}; + if (PCHRef) + Data[0] |= HasPCH; + if (ModuleMapRef) + Data[0] |= HasModuleMap; + + SmallVector Refs = {MainFileTree, FileList}; + if (PCHRef) + Refs.push_back(*PCHRef); + if (ModuleMapRef) + Refs.push_back(*ModuleMapRef); + + return IncludeTreeBase::create(DB, Refs, Data); } Expected IncludeTreeRoot::get(ObjectStore &DB, ObjectRef Ref) { @@ -234,7 +602,18 @@ Expected IncludeTreeRoot::get(ObjectStore &DB, ObjectRef Ref) { return IncludeTreeRoot(std::move(*Node)); } -llvm::Error IncludeFile::print(llvm::raw_ostream &OS, unsigned Indent) { +std::optional IncludeTreeRoot::getPCHRefIndex() const { + if (getData()[0] & HasPCH) + return 2; + return std::nullopt; +} +std::optional IncludeTreeRoot::getModuleMapRefIndex() const { + if (getData()[0] & HasModuleMap) + return (getData()[0] & HasPCH) ? 3u : 2u; + return std::nullopt; +} + +llvm::Error IncludeTree::File::print(llvm::raw_ostream &OS, unsigned Indent) { auto Filename = getFilename(); if (!Filename) return Filename.takeError(); @@ -251,6 +630,12 @@ llvm::Error IncludeTree::print(llvm::raw_ostream &OS, unsigned Indent) { if (llvm::Error E = IncludeBy->print(OS)) return E; + auto Submodule = getSubmoduleName(); + if (!Submodule) + return Submodule.takeError(); + if (*Submodule) + OS.indent(Indent) << "Submodule: " << **Submodule << '\n'; + llvm::SourceMgr SM; auto Blob = IncludeBy->getContents(); if (!Blob) @@ -258,22 +643,106 @@ llvm::Error IncludeTree::print(llvm::raw_ostream &OS, unsigned Indent) { auto MemBuf = llvm::MemoryBuffer::getMemBuffer(Blob->getData()); unsigned BufID = SM.AddNewSourceBuffer(std::move(MemBuf), llvm::SMLoc()); - return forEachInclude( - [&](std::pair Include) -> llvm::Error { - llvm::SMLoc Loc = llvm::SMLoc::getFromPointer( - SM.getMemoryBuffer(BufID)->getBufferStart() + Include.second); - auto LineCol = SM.getLineAndColumn(Loc); - OS.indent(Indent) << LineCol.first << ':' << LineCol.second << ' '; - return Include.first.print(OS, Indent + 2); - }); + return forEachInclude([&](std::pair Include) -> llvm::Error { + llvm::SMLoc Loc = llvm::SMLoc::getFromPointer( + SM.getMemoryBuffer(BufID)->getBufferStart() + Include.second); + auto LineCol = SM.getLineAndColumn(Loc); + OS.indent(Indent) << LineCol.first << ':' << LineCol.second << ' '; + return Include.first.print(OS, Indent + 2); + }); } -llvm::Error IncludeFileList::print(llvm::raw_ostream &OS, unsigned Indent) { - return forEachFile([&](cas::IncludeFile File, FileSizeTy) -> llvm::Error { +llvm::Error IncludeTree::FileList::print(llvm::raw_ostream &OS, + unsigned Indent) { + return forEachFile([&](File File, FileSizeTy) -> llvm::Error { return File.print(OS, Indent); }); } +llvm::Error IncludeTree::ModuleImport::print(llvm::raw_ostream &OS, + unsigned Indent) { + if (visibilityOnly()) + OS << "(Module for visibility only) "; + else + OS << "(Module) "; + OS << getModuleName() << '\n'; + return llvm::Error::success(); +} + +llvm::Error IncludeTree::Node::print(llvm::raw_ostream &OS, unsigned Indent) { + switch (K) { + case NodeKind::Tree: + return getIncludeTree().print(OS, Indent); + case NodeKind::ModuleImport: + return getModuleImport().print(OS, Indent); + } +} + +llvm::Error IncludeTree::Module::print(llvm::raw_ostream &OS, unsigned Indent) { + OS.indent(Indent) << getName(); + ModuleFlags Flags = getFlags(); + if (Flags.IsFramework) + OS << " (framework)"; + if (Flags.IsExplicit) + OS << " (explicit)"; + if (Flags.IsExternC) + OS << " (extern_c)"; + if (Flags.IsSystem) + OS << " (system)"; + OS << '\n'; + if (Flags.InferSubmodules) { + if (Flags.InferExplicitSubmodules) + OS << " explicit module *"; + else + OS << " module *"; + if (Flags.InferExportWildcard) + OS << " { export * }"; + OS << '\n'; + } + auto ExportList = getExports(); + if (!ExportList) + return ExportList.takeError(); + if (*ExportList) + if (llvm::Error E = (*ExportList)->print(OS, Indent + 2)) + return E; + auto LinkLibraries = getLinkLibraries(); + if (!LinkLibraries) + return LinkLibraries.takeError(); + if (*LinkLibraries) + if (llvm::Error E = (*LinkLibraries)->print(OS, Indent + 2)) + return E; + return forEachSubmodule( + [&](Module Sub) { return Sub.print(OS, Indent + 2); }); +} +llvm::Error IncludeTree::Module::ExportList::print(llvm::raw_ostream &OS, + unsigned Indent) { + if (hasGlobalWildcard()) + OS.indent(Indent) << "export *\n"; + return forEachExplicitExport([&](Export E) { + OS.indent(Indent) << "export " << E.ModuleName; + if (E.Wildcard) + OS << ".*"; + OS << '\n'; + return llvm::Error::success(); + }); +} + +llvm::Error IncludeTree::Module::LinkLibraryList::print(llvm::raw_ostream &OS, + unsigned Indent) { + return forEachLinkLibrary([&](LinkLibrary E) { + OS.indent(Indent) << "link " << E.Library; + if (E.IsFramework) + OS << " (framework)"; + OS << '\n'; + return llvm::Error::success(); + }); +} + +llvm::Error IncludeTree::ModuleMap::print(llvm::raw_ostream &OS, + unsigned Indent) { + return forEachModule([&](Module M) { return M.print(OS, Indent); }); +} + llvm::Error IncludeTreeRoot::print(llvm::raw_ostream &OS, unsigned Indent) { if (Optional PCHRef = getPCHRef()) { OS.indent(Indent) << "(PCH) "; @@ -285,8 +754,16 @@ llvm::Error IncludeTreeRoot::print(llvm::raw_ostream &OS, unsigned Indent) { return E; if (llvm::Error E = MainTree->print(OS.indent(Indent), Indent)) return E; + Optional ModuleMap; + if (llvm::Error E = getModuleMap().moveInto(ModuleMap)) + return E; + if (ModuleMap) { + OS.indent(Indent) << "Module Map:\n"; + if (llvm::Error E = ModuleMap->print(OS, Indent)) + return E; + } OS.indent(Indent) << "Files:\n"; - Optional List; + Optional List; if (llvm::Error E = getFileList().moveInto(List)) return E; return List->print(OS, Indent); @@ -333,7 +810,7 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { struct FileEntry { cas::ObjectRef ContentsRef; - IncludeFileList::FileSizeTy Size; + IncludeTree::FileList::FileSizeTy Size; llvm::sys::fs::UniqueID UniqueID; }; @@ -350,9 +827,8 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { Directories{Alloc}; llvm::ErrorOr status(const Twine &Path) override { - SmallString<128> FilenameBuffer; - StringRef Filename = Path.toStringRef(FilenameBuffer); - + SmallString<128> Filename; + getPath(Path, Filename); auto FileEntry = Files.find(Filename); if (FileEntry != Files.end()) { return makeStatus(Filename, FileEntry->second.Size, @@ -372,8 +848,8 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { llvm::ErrorOr> openFileForRead(const Twine &Path) override { - SmallString<128> FilenameBuffer; - StringRef Filename = Path.toStringRef(FilenameBuffer); + SmallString<128> Filename; + getPath(Path, Filename); auto MaterializedFile = materialize(Filename); if (!MaterializedFile) return llvm::errorToErrorCode(MaterializedFile.takeError()); @@ -396,6 +872,15 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { return MaterializedFile{ContentsBlob->getData(), Entry->second}; } + /// Produce a filename compatible with our StringMaps. See comment in + /// \c createIncludeTreeFileSystem. + void getPath(const Twine &Path, SmallVectorImpl &Out) { + Path.toVector(Out); + // Strip dots, but do not eliminate a path consisting only of '.' + if (Out.size() != 1) + llvm::sys::path::remove_dots(Out); + } + static llvm::vfs::Status makeStatus(StringRef Filename, uint64_t Size, llvm::sys::fs::UniqueID UniqueID, llvm::sys::fs::file_type Type) { @@ -427,20 +912,52 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { }; } // namespace +static llvm::Error diagnoseFileChange(IncludeTree::File F, ObjectRef Content) { + auto FilenameBlob = F.getFilename(); + if (!FilenameBlob) + return FilenameBlob.takeError(); + cas::ObjectStore &DB = F.getCAS(); + std::string Filename(FilenameBlob->getData()); + std::string OldID = DB.getID(Content).toString(); + std::string NewID = DB.getID(F.getContentsRef()).toString(); + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "file '%s' changed during build; include-tree " + "contents changed from %s to %s", + Filename.c_str(), OldID.c_str(), + NewID.c_str()); +} + Expected> cas::createIncludeTreeFileSystem(IncludeTreeRoot &Root) { auto FileList = Root.getFileList(); if (!FileList) return FileList.takeError(); + // Map from FilenameRef to ContentsRef. + llvm::DenseMap SeenContents; + IntrusiveRefCntPtr IncludeTreeFS = new IncludeTreeFileSystem(Root.getCAS()); llvm::Error E = FileList->forEachFile( - [&](IncludeFile File, IncludeFileList::FileSizeTy Size) -> llvm::Error { + [&](IncludeTree::File File, + IncludeTree::FileList::FileSizeTy Size) -> llvm::Error { + auto InsertPair = SeenContents.insert( + std::make_pair(File.getFilenameRef(), File.getContentsRef())); + if (!InsertPair.second) { + if (InsertPair.first->second != File.getContentsRef()) + return diagnoseFileChange(File, InsertPair.first->second); + return llvm::Error::success(); + } + auto FilenameBlob = File.getFilename(); if (!FilenameBlob) return FilenameBlob.takeError(); - StringRef Filename = FilenameBlob->getData(); + + SmallString<128> Filename(FilenameBlob->getData()); + // Strip './' in the filename to match the behaviour of ASTWriter; we + // also strip './' in IncludeTreeFileSystem::getPath. + assert(Filename != "."); + llvm::sys::path::remove_dots(Filename); StringRef DirName = llvm::sys::path::parent_path(Filename); if (DirName.empty()) diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index ec21cbfcfcb13..1008c1b376bc4 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -1190,7 +1190,8 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA, const Driver &D, const ArgList &Args, ArgStringList &CmdArgs, const InputInfo &Output, - const InputInfoList &Inputs) const { + const InputInfoList &Inputs, + std::optional &Sysroot) const { const bool IsIAMCU = getToolChain().getTriple().isOSIAMCU(); CheckPreprocessingOptions(D, Args); @@ -1428,10 +1429,15 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA, // -isysroot to the CC1 invocation. StringRef sysroot = C.getSysRoot(); if (sysroot != "") { - if (!Args.hasArg(options::OPT_isysroot)) { + if (Arg *A = Args.getLastArg(options::OPT_isysroot)) { + Sysroot = A->getValue(); + } else { CmdArgs.push_back("-isysroot"); CmdArgs.push_back(C.getArgs().MakeArgString(sysroot)); + Sysroot = sysroot; } + } else if (Arg *A = Args.getLastArg(options::OPT_isysroot)) { + Sysroot = A->getValue(); } // Parse additional include paths from environment variables. @@ -4524,6 +4530,52 @@ static void ProcessVSRuntimeLibrary(const ArgList &Args, } } +void Clang::AddPrefixMappingOptions(const ArgList &Args, ArgStringList &CmdArgs, + const Driver &D, + std::optional Sysroot) const { + auto IsPathApplicableAsPrefix = [](std::optional Path) { + return Path && llvm::sys::path::is_absolute(*Path) && + *Path != llvm::sys::path::root_path(*Path); + }; + + if (Arg *A = Args.getLastArg(options::OPT_fdepscan_prefix_map_sdk_EQ)) { + if (IsPathApplicableAsPrefix(Sysroot)) { + CmdArgs.push_back(Args.MakeArgString(Twine("-fdepscan-prefix-map=") + + *Sysroot + "=" + A->getValue())); + } else { + // FIXME: warning if we cannot infer sdk + } + } + + if (Arg *A = Args.getLastArg(options::OPT_fdepscan_prefix_map_toolchain_EQ)) { + StringRef ResourceDir = D.ResourceDir; + StringRef Guess = llvm::sys::path::parent_path(ResourceDir); + for (StringRef Dir : {"clang", "lib", "usr"}) { + if (llvm::sys::path::filename(Guess) != Dir) + break; + Guess = llvm::sys::path::parent_path(Guess); + } + if (IsPathApplicableAsPrefix(Guess)) { + CmdArgs.push_back(Args.MakeArgString(Twine("-fdepscan-prefix-map=") + + Guess + "=" + A->getValue())); + } else { + // FIXME: warning if we cannot infer toolchain + } + } + + for (const Arg *A : Args.filtered(options::OPT_fdepscan_prefix_map_EQ)) { + A->claim(); + StringRef Map = A->getValue(); + StringRef Prefix = Map.split('=').first; + if (Prefix.size() == Map.size() || !IsPathApplicableAsPrefix(Prefix)) { + D.Diag(diag::err_drv_invalid_argument_to_option) + << A->getValue() << A->getOption().getName(); + } else { + A->render(Args, CmdArgs); + } + } +} + void Clang::ConstructJob(Compilation &C, const JobAction &Job, const InputInfo &Output, const InputInfoList &Inputs, const ArgList &Args, const char *LinkingOutput) const { @@ -4640,19 +4692,15 @@ void Clang::ConstructJob(Compilation &C, const JobAction &Job, D.Diag(diag::err_drv_clang_unsupported) << "C++ for IAMCU"; { - const OptSpecifier DepScanOpts[] = { - options::OPT_fdepscan_EQ, - options::OPT_fdepscan_include_tree, - options::OPT_fdepscan_share_EQ, - options::OPT_fdepscan_share_identifier, - options::OPT_fdepscan_share_parent, - options::OPT_fdepscan_share_parent_EQ, - options::OPT_fno_depscan_share, - options::OPT_fdepscan_share_stop_EQ, - options::OPT_fdepscan_daemon_EQ, - options::OPT_fdepscan_prefix_map_EQ, - options::OPT_fdepscan_prefix_map_sdk_EQ, - options::OPT_fdepscan_prefix_map_toolchain_EQ}; + const OptSpecifier DepScanOpts[] = {options::OPT_fdepscan_EQ, + options::OPT_fdepscan_include_tree, + options::OPT_fdepscan_share_EQ, + options::OPT_fdepscan_share_identifier, + options::OPT_fdepscan_share_parent, + options::OPT_fdepscan_share_parent_EQ, + options::OPT_fno_depscan_share, + options::OPT_fdepscan_share_stop_EQ, + options::OPT_fdepscan_daemon_EQ}; // Handle depscan. if (Job.getKind() == Action::DepscanJobClass) { @@ -5890,8 +5938,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &Job, // preprocessor. // // FIXME: Support -fpreprocessed + std::optional Sysroot; if (types::getPreprocessedType(InputType) != types::TY_INVALID) - AddPreprocessingOptions(C, JA, D, Args, CmdArgs, Output, Inputs); + AddPreprocessingOptions(C, JA, D, Args, CmdArgs, Output, Inputs, Sysroot); + + // Handle -fdepscan-prefix-map-* options + AddPrefixMappingOptions(Args, CmdArgs, D, Sysroot); // Don't warn about "clang -c -DPIC -fPIC test.i" because libtool.m4 assumes // that "The compiler can only warn and ignore the option if not recognized". diff --git a/clang/lib/Driver/ToolChains/Clang.h b/clang/lib/Driver/ToolChains/Clang.h index d7550665539cc..e5940b45f6620 100644 --- a/clang/lib/Driver/ToolChains/Clang.h +++ b/clang/lib/Driver/ToolChains/Clang.h @@ -43,7 +43,8 @@ class LLVM_LIBRARY_VISIBILITY Clang : public Tool { const Driver &D, const llvm::opt::ArgList &Args, llvm::opt::ArgStringList &CmdArgs, const InputInfo &Output, - const InputInfoList &Inputs) const; + const InputInfoList &Inputs, + std::optional &Sysroot) const; void RenderTargetOptions(const llvm::Triple &EffectiveTriple, const llvm::opt::ArgList &Args, bool KernelOrKext, @@ -104,6 +105,11 @@ class LLVM_LIBRARY_VISIBILITY Clang : public Tool { StringRef Dir, Compilation &C, StringRef Target, const InputInfo &Output, const InputInfo &Input, const llvm::opt::ArgList &Args) const; + void AddPrefixMappingOptions(const llvm::opt::ArgList &Args, + llvm::opt::ArgStringList &CmdArgs, + const Driver &D, + std::optional Sysroot) const; + public: Clang(const ToolChain &TC, bool HasIntegratedBackend = true); ~Clang() override; diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 2ee4509d7eb42..96da881f2b0ea 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -640,10 +640,14 @@ class CompileCacheASTReaderHelper : public ASTReaderListener { : CAS(CAS), Cache(Cache), ModuleCache(ModuleCache), Diags(Diags) {} bool readCASFileSystemRootID(StringRef RootID, bool Complain) override; + bool readIncludeTreeID(StringRef ID, bool Complain) override; bool readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) override; private: + bool checkCASID(bool Complain, StringRef RootID, unsigned ParseDiagID, + unsigned MissingDiagID); + cas::ObjectStore &CAS; cas::ActionCache &Cache; InMemoryModuleCache &ModuleCache; @@ -2077,8 +2081,12 @@ CompilerInstance::loadModule(SourceLocation ImportLoc, PrivateModule, PP->getIdentifierInfo(Module->Name)->getTokenID()); PrivPath.push_back(std::make_pair(&II, Path[0].second)); + std::string FileName; + // If there is a modulemap module or prebuilt module, load it. if (PP->getHeaderSearchInfo().lookupModule(PrivateModule, ImportLoc, true, - !IsInclusionDirective)) + !IsInclusionDirective) || + selectModuleSource(nullptr, PrivateModule, FileName, BuiltModules, + PP->getHeaderSearchInfo()) != MS_ModuleNotFound) Sub = loadModule(ImportLoc, PrivPath, Visibility, IsInclusionDirective); if (Sub) { MapPrivateSubModToTopLevel = true; @@ -2431,21 +2439,36 @@ bool CompileCacheASTReaderHelper::readModuleCacheKey(StringRef ModuleName, Filename, CacheKey, "imported module", CAS, Cache, ModuleCache, Diags); } -bool CompileCacheASTReaderHelper::readCASFileSystemRootID(StringRef RootID, - bool Complain) { - // Verify that RootID is in the CAS. Otherwise the module cache probably was - // created with a different CAS. +/// Verify that ID is in the CAS. Otherwise the module cache probably was +/// created with a different CAS. +bool CompileCacheASTReaderHelper::checkCASID(bool Complain, StringRef RootID, + unsigned ParseDiagID, + unsigned MissingDiagID) { std::optional ID; if (errorToBool(CAS.parseID(RootID).moveInto(ID))) { if (Complain) - Diags.Report(diag::err_cas_cannot_parse_root_id) << RootID; + Diags.Report(ParseDiagID) << RootID; return true; } if (errorToBool(CAS.getProxy(*ID).takeError())) { if (Complain) { - Diags.Report(diag::err_cas_missing_root_id) << RootID; + Diags.Report(MissingDiagID) << RootID; } return true; } return false; } + +bool CompileCacheASTReaderHelper::readCASFileSystemRootID(StringRef RootID, + bool Complain) { + return checkCASID(Complain, RootID, diag::err_cas_cannot_parse_root_id, + diag::err_cas_missing_root_id); +} + +bool CompileCacheASTReaderHelper::readIncludeTreeID(StringRef ID, + bool Complain) { + // Verify that ID is in the CAS. Otherwise the module cache probably was + // created with a different CAS. + return checkCASID(Complain, ID, diag::err_cas_cannot_parse_include_tree_id, + diag::err_cas_missing_include_tree_id); +} diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 1502cfef57d1b..6ff91921007fa 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -3019,7 +3019,8 @@ static void GenerateFrontendArgs(const FrontendOptions &Opts, static void determineInputFromIncludeTree( StringRef IncludeTreeID, CASOptions &CASOpts, DiagnosticsEngine &Diags, - Optional &IncludeTree, StringRef &InputFilename) { + Optional &IncludeTree, + Optional &Buffer, StringRef &InputFilename) { assert(!IncludeTreeID.empty()); auto reportError = [&](llvm::Error &&E) { Diags.Report(diag::err_fe_unable_to_load_include_tree) @@ -3046,8 +3047,18 @@ static void determineInputFromIncludeTree( auto FilenameBlob = BaseFile->getFilename(); if (!FilenameBlob) return reportError(FilenameBlob.takeError()); + InputFilename = FilenameBlob->getData(); IncludeTree = *Root; + + if (InputFilename != Module::getModuleInputBufferName()) + return; + + // Handle buffer + auto Contents = BaseFile->getContents(); + if (!Contents) + return reportError(Contents.takeError()); + Buffer = llvm::MemoryBufferRef(Contents->getData(), InputFilename); } static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, @@ -3275,13 +3286,14 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, Opts.Inputs.clear(); Optional Tree; + Optional TreeInputBuffer; if (!Opts.CASIncludeTreeID.empty()) { if (!Inputs.empty()) { Diags.Report(diag::err_drv_inputs_and_include_tree); } StringRef InputFilename; determineInputFromIncludeTree(Opts.CASIncludeTreeID, CASOpts, Diags, Tree, - InputFilename); + TreeInputBuffer, InputFilename); if (!InputFilename.empty()) Inputs.push_back(InputFilename.str()); } @@ -3320,8 +3332,16 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, if (Tree) { FrontendInputFile &InputFile = Opts.Inputs.back(); - InputFile = FrontendInputFile(Tree->getRef(), InputFile.getFile(), - InputFile.getKind(), InputFile.isSystem()); + if (TreeInputBuffer) { + // This is automatically set to modulemap when building a module; revert + // to a source file for the module includes buffer. + auto Kind = InputFile.getKind().withFormat(InputKind::Source); + InputFile = FrontendInputFile(Tree->getRef(), *TreeInputBuffer, Kind, + InputFile.isSystem()); + } else { + InputFile = FrontendInputFile(Tree->getRef(), InputFile.getFile(), + InputFile.getKind(), InputFile.isSystem()); + } } Opts.DashX = DashX; diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index 2a03e521bd704..f91fc30e78242 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -546,6 +546,119 @@ static Module *prepareToBuildModule(CompilerInstance &CI, return M; } +static Expected makeIncludeTreeModule(CompilerInstance &CI, + cas::IncludeTree::Module Mod, + Module *Parent) { + ModuleMap &MMap = CI.getPreprocessor().getHeaderSearchInfo().getModuleMap(); + auto Flags = Mod.getFlags(); + Module *M = nullptr; + bool NewModule = false; + std::tie(M, NewModule) = MMap.findOrCreateModule( + Mod.getName(), Parent, Flags.IsFramework, Flags.IsExplicit); + assert(NewModule); + M->Kind = Module::IncludeTreeModuleMap; + M->IsExternC = Flags.IsExternC; + M->IsSystem = Flags.IsSystem; + M->InferSubmodules = Flags.InferSubmodules; + M->InferExplicitSubmodules = Flags.InferExplicitSubmodules; + M->InferExportWildcard = Flags.InferExportWildcard; + + auto ExportList = Mod.getExports(); + if (!ExportList) + return ExportList.takeError(); + if (*ExportList) { + if ((*ExportList)->hasGlobalWildcard()) + M->Exports.push_back(Module::ExportDecl(nullptr, true)); + + llvm::Error Err = (*ExportList)->forEachExplicitExport([&](auto Export) { + Module::UnresolvedExportDecl UED; + SmallVector ModuleComponents; + Export.ModuleName.split(ModuleComponents, '.'); + for (StringRef Name : ModuleComponents) + UED.Id.push_back({std::string(Name), SourceLocation()}); + UED.Wildcard = Export.Wildcard; + M->UnresolvedExports.push_back(std::move(UED)); + return llvm::Error::success(); + }); + if (Err) + return std::move(Err); + } + + auto LinkLibs = Mod.getLinkLibraries(); + if (!LinkLibs) + return LinkLibs.takeError(); + if (*LinkLibs) { + llvm::Error Err = (*LinkLibs)->forEachLinkLibrary([&](auto LL) { + M->LinkLibraries.emplace_back(std::string(LL.Library), LL.IsFramework); + return llvm::Error::success(); + }); + if (Err) + return std::move(Err); + } + + llvm::Error Err = Mod.forEachSubmodule([&](cas::IncludeTree::Module Sub) { + return makeIncludeTreeModule(CI, Sub, M).takeError(); + }); + if (Err) + return std::move(Err); + return M; +} + +/// Loads the include tree modulemap \p MM. +/// \returns true if there was an error. +static bool loadIncludeTreeModuleMap(CompilerInstance &CI, + cas::IncludeTree::ModuleMap MM) { + llvm::Error Err = MM.forEachModule([&](auto M) -> llvm::Error { + return makeIncludeTreeModule(CI, M, /*Parent=*/nullptr).takeError(); + }); + if (Err) { + CI.getDiagnostics().Report(diag::err_fe_unable_to_load_include_tree) + << CI.getFrontendOpts().CASIncludeTreeID << std::move(Err); + return true; + } + return false; +} + +static Module *prepareToBuildModule(CompilerInstance &CI, + cas::IncludeTree::ModuleMap MM) { + if (CI.getLangOpts().CurrentModule.empty()) { + CI.getDiagnostics().Report(diag::err_missing_module_name); + + // FIXME: Eventually, we could consider asking whether there was just + // a single module described in the module map, and use that as a + // default. Then it would be fairly trivial to just "compile" a module + // map with a single module (the common case). + return nullptr; + } + + if (loadIncludeTreeModuleMap(CI, MM)) + return nullptr; + + // Dig out the module definition. + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + Module *M = HS.lookupModule(CI.getLangOpts().CurrentModule, SourceLocation(), + /*AllowSearch=*/false); + if (!M) { + CI.getDiagnostics().Report(diag::err_missing_module_include_tree) + << CI.getLangOpts().CurrentModule + << CI.getFrontendOpts().CASIncludeTreeID; + + return nullptr; + } + + if (auto CacheKey = CI.getCompileJobCacheKey()) + M->setModuleCacheKey(CacheKey->toString()); + + // If we're being run from the command-line, the module build stack will not + // have been filled in yet, so complete it now in order to allow us to detect + // module cycles. + SourceManager &SourceMgr = CI.getSourceManager(); + if (SourceMgr.getModuleBuildStack().empty()) + SourceMgr.pushModuleBuildStack(CI.getLangOpts().CurrentModule, + FullSourceLoc(SourceLocation(), SourceMgr)); + return M; +} + /// Compute the input buffer that should be used to build the specified module. static std::unique_ptr getInputBufferForModule(CompilerInstance &CI, Module *M) { @@ -856,29 +969,47 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, CI.getLangOpts().CurrentModule = CI.getLangOpts().ModuleName; } + auto reportError = [&](llvm::Error &&E) -> bool { + std::string IncludeTreeID = + CI.getOrCreateObjectStore().getID(Input.getIncludeTree()).toString(); + CI.getDiagnostics().Report(diag::err_fe_unable_to_load_include_tree) + << IncludeTreeID << llvm::toString(std::move(E)); + return false; + }; + + Optional IncludeTreeRoot; Optional IncludeTreePCHBuffer; if (Input.isIncludeTree()) { - auto reportError = [&](llvm::Error &&E) -> bool { - std::string IncludeTreeID = - CI.getOrCreateObjectStore().getID(Input.getIncludeTree()).toString(); - CI.getDiagnostics().Report(diag::err_fe_unable_to_load_include_tree) - << IncludeTreeID << llvm::toString(std::move(E)); - return false; - }; - auto Root = cas::IncludeTreeRoot::get(CI.getOrCreateObjectStore(), - Input.getIncludeTree()); - if (!Root) - return reportError(Root.takeError()); + if (llvm::Error E = cas::IncludeTreeRoot::get(CI.getOrCreateObjectStore(), + Input.getIncludeTree()) + .moveInto(IncludeTreeRoot)) + return reportError(std::move(E)); Expected> PPCachedAct = - createPPActionsFromIncludeTree(*Root); + createPPActionsFromIncludeTree(*IncludeTreeRoot); if (!PPCachedAct) return reportError(PPCachedAct.takeError()); CI.getPreprocessor().setPPCachedActions(std::move(*PPCachedAct)); CI.getFrontendOpts().IncludeTimestamps = false; - if (llvm::Error E = Root->getPCHBuffer().moveInto(IncludeTreePCHBuffer)) + if (llvm::Error E = + IncludeTreeRoot->getPCHBuffer().moveInto(IncludeTreePCHBuffer)) return reportError(std::move(E)); + + auto ModMap = IncludeTreeRoot->getModuleMap(); + if (!ModMap) + return reportError(ModMap.takeError()); + if (*ModMap) { + if (CI.getFrontendOpts().ProgramAction == frontend::GenerateModule) { + auto *CurrentModule = prepareToBuildModule(CI, **ModMap); + if (!CurrentModule) + return false; + CI.getLangOpts().setCompilingModule(LangOptions::CMK_ModuleMap); + } else { + if (loadIncludeTreeModuleMap(CI, **ModMap)) + return false; + } + } } if (!CI.InitializeSourceManager(Input)) @@ -1049,6 +1180,10 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, } } + if (!CI.getFrontendOpts().CASIncludeTreeID.empty()) + CI.getASTContext().setCASIncludeTreeID( + CI.getFrontendOpts().CASIncludeTreeID); + CI.setASTConsumer(std::move(Consumer)); if (!CI.hasASTConsumer()) return false; diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index e86c683c204b4..aa1c048e94ee6 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -831,6 +831,8 @@ static StringRef ModuleKindName(Module::ModuleKind MK) { switch (MK) { case Module::ModuleMapModule: return "Module Map Module"; + case Module::IncludeTreeModuleMap: + return "Include Tree Module"; case Module::ModuleInterfaceUnit: return "Interface Unit"; case Module::ModulePartitionInterface: diff --git a/clang/lib/Frontend/IncludeTreePPActions.cpp b/clang/lib/Frontend/IncludeTreePPActions.cpp index d18908577677f..87db33b85d52a 100644 --- a/clang/lib/Frontend/IncludeTreePPActions.cpp +++ b/clang/lib/Frontend/IncludeTreePPActions.cpp @@ -61,7 +61,7 @@ class IncludeTreePPActions final : public PPCachedActions { IncludeStackInfo &IncludeInfo = IncludeStack.back(); Expected EnteredTree = - IncludeInfo.Tree.getInclude(IncludeInfo.CurIncludeIndex++); + IncludeInfo.Tree.getIncludeTree(IncludeInfo.CurIncludeIndex++); if (!EnteredTree) return reportError(EnteredTree.takeError()); auto FileInfo = EnteredTree->getBaseFileInfo(); @@ -84,33 +84,52 @@ class IncludeTreePPActions final : public PPCachedActions { return IncludeInfo.Tree.getCheckResult(Index); } - Optional + std::variant handleIncludeDirective(Preprocessor &PP, SourceLocation IncludeLoc, SourceLocation AfterDirectiveLoc) override { if (HasCASErrorOccurred) - return None; + return {}; IncludeStackInfo &IncludeInfo = IncludeStack.back(); if (IncludeInfo.CurIncludeIndex >= IncludeInfo.Tree.getNumIncludes()) - return None; + return {}; unsigned ExpectedOffset = IncludeInfo.Tree.getIncludeOffset(IncludeInfo.CurIncludeIndex); SourceLocation ExpectedLoc = IncludeInfo.FileStartLoc.getLocWithOffset(ExpectedOffset); if (ExpectedLoc != AfterDirectiveLoc) - return None; + return {}; - auto reportError = [&](llvm::Error &&E) -> Optional { + auto reportError = [&](llvm::Error &&E) -> std::monostate { this->reportError(PP, std::move(E)); - return None; + return {}; }; - Expected EnteredTree = - IncludeInfo.Tree.getInclude(IncludeInfo.CurIncludeIndex++); - if (!EnteredTree) - return reportError(EnteredTree.takeError()); - auto File = EnteredTree->getBaseFile(); + auto reportErrorTwine = [&](const llvm::Twine &T) -> std::monostate { + return reportError( + llvm::createStringError(llvm::inconvertibleErrorCode(), T)); + }; + + Expected Node = + IncludeInfo.Tree.getIncludeNode(IncludeInfo.CurIncludeIndex++); + if (!Node) + return reportError(Node.takeError()); + + if (Node->getKind() == cas::IncludeTree::NodeKind::ModuleImport) { + cas::IncludeTree::ModuleImport Import = Node->getModuleImport(); + SmallVector, 2> Path; + SmallVector ModuleComponents; + Import.getModuleName().split(ModuleComponents, '.'); + for (StringRef Component : ModuleComponents) + Path.emplace_back(PP.getIdentifierInfo(Component), IncludeLoc); + return IncludeModule{std::move(Path), Import.visibilityOnly()}; + } + + assert(Node->getKind() == cas::IncludeTree::NodeKind::Tree); + + cas::IncludeTree EnteredTree = Node->getIncludeTree(); + auto File = EnteredTree.getBaseFile(); if (!File) return reportError(File.takeError()); auto FilenameBlob = File->getFilename(); @@ -124,11 +143,39 @@ class IncludeTreePPActions final : public PPCachedActions { if (!FE) return reportError(FE.takeError()); FileID FID = - SM.createFileID(*FE, IncludeLoc, EnteredTree->getFileCharacteristic()); + SM.createFileID(*FE, IncludeLoc, EnteredTree.getFileCharacteristic()); PP.markIncluded(*FE); IncludeStack.push_back( - {std::move(*EnteredTree), SM.getLocForStartOfFile(FID)}); - return FID; + {std::move(EnteredTree), SM.getLocForStartOfFile(FID)}); + + Module *M = nullptr; + auto SubmoduleName = EnteredTree.getSubmoduleName(); + if (!SubmoduleName) + return reportError(SubmoduleName.takeError()); + if (*SubmoduleName) { + SmallVector ModuleComponents; + (*SubmoduleName)->split(ModuleComponents, '.'); + M = PP.getHeaderSearchInfo().lookupModule( + ModuleComponents[0], IncludeLoc, + /*AllowSearch=*/false, /*AllowExtraModuleMapSearch=*/false); + if (!M) + return reportErrorTwine(llvm::Twine("failed to find module '") + + ModuleComponents[0] + "'"); + for (StringRef Sub : ArrayRef(ModuleComponents).drop_front()) { + M = M->findOrInferSubmodule(Sub); + if (!M) + return reportErrorTwine( + llvm::Twine("failed to find or infer submodule '") + Sub + "'"); + } + + // Add to known headers for the module. + ModuleMap &MMap = PP.getHeaderSearchInfo().getModuleMap(); + Module::Header H; + H.Entry = *FE; + MMap.addHeader(M, std::move(H), ModuleMap::NormalHeader); + } + + return IncludeFile{FID, M}; } void exitedFile(Preprocessor &PP, FileID FID) override { diff --git a/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp b/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp index 8d5039935aa7f..a75ad2bbe066d 100644 --- a/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp +++ b/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp @@ -608,6 +608,9 @@ void SDiagsWriter::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel, return; } + // Call base class to update diagnostic counts. + DiagnosticConsumer::HandleDiagnostic(DiagLevel, Info); + // Enter the block for a non-note diagnostic immediately, rather than waiting // for beginDiagnostic, in case associated notes are emitted before we get // there. diff --git a/clang/lib/Frontend/SerializedDiagnosticReader.cpp b/clang/lib/Frontend/SerializedDiagnosticReader.cpp index aca3f77c2de3e..984d039fe7820 100644 --- a/clang/lib/Frontend/SerializedDiagnosticReader.cpp +++ b/clang/lib/Frontend/SerializedDiagnosticReader.cpp @@ -34,7 +34,12 @@ std::error_code SerializedDiagnosticReader::readDiagnostics(StringRef File) { if (!Buffer) return SDError::CouldNotLoad; - llvm::BitstreamCursor Stream(**Buffer); + return readDiagnostics(**Buffer); +} + +std::error_code +SerializedDiagnosticReader::readDiagnostics(llvm::MemoryBufferRef Buffer) { + llvm::BitstreamCursor Stream(Buffer); Optional BlockInfo; if (Stream.AtEndOfStream()) diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index 29f3790325c19..c2278ba0b5380 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -1973,39 +1973,87 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, return; } + // Verify that there is nothing after the filename, other than EOD. Note + // that we allow macros that expand to nothing after the filename, because + // this falls into the category of "#include pp-tokens new-line" specified + // in C99 6.10.2p4. + SourceLocation EndLoc = + CheckEndOfDirective(IncludeTok.getIdentifierInfo()->getNameStart(), true); + if (auto *CActions = getPPCachedActions()) { - DiscardUntilEndOfDirective(); SourceLocation IncludePos = FilenameTok.getLocation(); // If the filename string was the result of macro expansions, set the // include position on the file where it will be included and after the // expansions. if (IncludePos.isMacroID()) IncludePos = SourceMgr.getExpansionRange(IncludePos).getEnd(); - Optional FID = CActions->handleIncludeDirective( + auto Include = CActions->handleIncludeDirective( *this, IncludePos, CurLexer->getSourceLocation()); - if (!FID) { - // FIXME: Report \p Callbacks->FileSkipped? Note that it currently - // requires the resolved FileEntry for this particular #include. + + if (auto *File = std::get_if(&Include)) { + const FileEntry *FE = SourceMgr.getFileEntryForID(File->FID); + bool IsImport = + IncludeTok.getIdentifierInfo()->getPPKeywordID() == tok::pp_import; + if (FE && IsImport) { + HeaderInfo.getFileInfo(FE).isImport = true; + } + EnterSourceFile(File->FID, nullptr, FilenameTok.getLocation(), + /*IsFirstIncludeOfFile*/ true); + + if (Module *SM = File->Submodule) { + assert(!CurLexerSubmodule && + "should not have marked this as a module yet"); + CurLexerSubmodule = SM; + EnterSubmodule(SM, EndLoc, /*ForPragma=*/false); + EnterAnnotationToken(SourceRange(HashLoc, EndLoc), + tok::annot_module_begin, SM); + } return; } - const FileEntry *FE = SourceMgr.getFileEntryForID(*FID); - bool IsImport = - IncludeTok.getIdentifierInfo()->getPPKeywordID() == tok::pp_import; - if (FE && IsImport) { - HeaderInfo.getFileInfo(FE).isImport = true; + if (auto *Import = std::get_if(&Include)) { + ModuleLoadResult Imported; + if (Import->VisibilityOnly) { + ModuleMap &MMap = getHeaderSearchInfo().getModuleMap(); + Module *M = nullptr; + for (auto &NameLoc : Import->ImportPath) { + M = MMap.lookupModuleQualified(NameLoc.first->getName(), M); + if (!M) + break; + } + if (!M) { + Diags->Report(diag::err_pp_missing_module_include_tree) + << getLangOpts().CurrentModule; + + return; + } + Imported = M; + } else { + Imported = TheModuleLoader.loadModule( + IncludeTok.getLocation(), Import->ImportPath, Module::Hidden, + /*IsIncludeDirective=*/true); + if (!Imported) { + assert(hadModuleLoaderFatalFailure() && "unexpected failure kind"); + if (hadModuleLoaderFatalFailure()) { + IncludeTok.setKind(tok::eof); + CurLexer->cutOffLexing(); + } + return; + } + } + + makeModuleVisible(Imported, EndLoc); + if (IncludeTok.getIdentifierInfo()->getPPKeywordID() != + tok::pp___include_macros) + EnterAnnotationToken(SourceRange(HashLoc, EndLoc), + tok::annot_module_include, Imported); + return; } - EnterSourceFile(*FID, nullptr, FilenameTok.getLocation(), - /*IsFirstIncludeOfFile*/ true); + assert(std::holds_alternative(Include)); + // FIXME: Report \p Callbacks->FileSkipped? Note that it currently + // requires the resolved FileEntry for this particular #include. return; } - // Verify that there is nothing after the filename, other than EOD. Note - // that we allow macros that expand to nothing after the filename, because - // this falls into the category of "#include pp-tokens new-line" specified - // in C99 6.10.2p4. - SourceLocation EndLoc = - CheckEndOfDirective(IncludeTok.getIdentifierInfo()->getNameStart(), true); - auto Action = HandleHeaderIncludeOrImport(HashLoc, IncludeTok, FilenameTok, EndLoc, LookupFrom, LookupFromFile); switch (Action.Kind) { diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp index 4b01f109fc881..6284ab346ee64 100644 --- a/clang/lib/Sema/SemaModule.cpp +++ b/clang/lib/Sema/SemaModule.cpp @@ -366,6 +366,7 @@ Sema::ActOnPrivateModuleFragmentDecl(SourceLocation ModuleLoc, switch (ModuleScopes.empty() ? Module::GlobalModuleFragment : ModuleScopes.back().Module->Kind) { case Module::ModuleMapModule: + case Module::IncludeTreeModuleMap: case Module::GlobalModuleFragment: case Module::ModulePartitionImplementation: case Module::ModulePartitionInterface: diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index eb7333b167a72..cc53bbdfe2ada 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -257,7 +257,10 @@ bool ChainedASTReaderListener::readCASFileSystemRootID(StringRef RootID, return First->readCASFileSystemRootID(RootID, Complain) || Second->readCASFileSystemRootID(RootID, Complain); } - +bool ChainedASTReaderListener::readIncludeTreeID(StringRef ID, bool Complain) { + return First->readIncludeTreeID(ID, Complain) || + Second->readIncludeTreeID(ID, Complain); +} bool ChainedASTReaderListener::readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) { @@ -2984,6 +2987,15 @@ ASTReader::ReadControlBlock(ModuleFile &F, return OutOfDate; } break; + case CAS_INCLUDE_TREE_ID: + F.IncludeTreeID = Blob.str(); + if (Listener) { + bool Complain = + !canRecoverFromOutOfDate(F.FileName, ClientLoadCapabilities); + if (Listener->readIncludeTreeID(F.IncludeTreeID, Complain)) + return OutOfDate; + } + break; } } } @@ -5624,8 +5636,6 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F, CurrentModule->PresumedModuleMapFile = F.ModuleMapPath; if (!F.ModuleCacheKey.empty()) CurrentModule->setModuleCacheKey(F.ModuleCacheKey); - if (!F.CASFileSystemRootID.empty()) - CurrentModule->setCASFileSystemRootID(F.CASFileSystemRootID); } CurrentModule->Kind = Kind; diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp index f6334988071e5..f45c49220bcca 100644 --- a/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/clang/lib/Serialization/ASTReaderDecl.cpp @@ -2980,8 +2980,7 @@ static bool isConsumerInterestedIn(ASTContext &Ctx, Decl *D, bool HasBody) { // emitted when we import the relevant module. if (isPartOfPerModuleInitializer(D)) { auto *M = D->getImportedOwningModule(); - if (M && M->Kind == Module::ModuleMapModule && - Ctx.DeclMustBeEmitted(D)) + if (M && M->isModuleMapModule() && Ctx.DeclMustBeEmitted(D)) return false; } diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 419b7c109ecfd..fbb0f90804752 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -797,6 +797,7 @@ void ASTWriter::WriteBlockInfoBlock() { RECORD(INPUT_FILE_OFFSETS); RECORD(MODULE_CACHE_KEY); RECORD(CASFS_ROOT_ID); + RECORD(CAS_INCLUDE_TREE_ID); BLOCK(OPTIONS_BLOCK); RECORD(LANGUAGE_OPTIONS); @@ -1352,15 +1353,25 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context, RecordData::value_type Record[] = {MODULE_CACHE_KEY}; Stream.EmitRecordWithBlob(AbbrevCode, Record, *Key); } - // CAS filesystem root id, for the scanner. - if (auto ID = WritingModule->getCASFileSystemRootID()) { - auto Abbrev = std::make_shared(); - Abbrev->Add(BitCodeAbbrevOp(CASFS_ROOT_ID)); - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); - unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); - RecordData::value_type Record[] = {CASFS_ROOT_ID}; - Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); - } + } + + // CAS include-tree id, for the scanner. + if (auto ID = Context.getCASIncludeTreeID()) { + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(CAS_INCLUDE_TREE_ID)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + RecordData::value_type Record[] = {CAS_INCLUDE_TREE_ID}; + Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); + } + // CAS filesystem root id, for the scanner. + if (auto ID = Context.getCASFileSystemRootID()) { + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(CASFS_ROOT_ID)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + RecordData::value_type Record[] = {CASFS_ROOT_ID}; + Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); } // Imports diff --git a/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp b/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp index e6cd88115ceb8..7f940c6336a2b 100644 --- a/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp @@ -22,8 +22,7 @@ namespace { class CASFSActionController : public CallbackActionController { public: CASFSActionController(LookupModuleOutputCallback LookupModuleOutput, - llvm::cas::CachingOnDiskFileSystem &CacheFS, - DepscanPrefixMapping PrefixMapping); + llvm::cas::CachingOnDiskFileSystem &CacheFS); llvm::Error initialize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) override; @@ -38,7 +37,6 @@ class CASFSActionController : public CallbackActionController { private: llvm::cas::CachingOnDiskFileSystem &CacheFS; - DepscanPrefixMapping PrefixMapping; std::optional Mapper; CASOptions CASOpts; }; @@ -46,17 +44,15 @@ class CASFSActionController : public CallbackActionController { CASFSActionController::CASFSActionController( LookupModuleOutputCallback LookupModuleOutput, - llvm::cas::CachingOnDiskFileSystem &CacheFS, - DepscanPrefixMapping PrefixMapping) - : CallbackActionController(std::move(LookupModuleOutput)), CacheFS(CacheFS), - PrefixMapping(std::move(PrefixMapping)) {} + llvm::cas::CachingOnDiskFileSystem &CacheFS) + : CallbackActionController(std::move(LookupModuleOutput)), + CacheFS(CacheFS) {} Error CASFSActionController::initialize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { // Setup prefix mapping. Mapper.emplace(&CacheFS); - if (Error E = PrefixMapping.configurePrefixMapper(NewInvocation, *Mapper)) - return E; + DepscanPrefixMapping::configurePrefixMapper(NewInvocation, *Mapper); const PreprocessorOptions &PPOpts = ScanInstance.getPreprocessorOpts(); if (!PPOpts.Includes.empty() || !PPOpts.ImplicitPCHInclude.empty()) @@ -176,7 +172,7 @@ Error CASFSActionController::finalizeModuleBuild( Module *M = ModuleScanInstance.getPreprocessor().getCurrentModule(); assert(M && "finalizing without a module"); - M->setCASFileSystemRootID(RootID->toString()); + ModuleScanInstance.getASTContext().setCASFileSystemRootID(RootID->toString()); return Error::success(); } @@ -197,8 +193,6 @@ Error CASFSActionController::finalizeModuleInvocation(CompilerInvocation &CI, std::unique_ptr dependencies::createCASFSActionController( LookupModuleOutputCallback LookupModuleOutput, - llvm::cas::CachingOnDiskFileSystem &CacheFS, - DepscanPrefixMapping PrefixMapping) { - return std::make_unique(LookupModuleOutput, CacheFS, - std::move(PrefixMapping)); + llvm::cas::CachingOnDiskFileSystem &CacheFS) { + return std::make_unique(LookupModuleOutput, CacheFS); } diff --git a/clang/lib/Tooling/DependencyScanning/CachingActions.h b/clang/lib/Tooling/DependencyScanning/CachingActions.h index 4cbcdd87ddbdc..8cb39dc649641 100644 --- a/clang/lib/Tooling/DependencyScanning/CachingActions.h +++ b/clang/lib/Tooling/DependencyScanning/CachingActions.h @@ -19,13 +19,12 @@ class CachingOnDiskFileSystem; namespace clang::tooling::dependencies { std::unique_ptr -createIncludeTreeActionController(cas::ObjectStore &DB, - DepscanPrefixMapping PrefixMapping); +createIncludeTreeActionController(LookupModuleOutputCallback LookupModuleOutput, + cas::ObjectStore &DB); std::unique_ptr createCASFSActionController(LookupModuleOutputCallback LookupModuleOutput, - llvm::cas::CachingOnDiskFileSystem &CacheFS, - DepscanPrefixMapping PrefixMapping); + llvm::cas::CachingOnDiskFileSystem &CacheFS); /// The PCH recorded file paths with canonical paths, create a VFS that /// allows remapping back to the non-canonical source paths so that they are diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp index 5845ca5bd3f96..2097a16bbd350 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -162,11 +162,9 @@ class GetIncludeTree : public EmptyDependencyConsumer { llvm::Expected DependencyScanningTool::getDependencyTree( - const std::vector &CommandLine, StringRef CWD, - DepscanPrefixMapping PrefixMapping) { + const std::vector &CommandLine, StringRef CWD) { GetDependencyTree Consumer(*Worker.getCASFS()); - auto Controller = createCASFSActionController(nullptr, *Worker.getCASFS(), - std::move(PrefixMapping)); + auto Controller = createCASFSActionController(nullptr, *Worker.getCASFS()); auto Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller); if (Result) @@ -178,10 +176,9 @@ llvm::Expected DependencyScanningTool::getDependencyTreeFromCompilerInvocation( std::shared_ptr Invocation, StringRef CWD, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, - bool DiagGenerationAsCompilation, DepscanPrefixMapping PrefixMapping) { + bool DiagGenerationAsCompilation) { GetDependencyTree Consumer(*Worker.getCASFS()); - auto Controller = createCASFSActionController(nullptr, *Worker.getCASFS(), - std::move(PrefixMapping)); + auto Controller = createCASFSActionController(nullptr, *Worker.getCASFS()); Worker.computeDependenciesFromCompilerInvocation( std::move(Invocation), CWD, Consumer, *Controller, DiagsConsumer, VerboseOS, DiagGenerationAsCompilation); @@ -190,10 +187,9 @@ DependencyScanningTool::getDependencyTreeFromCompilerInvocation( Expected DependencyScanningTool::getIncludeTree( cas::ObjectStore &DB, const std::vector &CommandLine, - StringRef CWD, const DepscanPrefixMapping &PrefixMapping) { + StringRef CWD, LookupModuleOutputCallback LookupModuleOutput) { GetIncludeTree Consumer(DB); - auto Controller = - createIncludeTreeActionController(DB, std::move(PrefixMapping)); + auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB); llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller); if (Result) @@ -204,12 +200,11 @@ Expected DependencyScanningTool::getIncludeTree( Expected DependencyScanningTool::getIncludeTreeFromCompilerInvocation( cas::ObjectStore &DB, std::shared_ptr Invocation, - StringRef CWD, const DepscanPrefixMapping &PrefixMapping, + StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, bool DiagGenerationAsCompilation) { GetIncludeTree Consumer(DB); - auto Controller = - createIncludeTreeActionController(DB, std::move(PrefixMapping)); + auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB); Worker.computeDependenciesFromCompilerInvocation( std::move(Invocation), CWD, Consumer, *Controller, DiagsConsumer, VerboseOS, DiagGenerationAsCompilation); @@ -220,11 +215,9 @@ llvm::Expected DependencyScanningTool::getTranslationUnitDependencies( const std::vector &CommandLine, StringRef CWD, const llvm::StringSet<> &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping) { + LookupModuleOutputCallback LookupModuleOutput) { FullDependencyConsumer Consumer(AlreadySeen); - auto Controller = - createActionController(LookupModuleOutput, std::move(PrefixMapping)); + auto Controller = createActionController(LookupModuleOutput); llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller); if (Result) @@ -235,11 +228,9 @@ DependencyScanningTool::getTranslationUnitDependencies( llvm::Expected DependencyScanningTool::getModuleDependencies( StringRef ModuleName, const std::vector &CommandLine, StringRef CWD, const llvm::StringSet<> &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping) { + LookupModuleOutputCallback LookupModuleOutput) { FullDependencyConsumer Consumer(AlreadySeen); - auto Controller = - createActionController(LookupModuleOutput, std::move(PrefixMapping)); + auto Controller = createActionController(LookupModuleOutput); llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller, ModuleName); if (Result) @@ -255,6 +246,7 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() { TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps); TU.Commands = std::move(Commands); TU.CASFileSystemRootID = std::move(CASFileSystemRootID); + TU.IncludeTreeID = std::move(IncludeTreeID); for (auto &&M : ClangModuleDeps) { auto &MD = M.second; @@ -290,20 +282,17 @@ CallbackActionController::~CallbackActionController() {} std::unique_ptr DependencyScanningTool::createActionController( DependencyScanningWorker &Worker, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping) { + LookupModuleOutputCallback LookupModuleOutput) { if (Worker.getScanningFormat() == ScanningOutputFormat::FullIncludeTree) - return createIncludeTreeActionController(*Worker.getCAS(), std::move(PrefixMapping)); + return createIncludeTreeActionController(LookupModuleOutput, + *Worker.getCAS()); if (auto CacheFS = Worker.getCASFS()) - return createCASFSActionController(LookupModuleOutput, *CacheFS, - std::move(PrefixMapping)); + return createCASFSActionController(LookupModuleOutput, *CacheFS); return std::make_unique(LookupModuleOutput); } std::unique_ptr DependencyScanningTool::createActionController( - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping) { - return createActionController(Worker, std::move(LookupModuleOutput), - std::move(PrefixMapping)); + LookupModuleOutputCallback LookupModuleOutput) { + return createActionController(Worker, std::move(LookupModuleOutput)); } diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index d08e41e4b2fc8..fe63145171747 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -15,6 +15,7 @@ #include "clang/Driver/Driver.h" #include "clang/Driver/Job.h" #include "clang/Driver/Tool.h" +#include "clang/Frontend/CompileJobCacheKey.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/FrontendActions.h" @@ -176,17 +177,14 @@ class ReversePrefixMappingDependencyFileGenerator const DependencyOutputOptions &Opts) : DependencyFileGenerator(Opts) {} - Error initialize(const CompilerInvocation &CI, - const DepscanPrefixMapping &PrefixMapping) { + void initialize(const CompilerInvocation &CI) { llvm::PrefixMapper Mapper; - if (Error E = PrefixMapping.configurePrefixMapper(CI, Mapper)) - return E; + DepscanPrefixMapping::configurePrefixMapper(CI, Mapper); if (Mapper.empty()) - return Error::success(); + return; ReverseMapper.addInverseRange(Mapper.getMappings()); ReverseMapper.sort(); - return Error::success(); } void maybeAddDependency(StringRef Filename, bool FromModule, bool IsSystem, @@ -425,9 +423,7 @@ class DependencyScanningAction : public tooling::ToolAction { auto DFG = std::make_shared( *Opts); - if (auto *Mapping = Controller.getPrefixMapping()) - if (auto E = DFG->initialize(ScanInstance.getInvocation(), *Mapping)) - return reportError(std::move(E)); + DFG->initialize(ScanInstance.getInvocation()); ScanInstance.addDependencyCollector(std::move(DFG)); } @@ -435,15 +431,12 @@ class DependencyScanningAction : public tooling::ToolAction { std::move(Opts), ScanInstance, Consumer, Controller, OriginalInvocation, OptimizeArgs, EagerLoadModules); ScanInstance.addDependencyCollector(MDC); - if (CacheFS) { - ScanInstance.setGenModuleActionWrapper( - [CacheFS = CacheFS, &Controller = Controller]( - const FrontendOptions &Opts, - std::unique_ptr Wrapped) { - return std::make_unique( - std::move(Wrapped), Controller); - }); - } + ScanInstance.setGenModuleActionWrapper( + [&Controller = Controller](const FrontendOptions &Opts, + std::unique_ptr Wrapped) { + return std::make_unique( + std::move(Wrapped), Controller); + }); break; } @@ -461,6 +454,10 @@ class DependencyScanningAction : public tooling::ToolAction { else Action = std::make_unique(); + // Normally this would be handled by GeneratePCHAction + if (ScanInstance.getFrontendOpts().ProgramAction == frontend::GeneratePCH) + ScanInstance.getLangOpts().CompilingPCH = true; + if (Error E = Controller.initialize(ScanInstance, OriginalInvocation)) return reportError(std::move(E)); @@ -483,6 +480,13 @@ class DependencyScanningAction : public tooling::ToolAction { LastCC1Arguments = OriginalInvocation.getCC1CommandLine(); + if (ScanInstance.getFrontendOpts().CacheCompileJob) { + auto &CAS = ScanInstance.getOrCreateObjectStore(); + if (auto Key = createCompileJobCacheKey( + CAS, ScanInstance.getDiagnostics(), OriginalInvocation)) + TUCacheKey = Key->toString(); + } + return true; } @@ -497,6 +501,8 @@ class DependencyScanningAction : public tooling::ToolAction { return Result; } + const std::optional &getTUCacheKey() const { return TUCacheKey; } + IntrusiveRefCntPtr getDepScanFS() { if (DepFS) { assert(!DepCASFS && "CAS DepFS should not be set"); @@ -527,6 +533,7 @@ class DependencyScanningAction : public tooling::ToolAction { Optional ScanInstanceStorage; std::shared_ptr MDC; std::vector LastCC1Arguments; + std::optional TUCacheKey; bool Scanned = false; raw_ostream *VerboseOS; }; @@ -705,7 +712,8 @@ bool DependencyScanningWorker::computeDependencies( // consumer. Consumer.handleBuildCommand( {Cmd.getExecutable(), - {Cmd.getArguments().begin(), Cmd.getArguments().end()}}); + {Cmd.getArguments().begin(), Cmd.getArguments().end()}, + /*TUCacheKey=*/std::nullopt}); return true; } @@ -726,7 +734,8 @@ bool DependencyScanningWorker::computeDependencies( return false; std::vector Args = Action.takeLastCC1Arguments(); - Consumer.handleBuildCommand({Cmd.getExecutable(), std::move(Args)}); + Consumer.handleBuildCommand( + {Cmd.getExecutable(), std::move(Args), Action.getTUCacheKey()}); return true; }); diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 5d8a2ee36b334..1331e19be742e 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -20,50 +20,87 @@ using namespace dependencies; using llvm::Error; namespace { +class IncludeTreeBuilder; + class IncludeTreeActionController : public CallbackActionController { public: IncludeTreeActionController(cas::ObjectStore &DB, - DepscanPrefixMapping PrefixMapping) - : CallbackActionController(nullptr), DB(DB), - PrefixMapping(std::move(PrefixMapping)) {} + LookupModuleOutputCallback LookupOutput) + : CallbackActionController(LookupOutput), DB(DB) {} Expected getIncludeTree(); private: Error initialize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) override; + Error finalize(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation) override; - void enteredInclude(Preprocessor &PP, FileID FID) override; + Error initializeModuleBuild(CompilerInstance &ModuleScanInstance) override; + Error finalizeModuleBuild(CompilerInstance &ModuleScanInstance) override; + Error finalizeModuleInvocation(CompilerInvocation &CI, + const ModuleDeps &MD) override; - void exitedInclude(Preprocessor &PP, FileID IncludedBy, FileID Include, - SourceLocation ExitLoc) override; +private: + IncludeTreeBuilder ¤t() { + assert(!BuilderStack.empty()); + return *BuilderStack.back(); + } - void handleHasIncludeCheck(Preprocessor &PP, bool Result) override; +private: + cas::ObjectStore &DB; + CASOptions CASOpts; + llvm::PrefixMapper PrefixMapper; + // IncludeTreePPCallbacks keeps a pointer to the current builder, so use a + // pointer so the builder cannot move when resizing. + SmallVector> BuilderStack; + std::optional IncludeTreeResult; +}; - const DepscanPrefixMapping *getPrefixMapping() override { - return &PrefixMapping; - } +/// Callbacks for building an include-tree for a given translation unit or +/// module. The \c IncludeTreeActionController is responsiblee for pushing and +/// popping builders from the stack as modules are required. +class IncludeTreeBuilder { +public: + IncludeTreeBuilder(cas::ObjectStore &DB, llvm::PrefixMapper &PrefixMapper) + : DB(DB), PrefixMapper(PrefixMapper) {} - Error finalize(CompilerInstance &ScanInstance, - CompilerInvocation &NewInvocation) override; + Expected + finishIncludeTree(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation); - Expected getObjectForFile(Preprocessor &PP, FileID FID); - Expected - getObjectForFileNonCached(FileManager &FM, const SrcMgr::FileInfo &FI); - Expected getObjectForBuffer(const SrcMgr::FileInfo &FI); - Expected addToFileList(FileManager &FM, const FileEntry *FE); + void enteredInclude(Preprocessor &PP, FileID FID); + + void exitedInclude(Preprocessor &PP, FileID IncludedBy, FileID Include, + SourceLocation ExitLoc); + + void handleHasIncludeCheck(Preprocessor &PP, bool Result); + void moduleImport(Preprocessor &PP, const Module *M, SourceLocation EndLoc); + + void enteredSubmodule(Preprocessor &PP, Module *M, SourceLocation ImportLoc, + bool ForPragma); + void exitedSubmodule(Preprocessor &PP, Module *M, SourceLocation ImportLoc, + bool ForPragma); + +private: struct FilePPState { SrcMgr::CharacteristicKind FileCharacteristic; cas::ObjectRef File; - SmallVector, 6> Includes; + SmallVector Includes; + Optional SubmoduleName; llvm::SmallBitVector HasIncludeChecks; }; + Error addModuleInputs(ASTReader &Reader); + Expected getObjectForFile(Preprocessor &PP, FileID FID); + Expected + getObjectForFileNonCached(FileManager &FM, const SrcMgr::FileInfo &FI); + Expected getObjectForBuffer(const SrcMgr::FileInfo &FI); + Expected addToFileList(FileManager &FM, const FileEntry *FE); Expected getCASTreeForFileIncludes(FilePPState &&PPState); - - Expected createIncludeFile(StringRef Filename, - cas::ObjectRef Contents); + Expected createIncludeFile(StringRef Filename, + cas::ObjectRef Contents); bool hasErrorOccurred() const { return ErrorToReport.has_value(); } @@ -75,55 +112,75 @@ class IncludeTreeActionController : public CallbackActionController { return *E; } +private: cas::ObjectStore &DB; - CASOptions CASOpts; - DepscanPrefixMapping PrefixMapping; - llvm::PrefixMapper PrefixMapper; + llvm::PrefixMapper &PrefixMapper; + Optional PCHRef; bool StartedEnteringIncludes = false; // When a PCH is used this lists the filenames of the included files as they // are recorded in the PCH, ordered by \p FileEntry::UID index. SmallVector PreIncludedFileNames; llvm::BitVector SeenIncludeFiles; - SmallVector IncludedFiles; + SmallVector IncludedFiles; + SmallVector IncludedFileLists; Optional PredefinesBufferRef; + Optional ModuleIncludesBufferRef; + Optional ModuleMapRef; + /// When the builder is created from an existing tree, the main include tree. + Optional MainIncludeTreeRef; SmallVector IncludeStack; llvm::DenseMap> ObjectForFile; Optional ErrorToReport; }; -/// A utility for adding \c PPCallbacks to a compiler instance at the -/// appropriate time. -struct PPCallbacksDependencyCollector : public DependencyCollector { - using MakeCB = +/// A utility for adding \c PPCallbacks and/or \cASTReaderListener to a compiler +/// instance at the appropriate time. +struct AttachOnlyDependencyCollector : public DependencyCollector { + using MakePPCB = llvm::unique_function(Preprocessor &)>; - MakeCB Create; - PPCallbacksDependencyCollector(MakeCB Create) : Create(std::move(Create)) {} + using MakeASTReaderL = + llvm::unique_function(ASTReader &R)>; + MakePPCB CreatePPCB; + MakeASTReaderL CreateASTReaderL; + AttachOnlyDependencyCollector(MakePPCB CreatePPCB, MakeASTReaderL CreateL) + : CreatePPCB(std::move(CreatePPCB)), + CreateASTReaderL(std::move(CreateL)) {} + void attachToPreprocessor(Preprocessor &PP) final { - std::unique_ptr CB = Create(PP); - assert(CB); - PP.addPPCallbacks(std::move(CB)); + if (CreatePPCB) { + std::unique_ptr CB = CreatePPCB(PP); + assert(CB); + PP.addPPCallbacks(std::move(CB)); + } + } + + void attachToASTReader(ASTReader &R) final { + if (CreateASTReaderL) { + std::unique_ptr L = CreateASTReaderL(R); + assert(L); + R.addListener(std::move(L)); + } } }; struct IncludeTreePPCallbacks : public PPCallbacks { - DependencyActionController &Controller; + IncludeTreeBuilder &Builder; Preprocessor &PP; public: - IncludeTreePPCallbacks(DependencyActionController &Controller, - Preprocessor &PP) - : Controller(Controller), PP(PP) {} + IncludeTreePPCallbacks(IncludeTreeBuilder &Builder, Preprocessor &PP) + : Builder(Builder), PP(PP) {} void LexedFileChanged(FileID FID, LexedFileChangeReason Reason, SrcMgr::CharacteristicKind FileType, FileID PrevFID, SourceLocation Loc) override { switch (Reason) { case LexedFileChangeReason::EnterFile: - Controller.enteredInclude(PP, FID); + Builder.enteredInclude(PP, FID); break; case LexedFileChangeReason::ExitFile: { - Controller.exitedInclude(PP, FID, PrevFID, Loc); + Builder.exitedInclude(PP, FID, PrevFID, Loc); break; } } @@ -132,9 +189,80 @@ struct IncludeTreePPCallbacks : public PPCallbacks { void HasInclude(SourceLocation Loc, StringRef FileName, bool IsAngled, Optional File, SrcMgr::CharacteristicKind FileType) override { - Controller.handleHasIncludeCheck(PP, File.has_value()); + Builder.handleHasIncludeCheck(PP, File.has_value()); + } + + void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok, + StringRef FileName, bool IsAngled, + CharSourceRange FilenameRange, + Optional File, StringRef SearchPath, + StringRef RelativePath, const Module *Imported, + SrcMgr::CharacteristicKind FileType) override { + if (!Imported) + return; // File includes handled by LexedFileChanged. + + // Calculate EndLoc for the directive + // FIXME: pass EndLoc through PPCallbacks; it is already calculated + SourceManager &SM = PP.getSourceManager(); + std::pair LocInfo = SM.getDecomposedExpansionLoc(HashLoc); + StringRef Buffer = SM.getBufferData(LocInfo.first); + Lexer L(SM.getLocForStartOfFile(LocInfo.first), PP.getLangOpts(), + Buffer.begin(), Buffer.begin() + LocInfo.second, Buffer.end()); + L.setParsingPreprocessorDirective(true); + Token Tok; + do { + L.LexFromRawLexer(Tok); + } while (!Tok.isOneOf(tok::eod, tok::eof)); + SourceLocation EndLoc = L.getSourceLocation(); + + Builder.moduleImport(PP, Imported, EndLoc); + } + + void EnteredSubmodule(Module *M, SourceLocation ImportLoc, + bool ForPragma) override { + Builder.enteredSubmodule(PP, M, ImportLoc, ForPragma); + } + void LeftSubmodule(Module *M, SourceLocation ImportLoc, + bool ForPragma) override { + Builder.exitedSubmodule(PP, M, ImportLoc, ForPragma); } }; + +/// Utility to trigger module lookup in header search for modules loaded via +/// PCH. This causes dependency scanning via PCH to parse modulemap files at +/// roughly the same point they would with modulemap files embedded in the pcms, +/// which is disabled with include-tree modules. Without this, we can fail to +/// find modules that are in the same directory as a named import, since +/// it may be skipped during search (see \c loadFrameworkModule). +/// +/// The specific lookup we do matches what happens in ASTReader for the +/// MODULE_DIRECTORY record, and ignores the result. +class LookupPCHModulesListener : public ASTReaderListener { +public: + LookupPCHModulesListener(ASTReader &R) : Reader(R) {} + +private: + void visitModuleFile(StringRef Filename, + serialization::ModuleKind Kind) final { + // Any prebuilt or explicit modules seen during scanning are "full" modules + // rather than implicitly built scanner modules. + if (Kind == serialization::MK_PrebuiltModule || + Kind == serialization::MK_ExplicitModule) { + serialization::ModuleManager &Manager = Reader.getModuleManager(); + serialization::ModuleFile *MF = Manager.lookupByFileName(Filename); + assert(MF && "module file missing in visitModuleFile"); + // Match MODULE_DIRECTORY: allow full search and ignore failure to find + // the module. + HeaderSearch &HS = Reader.getPreprocessor().getHeaderSearchInfo(); + (void)HS.lookupModule(MF->ModuleName, SourceLocation(), + /*AllowSearch=*/true, + /*AllowExtraModuleMapSearch=*/true); + } + } + +private: + ASTReader &Reader; +}; } // namespace /// The PCH recorded file paths with canonical paths, create a VFS that @@ -152,11 +280,16 @@ void dependencies::addReversePrefixMappingFileSystem( ScanInstance.getFileManager().setVirtualFileSystem(std::move(FS)); } +Expected IncludeTreeActionController::getIncludeTree() { + if (IncludeTreeResult) + return *IncludeTreeResult; + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "failed to produce include-tree"); +} + Error IncludeTreeActionController::initialize( CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { - if (Error E = - PrefixMapping.configurePrefixMapper(NewInvocation, PrefixMapper)) - return E; + DepscanPrefixMapping::configurePrefixMapper(NewInvocation, PrefixMapper); auto ensurePathRemapping = [&]() { if (PrefixMapper.empty()) @@ -174,20 +307,99 @@ Error IncludeTreeActionController::initialize( }; ensurePathRemapping(); + BuilderStack.push_back( + std::make_unique(DB, PrefixMapper)); + // Attach callbacks for the IncludeTree of the TU. The preprocessor // does not exist yet, so we need to indirect this via DependencyCollector. - auto DC = std::make_shared( - [this](Preprocessor &PP) { - return std::make_unique(*this, PP); + auto DC = std::make_shared( + [&Builder = current()](Preprocessor &PP) { + return std::make_unique(Builder, PP); + }, + [](ASTReader &R) { + return std::make_unique(R); }); ScanInstance.addDependencyCollector(std::move(DC)); + // Enable caching in the resulting commands. + ScanInstance.getFrontendOpts().CacheCompileJob = true; CASOpts = ScanInstance.getCASOpts(); return Error::success(); } -void IncludeTreeActionController::enteredInclude(Preprocessor &PP, FileID FID) { +Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation) { + assert(!IncludeTreeResult); + assert(BuilderStack.size() == 1); + auto Builder = BuilderStack.pop_back_val(); + Error E = Builder->finishIncludeTree(ScanInstance, NewInvocation) + .moveInto(IncludeTreeResult); + if (E) + return E; + + configureInvocationForCaching(NewInvocation, CASOpts, + IncludeTreeResult->getID().toString(), + // FIXME: working dir? + /*CASFSWorkingDir=*/"", + /*ProduceIncludeTree=*/true); + + DepscanPrefixMapping::remapInvocationPaths(NewInvocation, PrefixMapper); + + return Error::success(); +} + +Error IncludeTreeActionController::initializeModuleBuild( + CompilerInstance &ModuleScanInstance) { + BuilderStack.push_back( + std::make_unique(DB, PrefixMapper)); + + // Attach callbacks for the IncludeTree of the module. The preprocessor + // does not exist yet, so we need to indirect this via DependencyCollector. + auto DC = std::make_shared( + [&Builder = current()](Preprocessor &PP) { + return std::make_unique(Builder, PP); + }, + [](ASTReader &R) { + return std::make_unique(R); + }); + ModuleScanInstance.addDependencyCollector(std::move(DC)); + + return Error::success(); +} + +Error IncludeTreeActionController::finalizeModuleBuild( + CompilerInstance &ModuleScanInstance) { + // FIXME: the scan invocation is incorrect here; we need the `NewInvocation` + // from `finalizeModuleInvocation` to finish the tree. + auto Builder = BuilderStack.pop_back_val(); + auto Tree = Builder->finishIncludeTree(ModuleScanInstance, + ModuleScanInstance.getInvocation()); + if (!Tree) + return Tree.takeError(); + + ModuleScanInstance.getASTContext().setCASIncludeTreeID( + Tree->getID().toString()); + + return Error::success(); +} + +Error IncludeTreeActionController::finalizeModuleInvocation( + CompilerInvocation &CI, const ModuleDeps &MD) { + if (!MD.IncludeTreeID) + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "missing include-tree for module '%s'", + MD.ID.ModuleName.c_str()); + + configureInvocationForCaching(CI, CASOpts, *MD.IncludeTreeID, + /*CASFSWorkingDir=*/"", + /*ProduceIncludeTree=*/true); + + DepscanPrefixMapping::remapInvocationPaths(CI, PrefixMapper); + return Error::success(); +} + +void IncludeTreeBuilder::enteredInclude(Preprocessor &PP, FileID FID) { if (hasErrorOccurred()) return; @@ -209,13 +421,11 @@ void IncludeTreeActionController::enteredInclude(Preprocessor &PP, FileID FID) { return; const SrcMgr::FileInfo &FI = PP.getSourceManager().getSLocEntry(FID).getFile(); - IncludeStack.push_back({FI.getFileCharacteristic(), *FileRef, {}, {}}); + IncludeStack.push_back({FI.getFileCharacteristic(), *FileRef, {}, {}, {}}); } -void IncludeTreeActionController::exitedInclude(Preprocessor &PP, - FileID IncludedBy, - FileID Include, - SourceLocation ExitLoc) { +void IncludeTreeBuilder::exitedInclude(Preprocessor &PP, FileID IncludedBy, + FileID Include, SourceLocation ExitLoc) { if (hasErrorOccurred()) return; @@ -227,20 +437,125 @@ void IncludeTreeActionController::exitedInclude(Preprocessor &PP, assert(*check(getObjectForFile(PP, IncludedBy)) == IncludeStack.back().File); SourceManager &SM = PP.getSourceManager(); std::pair LocInfo = SM.getDecomposedExpansionLoc(ExitLoc); - IncludeStack.back().Includes.push_back( - {IncludeTree->getRef(), LocInfo.second}); + IncludeStack.back().Includes.push_back({IncludeTree->getRef(), LocInfo.second, + cas::IncludeTree::NodeKind::Tree}); } -void IncludeTreeActionController::handleHasIncludeCheck(Preprocessor &PP, - bool Result) { +void IncludeTreeBuilder::handleHasIncludeCheck(Preprocessor &PP, bool Result) { if (hasErrorOccurred()) return; IncludeStack.back().HasIncludeChecks.push_back(Result); } -Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, - CompilerInvocation &NewInvocation) { +// FIXME: duplicates code in PPDirectives +static bool isForModuleBuilding(const Module *M, StringRef CurrentModule, + StringRef ModuleName) { + StringRef TopLevelName = M->getTopLevelModuleName(); + + // When building framework Foo, we wanna make sure that Foo *and* Foo_Private + // are textually included and no modules are built for both. + if (M->getTopLevelModule()->IsFramework && CurrentModule == ModuleName && + !CurrentModule.endswith("_Private") && TopLevelName.endswith("_Private")) + TopLevelName = TopLevelName.drop_back(8); + + return TopLevelName == CurrentModule; +} + +void IncludeTreeBuilder::moduleImport(Preprocessor &PP, const Module *M, + SourceLocation EndLoc) { + bool VisibilityOnly = isForModuleBuilding(M, PP.getLangOpts().CurrentModule, PP.getLangOpts().ModuleName); + auto Import = check(cas::IncludeTree::ModuleImport::create( + DB, M->getFullModuleName(), VisibilityOnly)); + if (!Import) + return; + + std::pair EndLocInfo = + PP.getSourceManager().getDecomposedExpansionLoc(EndLoc); + IncludeStack.back().Includes.push_back( + {Import->getRef(), EndLocInfo.second, + cas::IncludeTree::NodeKind::ModuleImport}); +} + +void IncludeTreeBuilder::enteredSubmodule(Preprocessor &PP, Module *M, + SourceLocation ImportLoc, + bool ForPragma) { + if (ForPragma) + return; // Will be parsed as normal. + if (hasErrorOccurred()) + return; + assert(!IncludeStack.back().SubmoduleName && "repeated enteredSubmodule"); + auto Ref = check(DB.storeFromString({}, M->getFullModuleName())); + IncludeStack.back().SubmoduleName = Ref; +} +void IncludeTreeBuilder::exitedSubmodule(Preprocessor &PP, Module *M, + SourceLocation ImportLoc, + bool ForPragma) { + // Submodule exit is handled automatically when leaving a modular file. +} + +static Expected +getIncludeTreeModule(cas::ObjectStore &DB, Module *M) { + using ITModule = cas::IncludeTree::Module; + SmallVector Submodules; + for (Module *Sub : M->submodules()) { + Expected SubTree = getIncludeTreeModule(DB, Sub); + if (!SubTree) + return SubTree.takeError(); + Submodules.push_back(SubTree->getRef()); + } + + ITModule::ModuleFlags Flags; + Flags.IsFramework = M->IsFramework; + Flags.IsExplicit = M->IsExplicit; + Flags.IsExternC = M->IsExternC; + Flags.IsSystem = M->IsSystem; + Flags.InferSubmodules = M->InferSubmodules; + Flags.InferExplicitSubmodules = M->InferExplicitSubmodules; + Flags.InferExportWildcard = M->InferExportWildcard; + + bool GlobalWildcardExport = false; + SmallVector Exports; + llvm::BumpPtrAllocator Alloc; + llvm::StringSaver Saver(Alloc); + for (Module::ExportDecl &Export : M->Exports) { + if (Export.getPointer() == nullptr && Export.getInt()) { + GlobalWildcardExport = true; + } else if (Export.getPointer()) { + StringRef Name = Saver.save(Export.getPointer()->getFullModuleName()); + Exports.push_back({Name, Export.getInt()}); + } + } + std::optional ExportList; + if (GlobalWildcardExport || !Exports.empty()) { + auto EL = ITModule::ExportList::create(DB, Exports, GlobalWildcardExport); + if (!EL) + return EL.takeError(); + ExportList = EL->getRef(); + } + + SmallVector Libraries; + for (Module::LinkLibrary &LL : M->LinkLibraries) { + Libraries.push_back({LL.Library, LL.IsFramework}); + } + std::optional LinkLibraries; + if (!Libraries.empty()) { + auto LL = ITModule::LinkLibraryList::create(DB, Libraries); + if (!LL) + return LL.takeError(); + LinkLibraries = LL->getRef(); + } + + return ITModule::create(DB, M->Name, Flags, Submodules, ExportList, + LinkLibraries); +} + +Expected +IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation) { + if (ErrorToReport) + return std::move(*ErrorToReport); + FileManager &FM = ScanInstance.getFileManager(); auto addFile = [&](StringRef FilePath, @@ -257,7 +572,7 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, for (StringRef FilePath : NewInvocation.getLangOpts()->NoSanitizeFiles) { if (Error E = addFile(FilePath)) - return E; + return std::move(E); } // Add profile files. // FIXME: Do not have the logic here to determine which path should be set @@ -265,13 +580,13 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, // checked the file needed exists. Just try load and ignore errors. if (Error E = addFile(NewInvocation.getCodeGenOpts().ProfileInstrumentUsePath, /*IgnoreFileError=*/true)) - return E; + return std::move(E); if (Error E = addFile(NewInvocation.getCodeGenOpts().SampleProfileFile, /*IgnoreFileError=*/true)) - return E; + return std::move(E); if (Error E = addFile(NewInvocation.getCodeGenOpts().ProfileRemappingFile, /*IgnoreFileError=*/true)) - return E; + return std::move(E); StringRef Sysroot = NewInvocation.getHeaderSearchOpts().Sysroot; if (!Sysroot.empty()) { @@ -280,35 +595,22 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, llvm::SmallString<256> FilePath = Sysroot; llvm::sys::path::append(FilePath, "SDKSettings.json"); if (Error E = addFile(FilePath, /*IgnoreFileError*/ true)) - return E; + return std::move(E); } auto FinishIncludeTree = [&]() -> Error { + IntrusiveRefCntPtr Reader = ScanInstance.getASTReader(); + if (!Reader) + return Error::success(); // no need for additional work. + + // Go through all the recorded input files. + if (Error E = addModuleInputs(*Reader)) + return E; + PreprocessorOptions &PPOpts = NewInvocation.getPreprocessorOpts(); if (PPOpts.ImplicitPCHInclude.empty()) return Error::success(); // no need for additional work. - // Go through all the recorded included files; we'll get additional files - // from the PCH that we need to include in the file list, in case they are - // referenced while replaying the include-tree. - SmallVector NotSeenIncludes; - for (const FileEntry *FE : - ScanInstance.getPreprocessor().getIncludedFiles()) { - if (FE->getUID() >= SeenIncludeFiles.size() || - !SeenIncludeFiles[FE->getUID()]) - NotSeenIncludes.push_back(FE); - } - // Sort so we can visit the files in deterministic order. - llvm::sort(NotSeenIncludes, [](const FileEntry *LHS, const FileEntry *RHS) { - return LHS->getUID() < RHS->getUID(); - }); - - for (const FileEntry *FE : NotSeenIncludes) { - auto FileNode = addToFileList(FM, FE); - if (!FileNode) - return FileNode.takeError(); - } - llvm::ErrorOr> CASContents = FM.getObjectRefForFileContent(PPOpts.ImplicitPCHInclude); if (!CASContents) @@ -319,24 +621,86 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, }; if (Error E = FinishIncludeTree()) - return E; + return std::move(E); - auto IncludeTreeRoot = getIncludeTree(); - if (!IncludeTreeRoot) - return IncludeTreeRoot.takeError(); + if (ErrorToReport) + return std::move(*ErrorToReport); - configureInvocationForCaching(NewInvocation, CASOpts, - IncludeTreeRoot->getID().toString(), - /*CASFSWorkingDir=*/"", - /*ProduceIncludeTree=*/true); + assert(IncludeStack.size() == 1); + Expected MainIncludeTree = + getCASTreeForFileIncludes(IncludeStack.pop_back_val()); + if (!MainIncludeTree) + return MainIncludeTree.takeError(); - DepscanPrefixMapping::remapInvocationPaths(NewInvocation, PrefixMapper); + if (!ScanInstance.getLangOpts().CurrentModule.empty()) { + SmallVector Modules; + auto AddModule = [&](Module *M) -> llvm::Error { + Expected Mod = getIncludeTreeModule(DB, M); + if (!Mod) + return Mod.takeError(); + Modules.push_back(Mod->getRef()); + return Error::success(); + }; + if (Module *M = ScanInstance.getPreprocessor().getCurrentModule()) { + if (Error E = AddModule(M)) + return std::move(E); + } else { + // When building a TU or PCH, we can have headers files that are part of + // both the public and private modules that are included textually. In + // that case we need both of those modules. + ModuleMap &MMap = + ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap(); + if (Module *M = MMap.findModule(ScanInstance.getLangOpts().CurrentModule)) + if (Error E = AddModule(M)) + return std::move(E); + if (Module *PM = + MMap.findModule(ScanInstance.getLangOpts().ModuleName + "_Private")) + if (Error E = AddModule(PM)) + return std::move(E); + } + + auto ModMap = cas::IncludeTree::ModuleMap::create(DB, Modules); + if (!ModMap) + return ModMap.takeError(); + ModuleMapRef = ModMap->getRef(); + } + + auto FileList = + cas::IncludeTree::FileList::create(DB, IncludedFiles, IncludedFileLists); + if (!FileList) + return FileList.takeError(); + + return cas::IncludeTreeRoot::create(DB, MainIncludeTree->getRef(), + FileList->getRef(), PCHRef, ModuleMapRef); +} + +Error IncludeTreeBuilder::addModuleInputs(ASTReader &Reader) { + for (serialization::ModuleFile &MF : Reader.getModuleManager()) { + // Only add direct imports to avoid duplication. Each include tree is a + // superset of its imported modules' include trees. + if (!MF.isDirectlyImported()) + continue; + + assert(!MF.IncludeTreeID.empty() && "missing include-tree for import"); + + Optional ID; + if (Error E = DB.parseID(MF.IncludeTreeID).moveInto(ID)) + return E; + Optional Ref = DB.getReference(*ID); + if (!Ref) + return DB.createUnknownObjectError(*ID); + Optional Root; + if (Error E = cas::IncludeTreeRoot::get(DB, *Ref).moveInto(Root)) + return E; + + IncludedFileLists.push_back(Root->getFileListRef()); + } return Error::success(); } -Expected -IncludeTreeActionController::getObjectForFile(Preprocessor &PP, FileID FID) { +Expected IncludeTreeBuilder::getObjectForFile(Preprocessor &PP, + FileID FID) { SourceManager &SM = PP.getSourceManager(); const SrcMgr::FileInfo &FI = SM.getSLocEntry(FID).getFile(); if (PP.getPredefinesFileID() == FID) { @@ -348,6 +712,15 @@ IncludeTreeActionController::getObjectForFile(Preprocessor &PP, FileID FID) { } return *PredefinesBufferRef; } + if (!FI.getContentCache().OrigEntry && + FI.getName() == Module::getModuleInputBufferName()) { + // Virtual buffer + if (!ModuleIncludesBufferRef) { + if (Error E = getObjectForBuffer(FI).moveInto(ModuleIncludesBufferRef)) + return std::move(E); + } + return *ModuleIncludesBufferRef; + } assert(FI.getContentCache().OrigEntry); auto &FileRef = ObjectForFile[FI.getContentCache().OrigEntry]; if (!FileRef) { @@ -359,8 +732,9 @@ IncludeTreeActionController::getObjectForFile(Preprocessor &PP, FileID FID) { return *FileRef; } -Expected IncludeTreeActionController::getObjectForFileNonCached( - FileManager &FM, const SrcMgr::FileInfo &FI) { +Expected +IncludeTreeBuilder::getObjectForFileNonCached(FileManager &FM, + const SrcMgr::FileInfo &FI) { const FileEntry *FE = FI.getContentCache().OrigEntry; assert(FE); @@ -373,21 +747,21 @@ Expected IncludeTreeActionController::getObjectForFileNonCached( } Expected -IncludeTreeActionController::getObjectForBuffer(const SrcMgr::FileInfo &FI) { +IncludeTreeBuilder::getObjectForBuffer(const SrcMgr::FileInfo &FI) { // This is a non-file buffer, like the predefines. auto Ref = DB.storeFromString( {}, FI.getContentCache().getBufferIfLoaded()->getBuffer()); if (!Ref) return Ref.takeError(); - Expected FileNode = createIncludeFile(FI.getName(), *Ref); + Expected FileNode = + createIncludeFile(FI.getName(), *Ref); if (!FileNode) return FileNode.takeError(); return FileNode->getRef(); } Expected -IncludeTreeActionController::addToFileList(FileManager &FM, - const FileEntry *FE) { +IncludeTreeBuilder::addToFileList(FileManager &FM, const FileEntry *FE) { StringRef Filename = FE->getName(); llvm::ErrorOr> CASContents = FM.getObjectRefForFileContent(Filename); @@ -402,7 +776,7 @@ IncludeTreeActionController::addToFileList(FileManager &FM, return FileNode.takeError(); IncludedFiles.push_back( {FileNode->getRef(), - static_cast(FE->getSize())}); + static_cast(FE->getSize())}); return FileNode->getRef(); }; @@ -420,42 +794,25 @@ IncludeTreeActionController::addToFileList(FileManager &FM, } Expected -IncludeTreeActionController::getCASTreeForFileIncludes(FilePPState &&PPState) { +IncludeTreeBuilder::getCASTreeForFileIncludes(FilePPState &&PPState) { return cas::IncludeTree::create(DB, PPState.FileCharacteristic, PPState.File, - PPState.Includes, PPState.HasIncludeChecks); + PPState.Includes, PPState.SubmoduleName, + PPState.HasIncludeChecks); } -Expected -IncludeTreeActionController::createIncludeFile(StringRef Filename, - cas::ObjectRef Contents) { +Expected +IncludeTreeBuilder::createIncludeFile(StringRef Filename, + cas::ObjectRef Contents) { SmallString<256> MappedPath; if (!PrefixMapper.empty()) { PrefixMapper.map(Filename, MappedPath); Filename = MappedPath; } - return cas::IncludeFile::create(DB, Filename, std::move(Contents)); -} - -Expected IncludeTreeActionController::getIncludeTree() { - if (ErrorToReport) - return std::move(*ErrorToReport); - - assert(IncludeStack.size() == 1); - Expected MainIncludeTree = - getCASTreeForFileIncludes(IncludeStack.pop_back_val()); - if (!MainIncludeTree) - return MainIncludeTree.takeError(); - auto FileList = cas::IncludeFileList::create(DB, IncludedFiles); - if (!FileList) - return FileList.takeError(); - - return cas::IncludeTreeRoot::create(DB, MainIncludeTree->getRef(), - FileList->getRef(), PCHRef); + return cas::IncludeTree::File::create(DB, Filename, std::move(Contents)); } std::unique_ptr dependencies::createIncludeTreeActionController( - cas::ObjectStore &DB, DepscanPrefixMapping PrefixMapping) { - return std::make_unique( - DB, std::move(PrefixMapping)); + LookupModuleOutputCallback LookupModuleOutput, cas::ObjectStore &DB) { + return std::make_unique(DB, LookupModuleOutput); } diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index 984582c08350d..960a3c7d7bfd9 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -106,6 +106,8 @@ ModuleDepCollector::makeInvocationForModuleBuildWithoutOutputs( if (!CI.getLangOpts()->ModulesCodegen) { CI.getCodeGenOpts().DebugCompilationDir.clear(); CI.getCodeGenOpts().CoverageCompilationDir.clear(); + CI.getCodeGenOpts().CoverageDataFile.clear(); + CI.getCodeGenOpts().CoverageNotesFile.clear(); } // Map output paths that affect behaviour to "-" so their existence is in the @@ -441,15 +443,6 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { MD.ImplicitModulePCMPath = std::string(M->getASTFile()->getName()); MD.IsSystem = M->IsSystem; - if (auto ID = M->getCASFileSystemRootID()) { - auto &CAS = MDC.ScanInstance.getOrCreateObjectStore(); - if (auto Err = CAS.parseID(*ID).moveInto(MD.CASFileSystemRootID)) { - MDC.ScanInstance.getDiagnostics().Report( - diag::err_cas_cannot_parse_root_id_for_module) - << *ID << MD.ID.ModuleName; - } - } - ModuleMap &ModMapInfo = MDC.ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap(); @@ -490,6 +483,19 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { MD.ModuleMapFileDeps.emplace_back(FE.getNameAsRequested()); }); + if (!MF->IncludeTreeID.empty()) + MD.IncludeTreeID = MF->IncludeTreeID; + + if (!MF->CASFileSystemRootID.empty()) { + auto &CAS = MDC.ScanInstance.getOrCreateObjectStore(); + if (auto Err = CAS.parseID(MF->CASFileSystemRootID) + .moveInto(MD.CASFileSystemRootID)) { + MDC.ScanInstance.getDiagnostics().Report( + diag::err_cas_cannot_parse_root_id_for_module) + << MF->CASFileSystemRootID << MD.ID.ModuleName; + } + } + CompilerInvocation CI = MDC.makeInvocationForModuleBuildWithoutOutputs( MD, [&](CompilerInvocation &BuildInvocation) { if (MDC.OptimizeArgs) diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index 43272f80290e1..88457d2f0a8e3 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -29,17 +29,30 @@ void tooling::dependencies::configureInvocationForCaching( // Clear this otherwise it defeats the purpose of making the compilation key // independent of certain arguments. CI.getCodeGenOpts().DwarfDebugFlags.clear(); + if (FrontendOpts.ProgramAction == frontend::GeneratePCH) { + // Clear paths that are emitted into binaries; they do not affect PCH. + // For modules this is handled in ModuleDepCollector. + CI.getCodeGenOpts().CoverageDataFile.clear(); + CI.getCodeGenOpts().CoverageNotesFile.clear(); + } // "Fix" the CAS options. auto &FileSystemOpts = CI.getFileSystemOpts(); if (ProduceIncludeTree) { FrontendOpts.CASIncludeTreeID = std::move(RootID); FrontendOpts.Inputs.clear(); + FrontendOpts.ModuleMapFiles.clear(); + HeaderSearchOptions &HSOpts = CI.getHeaderSearchOpts(); + HeaderSearchOptions OriginalHSOpts; + std::swap(HSOpts, OriginalHSOpts); // Preserve sysroot path to accommodate lookup for 'SDKSettings.json' during // availability checking. - std::string OriginalSysroot = CI.getHeaderSearchOpts().Sysroot; - CI.getHeaderSearchOpts() = HeaderSearchOptions(); - CI.getHeaderSearchOpts().Sysroot = OriginalSysroot; + HSOpts.Sysroot = std::move(OriginalHSOpts.Sysroot); + // Preserve resource-dir, which is added back by cc1_main if missing, and + // affects the cache key. + HSOpts.ResourceDir = std::move(OriginalHSOpts.ResourceDir); + // Preserve fmodule-file options. + HSOpts.PrebuiltModuleFiles = std::move(OriginalHSOpts.PrebuiltModuleFiles); auto &PPOpts = CI.getPreprocessorOpts(); // We don't need this because we save the contents of the PCH file in the // include tree root. @@ -164,68 +177,38 @@ void DepscanPrefixMapping::remapInvocationPaths(CompilerInvocation &Invocation, Mapper.mapInPlace(CodeGenOpts.ProfileInstrumentUsePath); Mapper.mapInPlace(CodeGenOpts.SampleProfileFile); Mapper.mapInPlace(CodeGenOpts.ProfileRemappingFile); + + // Dependency output options. + // Note: these are not in the cache key, but they are in the module context + // hash, which indirectly impacts the cache key when importing a module. + // In the future we may change how -fmodule-file-cache-key works when + // remapping to avoid needing this. + for (auto &ExtraDep : Invocation.getDependencyOutputOpts().ExtraDeps) + Mapper.mapInPlace(ExtraDep.first); } -Error DepscanPrefixMapping::configurePrefixMapper( - const CompilerInvocation &Invocation, llvm::PrefixMapper &Mapper) const { - auto isPathApplicableAsPrefix = [](StringRef Path) -> bool { - if (Path.empty()) - return false; - if (llvm::sys::path::is_relative(Path)) - return false; - if (Path == llvm::sys::path::root_path(Path)) - return false; - return true; - }; +void DepscanPrefixMapping::configurePrefixMapper(const CompilerInvocation &CI, + llvm::PrefixMapper &Mapper) { + return configurePrefixMapper(CI.getFrontendOpts().PathPrefixMappings, Mapper); +} - const HeaderSearchOptions &HSOpts = Invocation.getHeaderSearchOpts(); +void DepscanPrefixMapping::configurePrefixMapper( + ArrayRef PathPrefixMappings, llvm::PrefixMapper &Mapper) { + if (PathPrefixMappings.empty()) + return; - if (NewSDKPath) { - StringRef SDK = HSOpts.Sysroot; - if (isPathApplicableAsPrefix(SDK)) - // Need a new copy of the string since the invocation will be modified. - Mapper.add({SDK, *NewSDKPath}); - } - if (NewToolchainPath) { - // Look up for the toolchain, assuming resources are at - // /usr/lib/clang/. Return a shallower guess if the - // directories do not match. - // - // FIXME: Should this append ".." instead of calling parent_path? - StringRef ResourceDir = HSOpts.ResourceDir; - StringRef Guess = llvm::sys::path::parent_path(ResourceDir); - for (StringRef Dir : {"clang", "lib", "usr"}) { - if (llvm::sys::path::filename(Guess) != Dir) - break; - Guess = llvm::sys::path::parent_path(Guess); - } - if (isPathApplicableAsPrefix(Guess)) - // Need a new copy of the string since the invocation will be modified. - Mapper.add({Guess, *NewToolchainPath}); - } - if (!PrefixMap.empty()) { - llvm::SmallVector Split; - llvm::MappedPrefix::transformJoinedIfValid(PrefixMap, Split); - for (auto &MappedPrefix : Split) { - if (isPathApplicableAsPrefix(MappedPrefix.Old)) { - Mapper.add(MappedPrefix); - } else { - return createStringError(llvm::errc::invalid_argument, - "invalid prefix map: '" + MappedPrefix.Old + - "=" + MappedPrefix.New + "'"); - } - } - } + llvm::SmallVector Split; + llvm::MappedPrefix::transformJoinedIfValid(PathPrefixMappings, Split); + for (auto &MappedPrefix : Split) + Mapper.add(MappedPrefix); Mapper.sort(); - return Error::success(); } Expected clang::scanAndUpdateCC1InlineWithTool( DependencyScanningTool &Tool, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, CompilerInvocation &Invocation, - StringRef WorkingDirectory, const DepscanPrefixMapping &PrefixMapping, - llvm::cas::ObjectStore &DB) { + StringRef WorkingDirectory, llvm::cas::ObjectStore &DB) { // Override the CASOptions. They may match (the caller having sniffed them // out of InputArgs) but if they have been overridden we want the new ones. Invocation.getCASOpts() = Tool.getCASOpts(); @@ -242,8 +225,7 @@ Expected clang::scanAndUpdateCC1InlineWithTool( Tool.getCachingFileSystem()); } llvm::PrefixMapper &Mapper = *MapperPtr; - if (Error E = PrefixMapping.configurePrefixMapper(Invocation, Mapper)) - return std::move(E); + DepscanPrefixMapping::configurePrefixMapper(Invocation, Mapper); auto ScanInvocation = std::make_shared(Invocation); // An error during dep-scanning is treated as if the main compilation has @@ -252,17 +234,18 @@ Expected clang::scanAndUpdateCC1InlineWithTool( Optional Root; if (ProduceIncludeTree) { - if (Error E = Tool.getIncludeTreeFromCompilerInvocation( - DB, std::move(ScanInvocation), WorkingDirectory, - PrefixMapping, DiagsConsumer, VerboseOS, - /*DiagGenerationAsCompilation*/ true) - .moveInto(Root)) + if (Error E = + Tool.getIncludeTreeFromCompilerInvocation( + DB, std::move(ScanInvocation), WorkingDirectory, + /*LookupModuleOutput=*/nullptr, DiagsConsumer, VerboseOS, + /*DiagGenerationAsCompilation*/ true) + .moveInto(Root)) return std::move(E); } else { if (Error E = Tool.getDependencyTreeFromCompilerInvocation( std::move(ScanInvocation), WorkingDirectory, DiagsConsumer, VerboseOS, - /*DiagGenerationAsCompilation*/ true, PrefixMapping) + /*DiagGenerationAsCompilation*/ true) .moveInto(Root)) return std::move(E); } diff --git a/clang/test/CAS/cached-diagnostics.c b/clang/test/CAS/cached-diagnostics.c index 509417333d382..6f11adf48f88a 100644 --- a/clang/test/CAS/cached-diagnostics.c +++ b/clang/test/CAS/cached-diagnostics.c @@ -4,8 +4,8 @@ // RUN: %clang_cc1 -triple x86_64-apple-macos12 -fsyntax-only %t/src/main.c -I %t/src/inc -Wunknown-pragmas 2> %t/regular-diags1.txt -// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -fdepscan-prefix-map=%t/src=/^src -o %t/t1.rsp -cc1-args \ -// RUN: -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas \ +// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t/t1.rsp -cc1-args \ +// RUN: -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas -fdepscan-prefix-map=%t/src=/^src \ // RUN: -emit-obj %t/src/main.c -o %t/out/output.o -I %t/src/inc -Wunknown-pragmas // Compare diagnostics after a miss. @@ -41,8 +41,8 @@ // RUN: %clang_cc1 -triple x86_64-apple-macos12 -fsyntax-only %t/src2/main.c -I %t/src2/inc -Wunknown-pragmas 2> %t/regular-diags2.txt -// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -fdepscan-prefix-map=%t/src2=/^src -o %t/t2.rsp -cc1-args \ -// RUN: -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas \ +// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t/t2.rsp -cc1-args \ +// RUN: -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas -fdepscan-prefix-map=%t/src2=/^src \ // RUN: -emit-obj %t/src2/main.c -o %t/out2/output.o -I %t/src2/inc -Wunknown-pragmas // RUN: %clang @%t/t2.rsp -Rcompile-job-cache 2> %t/diags-hit2.txt diff --git a/clang/test/CAS/depscan-include-tree.c b/clang/test/CAS/depscan-include-tree.c index f78a027fc2944..ae3f79ea43428 100644 --- a/clang/test/CAS/depscan-include-tree.c +++ b/clang/test/CAS/depscan-include-tree.c @@ -2,7 +2,8 @@ // RUN: split-file %s %t // RUN: %clang -cc1depscan -o %t/inline.rsp -fdepscan=inline -fdepscan-include-tree -cc1-args -cc1 -triple x86_64-apple-macos11.0 \ -// RUN: -emit-obj %t/t.c -o %t/t.o -dwarf-ext-refs -fmodule-format=obj \ +// RUN: -emit-obj %t/t.c -o %t/t.o -dwarf-ext-refs -fmodule-format=obj \ +// RUN: -ftest-coverage -coverage-notes-file %t/t.gcno -coverage-data-file %t/t.gcda \ // RUN: -I %t/includes -isysroot %S/Inputs/SDK -fcas-path %t/cas -DSOME_MACRO -dependency-file %t/inline.d -MT deps // RUN: FileCheck %s -input-file %t/inline.rsp -DPREFIX=%t @@ -13,6 +14,9 @@ // CHECK: "-fcas-path" "[[PREFIX]]/cas" // CHECK: "-fcas-include-tree" // CHECK: "-isysroot" +// CHECK: "-ftest-coverage" +// CHECK: "-coverage-data-file" "[[PREFIX]]/t.gcda" +// CHECK: "-coverage-notes-file" "[[PREFIX]]/t.gcno" // SHOULD-NOT: "-fcas-fs" // SHOULD-NOT: "-fcas-fs-working-directory" // SHOULD-NOT: "-I" diff --git a/clang/test/CAS/depscan-prefix-map.c b/clang/test/CAS/depscan-prefix-map.c index 062d74b1f0742..25d1fa177b85b 100644 --- a/clang/test/CAS/depscan-prefix-map.c +++ b/clang/test/CAS/depscan-prefix-map.c @@ -5,52 +5,56 @@ // RUN: rm -rf %t.d // RUN: mkdir %t.d // RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=inline \ -// RUN: -fdepscan-prefix-map=%S=/^source \ -// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ -// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ -// RUN: -fdepscan-prefix-map-toolchain=/^toolchain \ -// RUN: -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ // RUN: -isysroot %S/Inputs/SDK \ // RUN: -resource-dir %S/Inputs/toolchain_dir/usr/lib/clang/1000 \ // RUN: -internal-isystem %S/Inputs/toolchain_dir/usr/lib/clang/1000/include \ // RUN: -working-directory %t.d \ // RUN: -fcas-path %t.d/cas \ +// RUN: -fdepscan-prefix-map=%S=/^source \ +// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ +// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ +// RUN: -fdepfile-entry=%t.d/extra \ // RUN: | FileCheck %s -DPREFIX=%t.d // RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=inline \ -// RUN: -fdepscan-prefix-map=%S=/^source \ -// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ -// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ -// RUN: -fdepscan-prefix-map-toolchain=/^toolchain \ -// RUN: -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ // RUN: -isysroot %S/Inputs/SDK \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 \ // RUN: -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -working-directory %t.d \ // RUN: -fcas-path %t.d/cas \ +// RUN: -fdepscan-prefix-map=%S=/^source \ +// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ +// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ +// RUN: -fdepfile-entry=%t.d/extra \ // RUN: | FileCheck %s -DPREFIX=%t.d // RUN: %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ // RUN: -cas-args -fcas-path %t.d/cas -- \ // RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=daemon \ // RUN: -fdepscan-daemon=%{clang-daemon-dir}/%basename_t \ -// RUN: -fdepscan-prefix-map=%S=/^source \ -// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ -// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ -// RUN: -fdepscan-prefix-map-toolchain=/^toolchain \ -// RUN: -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ // RUN: -isysroot %S/Inputs/SDK \ // RUN: -resource-dir %S/Inputs/toolchain_dir/usr/lib/clang/1000 \ // RUN: -internal-isystem %S/Inputs/toolchain_dir/usr/lib/clang/1000/include \ // RUN: -working-directory %t.d \ // RUN: -fcas-path %t.d/cas \ +// RUN: -fdepscan-prefix-map=%S=/^source \ +// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ +// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ +// RUN: -fdepfile-entry=%t.d/extra \ // RUN: | FileCheck %s -DPREFIX=%t.d // // CHECK: "-fcas-path" "[[PREFIX]]/cas" // CHECK-SAME: "-working-directory" "/^testdir" // CHECK-SAME: "-x" "c" "/^source/depscan-prefix-map.c" // CHECK-SAME: "-isysroot" "/^sdk" +// CHECK-SAME: "-fdepfile-entry=/^testdir/extra" // RUN: llvm-cas --cas %t.d/cas --ls-tree-recursive @%t.root \ // RUN: | FileCheck %s -check-prefix=CHECK-ROOT @@ -64,35 +68,5 @@ // CHECK-ROOT-NEXT: file {{.*}} /^source/depscan-prefix-map.c{{$}} // CHECK-ROOT-NEXT: file {{.*}} /^toolchain/usr/lib/clang/1000/include/stdarg.h{{$}} -// RUN: not %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ -// RUN: -cas-args -fcas-path %t.d/cas -- \ -// RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=daemon \ -// RUN: -fdepscan-daemon=%{clang-daemon-dir}/%basename_t \ -// RUN: -fdepscan-prefix-map=/=/^foo \ -// RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ -// RUN: -fcas-path %t.d/cas \ -// RUN: 2>&1 | FileCheck %s -DPREFIX=%t.d -check-prefix=ERROR_ROOT -// ERROR_ROOT: invalid prefix map: '/=/^foo' - -// RUN: not %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ -// RUN: -cas-args -fcas-path %t.d/cas -- \ -// RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=daemon \ -// RUN: -fdepscan-daemon=%{clang-daemon-dir}/%basename_t \ -// RUN: -fdepscan-prefix-map==/^foo \ -// RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ -// RUN: -fcas-path %t.d/cas \ -// RUN: 2>&1 | FileCheck %s -DPREFIX=%t.d -check-prefix=ERROR_EMPTY -// ERROR_EMPTY: invalid prefix map: '=/^foo' - -// RUN: not %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ -// RUN: -cas-args -fcas-path %t.d/cas -- \ -// RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=daemon \ -// RUN: -fdepscan-daemon=%{clang-daemon-dir}/%basename_t \ -// RUN: -fdepscan-prefix-map=relative=/^foo \ -// RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ -// RUN: -fcas-path %t.d/cas \ -// RUN: 2>&1 | FileCheck %s -DPREFIX=%t.d -check-prefix=ERROR_RELATIVE -// ERROR_RELATIVE: invalid prefix map: 'relative=/^foo' - #include int test() { return 0; } diff --git a/clang/test/CAS/driver-cache-launcher.c b/clang/test/CAS/driver-cache-launcher.c index e12f622830a5d..195e62c86d3ab 100644 --- a/clang/test/CAS/driver-cache-launcher.c +++ b/clang/test/CAS/driver-cache-launcher.c @@ -58,14 +58,15 @@ // RUN: cp -R %S/Inputs/cmake-build %t/cmake-build // RUN: pushd %t/cmake-build // RUN: cache-build-session -prefix-map-cmake -v echo 2>&1 | FileCheck %s -check-prefix=SESSION-CMAKE-PREFIX -// RUN: env LLVM_CACHE_CAS_PATH=%t/cas cache-build-session -prefix-map-cmake %clang-cache %clang -c %s -o %t.o -### 2>&1 | FileCheck %s -check-prefix=CLANG-CMAKE-PREFIX -DPREFIX=%t +// RUN: env LLVM_CACHE_CAS_PATH=%t/cas cache-build-session -prefix-map-cmake %clang-cache %clang -c %s -o %t.o -isysroot %S/Inputs/SDK -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -### 2>&1 | FileCheck %s -check-prefix=CLANG-CMAKE-PREFIX -DPREFIX=%t -DINPUTS=%S/Inputs // RUN: popd // SESSION-CMAKE-PREFIX: note: setting LLVM_CACHE_PREFIX_MAPS=/llvm/build=/^build;/llvm/llvm-project/llvm=/^src;/llvm/llvm-project/clang=/^src-clang;/llvm/llvm-project/clang-tools-extra=/^src-clang-tools-extra;/llvm/llvm-project/third-party/benchmark=/^src-benchmark;/llvm/llvm-project/other/benchmark=/^src-benchmark-1;/llvm/llvm-project/another/benchmark=/^src-benchmark-2{{$}} // SESSION-CMAKE-PREFIX: note: setting LLVM_CACHE_BUILD_SESSION_ID= // CLANG-CMAKE-PREFIX: "-cc1depscan" "-fdepscan=daemon" "-fdepscan-share-identifier" -// CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map-sdk=/^sdk" "-fdepscan-prefix-map-toolchain=/^toolchain" +// CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=[[INPUTS]]/SDK=/^sdk" +// CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=[[INPUTS]]/toolchain_dir=/^toolchain" // CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=/llvm/build=/^build" // CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=/llvm/llvm-project/llvm=/^src" // CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=/llvm/llvm-project/clang=/^src-clang" diff --git a/clang/test/CAS/fcas-include-tree-prefix-mapping.c b/clang/test/CAS/fcas-include-tree-prefix-mapping.c index d6de19be71139..088b8af61480d 100644 --- a/clang/test/CAS/fcas-include-tree-prefix-mapping.c +++ b/clang/test/CAS/fcas-include-tree-prefix-mapping.c @@ -3,11 +3,13 @@ // RUN: mkdir %t/out // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/t1.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out \ // RUN: -emit-llvm %t/src/main.c -o %t/out/output.ll -include %t/src/prefix.h -I %t/src/inc \ // RUN: -MT deps -dependency-file %t/t1.d @@ -41,11 +43,13 @@ // RUN: mkdir %t/out2 // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/t2.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out2 \ // RUN: -emit-llvm %t/src2/main.c -o %t/out2/output.ll -include %t/src2/prefix.h -I %t/src2/inc \ // RUN: -MT deps -dependency-file %t/t2.d @@ -77,22 +81,26 @@ // Check with PCH. // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/pch1.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out \ // RUN: -emit-pch -x c-header %t/src/prefix.h -o %t/out/prefix.h.pch -include %t/src/prefix.h -I %t/src/inc // RUN: %clang @%t/pch1.rsp // With different cas path to avoid cache hit. // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/pch2.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas2 -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out2 \ // RUN: -emit-pch -x c-header %t/src2/prefix.h -o %t/out2/prefix.h.pch -include %t/src2/prefix.h -I %t/src2/inc // RUN: %clang @%t/pch2.rsp @@ -100,11 +108,13 @@ // RUN: diff %t/out/prefix.h.pch %t/out2/prefix.h.pch // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/t3.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out \ // RUN: -emit-obj %t/src/main.c -o %t/out/main.o -include-pch %t/out/prefix.h.pch -I %t/src/inc \ // RUN: -MT deps -dependency-file %t/t1.pch.d @@ -115,11 +125,13 @@ // RUN: -e "s/' .*$//" > %t/cache-key3 // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/t4.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out2 \ // RUN: -emit-obj %t/src2/main.c -o %t/out2/main.o -include-pch %t/out2/prefix.h.pch -I %t/src2/inc \ // RUN: -MT deps -dependency-file %t/t2.pch.d diff --git a/clang/test/CAS/fcas-include-tree-with-pch.c b/clang/test/CAS/fcas-include-tree-with-pch.c index 8438e07977f2e..026e4fe785031 100644 --- a/clang/test/CAS/fcas-include-tree-with-pch.c +++ b/clang/test/CAS/fcas-include-tree-with-pch.c @@ -37,6 +37,14 @@ // RUN: %clang @tu2.rsp -emit-llvm -o tree-rel.ll // RUN: diff -u source-rel.ll tree-rel.ll +// Check that -coverage-notes-file and -coverage-data-file are stripped +// RUN: %clang -cc1depscan -o pch3.rsp -fdepscan=inline -fdepscan-include-tree -cc1-args \ +// RUN: -cc1 -emit-pch -x c-header prefix.h -I %t/inc -DCMD_MACRO=1 -fcas-path %t/cas \ +// RUN: -ftest-coverage -coverage-notes-file %t/pch.gcno -coverage-data-file %t/pch.gcda +// RUN: FileCheck %s -check-prefix=COVERAGE -input-file %t/pch3.rsp +// COVERAGE-NOT: -coverage-data-file +// COVERAGE-NOT: -coverage-notes-file + //--- t1.c #if S2_MACRO #include "s2-link.h" diff --git a/clang/test/CAS/pgo-profile.c b/clang/test/CAS/pgo-profile.c index b64716c34cf98..387b3e8fa7f70 100644 --- a/clang/test/CAS/pgo-profile.c +++ b/clang/test/CAS/pgo-profile.c @@ -37,9 +37,9 @@ // RUN: mkdir -p %t.dir/a && mkdir -p %t.dir/b // RUN: cp %t.profdata %t.dir/a/a.profdata // RUN: cp %t.profdata %t.dir/b/a.profdata -// RUN: %clang -cc1depscan -fdepscan=inline -o %t4.rsp -fdepscan-prefix-map=%t.dir/a=/^testdir -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache \ +// RUN: %clang -cc1depscan -fdepscan=inline -o %t4.rsp -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache -fdepscan-prefix-map=%t.dir/a=/^testdir \ // RUN: -x c %s -o %t.o -fcas-path %t.dir/cas -fprofile-instrument-use-path=%t.dir/a/a.profdata -// RUN: %clang -cc1depscan -fdepscan=inline -o %t5.rsp -fdepscan-prefix-map=%t.dir/b=/^testdir -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache \ +// RUN: %clang -cc1depscan -fdepscan=inline -o %t5.rsp -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache -fdepscan-prefix-map=%t.dir/b=/^testdir \ // RUN: -x c %s -o %t.o -fcas-path %t.dir/cas -fprofile-instrument-use-path=%t.dir/b/a.profdata // RUN: cat %t4.rsp | FileCheck %s --check-prefix=REMAP // RUN: %clang @%t4.rsp 2>&1 | FileCheck %s --check-prefix=CACHE-MISS @@ -54,9 +54,9 @@ // RUN: grep llvmcas %t.dir/cache-key1 // RUN: diff -u %t.dir/cache-key1 %t.dir/cache-key2 -// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t4.inc.rsp -fdepscan-prefix-map=%t.dir/a=/^testdir -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache \ +// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t4.inc.rsp -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache -fdepscan-prefix-map=%t.dir/a=/^testdir \ // RUN: -x c %s -o %t.o -fcas-path %t.dir/cas -fprofile-instrument-use-path=%t.dir/a/a.profdata -// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t5.inc.rsp -fdepscan-prefix-map=%t.dir/b=/^testdir -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache \ +// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t5.inc.rsp -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache -fdepscan-prefix-map=%t.dir/b=/^testdir \ // RUN: -x c %s -o %t.o -fcas-path %t.dir/cas -fprofile-instrument-use-path=%t.dir/b/a.profdata // RUN: cat %t4.inc.rsp | FileCheck %s --check-prefix=REMAP // RUN: %clang @%t4.inc.rsp 2>&1 | FileCheck %s --check-prefix=CACHE-MISS diff --git a/clang/test/ClangScanDeps/Inputs/removed-args/cdb.json.template b/clang/test/ClangScanDeps/Inputs/removed-args/cdb.json.template index 2bdc340a3fccd..7ae3c88aedd8d 100644 --- a/clang/test/ClangScanDeps/Inputs/removed-args/cdb.json.template +++ b/clang/test/ClangScanDeps/Inputs/removed-args/cdb.json.template @@ -1,7 +1,7 @@ [ { "directory": "DIR", - "command": "clang -fsyntax-only DIR/tu.c -fmodules -fimplicit-module-maps -fmodules-validate-once-per-build-session -fbuild-session-file=DIR/build-session -fmodules-prune-interval=123 -fmodules-prune-after=123 -fmodules-cache-path=DIR/cache -include DIR/header.h -grecord-command-line -fdebug-compilation-dir=DIR/debug -fcoverage-compilation-dir=DIR/coverage -o DIR/tu.o -serialize-diagnostics DIR/tu.diag -MT tu -MD -MF DIR/tu.d", + "command": "clang -fsyntax-only DIR/tu.c -fmodules -fimplicit-module-maps -fmodules-validate-once-per-build-session -fbuild-session-file=DIR/build-session -fmodules-prune-interval=123 -fmodules-prune-after=123 -fmodules-cache-path=DIR/cache -include DIR/header.h -grecord-command-line -fdebug-compilation-dir=DIR/debug -fcoverage-compilation-dir=DIR/coverage -ftest-coverage -o DIR/tu.o -serialize-diagnostics DIR/tu.diag -MT tu -MD -MF DIR/tu.d", "file": "DIR/tu.c" } ] diff --git a/clang/test/ClangScanDeps/cas-trees.c b/clang/test/ClangScanDeps/cas-trees.c index c54f9a276e8de..47dbac6c47e44 100644 --- a/clang/test/ClangScanDeps/cas-trees.c +++ b/clang/test/ClangScanDeps/cas-trees.c @@ -18,7 +18,8 @@ // FULL-TREE-NEXT: "modules": [], // FULL-TREE-NEXT: "translation-units": [ // FULL-TREE-NEXT: { -// FULL-TREE: "casfs-root-id": "[[T1_ROOT_ID:llvmcas://[[:xdigit:]]+]]" +// FULL-TREE: "cache-key": "[[T1_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// FULL-TREE-NEXT: "casfs-root-id": "[[T1_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // FULL-TREE-NEXT: "clang-context-hash": "{{[A-Z0-9]+}}", // FULL-TREE-NEXT: "clang-module-deps": [], // FULL-TREE-NEXT: "command-line": [ @@ -36,7 +37,8 @@ // FULL-TREE-NEXT: "input-file": "[[PREFIX]]/t1.c" // FULL-TREE-NEXT: } // FULL-TREE: { -// FULL-TREE: "casfs-root-id": "[[T2_ROOT_ID:llvmcas://[[:xdigit:]]+]]" +// FULL-TREE: "cache-key": "[[T2_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// FULL-TREE-NEXT: "casfs-root-id": "[[T2_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // FULL-TREE-NEXT: "clang-context-hash": "{{[A-Z0-9]+}}", // FULL-TREE-NEXT: "clang-module-deps": [], // FULL-TREE-NEXT: "command-line": [ @@ -56,13 +58,31 @@ // Build with caching // RUN: %deps-to-rsp %t/full_result.json --tu-index 0 > %t/t1.cc1.rsp // RUN: %deps-to-rsp %t/full_result.json --tu-index 1 > %t/t2.cc1.rsp -// RUN: %clang @%t/t1.cc1.rsp -Rcompile-job-cache 2>&1 | FileCheck %s -check-prefix=CACHE-MISS +// RUN: %clang @%t/t1.cc1.rsp -Rcompile-job-cache 2> %t/t1-miss.err +// RUN: FileCheck %s -input-file=%t/t1-miss.err -check-prefix=CACHE-MISS // RUN: %clang @%t/t1.cc1.rsp -Rcompile-job-cache 2>&1 | FileCheck %s -check-prefix=CACHE-HIT -// RUN: %clang @%t/t2.cc1.rsp -Rcompile-job-cache 2>&1 | FileCheck %s -check-prefix=CACHE-MISS +// RUN: %clang @%t/t2.cc1.rsp -Rcompile-job-cache 2> %t/t2-miss.err +// RUN: FileCheck %s -input-file=%t/t2-miss.err -check-prefix=CACHE-MISS // RUN: %clang @%t/t2.cc1.rsp -Rcompile-job-cache 2>&1 | FileCheck %s -check-prefix=CACHE-HIT // CACHE-HIT: remark: compile job cache hit // CACHE-MISS: remark: compile job cache miss +// Check cache keys. +// RUN: cp %t/full_result.json %t/combined.txt +// RUN: cat %t/t1-miss.err >> %t/combined.txt +// RUN: cat %t/t2-miss.err >> %t/combined.txt +// RUN: FileCheck %s -input-file=%t/combined.txt -check-prefix=COMBINED + +// COMBINED: "commands": [ +// COMBINED-NEXT: { +// COMBINED-NEXT: "cache-key": "[[T1_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// COMBINED: } +// COMBINED: "commands": [ +// COMBINED: { +// COMBINED-NEXT: "cache-key": "[[T2_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// COMBINED: remark: compile job cache miss for '[[T1_CACHE_KEY]]' +// COMBINED-NEXT: remark: compile job cache miss for '[[T2_CACHE_KEY]]' + // RUN: clang-scan-deps -compilation-database %t/cdb.json -cas-path %t/cas -format experimental-tree -emit-cas-compdb | FileCheck %s -DPREFIX=%/t -check-prefix=COMPDB // COMPDB: [ // COMPDB: { diff --git a/clang/test/ClangScanDeps/include-tree.c b/clang/test/ClangScanDeps/include-tree.c index eed9b3c784015..d21c408650156 100644 --- a/clang/test/ClangScanDeps/include-tree.c +++ b/clang/test/ClangScanDeps/include-tree.c @@ -48,6 +48,7 @@ // FULL-NEXT: { // FULL-NEXT: "commands": [ // FULL-NEXT: { +// FULL-NEXT: "cache-key": "[[TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // FULL: "clang-module-deps": [] // FULL: "command-line": [ // FULL-NEXT: "-cc1" @@ -84,7 +85,17 @@ // Build the include-tree command // RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp -// RUN: %clang @%t/tu.rsp +// RUN: %clang @%t/tu.rsp -Rcompile-job-cache 2> %t/t.err + +// Check cache key. +// RUN: cp %t/full.txt %t/combined.txt +// RUN: cat %t/t.err >> %t/combined.txt +// RUN: FileCheck %s -input-file=%t/combined.txt -check-prefix=COMBINED + +// COMBINED: "commands": [ +// COMBINED-NEXT: { +// COMBINED-NEXT: "cache-key": "[[TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// COMBINED: remark: compile job cache miss for '[[TU_CACHE_KEY]]' //--- cdb.json.template [ diff --git a/clang/test/ClangScanDeps/modules-cas-full-by-mod-name.c b/clang/test/ClangScanDeps/modules-cas-full-by-mod-name.c index 11a32b40cb1b0..c6ad2068c6c97 100644 --- a/clang/test/ClangScanDeps/modules-cas-full-by-mod-name.c +++ b/clang/test/ClangScanDeps/modules-cas-full-by-mod-name.c @@ -35,6 +35,7 @@ module transitive { header "transitive.h" } // CHECK: { // CHECK-NEXT: "modules": [ // CHECK-NEXT: { +// CHECK-NEXT: "cache-key": "[[DIRECT_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK-NEXT: "casfs-root-id": "[[LEFT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK-NEXT: "clang-module-deps": [ // CHECK-NEXT: { @@ -44,6 +45,9 @@ module transitive { header "transitive.h" } // CHECK-NEXT: ], // CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", // CHECK-NEXT: "command-line": [ +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "{{.*transitive-.*\.pcm}}" +// CHECK-NEXT: "[[TRANSITIVE_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: ], // CHECK-NEXT: "context-hash": "{{.*}}", // CHECK-NEXT: "file-deps": [ @@ -53,7 +57,8 @@ module transitive { header "transitive.h" } // CHECK-NEXT: "name": "direct" // CHECK-NEXT: }, // CHECK-NEXT: { -// CHECK-NEXT: "casfs-root-id": "[[LEFT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" +// CHECK-NEXT: "cache-key": "[[ROOT_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// CHECK-NEXT: "casfs-root-id": "[[ROOT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK-NEXT: "clang-module-deps": [ // CHECK-NEXT: { // CHECK-NEXT: "context-hash": "{{.*}}", @@ -62,6 +67,9 @@ module transitive { header "transitive.h" } // CHECK-NEXT: ], // CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", // CHECK-NEXT: "command-line": [ +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "{{.*direct-.*\.pcm}}" +// CHECK-NEXT: "[[DIRECT_CACHE_KEY]]" // CHECK: ], // CHECK-NEXT: "context-hash": "{{.*}}", // CHECK-NEXT: "file-deps": [ @@ -72,7 +80,8 @@ module transitive { header "transitive.h" } // CHECK-NEXT: "name": "root" // CHECK-NEXT: }, // CHECK-NEXT: { -// CHECK-NEXT: "casfs-root-id": "[[LEFT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" +// CHECK-NEXT: "cache-key": "[[TRANSITIVE_CACHE_KEY]]" +// CHECK-NEXT: "casfs-root-id": "[[LEFT_ROOT_ID]]" // CHECK-NEXT: "clang-module-deps": [], // CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", // CHECK-NEXT: "command-line": [ diff --git a/clang/test/ClangScanDeps/modules-cas-trees.c b/clang/test/ClangScanDeps/modules-cas-trees.c index f3d6cf578e01c..9aad19541aaef 100644 --- a/clang/test/ClangScanDeps/modules-cas-trees.c +++ b/clang/test/ClangScanDeps/modules-cas-trees.c @@ -64,6 +64,7 @@ // CHECK: { // CHECK-NEXT: "modules": [ // CHECK-NEXT: { +// CHECK: "cache-key": "[[LEFT_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: "casfs-root-id": "[[LEFT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK: "clang-module-deps": [ // CHECK: { @@ -92,6 +93,7 @@ // CHECK: "name": "Left" // CHECK: } // CHECK-NEXT: { +// CHECK: "cache-key": "[[RIGHT_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: "casfs-root-id": "[[RIGHT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK: "clang-module-deps": [ // CHECK: { @@ -110,7 +112,7 @@ // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" // CHECK: "[[TOP_PCM]]" -// CHECK: "[[TOP_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// CHECK: "[[TOP_CACHE_KEY]]" // CHECK: "-fmodule-file={{(Top=)?}}[[TOP_PCM]]" // CHECK: ] // CHECK: "file-deps": [ @@ -120,6 +122,7 @@ // CHECK: "name": "Right" // CHECK: } // CHECK-NEXT: { +// CHECK: "cache-key": "[[TOP_CACHE_KEY]]" // CHECK: "casfs-root-id": "[[TOP_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK: "clang-module-deps": [] // CHECK: "command-line": [ @@ -144,6 +147,7 @@ // CHECK: { // CHECK: "commands": [ // CHECK: { +// CHECK: "cache-key": "[[TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: "casfs-root-id": "[[TU_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK: "clang-module-deps": [ // CHECK: { @@ -162,10 +166,10 @@ // CHECK: "-fcache-compile-job" // CHECK: "-fmodule-file-cache-key" // CHECK: "[[LEFT_PCM]]" -// CHECK: "{{llvmcas://[[:xdigit:]]+}}" +// CHECK: "[[LEFT_CACHE_KEY]]" // CHECK: "-fmodule-file-cache-key" // CHECK: "[[RIGHT_PCM]]" -// CHECK: "{{llvmcas://[[:xdigit:]]+}}" +// CHECK: "[[RIGHT_CACHE_KEY]]" // CHECK: "-fmodule-file={{(Left=)?}}[[LEFT_PCM]]" // CHECK: "-fmodule-file={{(Right=)?}}[[RIGHT_PCM]]" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-include-tree-exports.c b/clang/test/ClangScanDeps/modules-include-tree-exports.c new file mode 100644 index 0000000000000..381854d4b94f3 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-exports.c @@ -0,0 +1,128 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// Extract the include-tree commands +// RUN: %deps-to-rsp %t/deps.json --module-name TwoSubs > %t/TwoSubs.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ExportExplicit > %t/ExportExplicit.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ExportWildcard > %t/ExportWildcard.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ExportGlobalWildcard > %t/ExportGlobalWildcard.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name NoExports > %t/NoExports.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu_export_explicit.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 1 > %t/tu_export_global_wildcard.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 2 > %t/tu_export_none.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 3 > %t/tu_export_wildcard.rsp + +// Build +// RUN: %clang @%t/TwoSubs.rsp +// RUN: %clang @%t/ExportExplicit.rsp +// RUN: %clang @%t/ExportWildcard.rsp +// RUN: %clang @%t/ExportGlobalWildcard.rsp +// RUN: %clang @%t/NoExports.rsp +// RUN: not %clang @%t/tu_export_explicit.rsp 2>&1 | FileCheck %s -check-prefix=tu_export_explicit +// RUN: %clang @%t/tu_export_wildcard.rsp 2>&1 | FileCheck %s -check-prefix=tu_export_wildcard -allow-empty +// RUN: %clang @%t/tu_export_global_wildcard.rsp 2>&1 | FileCheck %s -check-prefix=tu_export_global_wildcard -allow-empty +// RUN: not %clang @%t/tu_export_none.rsp 2>&1 | FileCheck %s -check-prefix=tu_export_none + +//--- cdb.json.template +[ +{ + "file": "DIR/tu_export_explicit.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_export_explicit.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +{ + "file": "DIR/tu_export_wildcard.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_export_wildcard.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +{ + "file": "DIR/tu_export_global_wildcard.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_export_global_wildcard.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +{ + "file": "DIR/tu_export_none.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_export_none.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +] + +//--- module.modulemap +module TwoSubs { + module Sub1 { header "Sub1.h" } + module Sub2 { header "Sub2.h" } +} + +module ExportExplicit { + header "Import.h" + export TwoSubs.Sub2 +} + +module ExportWildcard { + header "Import.h" + export TwoSubs.* +} + +module ExportGlobalWildcard { + header "Import.h" + export * +} + +module NoExports { + header "Import.h" +} + +//--- Sub1.h +void sub1(void); + +//--- Sub2.h +void sub2(void); + +//--- Import.h +#include "Sub1.h" +#include "Sub2.h" + +//--- tu_export_explicit.c +#pragma clang module import ExportExplicit +void tu1(void) { + sub2(); + // tu_export_explicit-NOT: error + sub1(); + // tu_export_explicit: error: call to undeclared function 'sub1' + // tu_export_explicit: error: missing '#include "Sub1.h"' +} + +//--- tu_export_wildcard.c +#pragma clang module import ExportWildcard +void tu1(void) { + sub1(); + sub2(); + // tu_export_wildcard-NOT: error +} + +//--- tu_export_global_wildcard.c +#pragma clang module import ExportGlobalWildcard +void tu1(void) { + sub1(); + sub2(); + // tu_export_global_wildcard-NOT: error +} + +//--- tu_export_none.c +#pragma clang module import NoExports +void tu1(void) { + sub1(); + // tu_export_none: error: call to undeclared function 'sub1' + // tu_export_none: error: missing '#include "Sub1.h"' + sub2(); + // tu_export_none: error: call to undeclared function 'sub2' + // tu_export_none: error: missing '#include "Sub2.h"' +} diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c new file mode 100644 index 0000000000000..279a88fcf3d06 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c @@ -0,0 +1,67 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: FileCheck %s -input-file %t/deps.json -check-prefix=NO_MODULES +// NO_MODULES: "modules": [] + +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid | FileCheck %s -DPREFIX=%/t +// RUN: %clang @%t/tu.rsp + +// CHECK: [[PREFIX]]/tu.m llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// +// CHECK: Submodule: Mod +// CHECK: 3:1 [[PREFIX]]/Mod.framework/PrivateHeaders/Priv.h llvmcas:// +// CHECK: Submodule: Mod_Private +// CHECK: 4:1 (Module for visibility only) Mod +// CHECK: 5:1 (Module for visibility only) Mod_Private +// CHECK: Module Map: +// CHECK: Mod (framework) +// CHECK: Mod_Private (framework) + +// CHECK: Files: +// CHECK: [[PREFIX]]/tu.m llvmcas:// +// CHECK-NOT: [[PREFIX]]/module.modulemap +// CHECK: [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// +// CHECK-NOT: [[PREFIX]]/module.modulemap +// CHECK: [[PREFIX]]/Mod.framework/PrivateHeaders/Priv.h llvmcas:// +// CHECK-NOT: [[PREFIX]]/module.modulemap + +//--- cdb.json.template +[{ + "file": "DIR/tu.m", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -F DIR -fmodule-name=Mod -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- Mod.framework/Modules/module.modulemap +framework module Mod { header "Mod.h" } + +//--- Mod.framework/Modules/module.private.modulemap +framework module Mod_Private { header "Priv.h" } + +//--- Mod.framework/Headers/Mod.h +void pub(void); + +//--- Mod.framework/PrivateHeaders/Priv.h +void priv(void); + +//--- tu.m +#import +#import +#import +#import +void tu(void) { + pub(); + priv(); +} diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation.c b/clang/test/ClangScanDeps/modules-include-tree-implementation.c new file mode 100644 index 0000000000000..282742ad803f1 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation.c @@ -0,0 +1,70 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: FileCheck %s -input-file %t/deps.json -check-prefix=NO_MODULES +// NO_MODULES: "modules": [] + +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid | FileCheck %s -DPREFIX=%/t +// RUN: %clang @%t/tu.rsp + +// CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} + +// Note: this is surprising, but correct: when building the implementation files +// of a module, the first include is textual but still uses the submodule +// machinery. The second include is treated as a module import (unless in a PCH) +// but will not actually import the module only trigger visibility changes. + +// CHECK: 2:1 [[PREFIX]]/Mod.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: Mod +// CHECK: 3:1 (Module for visibility only) Mod + +// CHECK: Files: +// CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap +// CHECK: [[PREFIX]]/Mod.h llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap + +// RUN: %deps-to-rsp %t/deps.json --tu-index 1 > %t/tu_missing_module.rsp +// RUN: %clang @%t/tu_missing_module.rsp + +//--- cdb.json.template +[ +{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -I DIR -fmodule-name=Mod -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +{ + "file": "DIR/tu_missing_module.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_missing_module.c -I DIR -fmodule-name=NonExistent -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +} +] + +//--- module.modulemap +module Mod { header "Mod.h" } + +//--- Mod.h +#pragma once +void top(void); + +//--- tu.c +#include "Mod.h" +#include "Mod.h" +void tu(void) { + top(); +} + +//--- tu_missing_module.c + diff --git a/clang/test/ClangScanDeps/modules-include-tree-inferred.m b/clang/test/ClangScanDeps/modules-include-tree-inferred.m new file mode 100644 index 0000000000000..f42847ef69004 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-inferred.m @@ -0,0 +1,47 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: %deps-to-rsp %t/deps.json --module-name Mod > %t/Mod.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: cat %t/Mod.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Mod.casid +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Mod.casid | FileCheck %s -DPREFIX=%/t +// RUN: %clang @%t/Mod.rsp +// RUN: %clang @%t/tu.rsp + +// CHECK: llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// +// CHECK: Submodule: Mod +// CHECK: Module Map: +// CHECK: Mod (framework) +// CHECK: Files: +// CHECK-NOT: [[PREFIX]]/module.modulemap +// CHECK: [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// +// CHECK-NOT: [[PREFIX]]/module.modulemap + +//--- cdb.json.template +[{ + "file": "DIR/tu.m", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- module.modulemap +framework module * {} + +//--- Mod.framework/Headers/Mod.h +void pub(void); + +//--- tu.m +#import +void tu(void) { + pub(); +} diff --git a/clang/test/ClangScanDeps/modules-include-tree-missing-submodule.c b/clang/test/ClangScanDeps/modules-include-tree-missing-submodule.c new file mode 100644 index 0000000000000..f599b6bb94687 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-missing-submodule.c @@ -0,0 +1,92 @@ +// Ensure we fallback to textual inclusion for headers in incomplete umbrellas. + +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json + +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch.json + +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Foo > %t/Foo.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/Foo.rsp +// RUN: %clang @%t/pch.rsp + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp + +// Extract include-tree casids +// RUN: cat %t/Foo.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Foo.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE Foo" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Foo.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// CHECK-LABEL: MODULE Foo +// CHECK: llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/Foo.framework/Headers/Foo.h llvmcas:// +// CHECK: Submodule: Foo +// CHECK-NOT: Bar +// CHECK: Module Map: +// CHECK: Foo (framework) +// CHECK-NOT: Bar +// CHECK: module * +// CHECK-NOT: Bar + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: (PCH) llvmcas:// +// CHECK: [[PREFIX]]/tu.c llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/Foo.framework/Headers/Bar.h llvmcas:// + +// RUN: %clang @%t/tu.rsp + +//--- cdb_pch.json.template +[{ + "file": "DIR/prefix.h", + "directory": "DIR", + "command": "clang -x c-header DIR/prefix.h -o DIR/prefix.h.pch -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" +}] + +//--- cdb.json.template +[{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -include prefix.h -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" +}] + +//--- Foo.framework/Modules/module.modulemap +framework module Foo { + umbrella header "Foo.h" + module * { export * } +} + +//--- Foo.framework/Headers/Foo.h +// Do not import Bar.h +void foo(void); + +//--- Foo.framework/Headers/Bar.h +void bar(void); + +//--- prefix.h +#include + +//--- tu.c +#include +// FIXME: -Wincomplete-umbrella warning +void tu(void) { + bar(); +} diff --git a/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c b/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c new file mode 100644 index 0000000000000..1f24620484ace --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c @@ -0,0 +1,122 @@ +// Test importing a private module whose public module was previously imported +// via a PCH. + +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json + +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch.json + +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Mod > %t/Mod.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Indirect1 > %t/Indirect1.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/Mod.rsp +// RUN: %clang @%t/Indirect1.rsp +// RUN: %clang @%t/pch.rsp + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: %deps-to-rsp %t/deps.json --module-name Mod_Private > %t/Mod_Private.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Indirect2 > %t/Indirect2.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/Mod_Private.rsp +// RUN: %clang @%t/Indirect2.rsp +// RUN: %clang @%t/tu.rsp + +// Extract include-tree casids +// RUN: cat %t/Indirect2.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Indirect.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE Indirect2" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Indirect.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt + +// Explicitly check that Mod_Private is imported as a module and not a header. +// RUN: FileCheck %s -DPREFIX=%/t -input-file %t/result.txt + +// CHECK-LABEL: MODULE Indirect2 +// CHECK: llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/indirect2.h llvmcas:// +// CHECK: Submodule: Indirect2 +// CHECK: 2:1 (Module) Indirect1 +// CHECK: 3:1 (Module) Mod_Private + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: (PCH) llvmcas:// +// CHECK: [[PREFIX]]/tu.m llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 (Module) Mod_Private +// CHECK: 3:1 (Module) Indirect2 + +//--- cdb.json.template +[{ + "file": "DIR/tu.m", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -include prefix.h -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- cdb_pch.json.template +[{ + "file": "DIR/prefix.h", + "directory": "DIR", + "command": "clang -x objective-c-header DIR/prefix.h -o DIR/prefix.h.pch -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- Mod.framework/Modules/module.modulemap +framework module Mod { header "Mod.h" } + +//--- Mod.framework/Modules/module.private.modulemap +framework module Mod_Private { header "Priv.h" } + +//--- Mod.framework/Headers/Mod.h +void pub(void); + +//--- Mod.framework/PrivateHeaders/Priv.h +void priv(void); + +//--- module.modulemap +module Indirect1 { + header "indirect1.h" + export * +} +module Indirect2 { + header "indirect2.h" + export * +} + +//--- indirect1.h +#import + +//--- indirect2.h +#import "indirect1.h" +#import + +static inline void indirect(void) { + pub(); + priv(); +} + +//--- prefix.h +#import +#import "indirect1.h" + +//--- tu.m +#import +#import "indirect2.h" + +void tu(void) { + pub(); + priv(); + indirect(); +} diff --git a/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c b/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c new file mode 100644 index 0000000000000..d96f6b21233b8 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c @@ -0,0 +1,305 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t/dir1 +// RUN: cp -r %t/dir1 %t/dir2 +// RUN: sed "s|DIR|%/t/dir1|g" %t/dir1/cdb.json.template > %t/cdb1.json +// RUN: sed "s|DIR|%/t/dir2|g" %t/dir1/cdb.json.template > %t/cdb2.json + +// RUN: clang-scan-deps -compilation-database %t/cdb1.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/dir1/outputs \ +// RUN: -prefix-map=%t/dir1/outputs=/^modules -prefix-map=%t/dir1=/^src -prefix-map-sdk=/^sdk -prefix-map-toolchain=/^tc \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// Extract the include-tree commands +// RUN: %deps-to-rsp %t/deps.json --module-name Top > %t/Top.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Left > %t/Left.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp + +// Extract include-tree casids +// RUN: cat %t/Top.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Top.casid +// RUN: cat %t/Left.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Left.casid +// RUN: cat %t/Right.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Right.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE Top" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Top.casid >> %t/result.txt +// RUN: echo "MODULE Left" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Left.casid >> %t/result.txt +// RUN: echo "MODULE Right" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Right.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t -check-prefix=NO_PATHS +// NO_PATHS-NOT: [[PREFIX]] + +// RUN: cat %t/deps.json >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// CHECK-LABEL: MODULE Top +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 /^src/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Module Map: +// CHECK: Top +// CHECK: export * +// CHECK: Files: +// CHECK-NOT: module.modulemap +// CHECK: /^src/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: module.modulemap + +// CHECK-LABEL: MODULE Left +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 /^src/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Top +// CHECK: Module Map: +// CHECK: Left +// CHECK: export * +// CHECK: Files: +// CHECK-NOT: module.modulemap +// CHECK: /^src/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: module.modulemap +// CHECK: /^src/Top.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: MODULE Right +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 /^src/Right.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Top +// CHECK: Module Map: +// CHECK: Right +// CHECK: export * +// CHECK: Files: +// CHECK-NOT: module.modulemap +// CHECK: /^src/Right.h llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: module.modulemap +// CHECK: /^src/Top.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: /^src/tu.m llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Left +// CHECK: 3:1 (Module) Right + +// Note: the modules with explicit imports are imported via parser and are not +// recorded in the include-tree; it's handled entirely by fmodule-map-file, +// fmodule-file, and fmodule-file-cache-key options. + +// CHECK-NOT: Module Map +// CHECK: Files: +// CHECK-NOT: module.modulemap +// CHECK: /^src/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: /^src/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: /^src/Right.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK: { +// CHECK-NEXT "modules": [ +// CHECK-NEXT { +// CHECK: "cas-include-tree-id": "[[LEFT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK: { +// CHECK: "module-name": "Top" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/dir1/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/dir1/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[LEFT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file=Top=/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Left" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/dir1/Left.h" +// CHECK-NEXT: "[[PREFIX]]/dir1/module.modulemap" +// CHECK: ] +// CHECK: "name": "Left" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Top" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/dir1/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/dir1/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[RIGHT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file-cache-key +// CHECK-NEXT: "/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file=Top=/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Right" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/dir1/Right.h" +// CHECK-NEXT: "[[PREFIX]]/dir1/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Right" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[TOP_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/dir1/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/dir1/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[TOP_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Top" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/dir1/Top.h" +// CHECK-NEXT: "[[PREFIX]]/dir1/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Top" +// CHECK: } +// CHECK: ] +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "commands": [ +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[TU_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Left" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "module-name": "Right" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file +// CHECK: "-disable-free" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[TU_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-fsyntax-only" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "/^modules/{{.*}}/Left-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "/^modules/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file=Left=/^modules/{{.*}}/Left-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=/^modules/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/dir1/tu.m" +// CHECK-NEXT: ] +// CHECK: "input-file": "[[PREFIX]]/dir1/tu.m" +// CHECK: } +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } + +// Build the include-tree commands +// RUN: %clang @%t/Top.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: %clang @%t/Left.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: %clang @%t/Right.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: %clang @%t/tu.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS + +// Scan in a different directory +// RUN: clang-scan-deps -compilation-database %t/cdb2.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/dir2/outputs \ +// RUN: -prefix-map=%t/dir2/outputs=/^modules -prefix-map=%t/dir2=/^src -prefix-map-sdk=/^sdk -prefix-map-toolchain=/^tc \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps2.json + +// RUN: %deps-to-rsp %t/deps2.json --module-name Top > %t/Top2.rsp +// RUN: %deps-to-rsp %t/deps2.json --module-name Left > %t/Left2.rsp +// RUN: %deps-to-rsp %t/deps2.json --module-name Right > %t/Right2.rsp +// RUN: %deps-to-rsp %t/deps2.json --tu-index 0 > %t/tu2.rsp + +// Check cache hits +// RUN: %clang @%t/Top2.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/Left2.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/Right2.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/tu2.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT + +// CACHE_MISS: compile job cache miss +// CACHE_HIT: compile job cache hit + +//--- cdb.json.template +[{ + "file": "DIR/tu.m", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" +}] + +//--- module.modulemap +module Top { header "Top.h" export *} +module Left { header "Left.h" export *} +module Right { header "Right.h" export *} + +//--- Top.h +#pragma once +struct Top { + int x; +}; +void top(void); + +//--- Left.h +#include "Top.h" +void left(void); + +//--- Right.h +#include "Top.h" +void right(void); + +//--- tu.m +#import "Left.h" +#import + +void tu(void) { + top(); + left(); + right(); +} diff --git a/clang/test/ClangScanDeps/modules-include-tree-submodules.c b/clang/test/ClangScanDeps/modules-include-tree-submodules.c new file mode 100644 index 0000000000000..91b92443258f3 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-submodules.c @@ -0,0 +1,147 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// Extract the include-tree commands +// RUN: %deps-to-rsp %t/deps.json --module-name TwoSubs > %t/TwoSubs.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name WithExplicit > %t/WithExplicit.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu1.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 1 > %t/tu2.rsp + +// Extract include-tree casids +// RUN: cat %t/TwoSubs.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/TwoSubs.casid +// RUN: cat %t/WithExplicit.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/WithExplicit.casid +// RUN: cat %t/tu1.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu1.casid +// RUN: cat %t/tu2.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu2.casid + +// RUN: echo "MODULE TwoSubs" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/TwoSubs.casid >> %t/result.txt +// RUN: echo "MODULE WithExplicit" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/WithExplicit.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT 1" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu1.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT 2" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu2.casid >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// Build the include-tree commands +// RUN: %clang @%t/TwoSubs.rsp +// RUN: %clang @%t/WithExplicit.rsp +// RUN: not %clang @%t/tu1.rsp 2>&1 | FileCheck %s -check-prefix=TU1 +// RUN: not %clang @%t/tu2.rsp 2>&1 | FileCheck %s -check-prefix=TU2 + +// CHECK: MODULE TwoSubs +// CHECK: 2:1 [[PREFIX]]/Sub1.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: TwoSubs.Sub1 +// CHECK: 3:1 [[PREFIX]]/Sub2.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: TwoSubs.Sub2 + +// CHECK: MODULE WithExplicit +// CHECK: 2:1 [[PREFIX]]/TopLevel.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: WithExplicit +// CHECK: 3:1 [[PREFIX]]/Implicit.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: WithExplicit.Implicit +// CHECK: 4:1 [[PREFIX]]/Explicit.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: WithExplicit.Explicit + +// CHECK: TRANSLATION UNIT 1 +// CHECK: [[PREFIX]]/tu1.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) TwoSubs.Sub1 +// CHECK: 3:1 (Module) WithExplicit + +// CHECK: TRANSLATION UNIT 2 +// CHECK: [[PREFIX]]/tu2.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) TwoSubs.Sub2 +// CHECK: 3:1 (Module) WithExplicit.Explicit + +//--- cdb.json.template +[ +{ + "file": "DIR/tu1.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu1.c -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +{ + "file": "DIR/tu2.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu2.c -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +] + +//--- module.modulemap +module TwoSubs { + module Sub1 { header "Sub1.h" } + module Sub2 { header "Sub2.h" } +} + +module WithExplicit { + header "TopLevel.h" + module Implicit { header "Implicit.h" } + explicit module Explicit { header "Explicit.h" } +} + +//--- Sub1.h +void sub1(void); + +//--- Sub2.h +void sub2(void); + +//--- TopLevel.h +void top(void); + +//--- Implicit.h +void implicit(void); + +//--- Explicit.h +void explicit(void); + +//--- tu1.c +#include "Sub1.h" +#include "TopLevel.h" + +void tu1(void) { + top(); + sub1(); + implicit(); + sub2(); +// TU1-NOT: error: +// TU1: error: call to undeclared function 'sub2' +// TU1: error: missing '#include "Sub2.h"' +// TU1: note: declaration here is not visible + explicit(); +// TU1: error: call to undeclared function 'explicit' +// TU1: error: missing '#include "Explicit.h"' +// TU1: Explicit.h:1:6: note: declaration here is not visible +} + +//--- tu2.c +#include "Sub2.h" +#include "Explicit.h" + +void tu2(void) { + sub2(); + explicit(); +// TU2-NOT: error: + top(); +// TU2: error: call to undeclared function 'top' +// TU2: error: missing '#include "TopLevel.h"' +// TU2: note: declaration here is not visible + sub1(); +// TU2: error: call to undeclared function 'sub1' +// TU2: error: missing '#include "Sub1.h"' +// TU2: note: declaration here is not visible + implicit(); +// TU2: error: call to undeclared function 'implicit' +// TU2: error: missing '#include "Implicit.h"' +// TU2: note: declaration here is not visible +} diff --git a/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c b/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c new file mode 100644 index 0000000000000..59589ed535a58 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c @@ -0,0 +1,98 @@ +// Check include-tree-based caching works with vfsoverlay files. + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed -e "s|DIR|%/t|g" %t/vfs.yaml.template > %t/vfs.yaml + +// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: -cas-path %t/cas > %t/deps.json + +// RUN: %deps-to-rsp %t/deps.json --module-name=A > %t/A.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/A.rsp +// RUN: %clang @%t/tu.rsp + +// Extract include-tree casids +// RUN: cat %t/A.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/A.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE A" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/A.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// CHECK-LABEL: MODULE A +// CHECK: llvmcas:// +// CHECK: 2:1 [[PREFIX]]/elsewhere2/A.h llvmcas:// +// CHECK: Submodule: A +// CHECK: Module Map: +// CHECK: A (framework) +// CHECK: Files: +// CHECK-NOT: modulemap +// CHECK: [[PREFIX]]/elsewhere2/A.h llvmcas:// +// CHECK-NOT: modulemap + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: Files: +// CHECK-NOT: .modulemap +// CHECK-NOT: .yaml +// CHECK: [[PREFIX]]/elsewhere2/A.h llvmcas:// +// CHECK-NOT: .modulemap +// CHECK-NOT: .yaml + +//--- cdb.json.template +[{ + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -ivfsoverlay DIR/vfs.yaml -F DIR", + "file": "DIR/tu.c" +}] + +//--- vfs.yaml.template +{ + "version": 0, + "case-sensitive": "false", + "roots": [ + { + "name": "DIR/A.framework", + "type": "directory" + "contents": [ + { + "name": "Modules", + "type": "directory" + "contents": [ + { + "external-contents": "DIR/elsewhere1/A.modulemap", + "name": "module.modulemap", + "type": "file" + } + ] + }, + { + "name": "Headers", + "type": "directory" + "contents": [ + { + "external-contents": "DIR/elsewhere2/A.h", + "name": "A.h", + "type": "file" + } + ] + } + ] + } + ] +} + +//--- elsewhere1/A.modulemap +framework module A { header "A.h" } + +//--- elsewhere2/A.h +typedef int A_t; + +//--- tu.c +#include "A/A.h" +A_t a = 0; diff --git a/clang/test/ClangScanDeps/modules-include-tree-with-pch.c b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c new file mode 100644 index 0000000000000..60b2ba3daf851 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c @@ -0,0 +1,156 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json + +// Scan PCH +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch.json + +// Build PCH +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Top > %t/Top.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Left > %t/Left.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/Top.rsp +// RUN: %clang @%t/Left.rsp +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/pch.rsp +// RUN: rm -rf %t/outputs + +// Scan TU with PCH +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// Build TU +// RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/Right.rsp +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/tu.rsp + +// RUN: FileCheck %s -input-file %t/deps.json -DPREFIX=%/t + +// CHECK: { +// CHECK-NEXT: "modules": [ +// CHECK-NEXT: { +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Right" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Right.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Right" +// CHECK: } +// CHECK-NOT: "clang-modulemap-file" +// CHECK: ] +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "commands": [ +// CHECK-NEXT: { +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Right" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file= +// CHECK: "-disable-free" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fcache-compile-job" +// CHECK: "-fsyntax-only" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodule-file=Right=[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/tu.c" +// CHECK-NEXT: "[[PREFIX]]/prefix.h.pch" +// CHECK-NEXT: ] +// CHECK: "input-file": "[[PREFIX]]/tu.c" +// CHECK: } +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } + +//--- cdb_pch.json.template +[{ + "file": "DIR/prefix.h", + "directory": "DIR", + "command": "clang -x c-header DIR/prefix.h -o DIR/prefix.h.pch -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- cdb.json.template +[{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -include DIR/prefix.h -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- module.modulemap +module Top { header "Top.h" export *} +module Left { header "Left.h" export *} +module Right { header "Right.h" export *} + +//--- Top.h +#pragma once +struct Top { int x; }; + +//--- Left.h +#pragma once +#include "Top.h" +struct Left { struct Top top; }; + +//--- Right.h +#pragma once +#include "Top.h" +struct Right { struct Top top; }; + +//--- prefix.h +#include "Left.h" + +//--- tu.c +#include "Right.h" + +void tu(void) { + struct Left _left; + struct Right _right; + struct Top _top; +} diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c new file mode 100644 index 0000000000000..cb42ded1fc728 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -0,0 +1,368 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// Extract the include-tree commands +// RUN: %deps-to-rsp %t/deps.json --module-name Top > %t/Top.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Left > %t/Left.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ZAtImport > %t/ZAtImport.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ZPragmaImport > %t/ZPragmaImport.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp + +// Extract include-tree casids +// RUN: cat %t/Top.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Top.casid +// RUN: cat %t/Left.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Left.casid +// RUN: cat %t/Right.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Right.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE Top" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Top.casid >> %t/result.txt +// RUN: echo "MODULE Left" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Left.casid >> %t/result.txt +// RUN: echo "MODULE Right" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Right.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt +// RUN: cat %t/deps.json >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// CHECK-LABEL: MODULE Top +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Module Map: +// CHECK: Top +// CHECK: export * +// CHECK: Files: +// CHECK-NOT: [[PREFIX]]/module.modulemap +// CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap + +// CHECK-LABEL: MODULE Left +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Top +// CHECK: Module Map: +// CHECK: Left +// CHECK: export * +// CHECK: Files: +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: MODULE Right +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Top +// CHECK: Module Map: +// CHECK: Right +// CHECK: export * +// CHECK: Files: +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: [[PREFIX]]/tu.m llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Left +// CHECK: 3:1 (Module) Right + +// Note: the modules with explicit imports are imported via parser and are not +// recorded in the include-tree; it's handled entirely by fmodule-map-file, +// fmodule-file, and fmodule-file-cache-key options. + +// CHECK-NOT: Module Map +// CHECK: Files: +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK: { +// CHECK-NEXT "modules": [ +// CHECK-NEXT { +// CHECK: "cas-include-tree-id": "[[LEFT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK: { +// CHECK: "module-name": "Top" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[LEFT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Left" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Left.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK: ] +// CHECK: "name": "Left" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Top" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[RIGHT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file-cache-key +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Right" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Right.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Right" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[TOP_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[TOP_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Top" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Top.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Top" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[AT_IMPORT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/ZAtImport-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[AT_IMPORT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=ZAtImport" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/AtImport.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "ZAtImport" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[PRAGMA_IMPORT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/ZPragmaImport-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[PRAGMA_IMPORT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=ZPragmaImport" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/PragmaImport.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "ZPragmaImport" +// CHECK: } +// CHECK: ] +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "commands": [ +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[TU_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Left" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "module-name": "Right" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "module-name": "ZAtImport" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "module-name": "ZPragmaImport" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file +// CHECK: "-disable-free" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[TU_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-fsyntax-only" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file=Left=[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/tu.m" +// CHECK-NEXT: ] +// CHECK: "input-file": "[[PREFIX]]/tu.m" +// CHECK: } +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } + +// Build the include-tree commands +// RUN: %clang @%t/Top.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// Ensure the pcm comes from the action cache +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/Left.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/Right.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/ZAtImport.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/ZPragmaImport.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/tu.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS + +// Check cache hits +// RUN: %clang @%t/Top.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/Left.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/Right.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/ZAtImport.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/ZPragmaImport.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/tu.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT + +// CACHE_MISS: compile job cache miss +// CACHE_HIT: compile job cache hit + +//--- cdb.json.template +[{ + "file": "DIR/tu.m", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" +}] + +//--- module.modulemap +module Top { header "Top.h" export *} +module Left { header "Left.h" export *} +module Right { header "Right.h" export *} +module ZAtImport { header "AtImport.h" } +module ZPragmaImport { header "PragmaImport.h" } + +//--- Top.h +#pragma once +struct Top { + int x; +}; +void top(void); + +//--- Left.h +#include "Top.h" +void left(void); + +//--- Right.h +#include "Top.h" +void right(void); + +//--- AtImport.h +void at_import(void); + +//--- PragmaImport.h +void pragma_import(void); + +//--- tu.m +#import "Left.h" +#import +@import ZAtImport; +#pragma clang module import ZPragmaImport + +void tu(void) { + top(); + left(); + right(); + at_import(); + pragma_import(); +} diff --git a/clang/test/ClangScanDeps/removed-args.c b/clang/test/ClangScanDeps/removed-args.c index 02fbb8cacff2c..9a4ef25838e46 100644 --- a/clang/test/ClangScanDeps/removed-args.c +++ b/clang/test/ClangScanDeps/removed-args.c @@ -23,6 +23,8 @@ // CHECK-NEXT: "-cc1" // CHECK-NOT: "-fdebug-compilation-dir=" // CHECK-NOT: "-fcoverage-compilation-dir=" +// CHECK-NOT: "-coverage-notes-file +// CHECK-NOT: "-coverage-data-file // CHECK-NOT: "-dwarf-debug-flags" // CHECK-NOT: "-main-file-name" // CHECK-NOT: "-include" @@ -46,6 +48,8 @@ // CHECK-NEXT: "-cc1" // CHECK-NOT: "-fdebug-compilation-dir= // CHECK-NOT: "-fcoverage-compilation-dir= +// CHECK-NOT: "-coverage-notes-file +// CHECK-NOT: "-coverage-data-file // CHECK-NOT: "-dwarf-debug-flags" // CHECK-NOT: "-main-file-name" // CHECK-NOT: "-include" diff --git a/clang/test/Driver/fdepscan-prefix-map-sdk.c b/clang/test/Driver/fdepscan-prefix-map-sdk.c new file mode 100644 index 0000000000000..421148c1a1a6d --- /dev/null +++ b/clang/test/Driver/fdepscan-prefix-map-sdk.c @@ -0,0 +1,8 @@ +// RUN: %clang -fdepscan-prefix-map-sdk=/^sdk -### %s 2>&1 | FileCheck %s -check-prefix=NONE +// RUN: %clang -fdepscan-prefix-map-sdk=/^sdk -isysroot relative -### %s 2>&1 | FileCheck %s -check-prefix=NONE + +// NONE-NOT: -fdepscan-prefix-map + +// RUN: %clang -fdepscan-prefix-map-sdk=/^sdk -isysroot /sys/path -### %s 2>&1 | FileCheck %s +// RUN: %clang -fdepscan-prefix-map-sdk=/^sdk --sysroot /sys/path -### %s 2>&1 | FileCheck %s +// CHECK: -fdepscan-prefix-map=/sys/path=/^sdk diff --git a/clang/test/Driver/fdepscan-prefix-map-toolchain.c b/clang/test/Driver/fdepscan-prefix-map-toolchain.c new file mode 100644 index 0000000000000..1bdbdfe1ccd78 --- /dev/null +++ b/clang/test/Driver/fdepscan-prefix-map-toolchain.c @@ -0,0 +1,15 @@ +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir '' -### %s 2>&1 | FileCheck %s -check-prefix=NONE +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir relative -### %s 2>&1 | FileCheck %s -check-prefix=NONE +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir /lib/clang/10 -### %s 2>&1 | FileCheck %s -check-prefix=NONE + +// NONE-NOT: -fdepscan-prefix-map + +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir /tc/10 -### %s 2>&1 | FileCheck %s +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir /tc/lib/clang/10 -### %s 2>&1 | FileCheck %s +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir /tc/usr/lib/clang/10 -### %s 2>&1 | FileCheck %s + +// CHECK: -fdepscan-prefix-map=/tc=/^tc + +// Implicit resource-dir +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -### %s 2>&1 | FileCheck %s -check-prefix=CHECK_IMPLICIT +// CHECK_IMPLICIT: -fdepscan-prefix-map={{.*}}=/^tc diff --git a/clang/test/Driver/fdepscan-prefix-map.c b/clang/test/Driver/fdepscan-prefix-map.c new file mode 100644 index 0000000000000..4c5c34bb98bfd --- /dev/null +++ b/clang/test/Driver/fdepscan-prefix-map.c @@ -0,0 +1,8 @@ +// RUN: %clang -fdepscan-prefix-map=/^bad -### %s 2>&1 | FileCheck %s -check-prefix=INVALID +// RUN: %clang -fdepscan-prefix-map==/^bad -### %s 2>&1 | FileCheck %s -check-prefix=INVALID +// RUN: %clang -fdepscan-prefix-map=relative=/^bad -### %s 2>&1 | FileCheck %s -check-prefix=INVALID +// RUN: %clang -fdepscan-prefix-map=/=/^bad -### %s 2>&1 | FileCheck %s -check-prefix=INVALID +// INVALID: error: invalid argument '{{.*}}/^bad' to -fdepscan-prefix-map= + +// RUN: %clang -fdepscan-prefix-map=/good=/^good -### %s 2>&1 | FileCheck %s +// CHECK: "-fdepscan-prefix-map=/good=/^good" diff --git a/clang/test/Index/Core/scan-deps-cas.m b/clang/test/Index/Core/scan-deps-cas.m index 54b13da1e025f..afeba129d9ecc 100644 --- a/clang/test/Index/Core/scan-deps-cas.m +++ b/clang/test/Index/Core/scan-deps-cas.m @@ -1,26 +1,26 @@ -// RUN: rm -rf %t.mcp %t +// RUN: rm -rf %t // RUN: c-index-test core --scan-deps %S -output-dir=%t -cas-path %t/cas \ // RUN: -- %clang -c -I %S/Inputs/module \ -// RUN: -fmodules -fmodules-cache-path=%t.mcp \ +// RUN: -fmodules -fmodules-cache-path=%t/mcpit \ // RUN: -o FoE.o -x objective-c %s > %t.result -// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t +// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t -check-prefix=INCLUDE_TREE // RUN: env CLANG_CACHE_USE_CASFS_DEPSCAN=1 c-index-test core --scan-deps %S -output-dir=%t -cas-path %t/cas \ // RUN: -- %clang -c -I %S/Inputs/module \ -// RUN: -fmodules -fmodules-cache-path=%t.mcp \ +// RUN: -fmodules -fmodules-cache-path=%t/mcp \ // RUN: -o FoE.o -x objective-c %s > %t.casfs.result // RUN: cat %t.casfs.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t -// FIXME: enable modules when supported. // RUN: env CLANG_CACHE_USE_INCLUDE_TREE=1 c-index-test core --scan-deps %S -output-dir=%t -cas-path %t/cas \ // RUN: -- %clang -c -I %S/Inputs/module \ +// RUN: -fmodules -fmodules-cache-path=%t/mcpit \ // RUN: -o FoE.o -x objective-c %s > %t.includetree.result // RUN: cat %t.includetree.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t -check-prefix=INCLUDE_TREE // RUN: c-index-test core --scan-deps %S -output-dir=%t \ // RUN: -- %clang -c -I %S/Inputs/module \ -// RUN: -fmodules -fmodules-cache-path=%t.mcp \ +// RUN: -fmodules -fmodules-cache-path=%t/mcp \ // RUN: -o FoE.o -x objective-c %s | FileCheck %s -check-prefix=NO_CAS // NO_CAS-NOT: fcas // NO_CAS-NOT: faction-cache @@ -33,6 +33,7 @@ // CHECK-NEXT: name: ModA // CHECK-NEXT: context-hash: [[HASH_MOD_A:[A-Z0-9]+]] // CHECK-NEXT: module-map-path: [[PREFIX]]/Inputs/module/module.modulemap +// CHECK-NEXT: cache-key: [[CASFS_MODA_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // CHECK-NEXT: module-deps: // CHECK-NEXT: file-deps: // CHECK-NEXT: [[PREFIX]]/Inputs/module/ModA.h @@ -51,6 +52,7 @@ // CHECK-NEXT: dependencies: // CHECK-NEXT: command 0: // CHECK-NEXT: context-hash: [[HASH_TU:[A-Z0-9]+]] +// CHECK-NEXT: cache-key: [[CASFS_TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // CHECK-NEXT: module-deps: // CHECK-NEXT: ModA:[[HASH_MOD_A]] // CHECK-NEXT: file-deps: @@ -63,15 +65,37 @@ // CHECK-SAME: -fmodule-file-cache-key [[PCM:.*ModA_.*pcm]] llvmcas://{{[[:xdigit:]]+}} // CHECK-SAME: -fmodule-file={{(ModA=)?}}[[PCM]] + +// INCLUDE_TREE: modules: +// INCLUDE_TREE-NEXT: module: +// INCLUDE_TREE-NEXT: name: ModA +// INCLUDE_TREE-NEXT: context-hash: [[HASH_MOD_A:[A-Z0-9]+]] +// INCLUDE_TREE-NEXT: module-map-path: [[PREFIX]]/Inputs/module/module.modulemap +// INCLUDE_TREE-NEXT: cache-key: [[ModA_CACHE_KEY:llvmcas://[[:xdigit:]]+]] +// INCLUDE_TREE-NEXT: module-deps: +// INCLUDE_TREE-NEXT: file-deps: +// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/ModA.h +// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/SubModA.h +// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/SubSubModA.h +// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/module.modulemap +// INCLUDE_TREE-NEXT: build-args: +// INCLUDE_TREE-SAME: -cc1 +// INCLUDE_TREE-SAME: -fcas-path +// INCLUDE_TREE-SAME: -fcas-include-tree llvmcas://{{[[:xdigit:]]+}} +// INCLUDE_TREE-SAME: -fcache-compile-job + // INCLUDE_TREE: dependencies: // INCLUDE_TREE-NEXT: command 0: // INCLUDE_TREE-NEXT: context-hash: [[HASH_TU:[A-Z0-9]+]] +// INCLUDE_TREE-NEXT: cache-key: [[INC_TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // INCLUDE_TREE-NEXT: module-deps: +// INCLUDE_TREE-NEXT: ModA:[[HASH_MOD_A]] // INCLUDE_TREE-NEXT: file-deps: // INCLUDE_TREE-NEXT: [[PREFIX]]/scan-deps-cas.m -// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/ModA.h // INCLUDE_TREE-NEXT: build-args: // INCLUDE_TREE-SAME: -cc1 // INCLUDE_TREE-SAME: -fcas-path // INCLUDE_TREE-SAME: -fcas-include-tree llvmcas://{{[[:xdigit:]]+}} // INCLUDE_TREE-SAME: -fcache-compile-job +// INCLUDE_TREE-SAME: -fmodule-file-cache-key [[PCM:.*ModA_.*pcm]] [[ModA_CACHE_KEY]] +// INCLUDE_TREE-SAME: -fmodule-file={{(ModA=)?}}[[PCM]] \ No newline at end of file diff --git a/clang/test/Index/Core/scan-deps-with-diags.m b/clang/test/Index/Core/scan-deps-with-diags.m new file mode 100644 index 0000000000000..a47d3646385ff --- /dev/null +++ b/clang/test/Index/Core/scan-deps-with-diags.m @@ -0,0 +1,6 @@ +// RUN: not c-index-test core --scan-deps %S -output-dir=%t -- \ +// RUN: %clang -c %s -o %t/t.o 2> %t.err.txt +// RUN: FileCheck -input-file=%t.err.txt %s + +// CHECK: [[@LINE+1]]:10: fatal error: 'not-existent.h' file not found +#include "not-existent.h" diff --git a/clang/test/Modules/implicit-private-with-submodule-explicit.m b/clang/test/Modules/implicit-private-with-submodule-explicit.m new file mode 100644 index 0000000000000..a2e9950ec3181 --- /dev/null +++ b/clang/test/Modules/implicit-private-with-submodule-explicit.m @@ -0,0 +1,31 @@ +// Checks that the use of .Private to refer to _Private modules works with an +// explicit module. + +// RUN: rm -rf %t +// RUN: split-file %s %t + +// RUN: %clang_cc1 -x objective-c -fmodules -fno-implicit-modules -emit-module -fmodule-name=A %t/module.modulemap -o %t/A.pcm +// RUN: %clang_cc1 -x objective-c -fmodules -fno-implicit-modules -emit-module -fmodule-name=A_Private %t/module.modulemap -o %t/A_Private.pcm + +// Check lazily-loaded module +// RUN: %clang_cc1 -x objective-c -verify -fmodules -fno-implicit-modules -fmodule-file=A=%t/A.pcm -fmodule-file=A_Private=%t/A_Private.pcm -fsyntax-only %t/tu.m + +// Check eagerly-loaded module +// RUN: %clang_cc1 -x objective-c -verify -fmodules -fno-implicit-modules -fmodule-file=%t/A.pcm -fmodule-file=%t/A_Private.pcm -fsyntax-only %t/tu.m + +//--- module.modulemap +module A { header "a.h" } +module A_Private { header "priv.h" } + +//--- a.h + +//--- priv.h +void priv(void); + +//--- tu.m +@import A.Private; // expected-warning{{no submodule named 'Private' in module 'A'; using top level 'A_Private'}} +// expected-note@*:* {{defined here}} + +void tu(void) { + priv(); +} \ No newline at end of file diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index 2fa66355c8daf..d9aa02fce3228 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -719,35 +719,6 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, clang_experimental_DependencyScannerService_dispose_v0(Service); }); - auto Callback = [&](CXModuleDependencySet *MDS) { - llvm::outs() << "modules:\n"; - for (const auto &M : llvm::makeArrayRef(MDS->Modules, MDS->Count)) { - llvm::outs() << " module:\n" - << " name: " << clang_getCString(M.Name) << "\n" - << " context-hash: " << clang_getCString(M.ContextHash) - << "\n" - << " module-map-path: " - << clang_getCString(M.ModuleMapPath) << "\n" - << " module-deps:\n"; - for (const auto &ModuleName : - llvm::makeArrayRef(M.ModuleDeps->Strings, M.ModuleDeps->Count)) - llvm::outs() << " " << clang_getCString(ModuleName) << "\n"; - llvm::outs() << " file-deps:\n"; - for (const auto &FileName : - llvm::makeArrayRef(M.FileDeps->Strings, M.FileDeps->Count)) - llvm::outs() << " " << clang_getCString(FileName) << "\n"; - llvm::outs() << " build-args:"; - for (const auto &Arg : llvm::makeArrayRef(M.BuildArguments->Strings, - M.BuildArguments->Count)) - llvm::outs() << " " << clang_getCString(Arg); - llvm::outs() << "\n"; - } - clang_experimental_ModuleDependencySet_dispose(MDS); - }; - - auto CB = - functionObjectToCCallbackRef(Callback); - auto LookupOutput = [&](const char *ModuleName, const char *ContextHash, CXOutputKind Kind, char *Output, size_t MaxLen) { std::string Out = OutputPath + "/" + ModuleName + "_" + ContextHash; @@ -781,49 +752,111 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, char *Output, size_t MaxLen)>(LookupOutput); unsigned CommandIndex = 0; - auto HandleCommand = [&](CXString ContextHash, CXStringSet *ModuleDeps, - CXStringSet *FileDeps, CXStringSet *Args) { + auto HandleCommand = [&](const char *ContextHash, CXCStringArray ModuleDeps, + CXCStringArray FileDeps, CXCStringArray Args, + const char *CacheKey) { llvm::outs() << " command " << CommandIndex++ << ":\n"; - llvm::outs() << " context-hash: " << clang_getCString(ContextHash) - << "\n" - << " module-deps:\n"; + llvm::outs() << " context-hash: " << ContextHash << "\n"; + if (CacheKey) + llvm::outs() << " cache-key: " << CacheKey << "\n"; + llvm::outs() << " module-deps:\n"; for (const auto &ModuleName : - llvm::makeArrayRef(ModuleDeps->Strings, ModuleDeps->Count)) - llvm::outs() << " " << clang_getCString(ModuleName) << "\n"; + llvm::makeArrayRef(ModuleDeps.Strings, ModuleDeps.Count)) + llvm::outs() << " " << ModuleName << "\n"; llvm::outs() << " file-deps:\n"; - for (const auto &FileName : - llvm::makeArrayRef(FileDeps->Strings, FileDeps->Count)) - llvm::outs() << " " << clang_getCString(FileName) << "\n"; + for (const auto &FileName : ArrayRef(FileDeps.Strings, FileDeps.Count)) + llvm::outs() << " " << FileName << "\n"; llvm::outs() << " build-args:"; - for (const auto &Arg : llvm::makeArrayRef(Args->Strings, Args->Count)) - llvm::outs() << " " << clang_getCString(Arg); + for (const auto &Arg : ArrayRef(Args.Strings, Args.Count)) + llvm::outs() << " " << Arg; llvm::outs() << "\n"; }; - CXFileDependenciesList *Result = nullptr; - CXDiagnosticSet Diags; - auto DisposeDiagnosticSet = - llvm::make_scope_exit([&]() { clang_disposeDiagnosticSet(Diags); }); - CXErrorCode Err = - clang_experimental_DependencyScannerWorker_getFileDependencies_v5( - Worker, Args.size(), Args.data(), - ModuleName ? ModuleName->c_str() : nullptr, WorkingDirectory.c_str(), - CB.Context, CB.Callback, LookupOutputCB.Context, - LookupOutputCB.Callback, - /*Options=*/0, &Result, &Diags); + CXDependencyScannerWorkerScanSettings ScanSettings = + clang_experimental_DependencyScannerWorkerScanSettings_create( + Args.size(), Args.data(), ModuleName ? ModuleName->c_str() : nullptr, + WorkingDirectory.c_str(), LookupOutputCB.Context, + LookupOutputCB.Callback); + auto DisposeScanSettings = llvm::make_scope_exit([&]() { + clang_experimental_DependencyScannerWorkerScanSettings_dispose( + ScanSettings); + }); + CXDepGraph Graph = nullptr; + auto DisposeDepGraph = llvm::make_scope_exit( + [&]() { clang_experimental_DepGraph_dispose(Graph); }); + CXErrorCode Err = clang_experimental_DependencyScannerWorker_getDepGraph( + Worker, ScanSettings, &Graph); + if (Err == CXError_Success) { + llvm::outs() << "modules:\n"; + for (size_t I = 0, E = clang_experimental_DepGraph_getNumModules(Graph); + I < E; ++I) { + CXDepGraphModule Mod = clang_experimental_DepGraph_getModule(Graph, I); + const char *Name = clang_experimental_DepGraphModule_getName(Mod); + const char *ContextHash = + clang_experimental_DepGraphModule_getContextHash(Mod); + const char *ModuleMapPath = + clang_experimental_DepGraphModule_getModuleMapPath(Mod); + const char *ModuleCacheKey = + clang_experimental_DepGraphModule_getCacheKey(Mod); + CXCStringArray ModuleDeps = + clang_experimental_DepGraphModule_getModuleDeps(Mod); + CXCStringArray FileDeps = + clang_experimental_DepGraphModule_getFileDeps(Mod); + CXCStringArray BuildArguments = + clang_experimental_DepGraphModule_getBuildArguments(Mod); + auto Dispose = llvm::make_scope_exit( + [&]() { clang_experimental_DepGraphModule_dispose(Mod); }); + llvm::outs() << " module:\n" + << " name: " << Name << "\n" + << " context-hash: " << ContextHash << "\n" + << " module-map-path: " + << (ModuleMapPath ? ModuleMapPath : "") << "\n"; + if (ModuleCacheKey) + llvm::outs() << " cache-key: " << ModuleCacheKey << "\n"; + llvm::outs() << " module-deps:\n"; + for (const auto &ModuleName : + ArrayRef(ModuleDeps.Strings, ModuleDeps.Count)) + llvm::outs() << " " << ModuleName << "\n"; + llvm::outs() << " file-deps:\n"; + for (const auto &FileName : ArrayRef(FileDeps.Strings, FileDeps.Count)) + llvm::outs() << " " << FileName << "\n"; + llvm::outs() << " build-args:"; + for (const auto &Arg : + ArrayRef(BuildArguments.Strings, BuildArguments.Count)) + llvm::outs() << " " << Arg; + llvm::outs() << "\n"; + } + llvm::outs() << "dependencies:\n"; - for (size_t I = 0; I < Result->NumCommands; ++I) - HandleCommand( - Result->Commands[I].ContextHash, Result->Commands[I].ModuleDeps, - Result->Commands[I].FileDeps, Result->Commands[I].BuildArguments); - clang_experimental_FileDependenciesList_dispose(Result); + const char *TUContextHash = + clang_experimental_DepGraph_getTUContextHash(Graph); + CXCStringArray TUModuleDeps = + clang_experimental_DepGraph_getTUModuleDeps(Graph); + CXCStringArray TUFileDeps = + clang_experimental_DepGraph_getTUFileDeps(Graph); + for (size_t I = 0, E = clang_experimental_DepGraph_getNumTUCommands(Graph); + I < E; ++I) { + CXDepGraphTUCommand Cmd = + clang_experimental_DepGraph_getTUCommand(Graph, I); + CXCStringArray Args = + clang_experimental_DepGraphTUCommand_getBuildArguments(Cmd); + const char *CacheKey = + clang_experimental_DepGraphTUCommand_getCacheKey(Cmd); + auto Dispose = llvm::make_scope_exit( + [&]() { clang_experimental_DepGraphTUCommand_dispose(Cmd); }); + HandleCommand(TUContextHash, TUModuleDeps, TUFileDeps, Args, CacheKey); + } return 0; } llvm::errs() << "error: failed to get dependencies\n"; + CXDiagnosticSet Diags = clang_experimental_DepGraph_getDiagnostics(Graph); + auto DisposeDiagnosticSet = + llvm::make_scope_exit([&]() { clang_disposeDiagnosticSet(Diags); }); for (unsigned I = 0, N = clang_getNumDiagnosticsInSet(Diags); I < N; ++I) { CXDiagnostic Diag = clang_getDiagnosticInSet(Diags, I); - CXString Spelling = clang_getDiagnosticSpelling(Diag); + CXString Spelling = + clang_formatDiagnostic(Diag, clang_defaultDiagnosticDisplayOptions()); llvm::errs() << clang_getCString(Spelling) << "\n"; clang_disposeString(Spelling); clang_disposeDiagnostic(Diag); diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index cfe6d22aef6f1..9858575bbbb20 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -265,10 +265,8 @@ llvm::cl::opt Verbose("v", llvm::cl::Optional, static bool emitCompilationDBWithCASTreeArguments( std::shared_ptr DB, std::vector Inputs, - DiagnosticConsumer &DiagsConsumer, - const DepscanPrefixMapping &PrefixMapping, - DependencyScanningService &Service, llvm::ThreadPool &Pool, - llvm::raw_ostream &OS) { + DiagnosticConsumer &DiagsConsumer, DependencyScanningService &Service, + llvm::ThreadPool &Pool, llvm::raw_ostream &OS) { // Follow `-cc1depscan` and also ignore diagnostics. // FIXME: Seems not a good idea to do this.. @@ -329,7 +327,6 @@ static bool emitCompilationDBWithCASTreeArguments( tooling::dependencies::DependencyScanningTool &WorkerTool; DiagnosticConsumer &DiagsConsumer; StringRef CWD; - const DepscanPrefixMapping &PrefixMapping; SmallVectorImpl &OutputArgs; llvm::StringSaver &Saver; @@ -338,12 +335,10 @@ static bool emitCompilationDBWithCASTreeArguments( llvm::cas::ObjectStore &DB, tooling::dependencies::DependencyScanningTool &WorkerTool, DiagnosticConsumer &DiagsConsumer, StringRef CWD, - const DepscanPrefixMapping &PrefixMapping, SmallVectorImpl &OutputArgs, llvm::StringSaver &Saver) : DB(DB), WorkerTool(WorkerTool), DiagsConsumer(DiagsConsumer), - CWD(CWD), PrefixMapping(PrefixMapping), OutputArgs(OutputArgs), - Saver(Saver) {} + CWD(CWD), OutputArgs(OutputArgs), Saver(Saver) {} bool runInvocation(std::shared_ptr Invocation, @@ -352,7 +347,7 @@ static bool emitCompilationDBWithCASTreeArguments( DiagnosticConsumer *DiagConsumer) override { Expected Root = scanAndUpdateCC1InlineWithTool( WorkerTool, DiagsConsumer, /*VerboseOS*/ nullptr, *Invocation, - CWD, PrefixMapping, DB); + CWD, DB); if (!Root) { llvm::consumeError(Root.takeError()); return false; @@ -369,7 +364,7 @@ static bool emitCompilationDBWithCASTreeArguments( llvm::StringSaver &Saver = PerThreadStates[I]->Saver; OutputArgs.push_back(Saver.save(Input->CommandLine.front()).data()); ScanForCC1Action Action(*DB, WorkerTool, *IgnoringDiagsConsumer, CWD, - PrefixMapping, OutputArgs, Saver); + OutputArgs, Saver); llvm::IntrusiveRefCntPtr FileMgr = WorkerTool.getOrCreateFileManager(); @@ -544,6 +539,7 @@ class FullDeps { ID.FileDeps = std::move(TUDeps.FileDeps); ID.ModuleDeps = std::move(TUDeps.ClangModuleDeps); ID.CASFileSystemRootID = std::move(TUDeps.CASFileSystemRootID); + ID.IncludeTreeID = std::move(TUDeps.IncludeTreeID); ID.DriverCommandLine = std::move(TUDeps.DriverCommandLine); ID.Commands = std::move(TUDeps.Commands); @@ -622,8 +618,12 @@ class FullDeps { {"clang-modulemap-file", MD.ClangModuleMapFile}, {"command-line", MD.BuildArguments}, }; + if (MD.ModuleCacheKey) + O.try_emplace("cache-key", MD.ModuleCacheKey); if (MD.CASFileSystemRootID) O.try_emplace("casfs-root-id", MD.CASFileSystemRootID->toString()); + if (MD.IncludeTreeID) + O.try_emplace("cas-include-tree-id", MD.IncludeTreeID); OutModules.push_back(std::move(O)); } @@ -640,8 +640,12 @@ class FullDeps { {"executable", Cmd.Executable}, {"command-line", Cmd.Arguments}, }; + if (Cmd.TUCacheKey) + O.try_emplace("cache-key", *Cmd.TUCacheKey); if (I.CASFileSystemRootID) O.try_emplace("casfs-root-id", I.CASFileSystemRootID); + if (I.IncludeTreeID) + O.try_emplace("cas-include-tree-id", I.IncludeTreeID); Commands.push_back(std::move(O)); } } else { @@ -655,6 +659,8 @@ class FullDeps { }; if (I.CASFileSystemRootID) O.try_emplace("casfs-root-id", I.CASFileSystemRootID); + if (I.IncludeTreeID) + O.try_emplace("cas-include-tree-id", I.IncludeTreeID); Commands.push_back(std::move(O)); } TUs.push_back(Object{ @@ -694,7 +700,8 @@ class FullDeps { std::string ContextHash; std::vector FileDeps; std::vector ModuleDeps; - llvm::Optional CASFileSystemRootID; + Optional CASFileSystemRootID; + Optional IncludeTreeID; std::vector DriverCommandLine; std::vector Commands; }; @@ -859,6 +866,14 @@ int main(int argc, const char **argv) { } } AdjustedArgs.insert(AdjustedArgs.end(), FlagsEnd, Args.end()); + if (!PrefixMapToolchain.empty()) + AdjustedArgs.push_back("-fdepscan-prefix-map-toolchain=" + + PrefixMapToolchain); + if (!PrefixMapSDK.empty()) + AdjustedArgs.push_back("-fdepscan-prefix-map-sdk=" + PrefixMapSDK); + for (StringRef Map : PrefixMaps) { + AdjustedArgs.push_back("-fdepscan-prefix-map=" + std::string(Map)); + } return AdjustedArgs; }); @@ -890,13 +905,6 @@ int main(int argc, const char **argv) { FS = llvm::cantFail(llvm::cas::createCachingOnDiskFileSystem(*CAS)); } - DepscanPrefixMapping PrefixMapping; - if (!PrefixMapToolchain.empty()) - PrefixMapping.NewToolchainPath = PrefixMapToolchain; - if (!PrefixMapSDK.empty()) - PrefixMapping.NewSDKPath = PrefixMapSDK; - PrefixMapping.PrefixMap.append(PrefixMaps.begin(), PrefixMaps.end()); - DependencyScanningService Service(ScanMode, Format, CASOpts, CAS, Cache, FS, OptimizeArgs, EagerLoadModules); llvm::ThreadPool Pool(llvm::hardware_concurrency(NumThreads)); @@ -908,7 +916,7 @@ int main(int argc, const char **argv) { } return emitCompilationDBWithCASTreeArguments( CAS, AdjustingCompilations->getAllCompileCommands(), *DiagsConsumer, - PrefixMapping, Service, Pool, llvm::outs()); + Service, Pool, llvm::outs()); } std::vector> WorkerTools; @@ -951,7 +959,7 @@ int main(int argc, const char **argv) { << " files using " << Pool.getThreadCount() << " workers\n"; } for (unsigned I = 0; I < Pool.getThreadCount(); ++I) { - Pool.async([I, &CAS, &PrefixMapping, &Lock, &Index, &Inputs, &TreeResults, + Pool.async([I, &CAS, &Lock, &Index, &Inputs, &TreeResults, &HadErrors, &FD, &WorkerTools, &DependencyOS, &Errs]() { llvm::StringSet<> AlreadySeenModules; while (true) { @@ -988,28 +996,27 @@ int main(int argc, const char **argv) { Errs)) HadErrors = true; } else if (Format == ScanningOutputFormat::Tree) { - auto MaybeTree = WorkerTools[I]->getDependencyTree( - Input->CommandLine, CWD, PrefixMapping); + auto MaybeTree = + WorkerTools[I]->getDependencyTree(Input->CommandLine, CWD); std::unique_lock LockGuard(Lock); TreeResults.emplace_back(LocalIndex, std::move(Filename), std::move(MaybeTree)); } else if (Format == ScanningOutputFormat::IncludeTree) { auto MaybeTree = WorkerTools[I]->getIncludeTree( - *CAS, Input->CommandLine, CWD, PrefixMapping); + *CAS, Input->CommandLine, CWD, LookupOutput); std::unique_lock LockGuard(Lock); TreeResults.emplace_back(LocalIndex, std::move(Filename), std::move(MaybeTree)); } else if (MaybeModuleName) { auto MaybeFullDeps = WorkerTools[I]->getModuleDependencies( *MaybeModuleName, Input->CommandLine, CWD, AlreadySeenModules, - LookupOutput, PrefixMapping); + LookupOutput); if (handleModuleResult(Filename, MaybeFullDeps, FD, LocalIndex, DependencyOS, Errs)) HadErrors = true; } else { auto MaybeTUDeps = WorkerTools[I]->getTranslationUnitDependencies( - Input->CommandLine, CWD, AlreadySeenModules, LookupOutput, - PrefixMapping); + Input->CommandLine, CWD, AlreadySeenModules, LookupOutput); if (handleTranslationUnitResult(Filename, MaybeTUDeps, FD, LocalIndex, DependencyOS, Errs)) HadErrors = true; diff --git a/clang/tools/driver/cc1depscanProtocol.cpp b/clang/tools/driver/cc1depscanProtocol.cpp index b65d77682c9c9..7e77f4b483f4f 100644 --- a/clang/tools/driver/cc1depscanProtocol.cpp +++ b/clang/tools/driver/cc1depscanProtocol.cpp @@ -280,73 +280,23 @@ llvm::Error CC1DepScanDProtocol::getArgs(llvm::StringSaver &Saver, return Error::success(); } -llvm::Error -CC1DepScanDProtocol::putCommand(StringRef WorkingDirectory, - ArrayRef Args, - const DepscanPrefixMapping &Mapping) { +llvm::Error CC1DepScanDProtocol::putCommand(StringRef WorkingDirectory, + ArrayRef Args) { if (llvm::Error E = putString(WorkingDirectory)) return E; if (llvm::Error E = putArgs(Args)) return E; - return putDepscanPrefixMapping(Mapping); + return llvm::Error::success(); } -llvm::Error CC1DepScanDProtocol::getCommand(llvm::StringSaver &Saver, - StringRef &WorkingDirectory, - SmallVectorImpl &Args, - DepscanPrefixMapping &Mapping) { +llvm::Error +CC1DepScanDProtocol::getCommand(llvm::StringSaver &Saver, + StringRef &WorkingDirectory, + SmallVectorImpl &Args) { if (llvm::Error E = getString(Saver, WorkingDirectory)) return E; if (llvm::Error E = getArgs(Saver, Args)) return E; - return getDepscanPrefixMapping(Saver, Mapping); -} - -llvm::Error CC1DepScanDProtocol::putDepscanPrefixMapping( - const DepscanPrefixMapping &Mapping) { - // Construct the message. - SmallString<256> FullMapping; - if (Mapping.NewSDKPath) - FullMapping.append(*Mapping.NewSDKPath); - FullMapping.push_back(0); - if (Mapping.NewToolchainPath) - FullMapping.append(*Mapping.NewToolchainPath); - FullMapping.push_back(0); - for (StringRef Map : Mapping.PrefixMap) { - FullMapping.append(Map); - FullMapping.push_back(0); - } - return putString(FullMapping); -} - -llvm::Error -CC1DepScanDProtocol::getDepscanPrefixMapping(llvm::StringSaver &Saver, - DepscanPrefixMapping &Mapping) { - StringRef FullMapping; - if (llvm::Error E = getString(Saver, FullMapping)) - return E; - - // Parse the mapping. - size_t Count = 0; - for (auto I = FullMapping.begin(), B = I, E = FullMapping.end(); I != E; - ++I) { - if (I != B && I[-1]) - continue; // Wait for null-terminator. - StringRef Map = I; - switch (Count++) { - case 0: - if (!Map.empty()) - Mapping.NewSDKPath = std::string(Map); - break; - case 1: - if (!Map.empty()) - Mapping.NewToolchainPath = std::string(Map); - break; - default: - Mapping.PrefixMap.push_back(std::string(Map)); - break; - } - } return llvm::Error::success(); } diff --git a/clang/tools/driver/cc1depscanProtocol.h b/clang/tools/driver/cc1depscanProtocol.h index 575ee4f6f36b6..2600306d391c9 100644 --- a/clang/tools/driver/cc1depscanProtocol.h +++ b/clang/tools/driver/cc1depscanProtocol.h @@ -21,14 +21,7 @@ #include // FIXME: Unix-only. Not portable. #include // FIXME: Unix-only. Not portable. -namespace clang { -namespace tooling::dependencies { -struct DepscanPrefixMapping; -} - -namespace cc1depscand { -using tooling::dependencies::DepscanPrefixMapping; - +namespace clang::cc1depscand { struct DepscanSharing { bool OnlyShareParent = false; bool ShareViaIdentifier = false; @@ -137,20 +130,14 @@ class CC1DepScanDProtocol { return putMessage(String.size(), String.begin()); } - llvm::Error putDepscanPrefixMapping(const DepscanPrefixMapping &Mapping); - llvm::Error getDepscanPrefixMapping(llvm::StringSaver &Saver, - DepscanPrefixMapping &Mapping); - llvm::Error putArgs(ArrayRef Args); llvm::Error getArgs(llvm::StringSaver &Saver, SmallVectorImpl &Args); llvm::Error putCommand(StringRef WorkingDirectory, - ArrayRef Args, - const DepscanPrefixMapping &Mapping); + ArrayRef Args); llvm::Error getCommand(llvm::StringSaver &Saver, StringRef &WorkingDirectory, - SmallVectorImpl &Args, - DepscanPrefixMapping &Mapping); + SmallVectorImpl &Args); llvm::Error putScanResultFailed(StringRef Reason); llvm::Error putScanResultSuccess(StringRef RootID, @@ -240,8 +227,7 @@ class ScanDaemon : public OpenSocket { explicit ScanDaemon(OpenSocket S) : OpenSocket(std::move(S)) {} }; -} // namespace cc1depscand -} // namespace clang +} // namespace clang::cc1depscand #endif /* LLVM_ON_UNIX */ #endif // LLVM_CLANG_TOOLS_DRIVER_CC1DEPSCANPROTOCOL_H diff --git a/clang/tools/driver/cc1depscan_main.cpp b/clang/tools/driver/cc1depscan_main.cpp index 030fb270fa02f..85abbe55e31dd 100644 --- a/clang/tools/driver/cc1depscan_main.cpp +++ b/clang/tools/driver/cc1depscan_main.cpp @@ -64,7 +64,6 @@ using namespace clang; using namespace llvm::opt; using cc1depscand::DepscanSharing; -using clang::tooling::dependencies::DepscanPrefixMapping; using llvm::Error; #define DEBUG_TYPE "cc1depscand" @@ -357,7 +356,6 @@ static Expected scanAndUpdateCC1Inline( const char *Exec, ArrayRef InputArgs, StringRef WorkingDirectory, SmallVectorImpl &OutputArgs, bool ProduceIncludeTree, bool &DiagnosticErrorOccurred, - const DepscanPrefixMapping &PrefixMapping, llvm::function_ref SaveArg, const CASOptions &CASOpts, std::shared_ptr DB, std::shared_ptr Cache); @@ -366,15 +364,14 @@ static Expected scanAndUpdateCC1InlineWithTool( tooling::dependencies::DependencyScanningTool &Tool, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, const char *Exec, ArrayRef InputArgs, StringRef WorkingDirectory, - SmallVectorImpl &OutputArgs, - const DepscanPrefixMapping &PrefixMapping, llvm::cas::ObjectStore &DB, + SmallVectorImpl &OutputArgs, llvm::cas::ObjectStore &DB, llvm::function_ref SaveArg); static llvm::Expected scanAndUpdateCC1UsingDaemon( const char *Exec, ArrayRef OldArgs, StringRef WorkingDirectory, SmallVectorImpl &NewArgs, - bool &DiagnosticErrorOccurred, const DepscanPrefixMapping &Mapping, - StringRef Path, const DepscanSharing &Sharing, + bool &DiagnosticErrorOccurred, StringRef Path, + const DepscanSharing &Sharing, llvm::function_ref SaveArg, llvm::cas::ObjectStore &CAS) { using namespace clang::cc1depscand; @@ -391,7 +388,7 @@ static llvm::Expected scanAndUpdateCC1UsingDaemon( CC1DepScanDProtocol Comms(*Daemon); // llvm::dbgs() << "sending request...\n"; - if (auto E = Comms.putCommand(WorkingDirectory, OldArgs, Mapping)) + if (auto E = Comms.putCommand(WorkingDirectory, OldArgs)) return std::move(E); llvm::BumpPtrAllocator Alloc; @@ -441,29 +438,6 @@ static void writeResponseFile(raw_ostream &OS, OS << "\n"; } -static DepscanPrefixMapping -parseCASFSAutoPrefixMappings(DiagnosticsEngine &Diag, const ArgList &Args) { - using namespace clang::driver; - DepscanPrefixMapping Mapping; - for (const Arg *A : Args.filtered(options::OPT_fdepscan_prefix_map_EQ)) { - StringRef Map = A->getValue(); - size_t Equals = Map.find('='); - if (Equals == StringRef::npos) - Diag.Report(diag::err_drv_invalid_argument_to_option) - << Map << A->getOption().getName(); - else - Mapping.PrefixMap.push_back(std::string(Map)); - A->claim(); - } - if (const Arg *A = Args.getLastArg(options::OPT_fdepscan_prefix_map_sdk_EQ)) - Mapping.NewSDKPath = A->getValue(); - if (const Arg *A = - Args.getLastArg(options::OPT_fdepscan_prefix_map_toolchain_EQ)) - Mapping.NewToolchainPath = A->getValue(); - - return Mapping; -} - static int scanAndUpdateCC1(const char *Exec, ArrayRef OldArgs, SmallVectorImpl &NewArgs, DiagnosticsEngine &Diag, @@ -531,8 +505,6 @@ static int scanAndUpdateCC1(const char *Exec, ArrayRef OldArgs, bool ProduceIncludeTree = Args.hasArg(options::OPT_fdepscan_include_tree); - DepscanPrefixMapping PrefixMapping = parseCASFSAutoPrefixMappings(Diag, Args); - auto SaveArg = [&Args](const Twine &T) { return Args.MakeArgString(T); }; CompilerInvocation::GenerateCASArgs(CASOpts, Sharing.CASArgs, SaveArg); if (ProduceIncludeTree) @@ -543,10 +515,10 @@ static int scanAndUpdateCC1(const char *Exec, ArrayRef OldArgs, if (Optional DaemonPath = makeDepscanDaemonPath(Mode, Sharing)) return scanAndUpdateCC1UsingDaemon( Exec, OldArgs, WorkingDirectory, NewArgs, DiagnosticErrorOccurred, - PrefixMapping, *DaemonPath, Sharing, SaveArg, *DB); + *DaemonPath, Sharing, SaveArg, *DB); return scanAndUpdateCC1Inline(Exec, OldArgs, WorkingDirectory, NewArgs, ProduceIncludeTree, DiagnosticErrorOccurred, - PrefixMapping, SaveArg, CASOpts, DB, Cache); + SaveArg, CASOpts, DB, Cache); }; if (llvm::Error E = ScanAndUpdate().moveInto(RootID)) { Diag.Report(diag::err_cas_depscan_failed) << std::move(E); @@ -943,9 +915,7 @@ int ScanServer::listen() { llvm::StringSaver Saver(Alloc); StringRef WorkingDirectory; SmallVector Args; - DepscanPrefixMapping PrefixMapping; - if (llvm::Error E = - Comms.getCommand(Saver, WorkingDirectory, Args, PrefixMapping)) { + if (llvm::Error E = Comms.getCommand(Saver, WorkingDirectory, Args)) { SharedOS.applyLocked([&](raw_ostream &OS) { OS << I << ": failed to get command\n"; logAllUnhandledErrors(std::move(E), OS); @@ -988,8 +958,7 @@ int ScanServer::listen() { SmallVector NewArgs; auto RootID = scanAndUpdateCC1InlineWithTool( *Tool, *DiagsConsumer, &DiagsOS, Argv0, Args, WorkingDirectory, - NewArgs, PrefixMapping, *CAS, - [&](const Twine &T) { return Saver.save(T).data(); }); + NewArgs, *CAS, [&](const Twine &T) { return Saver.save(T).data(); }); if (!RootID) { consumeError(Comms.putScanResultFailed(toString(RootID.takeError()))); SharedOS.applyLocked([&](raw_ostream &OS) { @@ -1071,8 +1040,7 @@ static Expected scanAndUpdateCC1InlineWithTool( tooling::dependencies::DependencyScanningTool &Tool, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, const char *Exec, ArrayRef InputArgs, StringRef WorkingDirectory, - SmallVectorImpl &OutputArgs, - const DepscanPrefixMapping &PrefixMapping, llvm::cas::ObjectStore &DB, + SmallVectorImpl &OutputArgs, llvm::cas::ObjectStore &DB, llvm::function_ref SaveArg) { DiagnosticsEngine Diags(new DiagnosticIDs(), new DiagnosticOptions()); Diags.setClient(&DiagsConsumer, /*ShouldOwnClient=*/false); @@ -1082,8 +1050,7 @@ static Expected scanAndUpdateCC1InlineWithTool( "failed to create compiler invocation"); Expected Root = scanAndUpdateCC1InlineWithTool( - Tool, DiagsConsumer, VerboseOS, *Invocation, WorkingDirectory, - PrefixMapping, DB); + Tool, DiagsConsumer, VerboseOS, *Invocation, WorkingDirectory, DB); if (!Root) return Root; @@ -1097,7 +1064,6 @@ static Expected scanAndUpdateCC1Inline( const char *Exec, ArrayRef InputArgs, StringRef WorkingDirectory, SmallVectorImpl &OutputArgs, bool ProduceIncludeTree, bool &DiagnosticErrorOccurred, - const DepscanPrefixMapping &PrefixMapping, llvm::function_ref SaveArg, const CASOptions &CASOpts, std::shared_ptr DB, std::shared_ptr Cache) { @@ -1128,7 +1094,7 @@ static Expected scanAndUpdateCC1Inline( auto Result = scanAndUpdateCC1InlineWithTool( Tool, *DiagsConsumer, /*VerboseOS*/ nullptr, Exec, InputArgs, - WorkingDirectory, OutputArgs, PrefixMapping, *DB, SaveArg); + WorkingDirectory, OutputArgs, *DB, SaveArg); DiagnosticErrorOccurred = DiagsConsumer->getNumErrors() != 0; return Result; } diff --git a/clang/tools/driver/features.json b/clang/tools/driver/features.json index 770b73c613387..3989988e1bff0 100644 --- a/clang/tools/driver/features.json +++ b/clang/tools/driver/features.json @@ -15,6 +15,9 @@ { "name": "extract-api-ignores" }, + { + "name": "depscan-prefix-map" + }, { "name": "deployment-target-environment-variables", "value": [ diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 4e202d2153282..093b9cba6d952 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -13,14 +13,17 @@ #include "CASUtils.h" #include "CXDiagnosticSetDiagnosticConsumer.h" +#include "CXLoadedDiagnostic.h" #include "CXString.h" #include "clang-c/Dependencies.h" #include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/SerializedDiagnosticPrinter.h" #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "llvm/CAS/CASProvidingFileSystem.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/Support/Process.h" @@ -142,8 +145,8 @@ ScanningOutputFormat DependencyScannerServiceOptions::getFormat() const { if (llvm::sys::Process::GetEnv("CLANG_CACHE_USE_CASFS_DEPSCAN")) return ScanningOutputFormat::FullTree; - // Note: default caching behaviour is currently cas-fs. - return ConfiguredFormat; + // Use include-tree by default. + return ScanningOutputFormat::FullIncludeTree; } CXDependencyScannerService @@ -153,15 +156,18 @@ clang_experimental_DependencyScannerService_create_v1( std::shared_ptr CAS = unwrap(Opts)->CAS; std::shared_ptr Cache = unwrap(Opts)->Cache; IntrusiveRefCntPtr FS; - if (CAS && Cache) { + ScanningOutputFormat Format = unwrap(Opts)->getFormat(); + bool IsCASFSOutput = Format == ScanningOutputFormat::Tree || + Format == ScanningOutputFormat::FullTree; + if (CAS && Cache && IsCASFSOutput) { assert(unwrap(Opts)->CASOpts.getKind() != CASOptions::UnknownCAS && "CAS and ActionCache must match CASOptions"); FS = llvm::cantFail( llvm::cas::createCachingOnDiskFileSystem(CAS)); } return wrap(new DependencyScanningService( - ScanningMode::DependencyDirectivesScan, unwrap(Opts)->getFormat(), - unwrap(Opts)->CASOpts, std::move(CAS), std::move(Cache), std::move(FS), + ScanningMode::DependencyDirectivesScan, Format, unwrap(Opts)->CASOpts, + std::move(CAS), std::move(Cache), std::move(FS), /*ReuseFilemanager=*/false)); } @@ -193,8 +199,16 @@ void clang_experimental_FileDependenciesList_dispose( CXDependencyScannerWorker clang_experimental_DependencyScannerWorker_create_v0( CXDependencyScannerService Service) { - return wrap(new DependencyScanningWorker( - *unwrap(Service), llvm::vfs::createPhysicalFileSystem())); + ScanningOutputFormat Format = unwrap(Service)->getFormat(); + bool IsIncludeTreeOutput = Format == ScanningOutputFormat::IncludeTree || + Format == ScanningOutputFormat::FullIncludeTree; + llvm::IntrusiveRefCntPtr FS = + llvm::vfs::createPhysicalFileSystem(); + if (IsIncludeTreeOutput) + FS = llvm::cas::createCASProvidingFileSystem(unwrap(Service)->getCAS(), + std::move(FS)); + + return wrap(new DependencyScanningWorker(*unwrap(Service), FS)); } void clang_experimental_DependencyScannerWorker_dispose_v0( @@ -216,7 +230,7 @@ static CXErrorCode getFullDependencies(DependencyScanningWorker *Worker, llvm::StringSet<> AlreadySeen; FullDependencyConsumer DepConsumer(AlreadySeen); auto Controller = DependencyScanningTool::createActionController( - *Worker, std::move(LookupOutput), /*PrefixMapping=*/{}); + *Worker, std::move(LookupOutput)); bool HasDiagConsumer = DiagConsumer; bool HasError = Error; @@ -239,7 +253,7 @@ static CXErrorCode getFullDependencies(DependencyScanningWorker *Worker, TranslationUnitDeps TU = DepConsumer.takeTranslationUnitDeps(); - if (!TU.ModuleGraph.empty()) { + if (MDC && !TU.ModuleGraph.empty()) { CXModuleDependencySet *MDS = new CXModuleDependencySet; MDS->Count = TU.ModuleGraph.size(); MDS->Modules = new CXModuleDependency[MDS->Count]; @@ -342,12 +356,117 @@ CXErrorCode clang_experimental_DependencyScannerWorker_getFileDependencies_v4( return Result; } -CXErrorCode clang_experimental_DependencyScannerWorker_getFileDependencies_v5( - CXDependencyScannerWorker W, int argc, const char *const *argv, - const char *ModuleName, const char *WorkingDirectory, void *MDCContext, - CXModuleDiscoveredCallback *MDC, void *MLOContext, - CXModuleLookupOutputCallback *MLO, unsigned, CXFileDependenciesList **Out, - CXDiagnosticSet *OutDiags) { +namespace { + +struct DependencyScannerWorkerScanSettings { + int argc; + const char *const *argv; + const char *ModuleName; + const char *WorkingDirectory; + void *MLOContext; + CXModuleLookupOutputCallback *MLO; +}; + +struct CStringsManager { + SmallVector>> OwnedCStr; + SmallVector>> OwnedStdStr; + + /// Doesn't own the string contents. + CXCStringArray createCStringsRef(ArrayRef Strings) { + OwnedCStr.push_back(std::make_unique>()); + std::vector &CStrings = *OwnedCStr.back(); + CStrings.reserve(Strings.size()); + for (const auto &String : Strings) + CStrings.push_back(String.c_str()); + return {CStrings.data(), CStrings.size()}; + } + + /// Doesn't own the string contents. + CXCStringArray createCStringsRef(const llvm::StringSet<> &StringsUnordered) { + std::vector Strings; + + for (auto SI = StringsUnordered.begin(), SE = StringsUnordered.end(); + SI != SE; ++SI) + Strings.push_back(SI->getKey()); + + llvm::sort(Strings); + + OwnedCStr.push_back(std::make_unique>()); + std::vector &CStrings = *OwnedCStr.back(); + CStrings.reserve(Strings.size()); + for (const auto &String : Strings) + CStrings.push_back(String.data()); + return {CStrings.data(), CStrings.size()}; + } + + /// Gets ownership of string contents. + CXCStringArray createCStringsOwned(std::vector &&Strings) { + OwnedStdStr.push_back( + std::make_unique>(std::move(Strings))); + return createCStringsRef(*OwnedStdStr.back()); + } +}; + +struct DependencyGraph { + TranslationUnitDeps TUDeps; + SmallString<256> SerialDiagBuf; + CStringsManager StrMgr{}; + + CXDiagnosticSet getDiagnosticSet() const { + CXLoadDiag_Error Error; + CXString ErrorString; + CXDiagnosticSet DiagSet = loadCXDiagnosticsFromBuffer( + llvm::MemoryBufferRef(SerialDiagBuf, ""), &Error, &ErrorString); + assert(Error == CXLoadDiag_None); + clang_disposeString(ErrorString); + return DiagSet; + } +}; + +struct DependencyGraphModule { + ModuleDeps *ModDeps; + CStringsManager StrMgr{}; +}; + +struct DependencyGraphTUCommand { + Command *TUCmd; + CStringsManager StrMgr{}; +}; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerWorkerScanSettings, + CXDependencyScannerWorkerScanSettings) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyGraph, CXDepGraph) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyGraphModule, CXDepGraphModule) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyGraphTUCommand, + CXDepGraphTUCommand) + +} // end anonymous namespace + +CXDependencyScannerWorkerScanSettings +clang_experimental_DependencyScannerWorkerScanSettings_create( + int argc, const char *const *argv, const char *ModuleName, + const char *WorkingDirectory, void *MLOContext, + CXModuleLookupOutputCallback *MLO) { + return wrap(new DependencyScannerWorkerScanSettings{ + argc, argv, ModuleName, WorkingDirectory, MLOContext, MLO}); +} + +void clang_experimental_DependencyScannerWorkerScanSettings_dispose( + CXDependencyScannerWorkerScanSettings Settings) { + delete unwrap(Settings); +} + +enum CXErrorCode clang_experimental_DependencyScannerWorker_getDepGraph( + CXDependencyScannerWorker W, + CXDependencyScannerWorkerScanSettings CXSettings, CXDepGraph *Out) { + DependencyScannerWorkerScanSettings &Settings = *unwrap(CXSettings); + int argc = Settings.argc; + const char *const *argv = Settings.argv; + const char *ModuleName = Settings.ModuleName; + const char *WorkingDirectory = Settings.WorkingDirectory; + void *MLOContext = Settings.MLOContext; + CXModuleLookupOutputCallback *MLO = Settings.MLO; + OutputLookup OL(MLOContext, MLO); auto LookupOutputs = [&](const ModuleID &ID, ModuleOutputKind MOK) { return OL.lookupModuleOutput(ID, MOK); @@ -357,34 +476,154 @@ CXErrorCode clang_experimental_DependencyScannerWorker_getFileDependencies_v5( return CXError_InvalidArguments; *Out = nullptr; - CXDiagnosticSetDiagnosticConsumer DiagConsumer; + DependencyGraph *DepGraph = new DependencyGraph(); - CXErrorCode Result = getFileDependencies( - W, argc, argv, WorkingDirectory, MDC, MDCContext, nullptr, &DiagConsumer, - LookupOutputs, ModuleName ? Optional(ModuleName) : None, - [&](TranslationUnitDeps TU) { - assert(TU.DriverCommandLine.empty()); - std::vector Modules; - for (const ModuleID &MID : TU.ClangModuleDeps) - Modules.push_back(MID.ModuleName + ":" + MID.ContextHash); - auto *Commands = new CXTranslationUnitCommand[TU.Commands.size()]; - for (size_t I = 0, E = TU.Commands.size(); I < E; ++I) { - Commands[I].ContextHash = cxstring::createDup(TU.ID.ContextHash); - Commands[I].FileDeps = cxstring::createSet(TU.FileDeps); - Commands[I].ModuleDeps = cxstring::createSet(Modules); - Commands[I].Executable = - cxstring::createDup(TU.Commands[I].Executable); - Commands[I].BuildArguments = - cxstring::createSet(TU.Commands[I].Arguments); - } - *Out = new CXFileDependenciesList{TU.Commands.size(), Commands}; - }); + // We capture diagnostics as a serialized diagnostics buffer, so that we don't + // need to keep a valid SourceManager in order to access diagnostic locations. + auto DiagOpts = llvm::makeIntrusiveRefCnt(); + auto DiagOS = + std::make_unique(DepGraph->SerialDiagBuf); + std::unique_ptr SerialDiagConsumer = + serialized_diags::create("", DiagOpts.get(), + /*MergeChildRecords=*/false, std::move(DiagOS)); - *OutDiags = DiagConsumer.getDiagnosticSet(); + CXErrorCode Result = getFileDependencies( + W, argc, argv, WorkingDirectory, /*MDC=*/nullptr, /*MDCContext=*/nullptr, + /*Error=*/nullptr, SerialDiagConsumer.get(), LookupOutputs, + ModuleName ? Optional(ModuleName) : std::nullopt, + [&](TranslationUnitDeps TU) { DepGraph->TUDeps = std::move(TU); }); + *Out = wrap(DepGraph); return Result; } +void clang_experimental_DepGraph_dispose(CXDepGraph Graph) { + delete unwrap(Graph); +} + +size_t clang_experimental_DepGraph_getNumModules(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return TUDeps.ModuleGraph.size(); +} + +CXDepGraphModule clang_experimental_DepGraph_getModule(CXDepGraph Graph, + size_t Index) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return wrap(new DependencyGraphModule{&TUDeps.ModuleGraph[Index]}); +} + +void clang_experimental_DepGraphModule_dispose(CXDepGraphModule CXDepMod) { + delete unwrap(CXDepMod); +} + +const char * +clang_experimental_DepGraphModule_getName(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + return ModDeps.ID.ModuleName.c_str(); +} + +const char * +clang_experimental_DepGraphModule_getContextHash(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + return ModDeps.ID.ContextHash.c_str(); +} + +const char * +clang_experimental_DepGraphModule_getModuleMapPath(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + if (ModDeps.ClangModuleMapFile.empty()) + return nullptr; + return ModDeps.ClangModuleMapFile.c_str(); +} + +CXCStringArray +clang_experimental_DepGraphModule_getFileDeps(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + return unwrap(CXDepMod)->StrMgr.createCStringsRef(ModDeps.FileDeps); +} + +CXCStringArray +clang_experimental_DepGraphModule_getModuleDeps(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + std::vector Modules; + Modules.reserve(ModDeps.ClangModuleDeps.size()); + for (const ModuleID &MID : ModDeps.ClangModuleDeps) + Modules.push_back(MID.ModuleName + ":" + MID.ContextHash); + return unwrap(CXDepMod)->StrMgr.createCStringsOwned(std::move(Modules)); +} + +CXCStringArray +clang_experimental_DepGraphModule_getBuildArguments(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + return unwrap(CXDepMod)->StrMgr.createCStringsRef(ModDeps.BuildArguments); +} + +const char * +clang_experimental_DepGraphModule_getCacheKey(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + if (ModDeps.ModuleCacheKey) + return ModDeps.ModuleCacheKey->c_str(); + return nullptr; +} + +size_t clang_experimental_DepGraph_getNumTUCommands(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return TUDeps.Commands.size(); +} + +CXDepGraphTUCommand clang_experimental_DepGraph_getTUCommand(CXDepGraph Graph, + size_t Index) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return wrap(new DependencyGraphTUCommand{&TUDeps.Commands[Index]}); +} + +void clang_experimental_DepGraphTUCommand_dispose(CXDepGraphTUCommand CXCmd) { + delete unwrap(CXCmd); +} + +const char * +clang_experimental_DepGraphTUCommand_getExecutable(CXDepGraphTUCommand CXCmd) { + Command &TUCmd = *unwrap(CXCmd)->TUCmd; + return TUCmd.Executable.c_str(); +} + +CXCStringArray clang_experimental_DepGraphTUCommand_getBuildArguments( + CXDepGraphTUCommand CXCmd) { + Command &TUCmd = *unwrap(CXCmd)->TUCmd; + return unwrap(CXCmd)->StrMgr.createCStringsRef(TUCmd.Arguments); +} + +const char * +clang_experimental_DepGraphTUCommand_getCacheKey(CXDepGraphTUCommand CXCmd) { + Command &TUCmd = *unwrap(CXCmd)->TUCmd; + if (TUCmd.TUCacheKey) + return TUCmd.TUCacheKey->c_str(); + return nullptr; +} + +CXCStringArray clang_experimental_DepGraph_getTUFileDeps(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return unwrap(Graph)->StrMgr.createCStringsRef(TUDeps.FileDeps); +} + +CXCStringArray clang_experimental_DepGraph_getTUModuleDeps(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + std::vector Modules; + Modules.reserve(TUDeps.ClangModuleDeps.size()); + for (const ModuleID &MID : TUDeps.ClangModuleDeps) + Modules.push_back(MID.ModuleName + ":" + MID.ContextHash); + return unwrap(Graph)->StrMgr.createCStringsOwned(std::move(Modules)); +} + +const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return TUDeps.ID.ContextHash.c_str(); +} + +CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph Graph) { + return unwrap(Graph)->getDiagnosticSet(); +} + static std::string lookupModuleOutput(const ModuleID &ID, ModuleOutputKind MOK, void *MLOContext, CXModuleLookupOutputCallback *MLO) { diff --git a/clang/tools/libclang/CXLoadedDiagnostic.cpp b/clang/tools/libclang/CXLoadedDiagnostic.cpp index 3d2b3047f47b5..6481109a17824 100644 --- a/clang/tools/libclang/CXLoadedDiagnostic.cpp +++ b/clang/tools/libclang/CXLoadedDiagnostic.cpp @@ -249,6 +249,9 @@ class DiagLoader : serialized_diags::SerializedDiagnosticReader { } CXDiagnosticSet load(const char *file); + CXDiagnosticSet load(llvm::MemoryBufferRef Buffer); + + CXDiagnosticSet reportError(std::error_code EC); }; } // end anonymous namespace @@ -256,24 +259,38 @@ CXDiagnosticSet DiagLoader::load(const char *file) { TopDiags = std::make_unique(); std::error_code EC = readDiagnostics(file); - if (EC) { - switch (EC.value()) { - case static_cast(serialized_diags::SDError::HandlerFailed): - // We've already reported the problem. - break; - case static_cast(serialized_diags::SDError::CouldNotLoad): - reportBad(CXLoadDiag_CannotLoad, EC.message()); - break; - default: - reportInvalidFile(EC.message()); - break; - } - return nullptr; - } + if (EC) + return reportError(EC); + + return (CXDiagnosticSet)TopDiags.release(); +} + +CXDiagnosticSet DiagLoader::load(llvm::MemoryBufferRef Buffer) { + TopDiags = std::make_unique(); + + std::error_code EC = readDiagnostics(Buffer); + if (EC) + return reportError(EC); return (CXDiagnosticSet)TopDiags.release(); } +CXDiagnosticSet DiagLoader::reportError(std::error_code EC) { + assert(EC); + switch (EC.value()) { + case static_cast(serialized_diags::SDError::HandlerFailed): + // We've already reported the problem. + break; + case static_cast(serialized_diags::SDError::CouldNotLoad): + reportBad(CXLoadDiag_CannotLoad, EC.message()); + break; + default: + reportInvalidFile(EC.message()); + break; + } + return nullptr; +} + std::error_code DiagLoader::readLocation(const serialized_diags::Location &SDLoc, CXLoadedDiagnostic::Location &LoadedLoc) { @@ -420,3 +437,10 @@ CXDiagnosticSet clang_loadDiagnostics(const char *file, DiagLoader L(error, errorString); return L.load(file); } + +CXDiagnosticSet clang::loadCXDiagnosticsFromBuffer(llvm::MemoryBufferRef buffer, + enum CXLoadDiag_Error *error, + CXString *errorString) { + DiagLoader L(error, errorString); + return L.load(buffer); +} diff --git a/clang/tools/libclang/CXLoadedDiagnostic.h b/clang/tools/libclang/CXLoadedDiagnostic.h index 93995d7bb798c..eb76dfbfe11b5 100644 --- a/clang/tools/libclang/CXLoadedDiagnostic.h +++ b/clang/tools/libclang/CXLoadedDiagnostic.h @@ -19,6 +19,10 @@ #include "clang/Basic/LLVM.h" #include +namespace llvm { +class MemoryBufferRef; +} + namespace clang { class CXLoadedDiagnostic : public CXDiagnosticImpl { public: @@ -88,6 +92,13 @@ class CXLoadedDiagnostic : public CXDiagnosticImpl { unsigned severity; unsigned category; }; -} + +/// Read a serialized diagnostics \p buffer and create a \c CXDiagnosticSet +/// object for the loaded diagnostics. +CXDiagnosticSet loadCXDiagnosticsFromBuffer(llvm::MemoryBufferRef buffer, + enum CXLoadDiag_Error *error, + CXString *errorString); + +} // namespace clang #endif diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index 6ba53c57c1f15..1b7035d732ee1 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -489,7 +489,30 @@ LLVM_16 { clang_experimental_DependencyScannerServiceOptions_setCASDatabases; clang_experimental_DependencyScannerServiceOptions_setDependencyMode; clang_experimental_DependencyScannerServiceOptions_setObjectStore; - clang_experimental_DependencyScannerWorker_getFileDependencies_v5; + clang_experimental_DependencyScannerWorker_getDepGraph; + clang_experimental_DependencyScannerWorkerScanSettings_create; + clang_experimental_DependencyScannerWorkerScanSettings_dispose; + clang_experimental_DepGraph_dispose; + clang_experimental_DepGraph_getDiagnostics; + clang_experimental_DepGraph_getModule; + clang_experimental_DepGraph_getNumModules; + clang_experimental_DepGraph_getNumTUCommands; + clang_experimental_DepGraph_getTUCommand; + clang_experimental_DepGraph_getTUContextHash; + clang_experimental_DepGraph_getTUFileDeps; + clang_experimental_DepGraph_getTUModuleDeps; + clang_experimental_DepGraphModule_dispose; + clang_experimental_DepGraphModule_getBuildArguments; + clang_experimental_DepGraphModule_getCacheKey; + clang_experimental_DepGraphModule_getContextHash; + clang_experimental_DepGraphModule_getFileDeps; + clang_experimental_DepGraphModule_getModuleDeps; + clang_experimental_DepGraphModule_getModuleMapPath; + clang_experimental_DepGraphModule_getName; + clang_experimental_DepGraphTUCommand_dispose; + clang_experimental_DepGraphTUCommand_getBuildArguments; + clang_experimental_DepGraphTUCommand_getCacheKey; + clang_experimental_DepGraphTUCommand_getExecutable; clang_getUnqualifiedType; clang_getNonReferenceType; clang_CXXMethod_isDeleted; diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index 6ddba05cde1ca..46121711aedd8 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -4,6 +4,7 @@ #include "llvm/CAS/CASProvidingFileSystem.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/CAS/ObjectStore.h" +#include "llvm/Support/Error.h" #include "llvm/Testing/Support/Error.h" #include "gtest/gtest.h" @@ -53,16 +54,15 @@ TEST(IncludeTree, IncludeTreeScan) { "-o" "t.cpp.o"}; Optional Root; - DepscanPrefixMapping PrefixMapping; ASSERT_THAT_ERROR( - ScanTool.getIncludeTree(*DB, CommandLine, /*CWD*/ "", PrefixMapping) + ScanTool.getIncludeTree(*DB, CommandLine, /*CWD*/ "", nullptr) .moveInto(Root), llvm::Succeeded()); - Optional MainFile; - Optional A1File; - Optional B1File; - Optional SysFile; + Optional MainFile; + Optional A1File; + Optional B1File; + Optional SysFile; Optional Main; ASSERT_THAT_ERROR(Root->getMainFileTree().moveInto(Main), llvm::Succeeded()); @@ -70,7 +70,7 @@ TEST(IncludeTree, IncludeTreeScan) { ASSERT_THAT_ERROR(Main->getBaseFile().moveInto(MainFile), llvm::Succeeded()); EXPECT_EQ(Main->getFileCharacteristic(), SrcMgr::C_User); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(MainFile->getFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, "t.cpp"); EXPECT_EQ(FI.Contents, MainContents); @@ -78,23 +78,24 @@ TEST(IncludeTree, IncludeTreeScan) { ASSERT_EQ(Main->getNumIncludes(), uint32_t(3)); Optional Predef; - ASSERT_THAT_ERROR(Main->getInclude(0).moveInto(Predef), llvm::Succeeded()); + ASSERT_THAT_ERROR(Main->getIncludeTree(0).moveInto(Predef), + llvm::Succeeded()); EXPECT_EQ(Main->getIncludeOffset(0), uint32_t(0)); { EXPECT_EQ(Predef->getFileCharacteristic(), SrcMgr::C_User); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(Predef->getBaseFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, ""); } Optional A1; - ASSERT_THAT_ERROR(Main->getInclude(1).moveInto(A1), llvm::Succeeded()); + ASSERT_THAT_ERROR(Main->getIncludeTree(1).moveInto(A1), llvm::Succeeded()); EXPECT_EQ(Main->getIncludeOffset(1), uint32_t(21)); { ASSERT_THAT_ERROR(A1->getBaseFile().moveInto(A1File), llvm::Succeeded()); EXPECT_EQ(A1->getFileCharacteristic(), SrcMgr::C_User); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(A1File->getFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, "./a1.h"); EXPECT_EQ(FI.Contents, A1Contents); @@ -103,12 +104,12 @@ TEST(IncludeTree, IncludeTreeScan) { ASSERT_EQ(A1->getNumIncludes(), uint32_t(1)); Optional B1; - ASSERT_THAT_ERROR(A1->getInclude(0).moveInto(B1), llvm::Succeeded()); + ASSERT_THAT_ERROR(A1->getIncludeTree(0).moveInto(B1), llvm::Succeeded()); EXPECT_EQ(A1->getIncludeOffset(0), uint32_t(122)); { ASSERT_THAT_ERROR(B1->getBaseFile().moveInto(B1File), llvm::Succeeded()); EXPECT_EQ(B1->getFileCharacteristic(), SrcMgr::C_User); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(B1->getBaseFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, "./b1.h"); EXPECT_EQ(FI.Contents, ""); @@ -118,12 +119,12 @@ TEST(IncludeTree, IncludeTreeScan) { } Optional Sys; - ASSERT_THAT_ERROR(Main->getInclude(2).moveInto(Sys), llvm::Succeeded()); + ASSERT_THAT_ERROR(Main->getIncludeTree(2).moveInto(Sys), llvm::Succeeded()); EXPECT_EQ(Main->getIncludeOffset(2), uint32_t(42)); { ASSERT_THAT_ERROR(Sys->getBaseFile().moveInto(SysFile), llvm::Succeeded()); EXPECT_EQ(Sys->getFileCharacteristic(), SrcMgr::C_System); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(Sys->getBaseFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, "sys/sys.h"); EXPECT_EQ(FI.Contents, ""); @@ -131,31 +132,129 @@ TEST(IncludeTree, IncludeTreeScan) { ASSERT_EQ(Sys->getNumIncludes(), uint32_t(0)); } - Optional FileList; + Optional FileList; ASSERT_THAT_ERROR(Root->getFileList().moveInto(FileList), llvm::Succeeded()); - ASSERT_EQ(FileList->getNumFiles(), size_t(4)); - { - Optional File; - ASSERT_THAT_ERROR(FileList->getFile(0).moveInto(File), llvm::Succeeded()); - EXPECT_EQ(File->getRef(), MainFile->getRef()); - EXPECT_EQ(FileList->getFileSize(0), MainContents.size()); - } - { - Optional File; - ASSERT_THAT_ERROR(FileList->getFile(1).moveInto(File), llvm::Succeeded()); - EXPECT_EQ(File->getRef(), A1File->getRef()); - EXPECT_EQ(FileList->getFileSize(1), A1Contents.size()); - } - { - Optional File; - ASSERT_THAT_ERROR(FileList->getFile(2).moveInto(File), llvm::Succeeded()); - EXPECT_EQ(File->getRef(), B1File->getRef()); - EXPECT_EQ(FileList->getFileSize(2), IncludeFileList::FileSizeTy(0)); + + SmallVector> + Files; + ASSERT_THAT_ERROR(FileList->forEachFile([&](auto F, auto S) -> llvm::Error { + Files.push_back({F, S}); + return llvm::Error::success(); + }), + llvm::Succeeded()); + + ASSERT_EQ(Files.size(), size_t(4)); + EXPECT_EQ(Files[0].first.getRef(), MainFile->getRef()); + EXPECT_EQ(Files[0].second, MainContents.size()); + EXPECT_EQ(Files[1].first.getRef(), A1File->getRef()); + EXPECT_EQ(Files[1].second, A1Contents.size()); + EXPECT_EQ(Files[2].first.getRef(), B1File->getRef()); + EXPECT_EQ(Files[2].second, IncludeTree::FileList::FileSizeTy(0)); + EXPECT_EQ(Files[3].first.getRef(), SysFile->getRef()); + EXPECT_EQ(Files[3].second, IncludeTree::FileList::FileSizeTy(0)); +} + +TEST(IncludeTree, IncludeTreeFileList) { + std::shared_ptr DB = llvm::cas::createInMemoryCAS(); + SmallVector Files; + for (unsigned I = 0; I < 10; ++I) { + Optional File; + std::string Path = "/file" + std::to_string(I); + static constexpr StringRef Bytes = "123456789"; + Optional Content; + ASSERT_THAT_ERROR( + DB->storeFromString({}, Bytes.substr(0, I)).moveInto(Content), + llvm::Succeeded()); + ASSERT_THAT_ERROR( + IncludeTree::File::create(*DB, Path, *Content).moveInto(File), + llvm::Succeeded()); + Files.push_back(std::move(*File)); } - { - Optional File; - ASSERT_THAT_ERROR(FileList->getFile(3).moveInto(File), llvm::Succeeded()); - EXPECT_EQ(File->getRef(), SysFile->getRef()); - EXPECT_EQ(FileList->getFileSize(3), IncludeFileList::FileSizeTy(0)); + + auto MakeFileList = [&](unsigned Begin, unsigned End, + ArrayRef Lists) { + SmallVector Entries; + for (; Begin != End; ++Begin) + Entries.push_back({Files[Begin].getRef(), Begin}); + return IncludeTree::FileList::create(*DB, Entries, Lists); + }; + + Optional L89, L7, L29, L; + ASSERT_THAT_ERROR(MakeFileList(8, 10, {}).moveInto(L89), llvm::Succeeded()); + EXPECT_EQ(L89->getNumReferences(), 2u); + ASSERT_THAT_ERROR(MakeFileList(7, 8, {}).moveInto(L7), llvm::Succeeded()); + EXPECT_EQ(L7->getNumReferences(), 1u); + ASSERT_THAT_ERROR( + MakeFileList(2, 7, {L7->getRef(), L89->getRef()}).moveInto(L29), + llvm::Succeeded()); + EXPECT_EQ(L29->getNumReferences(), 7u); // 2,3,4,5,6, {7}, {8, 9} + ASSERT_THAT_ERROR(MakeFileList(0, 2, {L29->getRef()}).moveInto(L), + llvm::Succeeded()); + EXPECT_EQ(L->getNumReferences(), 3u); // 0, 1, {2, ...} + + size_t I = 0; + ASSERT_THAT_ERROR( + L->forEachFile([&](IncludeTree::File F, auto Size) -> llvm::Error { + if (I >= Files.size()) + return llvm::Error::success(); // diagnosed later. + EXPECT_EQ(F.getFilenameRef(), Files[I].getFilenameRef()) + << "filename mismatch at " << I; + EXPECT_EQ(F.getContentsRef(), Files[I].getContentsRef()) + << "contents mismatch at " << I; + EXPECT_EQ(Size, I) << "size mismatch at " << I; + I += 1; + return llvm::Error::success(); + }), + llvm::Succeeded()); + EXPECT_EQ(I, Files.size()); +} + +TEST(IncludeTree, IncludeTreeFileListDuplicates) { + std::shared_ptr DB = llvm::cas::createInMemoryCAS(); + SmallVector Files; + for (unsigned I = 0; I < 10; ++I) { + std::optional File; + std::string Path = "/file" + std::to_string(I); + static constexpr StringRef Bytes = "123456789"; + std::optional Content; + ASSERT_THAT_ERROR( + DB->storeFromString({}, Bytes.substr(0, I)).moveInto(Content), + llvm::Succeeded()); + ASSERT_THAT_ERROR( + IncludeTree::File::create(*DB, Path, *Content).moveInto(File), + llvm::Succeeded()); + Files.push_back(std::move(*File)); } + + auto MakeFileList = [&](unsigned Begin, unsigned End, + ArrayRef Lists) { + SmallVector Entries; + for (; Begin != End; ++Begin) + Entries.push_back({Files[Begin].getRef(), Begin}); + return IncludeTree::FileList::create(*DB, Entries, Lists); + }; + + std::optional L89, L; + ASSERT_THAT_ERROR(MakeFileList(8, 10, {}).moveInto(L89), llvm::Succeeded()); + EXPECT_EQ(L89->getNumReferences(), 2u); + ASSERT_THAT_ERROR( + MakeFileList(0, 9, {L89->getRef(), L89->getRef()}).moveInto(L), + llvm::Succeeded()); + EXPECT_EQ(L->getNumReferences(), 11u); // 0, 1, ..., 8, {8, 9}, {8, 9} + + size_t I = 0; + ASSERT_THAT_ERROR( + L->forEachFile([&](IncludeTree::File F, auto Size) -> llvm::Error { + if (I >= Files.size()) + return llvm::Error::success(); // diagnosed later. + EXPECT_EQ(F.getFilenameRef(), Files[I].getFilenameRef()) + << "filename mismatch at " << I; + EXPECT_EQ(F.getContentsRef(), Files[I].getContentsRef()) + << "contents mismatch at " << I; + EXPECT_EQ(Size, I) << "size mismatch at " << I; + I += 1; + return llvm::Error::success(); + }), + llvm::Succeeded()); + EXPECT_EQ(I, Files.size()); }