From 7d715799502a2351aeffc92acbafed2e6ada188b Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Wed, 13 Feb 2019 17:24:24 -0800 Subject: [PATCH 1/3] [Serialization] Allow either hash-based or mtime-based deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a bit to the module to determine whether the dependency’s stored bit pattern is a hash or an mtime. Prebuilt modules store a hash of their dependencies because we can’t be sure their dependencies will have the same modtime as when they were built. --- include/swift/Serialization/ModuleFormat.h | 11 +-- .../Serialization/SerializationOptions.h | 83 ++++++++++++++++++- lib/Frontend/ParseableInterfaceSupport.cpp | 69 +++++++++++---- lib/Serialization/ModuleFile.cpp | 15 +++- lib/Serialization/Serialization.cpp | 13 ++- 5 files changed, 160 insertions(+), 31 deletions(-) diff --git a/include/swift/Serialization/ModuleFormat.h b/include/swift/Serialization/ModuleFormat.h index 638fbd841f961..8fb86e1338e5a 100644 --- a/include/swift/Serialization/ModuleFormat.h +++ b/include/swift/Serialization/ModuleFormat.h @@ -52,7 +52,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0; /// describe what change you made. The content of this comment isn't important; /// it just ensures a conflict if two people change the module format. /// Don't worry about adhering to the 80-column limit for this line. -const uint16_t SWIFTMODULE_VERSION_MINOR = 475; // Last change: Generalize nested archetype serialization +const uint16_t SWIFTMODULE_VERSION_MINOR = 476; // Last change: prebuilt module cache using DeclIDField = BCFixed<31>; @@ -107,7 +107,7 @@ using CharOffset = BitOffset; using CharOffsetField = BitOffsetField; using FileSizeField = BCVBR<16>; -using FileModTimeField = BCVBR<16>; +using FileModTimeOrContentHashField = BCVBR<16>; using FileHashField = BCVBR<16>; // These IDs must \em not be renumbered or reordered without incrementing @@ -671,9 +671,10 @@ namespace input_block { using FileDependencyLayout = BCRecordLayout< FILE_DEPENDENCY, - FileSizeField, // file size (for validation) - FileModTimeField, // file mtime (for validation) - BCBlob // path + FileSizeField, // file size (for validation) + FileModTimeOrContentHashField, // mtime or content hash (for validation) + BCFixed<1>, // are we reading mtime (0) or hash (1)? + BCBlob // path >; } diff --git a/include/swift/Serialization/SerializationOptions.h b/include/swift/Serialization/SerializationOptions.h index 0d0f170ae1078..bd041f3b6542d 100644 --- a/include/swift/Serialization/SerializationOptions.h +++ b/include/swift/Serialization/SerializationOptions.h @@ -35,10 +35,87 @@ namespace swift { StringRef ModuleLinkName; ArrayRef ExtraClangOptions; - struct FileDependency { - uint64_t Size; - uint64_t ModificationTime; + /// Describes a single-file dependency for this module, along with the + /// appropriate strategy for how to verify if it's up-to-date. + class FileDependency { + /// The size of the file on disk, in bytes. + uint64_t Size : 63; + + /// A dependency can be either hash-based or modification-time-based. + bool IsHashBased : 1; + + union { + /// The last modification time of the file. + uint64_t ModificationTime; + + /// The xxHash of the full contents of the file. + uint64_t ContentHash; + }; + + /// The path to the dependency. std::string Path; + + FileDependency(uint64_t size, bool isHash, uint64_t hashOrModTime, + StringRef path): + Size(size), IsHashBased(isHash), ModificationTime(hashOrModTime), + Path(path) {} + public: + FileDependency() = delete; + + /// Creates a new hash-based file dependency. + static FileDependency + hashBased(StringRef path, uint64_t size, uint64_t hash) { + return FileDependency(size, /*isHash*/true, hash, path); + } + + /// Creates a new modification time-based file dependency. + static FileDependency + modTimeBased(StringRef path, uint64_t size, uint64_t mtime) { + return FileDependency(size, /*isHash*/false, mtime, path); + } + + /// Updates the last-modified time of this dependency. + /// If the dependency is a hash-based dependency, it becomes + /// modification time-based. + void setLastModificationTime(uint64_t mtime) { + IsHashBased = false; + ModificationTime = mtime; + } + + /// Updates the content hash of this dependency. + /// If the dependency is a modification time-based dependency, it becomes + /// hash-based. + void setContentHash(uint64_t hash) { + IsHashBased = true; + ContentHash = hash; + } + + /// Determines if this dependency is hash-based and should be validated + /// based on content hash. + bool isHashBased() const { return IsHashBased; } + + /// Determines if this dependency is hash-based and should be validated + /// based on modification time. + bool isModificationTimeBased() const { return !IsHashBased; } + + /// Gets the modification time, if this is a modification time-based + /// dependency. + uint64_t getModificationTime() const { + assert(isModificationTimeBased() && + "cannot get modification time for hash-based dependency"); + return ModificationTime; + } + + /// Gets the content hash, if this is a hash-based + /// dependency. + uint64_t getContentHash() const { + assert(isHashBased() && + "cannot get content hash for mtime-based dependency"); + return ContentHash; + } + + StringRef getPath() const { return Path; } + uint64_t getSize() const { return Size; } }; ArrayRef Dependencies; diff --git a/lib/Frontend/ParseableInterfaceSupport.cpp b/lib/Frontend/ParseableInterfaceSupport.cpp index 800354c1a4b97..94cea120b36fa 100644 --- a/lib/Frontend/ParseableInterfaceSupport.cpp +++ b/lib/Frontend/ParseableInterfaceSupport.cpp @@ -225,14 +225,40 @@ void ParseableInterfaceModuleLoader::configureSubInvocationInputsAndOutputs( } // Checks that a dependency read from the cached module is up to date compared -// to the interface file it represents. -static bool dependencyIsUpToDate(llvm::vfs::FileSystem &FS, FileDependency In, - StringRef ModulePath, DiagnosticEngine &Diags, - SourceLoc DiagLoc) { - auto Status = getStatusOfDependency(FS, ModulePath, In.Path, Diags, DiagLoc); - if (!Status) return false; - uint64_t mtime = Status->getLastModificationTime().time_since_epoch().count(); - return Status->getSize() == In.Size && mtime == In.ModificationTime; +// to the interface file it represents. If it's up-to-date, return its status, +// so we can switch hash-based dependencies to mtimes after we've validated +// the hashes. +static Optional +dependencyIsUpToDate(llvm::vfs::FileSystem &FS, FileDependency In, + StringRef ModulePath, DiagnosticEngine &Diags, + SourceLoc DiagLoc) { + auto Status = getStatusOfDependency(FS, ModulePath, In.getPath(), + Diags, DiagLoc); + if (!Status) return None; + + // If the sizes differ, then we know the file has changed. + if (Status->getSize() != In.getSize()) return None; + + // Otherwise, if this dependency is verified by modification time, check + // it vs. the modification time of the file. + uint64_t mtime = + Status->getLastModificationTime().time_since_epoch().count(); + + if (In.isModificationTimeBased()) + return mtime == In.getModificationTime() ? Status : None; + + // Slow path: if the dependency is verified by content hash, check it vs. the + // hash of the file. + auto buf = getBufferOfDependency(FS, ModulePath, In.getPath(), + Diags, DiagLoc); + if (!buf) return None; + + if (xxHash64(buf->getBuffer()) == In.getContentHash()) + return Status; + + return None; +} + } // Check that the output .swiftmodule file is at least as new as all the @@ -289,6 +315,7 @@ collectDepsForSerialization(llvm::vfs::FileSystem &FS, CompilerInstance &SubInstance, StringRef InPath, StringRef ModuleCachePath, SmallVectorImpl &Deps, + bool IsHashBased, DiagnosticEngine &Diags, SourceLoc DiagLoc, DependencyTracker *OuterTracker) { auto DTDeps = SubInstance.getDependencyTracker()->getDependencies(); @@ -297,17 +324,25 @@ collectDepsForSerialization(llvm::vfs::FileSystem &FS, llvm::StringSet<> AllDepNames; for (auto const &DepName : InitialDepNames) { if (AllDepNames.insert(DepName).second && OuterTracker) { - OuterTracker->addDependency(DepName, /*IsSystem=*/false); + OuterTracker->addDependency(DepName, /*IsSystem=*/false); } - auto Status = getStatusOfDependency(FS, InPath, DepName, Diags, DiagLoc); - if (!Status) - return true; auto DepBuf = getBufferOfDependency(FS, InPath, DepName, Diags, DiagLoc); if (!DepBuf) return true; - uint64_t mtime = - Status->getLastModificationTime().time_since_epoch().count(); - Deps.push_back(FileDependency{Status->getSize(), mtime, DepName}); + auto Status = getStatusOfDependency(FS, InPath, DepName, Diags, DiagLoc); + if (!Status) + return true; + + if (IsHashBased) { + uint64_t hash = xxHash64(DepBuf->getBuffer()); + Deps.push_back( + FileDependency::hashBased(DepName, Status->getSize(), hash)); + } else { + uint64_t mtime = + Status->getLastModificationTime().time_since_epoch().count(); + Deps.push_back( + FileDependency::modTimeBased(DepName, Status->getSize(), mtime)); + } if (ModuleCachePath.empty()) continue; @@ -331,10 +366,10 @@ collectDepsForSerialization(llvm::vfs::FileSystem &FS, return true; } for (auto const &SubDep : SubDeps) { - if (AllDepNames.insert(SubDep.Path).second) { + if (AllDepNames.insert(SubDep.getPath()).second) { Deps.push_back(SubDep); if (OuterTracker) - OuterTracker->addDependency(SubDep.Path, /*IsSystem=*/false); + OuterTracker->addDependency(SubDep.getPath(), /*IsSystem=*/false); } } } diff --git a/lib/Serialization/ModuleFile.cpp b/lib/Serialization/ModuleFile.cpp index f8005e992f0e8..2582b9b9bd249 100644 --- a/lib/Serialization/ModuleFile.cpp +++ b/lib/Serialization/ModuleFile.cpp @@ -252,10 +252,19 @@ static bool validateInputBlock( StringRef blobData; unsigned kind = cursor.readRecord(entry.ID, scratch, &blobData); switch (kind) { - case input_block::FILE_DEPENDENCY: - dependencies.push_back(SerializationOptions::FileDependency{ - scratch[0], scratch[1], blobData}); + case input_block::FILE_DEPENDENCY: { + bool isHashBased = scratch[2] != 0; + if (isHashBased) { + dependencies.push_back( + SerializationOptions::FileDependency::hashBased( + blobData, scratch[0], scratch[1])); + } else { + dependencies.push_back( + SerializationOptions::FileDependency::modTimeBased( + blobData, scratch[0], scratch[1])); + } break; + } default: // Unknown metadata record, possibly for use by a future version of the // module format. diff --git a/lib/Serialization/Serialization.cpp b/lib/Serialization/Serialization.cpp index d4f9b6d9be39a..7f21057684bc5 100644 --- a/lib/Serialization/Serialization.cpp +++ b/lib/Serialization/Serialization.cpp @@ -1037,6 +1037,11 @@ static void flattenImportPath(const ModuleDecl::ImportedModule &import, out.append(accessPathElem.first.str()); } +uint64_t getRawModTimeOrHash(const SerializationOptions::FileDependency &dep) { + if (dep.isHashBased()) return dep.getContentHash(); + return dep.getModificationTime(); +} + void Serializer::writeInputBlock(const SerializationOptions &options) { BCBlockRAII restoreBlock(Out, INPUT_BLOCK_ID, 4); input_block::ImportedModuleLayout ImportedModule(Out); @@ -1058,9 +1063,11 @@ void Serializer::writeInputBlock(const SerializationOptions &options) { } for (auto const &dep : options.Dependencies) { - FileDependency.emit(ScratchRecord, dep.Size, - dep.ModificationTime, - dep.Path); + FileDependency.emit(ScratchRecord, + dep.getSize(), + getRawModTimeOrHash(dep), + dep.isHashBased(), + dep.getPath()); } SmallVector allImports; From 67a447f15fe9a9d533629f4aefac502c99bed5db Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Wed, 13 Feb 2019 17:32:38 -0800 Subject: [PATCH 2/3] [Frontend] Add flag to use hash-based dependencies Since prebuilt modules are going to use hashes for their dependencies, we need a flag to turn that behavior on. By default, we use modification times. --- include/swift/Frontend/FrontendOptions.h | 4 ++++ include/swift/Option/FrontendOptions.td | 4 ++++ lib/Frontend/ArgsToFrontendOptionsConverter.cpp | 3 +++ lib/Frontend/ParseableInterfaceSupport.cpp | 6 ++++-- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h index 460ae3bc3f31f..d318a7e99f1ae 100644 --- a/include/swift/Frontend/FrontendOptions.h +++ b/include/swift/Frontend/FrontendOptions.h @@ -267,6 +267,10 @@ class FrontendOptions { /// dependencies as well. bool TrackSystemDeps = false; + /// Should we serialize the hashes of dependencies (vs. the modification + /// times) when compiling a parseable module interface? + bool SerializeParseableModuleInterfaceDependencyHashes = false; + /// The different modes for validating TBD against the LLVM IR. enum class TBDValidationMode { Default, ///< Do the default validation for the current platform. diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index f792e5bc6f451..21cb8501eebd7 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -64,6 +64,10 @@ def emit_interface_path : Separate<["-"], "emit-interface-path">, Alias; +def serialize_parseable_module_interface_dependency_hashes + : Flag<["-"], "serialize-parseable-module-interface-dependency-hashes">, + Flags<[HelpHidden]>; + def tbd_install_name : Separate<["-"], "tbd-install_name">, MetaVarName<"">, HelpText<"The install_name to use in an emitted TBD file">; diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index 33469b654f885..705881f8bf028 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -76,6 +76,9 @@ bool ArgsToFrontendOptionsConverter::convert( Opts.TrackSystemDeps |= Args.hasArg(OPT_track_system_dependencies); + Opts.SerializeParseableModuleInterfaceDependencyHashes |= + Args.hasArg(OPT_serialize_parseable_module_interface_dependency_hashes); + computePrintStatsOptions(); computeDebugTimeOptions(); computeTBDOptions(); diff --git a/lib/Frontend/ParseableInterfaceSupport.cpp b/lib/Frontend/ParseableInterfaceSupport.cpp index 94cea120b36fa..ba0c3599e5e17 100644 --- a/lib/Frontend/ParseableInterfaceSupport.cpp +++ b/lib/Frontend/ParseableInterfaceSupport.cpp @@ -480,8 +480,10 @@ bool ParseableInterfaceModuleLoader::buildSwiftModuleFromSwiftInterface( SerializationOpts.OutputPath = OutPathStr.c_str(); SerializationOpts.ModuleLinkName = FEOpts.ModuleLinkName; SmallVector Deps; - if (collectDepsForSerialization(FS, SubInstance, InPath, ModuleCachePath, - Deps, Diags, DiagLoc, OuterTracker)) { + if (collectDepsForSerialization( + FS, SubInstance, InPath, ModuleCachePath, Deps, + FEOpts.SerializeParseableModuleInterfaceDependencyHashes, + Diags, DiagLoc, OuterTracker)) { SubError = true; return; } From 366bbf48b95f13f5c8fcab5a27e1d9fc9bf921f0 Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Wed, 13 Feb 2019 17:36:55 -0800 Subject: [PATCH 3/3] =?UTF-8?q?[ParseableInterface]=20Add=20=E2=80=98forwa?= =?UTF-8?q?rding=20modules=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A ‘forwarding module’ is a YAML file that’s meant to stand in for a .swiftmodule file and provide an up-to-date description of its dependencies, always using modification times. When a ‘prebuilt module’ is first loaded, we verify that it’s up-to-date by hashing all of its dependencies. Since this is orders of magnitude slower than reading mtimes, we’ll install a `forwarding module` containing the mtimes of the now-validated dependencies. --- .../swift/Frontend/FrontendInputsAndOutputs.h | 2 +- .../Frontend/ParseableInterfaceModuleLoader.h | 164 +++ .../Frontend/ParseableInterfaceSupport.h | 75 +- lib/Frontend/CMakeLists.txt | 1 + lib/Frontend/Frontend.cpp | 1 + .../ParseableInterfaceModuleLoader.cpp | 1026 +++++++++++++++++ lib/Frontend/ParseableInterfaceSupport.cpp | 634 +--------- lib/FrontendTool/FrontendTool.cpp | 4 +- .../Inputs/check-is-forwarding-module.py | 7 + .../force-module-loading-mode-archs.swift | 14 + .../force-module-loading-mode-framework.swift | 15 + .../force-module-loading-mode.swift | 15 + .../prebuilt-module-cache-archs.swift | 4 +- .../prebuilt-module-cache-forwarding.swift | 46 + .../prebuilt-module-cache-recursive.swift | 4 +- .../ModuleCache/prebuilt-module-cache.swift | 4 +- .../SmokeTest.swiftinterface | 2 +- 17 files changed, 1315 insertions(+), 703 deletions(-) create mode 100644 include/swift/Frontend/ParseableInterfaceModuleLoader.h create mode 100644 lib/Frontend/ParseableInterfaceModuleLoader.cpp create mode 100644 test/ParseableInterface/ModuleCache/Inputs/check-is-forwarding-module.py create mode 100644 test/ParseableInterface/ModuleCache/prebuilt-module-cache-forwarding.swift diff --git a/include/swift/Frontend/FrontendInputsAndOutputs.h b/include/swift/Frontend/FrontendInputsAndOutputs.h index 965dc3fda5fa9..fdbfdc31f4b46 100644 --- a/include/swift/Frontend/FrontendInputsAndOutputs.h +++ b/include/swift/Frontend/FrontendInputsAndOutputs.h @@ -173,7 +173,7 @@ class FrontendInputsAndOutputs { private: friend class ArgsToFrontendOptionsConverter; - friend class ParseableInterfaceModuleLoader; + friend class ParseableInterfaceBuilder; void setMainAndSupplementaryOutputs( ArrayRef outputFiles, ArrayRef supplementaryOutputs); diff --git a/include/swift/Frontend/ParseableInterfaceModuleLoader.h b/include/swift/Frontend/ParseableInterfaceModuleLoader.h new file mode 100644 index 0000000000000..757f169593f33 --- /dev/null +++ b/include/swift/Frontend/ParseableInterfaceModuleLoader.h @@ -0,0 +1,164 @@ +//===--- ParseableInterfaceModuleLoader.h - Loads .swiftinterface files ---===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2019 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +/// \file This implements the logic for loading and building parseable module +/// interfaces. +/// +/// === Loading Parseable Modules === +/// +/// If there is a .swiftinterface file corresponding to a given module name +/// present in the frontend's search paths, then this module loader will look in +/// the following places for a module: +/// +/// - First, look in the module cache (specified by -module-cache-path) +/// - We check here first because an existing .swiftmodule might be +/// out-of-date, necessitating a rebuild. If a cached module is out-of-date, +/// it's simply rebuilt. +/// - Next, look adjacent to the .swiftinterface. If we find a module that's +/// either loadable by this compiler, valid, and up-to-date, or totally +/// unreadable, then delegate to the serialized module loader to either load +/// or diagnose it. +/// - Finally, look in the prebuilt module cache (specified +/// by -prebuilt-module-cache-path) +/// +/// If we can't find an appropriate module to load, we can always fall back and +/// recompile the .swiftinterface file. +/// +/// === Dependency Checking === +/// +/// Cached modules keep track of their dependencies' last modification time and +/// file size. This means that checking if a module is up-to-date requires +/// `stat`ing the dependencies and comparing the results from the filesystem +/// with the results in the module. +/// +/// Prebuilt modules, on the other hand, won't have a reliable modification +/// time, as their dependencies live in the SDK. Prebuilt modules will instead +/// keep track of the size and content hash of their dependencies. +/// In order to avoid constantly re-hashing the dependencies, however, we will +/// install a "forwarding module" in the regular cache. +/// This "forwarding module" +/// - Points to the prebuilt module on disk, and +/// - Lists modification times from the last time we verified the content +/// +/// So, to recap, there are 4 kinds of modules: +/// ┌───────────────────────────────┐ +/// │ ┌───┐ ┌───┐ │ +/// │ │ I │ │ M │ │ +/// │ └───┘ └───┘ │ +/// │ .swiftinterface .swiftmodule │ +/// │ ┌───┐ ┌───┐ │ +/// │ │ P │ │ F │ │ +/// │ └───┘ └───┘ │ +/// │ Prebuilt Forwarding │ +/// │ .swiftmodule .swiftmodule │ +/// └───────────────────────────────┘ +/// +/// - Prebuilt modules have hash-based dependencies, cached modules have +/// mod-time-based dependencies +/// - Forwarding modules point to prebuilt modules and augment them with +/// modification times +/// +/// === Example Cache === +/// +/// Here's an example of what's in a prebuilt cache or module cache. +/// +/// Say there are 4 frameworks, each exposing a .swiftinterface file. +/// Then, we pre-build 2 of those frameworks and put them in the prebuilt cache. +/// Finally, we import all 4 of those frameworks while building a project. +/// +/// For the 2 frameworks with modules in the prebuilt cache, we'll have +/// installed 2 forwarding modules. For the other 2 frameworks, we'll have +/// compiled the interfaces and put them in the module cache. +/// +/// ┌─────┐ +/// ┌─────────────────────────────┤ SDK ├─────────────────────────┐ +/// │ ┌────────────────┐ └─────┘ ┌────────────────┐ │ +/// │ ┌───────┤ Framework Dirs ├────────┐ ┌┤ Prebuilt Cache ├┐ │ +/// │ │ └────────────────┘ │ │└────────────────┘│ │ +/// │ │ ┌───┐ ┌───┐ ┌───┐ ┌───┐ │ │ ┌───┐ ┌───┐ │ │ +/// │ │ │ I │ │ I │ │ I │ │ I │◀─┼───┼───│ P │ │ P │◀═╗│ │ +/// │ │ └───┘ └───┘ └───┘ └───┘ │ │ └───┘ └───┘ ║│ │ +/// │ │ ▲ ▲ ▲ │ │ ▲ │ ║│ │ +/// │ └────┼───────┼───────┼────────────┘ └─────╫──────┼────╫┘ │ +/// │ │ │ └──────────────────────╫──────┘ ║ │ +/// └───────┼───────┼──────────────────────────────╫───────────╫──┘ +/// │ │ ┌───────────────┐ ║ ║ +/// │ ┌────┼───┤ Module Cache ├────────┐ ║ ║ +/// │ │ │ └───────────────┘ │ ║ ║ +/// │ │ ┌───┐ ┌───┐ ┌───┐ ┌───┐ │ ║ ║ +/// │ │ │ M │ │ M │ │ F │ │ F │ │ ║ ║ +/// │ │ └───┘ └───┘ └───┘ └───┘ │ ║ ║ +/// │ │ │ ║ ╚════╪═╝ ║ +/// │ └────────────┼───────╫────────────┘ ║ +/// └───────────────┘ ╚══════════════════════════╝ +/// +//===----------------------------------------------------------------------===// + +#include "swift/Basic/LLVM.h" +#include "swift/Frontend/ParseableInterfaceSupport.h" +#include "swift/Serialization/SerializedModuleLoader.h" + +namespace clang { + class CompilerInstance; +} + +namespace swift { + +/// A ModuleLoader that runs a subordinate \c CompilerInvocation and +/// \c CompilerInstance to convert .swiftinterface files to .swiftmodule +/// files on the fly, caching the resulting .swiftmodules in the module cache +/// directory, and loading the serialized .swiftmodules from there. +class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase { + explicit ParseableInterfaceModuleLoader(ASTContext &ctx, StringRef cacheDir, + StringRef prebuiltCacheDir, + DependencyTracker *tracker, + ModuleLoadingMode loadMode) + : SerializedModuleLoaderBase(ctx, tracker, loadMode), + CacheDir(cacheDir), PrebuiltCacheDir(prebuiltCacheDir) + {} + + std::string CacheDir; + std::string PrebuiltCacheDir; + + std::error_code findModuleFilesInDirectory( + AccessPathElem ModuleID, StringRef DirPath, StringRef ModuleFilename, + StringRef ModuleDocFilename, + std::unique_ptr *ModuleBuffer, + std::unique_ptr *ModuleDocBuffer) override; + + +public: + static std::unique_ptr + create(ASTContext &ctx, StringRef cacheDir, StringRef prebuiltCacheDir, + DependencyTracker *tracker, ModuleLoadingMode loadMode) { + return std::unique_ptr( + new ParseableInterfaceModuleLoader(ctx, cacheDir, prebuiltCacheDir, + tracker, loadMode)); + } + + /// Unconditionally build \p InPath (a swiftinterface file) to \p OutPath (as + /// a swiftmodule file). + /// + /// A simplified version of the core logic in #openModuleFiles, mostly for + /// testing purposes. + static bool buildSwiftModuleFromSwiftInterface( + ASTContext &Ctx, StringRef CacheDir, StringRef PrebuiltCacheDir, + StringRef ModuleName, StringRef InPath, StringRef OutPath, + bool SerializeDependencyHashes); +}; + +/// Extract the specified-or-defaulted -module-cache-path that winds up in +/// the clang importer, for reuse as the .swiftmodule cache path when +/// building a ParseableInterfaceModuleLoader. +std::string +getModuleCachePathFromClang(const clang::CompilerInstance &Instance); + +} diff --git a/include/swift/Frontend/ParseableInterfaceSupport.h b/include/swift/Frontend/ParseableInterfaceSupport.h index 1829ee21e2238..f0f819521e122 100644 --- a/include/swift/Frontend/ParseableInterfaceSupport.h +++ b/include/swift/Frontend/ParseableInterfaceSupport.h @@ -2,7 +2,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2018 Apple Inc. and the Swift project authors +// Copyright (c) 2019 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -14,9 +14,13 @@ #define SWIFT_FRONTEND_PARSEABLEINTERFACESUPPORT_H #include "swift/Basic/LLVM.h" -#include "swift/Serialization/SerializedModuleLoader.h" +#include "swift/Basic/Version.h" #include "llvm/Support/Regex.h" +#define SWIFT_INTERFACE_FORMAT_VERSION_KEY "swift-interface-format-version" +#define SWIFT_TOOLS_VERSION_KEY "swift-tools-version" +#define SWIFT_MODULE_FLAGS_KEY "swift-module-flags" + namespace swift { class ModuleDecl; @@ -29,6 +33,8 @@ struct ParseableInterfaceOptions { std::string ParseableInterfaceFlags; }; +extern version::Version InterfaceFormatVersion; + llvm::Regex getSwiftInterfaceFormatVersionRegex(); llvm::Regex getSwiftInterfaceModuleFlagsRegex(); @@ -49,71 +55,6 @@ bool emitParseableInterface(raw_ostream &out, ParseableInterfaceOptions const &Opts, ModuleDecl *M); -/// Extract the specified-or-defaulted -module-cache-path that winds up in -/// the clang importer, for reuse as the .swiftmodule cache path when -/// building a ParseableInterfaceModuleLoader. -std::string -getModuleCachePathFromClang(const clang::CompilerInstance &Instance); - -/// A ModuleLoader that runs a subordinate \c CompilerInvocation and \c -/// CompilerInstance to convert .swiftinterface files to .swiftmodule -/// files on the fly, caching the resulting .swiftmodules in the module cache -/// directory, and loading the serialized .swiftmodules from there. -class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase { - explicit ParseableInterfaceModuleLoader(ASTContext &ctx, StringRef cacheDir, - StringRef prebuiltCacheDir, - DependencyTracker *tracker, - ModuleLoadingMode loadMode) - : SerializedModuleLoaderBase(ctx, tracker, loadMode), - CacheDir(cacheDir), PrebuiltCacheDir(prebuiltCacheDir) - {} - - std::string CacheDir; - std::string PrebuiltCacheDir; - - /// Wire up the SubInvocation's InputsAndOutputs to contain both input and - /// output filenames. - /// - /// This is a method rather than a helper function in the implementation file - /// because it accesses non-public bits of FrontendInputsAndOutputs. - static void configureSubInvocationInputsAndOutputs( - CompilerInvocation &SubInvocation, StringRef InPath, StringRef OutPath); - - static bool buildSwiftModuleFromSwiftInterface( - llvm::vfs::FileSystem &FS, DiagnosticEngine &Diags, SourceLoc DiagLoc, - CompilerInvocation &SubInvocation, StringRef InPath, StringRef OutPath, - StringRef ModuleCachePath, DependencyTracker *OuterTracker, - bool ShouldSerializeDeps); - - std::error_code findModuleFilesInDirectory( - AccessPathElem ModuleID, StringRef DirPath, StringRef ModuleFilename, - StringRef ModuleDocFilename, - std::unique_ptr *ModuleBuffer, - std::unique_ptr *ModuleDocBuffer) override; - -public: - static std::unique_ptr - create(ASTContext &ctx, StringRef cacheDir, StringRef prebuiltCacheDir, - DependencyTracker *tracker, ModuleLoadingMode loadMode) { - return std::unique_ptr( - new ParseableInterfaceModuleLoader(ctx, cacheDir, prebuiltCacheDir, - tracker, loadMode)); - } - - /// Unconditionally build \p InPath (a swiftinterface file) to \p OutPath (as - /// a swiftmodule file). - /// - /// A simplified version of the core logic in #openModuleFiles, mostly for - /// testing purposes. - static bool buildSwiftModuleFromSwiftInterface(ASTContext &Ctx, - StringRef CacheDir, - StringRef PrebuiltCacheDir, - StringRef ModuleName, - StringRef InPath, - StringRef OutPath); -}; - - } // end namespace swift #endif diff --git a/lib/Frontend/CMakeLists.txt b/lib/Frontend/CMakeLists.txt index 575bf7320043b..9a311be9c97d1 100644 --- a/lib/Frontend/CMakeLists.txt +++ b/lib/Frontend/CMakeLists.txt @@ -7,6 +7,7 @@ add_swift_host_library(swiftFrontend STATIC Frontend.cpp FrontendInputsAndOutputs.cpp FrontendOptions.cpp + ParseableInterfaceModuleLoader.cpp ParseableInterfaceSupport.cpp PrintingDiagnosticConsumer.cpp SerializedDiagnosticConsumer.cpp diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index 6881fc6f3d6da..388db393c5c89 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -24,6 +24,7 @@ #include "swift/Basic/SourceManager.h" #include "swift/Basic/Statistic.h" #include "swift/DWARFImporter/DWARFImporter.h" +#include "swift/Frontend/ParseableInterfaceModuleLoader.h" #include "swift/Parse/DelayedParsingCallbacks.h" #include "swift/Parse/Lexer.h" #include "swift/SIL/SILModule.h" diff --git a/lib/Frontend/ParseableInterfaceModuleLoader.cpp b/lib/Frontend/ParseableInterfaceModuleLoader.cpp new file mode 100644 index 0000000000000..4de66307c8aec --- /dev/null +++ b/lib/Frontend/ParseableInterfaceModuleLoader.cpp @@ -0,0 +1,1026 @@ +//===-- ParseableInterfaceModuleLoader.cpp - Loads .swiftinterface files --===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2019 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "textual-module-interface" +#include "swift/Frontend/ParseableInterfaceModuleLoader.h" +#include "swift/AST/ASTContext.h" +#include "swift/AST/DiagnosticsFrontend.h" +#include "swift/AST/DiagnosticsSema.h" +#include "swift/AST/FileSystem.h" +#include "swift/AST/Module.h" +#include "swift/AST/ProtocolConformance.h" +#include "swift/Basic/Lazy.h" +#include "swift/Basic/STLExtras.h" +#include "swift/Frontend/Frontend.h" +#include "swift/Frontend/ParseableInterfaceSupport.h" +#include "swift/Frontend/PrintingDiagnosticConsumer.h" +#include "swift/SILOptimizer/PassManager/Passes.h" +#include "swift/Serialization/ModuleFormat.h" +#include "swift/Serialization/SerializationOptions.h" +#include "swift/Serialization/Validation.h" +#include "clang/Basic/Module.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Lex/PreprocessorOptions.h" +#include "clang/Lex/HeaderSearch.h" +#include "clang/Frontend/CompilerInstance.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/xxhash.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/CrashRecoveryContext.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/StringSaver.h" +#include "llvm/Support/YAMLTraits.h" + +using namespace swift; +using FileDependency = SerializationOptions::FileDependency; + +/// Extract the specified-or-defaulted -module-cache-path that winds up in +/// the clang importer, for reuse as the .swiftmodule cache path when +/// building a ParseableInterfaceModuleLoader. +std::string +swift::getModuleCachePathFromClang(const clang::CompilerInstance &Clang) { + if (!Clang.hasPreprocessor()) + return ""; + std::string SpecificModuleCachePath = Clang.getPreprocessor() + .getHeaderSearchInfo() + .getModuleCachePath(); + + // The returned-from-clang module cache path includes a suffix directory + // that is specific to the clang version and invocation; we want the + // directory above that. + return llvm::sys::path::parent_path(SpecificModuleCachePath); +} + +#pragma mark - Forwarding Modules + +namespace { + +/// Describes a "forwarding module", that is, a .swiftmodule that's actually +/// a YAML file inside, pointing to a the original .swiftmodule but describing +/// a different dependency resolution strategy. +struct ForwardingModule { + /// The path to the original .swiftmodule in the prebuilt cache. + std::string underlyingModulePath; + + /// Describes a set of file-based dependencies with their size and + /// modification time stored. This is slightly different from + /// \c SerializationOptions::FileDependency, because this type needs to be + /// serializable to and from YAML. + struct Dependency { + std::string path; + uint64_t size; + uint64_t lastModificationTime; + }; + std::vector dependencies; + unsigned version = 1; + + ForwardingModule() = default; + ForwardingModule(StringRef underlyingModulePath) + : underlyingModulePath(underlyingModulePath) {} + + /// Loads the contents of the forwarding module whose contents lie in + /// the provided buffer, and returns a new \c ForwardingModule, or an error + /// if the YAML could not be parsed. + static llvm::ErrorOr load(const llvm::MemoryBuffer &buf); + + /// Adds a given dependency to the dependencies list. + void addDependency(StringRef path, uint64_t size, uint64_t modTime) { + dependencies.push_back({path.str(), size, modTime}); + } +}; + +} // end anonymous namespace + +#pragma mark - YAML Serialization + +namespace llvm { + namespace yaml { + template <> + struct MappingTraits { + static void mapping(IO &io, ForwardingModule::Dependency &dep) { + io.mapRequired("mtime", dep.lastModificationTime); + io.mapRequired("path", dep.path); + io.mapRequired("size", dep.size); + } + }; + + template <> + struct SequenceElementTraits { + static const bool flow = false; + }; + + template <> + struct MappingTraits { + static void mapping(IO &io, ForwardingModule &module) { + io.mapRequired("path", module.underlyingModulePath); + io.mapRequired("dependencies", module.dependencies); + io.mapRequired("version", module.version); + } + }; + } +} // end namespace llvm + +llvm::ErrorOr +ForwardingModule::load(const llvm::MemoryBuffer &buf) { + llvm::yaml::Input yamlIn(buf.getBuffer()); + ForwardingModule fwd; + yamlIn >> fwd; + if (yamlIn.error()) + return yamlIn.error(); + + // We only currently support Version 1 of the forwarding module format. + if (fwd.version != 1) + return std::make_error_code(std::errc::not_supported); + return std::move(fwd); +} + +#pragma mark - Module Discovery + +namespace { + +/// The result of a search for a module either alongside an interface, in the +/// module cache, or in the prebuilt module cache. +class DiscoveredModule { + /// The kind of module we've found. + enum class Kind { + /// A module that's either alongside the swiftinterface or in the + /// module cache. + Normal, + + /// A module that resides in the prebuilt cache, and has hash-based + /// dependencies. + Prebuilt, + + /// A 'forwarded' module. This is a module in the prebuilt cache, but whose + /// dependencies live in a forwarding module. + /// \sa ForwardingModule. + Forwarded + }; + + /// The kind of module that's been discovered. + const Kind kind; + + DiscoveredModule(StringRef path, Kind kind): kind(kind), path(path) {} +public: + /// The path to the discovered serialized .swiftmodule on disk. + const std::string path; + + /// Creates a \c Normal discovered module. + static DiscoveredModule normal(StringRef path) { + return { path, Kind::Normal }; + } + + /// Creates a \c Prebuilt discovered module. + static DiscoveredModule prebuilt(StringRef path) { + return { path, Kind::Prebuilt }; + } + + /// Creates a \c Forwarded discovered module, whose dependencies have been + /// externally validated by a \c ForwardingModule. + static DiscoveredModule forwarded(StringRef path) { + return { path, Kind::Forwarded }; + } + + bool isNormal() const { return kind == Kind::Normal; } + bool isPrebuilt() const { return kind == Kind::Prebuilt; } + bool isForwarding() const { return kind == Kind::Forwarded; } +}; + +} // end anonymous namespace + +#pragma mark - Common utilities + +namespace path = llvm::sys::path; + +static bool serializedASTLooksValid(const llvm::MemoryBuffer &buf) { + auto VI = serialization::validateSerializedAST(buf.getBuffer()); + return VI.status == serialization::Status::Valid; +} + +static std::unique_ptr getBufferOfDependency( + llvm::vfs::FileSystem &fs, StringRef depPath, StringRef interfacePath, + DiagnosticEngine &diags, SourceLoc diagnosticLoc) { + auto depBuf = fs.getBufferForFile(depPath, /*FileSize=*/-1, + /*RequiresNullTerminator=*/false); + if (!depBuf) { + diags.diagnose(diagnosticLoc, + diag::missing_dependency_of_parseable_module_interface, + depPath, interfacePath, depBuf.getError().message()); + return nullptr; + } + return std::move(depBuf.get()); +} + +static Optional getStatusOfDependency( + llvm::vfs::FileSystem &fs, StringRef depPath, StringRef interfacePath, + DiagnosticEngine &diags, SourceLoc diagnosticLoc) { + auto status = fs.status(depPath); + if (!status) { + diags.diagnose(diagnosticLoc, + diag::missing_dependency_of_parseable_module_interface, + depPath, interfacePath, status.getError().message()); + return None; + } + return status.get(); +} + +#pragma mark - Module Building + +/// Builds a parseable module interface into a .swiftmodule at the provided +/// output path. +/// \note Needs to be in the swift namespace so CompilerInvocation can see it. +class swift::ParseableInterfaceBuilder { + ASTContext &ctx; + llvm::vfs::FileSystem &fs; + DiagnosticEngine &diags; + const StringRef interfacePath; + const StringRef moduleName; + const StringRef moduleCachePath; + const StringRef prebuiltCachePath; + const bool serializeDependencyHashes; + const SourceLoc diagnosticLoc; + DependencyTracker *const dependencyTracker; + CompilerInvocation subInvocation; + + void configureSubInvocationInputsAndOutputs(StringRef OutPath) { + auto &SubFEOpts = subInvocation.getFrontendOptions(); + SubFEOpts.RequestedAction = FrontendOptions::ActionType::EmitModuleOnly; + SubFEOpts.EnableParseableModuleInterface = true; + SubFEOpts.InputsAndOutputs.addPrimaryInputFile(interfacePath); + SupplementaryOutputPaths SOPs; + SOPs.ModuleOutputPath = OutPath.str(); + + // Pick a primary output path that will cause problems to use. + StringRef MainOut = "/"; + SubFEOpts.InputsAndOutputs + .setMainAndSupplementaryOutputs({MainOut}, {SOPs}); + } + + void configureSubInvocation() { + auto &SearchPathOpts = ctx.SearchPathOpts; + auto &LangOpts = ctx.LangOpts; + + // Start with a SubInvocation that copies various state from our + // invoking ASTContext. + subInvocation.setImportSearchPaths(SearchPathOpts.ImportSearchPaths); + subInvocation.setFrameworkSearchPaths(SearchPathOpts.FrameworkSearchPaths); + subInvocation.setSDKPath(SearchPathOpts.SDKPath); + subInvocation.setInputKind(InputFileKind::SwiftModuleInterface); + subInvocation.setRuntimeResourcePath(SearchPathOpts.RuntimeResourcePath); + subInvocation.setTargetTriple(LangOpts.Target); + + subInvocation.setModuleName(moduleName); + subInvocation.setClangModuleCachePath(moduleCachePath); + subInvocation.getFrontendOptions().PrebuiltModuleCachePath = + prebuiltCachePath; + + // Respect the detailed-record preprocessor setting of the parent context. + // This, and the "raw" clang module format it implicitly enables, are + // required by sourcekitd. + if (auto *ClangLoader = ctx.getClangModuleLoader()) { + auto &Opts = ClangLoader->getClangInstance().getPreprocessorOpts(); + if (Opts.DetailedRecord) { + subInvocation.getClangImporterOptions().DetailedPreprocessingRecord = true; + } + } + + // Inhibit warnings from the SubInvocation since we are assuming the user + // is not in a position to fix them. + subInvocation.getDiagnosticOptions().SuppressWarnings = true; + + // Inherit this setting down so that it can affect error diagnostics (mostly + // by making them non-fatal). + subInvocation.getLangOptions().DebuggerSupport = LangOpts.DebuggerSupport; + + // Disable this; deinitializers always get printed with `@objc` even in + // modules that don't import Foundation. + subInvocation.getLangOptions().EnableObjCAttrRequiresFoundation = false; + + // Tell the subinvocation to serialize dependency hashes if asked to do so. + auto &frontendOpts = subInvocation.getFrontendOptions(); + frontendOpts.SerializeParseableModuleInterfaceDependencyHashes = + serializeDependencyHashes; + } + + bool extractSwiftInterfaceVersionAndArgs( + swift::version::Version &Vers, llvm::StringSaver &SubArgSaver, + SmallVectorImpl &SubArgs) { + auto FileOrError = swift::vfs::getFileOrSTDIN(fs, interfacePath); + if (!FileOrError) { + diags.diagnose(diagnosticLoc, diag::error_open_input_file, + interfacePath, FileOrError.getError().message()); + return true; + } + auto SB = FileOrError.get()->getBuffer(); + auto VersRe = getSwiftInterfaceFormatVersionRegex(); + auto FlagRe = getSwiftInterfaceModuleFlagsRegex(); + SmallVector VersMatches, FlagMatches; + if (!VersRe.match(SB, &VersMatches)) { + diags.diagnose(diagnosticLoc, + diag::error_extracting_version_from_parseable_interface); + return true; + } + if (!FlagRe.match(SB, &FlagMatches)) { + diags.diagnose(diagnosticLoc, + diag::error_extracting_flags_from_parseable_interface); + return true; + } + assert(VersMatches.size() == 2); + assert(FlagMatches.size() == 2); + Vers = swift::version::Version(VersMatches[1], SourceLoc(), &diags); + llvm::cl::TokenizeGNUCommandLine(FlagMatches[1], SubArgSaver, SubArgs); + return false; + } + + /// Populate the provided \p Deps with \c FileDependency entries including: + /// + /// - All the dependencies mentioned by \p SubInstance's DependencyTracker, + /// that were read while compiling the module. + /// + /// - For any file in the latter set that is itself a .swiftmodule + /// living in \p cachePath, all of _its_ dependencies, copied + /// out to avoid having to do recursive scanning when rechecking this + /// dependency in the future. + bool collectDepsForSerialization(CompilerInstance &SubInstance, + SmallVectorImpl &Deps, + bool IsHashBased) { + auto DTDeps = SubInstance.getDependencyTracker()->getDependencies(); + SmallVector InitialDepNames(DTDeps.begin(), DTDeps.end()); + InitialDepNames.push_back(interfacePath); + llvm::StringSet<> AllDepNames; + for (auto const &DepName : InitialDepNames) { + if (AllDepNames.insert(DepName).second && dependencyTracker) { + dependencyTracker->addDependency(DepName, /*isSystem*/false); + } + + /// Lazily load the dependency buffer if we need it. If we're not + /// dealing with a hash-based dependencies, and if the dependency is + /// not a .swiftmodule, we can avoid opening the buffer. + std::unique_ptr DepBuf = nullptr; + auto getDepBuf = [&]() -> llvm::MemoryBuffer * { + if (DepBuf) return DepBuf.get(); + if (auto Buf = getBufferOfDependency(fs, DepName, interfacePath, + diags, diagnosticLoc)) { + DepBuf = std::move(Buf); + return DepBuf.get(); + } + return nullptr; + }; + + auto Status = getStatusOfDependency(fs, DepName, interfacePath, + diags, diagnosticLoc); + if (!Status) + return true; + + if (IsHashBased) { + auto buf = getDepBuf(); + if (!buf) return true; + uint64_t hash = xxHash64(buf->getBuffer()); + Deps.push_back( + FileDependency::hashBased(DepName, Status->getSize(), hash)); + } else { + uint64_t mtime = + Status->getLastModificationTime().time_since_epoch().count(); + Deps.push_back( + FileDependency::modTimeBased(DepName, Status->getSize(), mtime)); + } + + if (moduleCachePath.empty()) + continue; + + // If Dep is itself a .swiftmodule in the cache dir, pull out its deps + // and include them in our own, so we have a single-file view of + // transitive deps: removes redundancies, and avoids opening and reading + // multiple swiftmodules during future loads. + auto Ext = llvm::sys::path::extension(DepName); + auto Ty = file_types::lookupTypeForExtension(Ext); + if (Ty == file_types::TY_SwiftModuleFile && + DepName.startswith(moduleCachePath)) { + auto buf = getDepBuf(); + if (!buf) return true; + SmallVector SubDeps; + auto VI = serialization::validateSerializedAST(buf->getBuffer(), + /*ExtendedValidationInfo=*/nullptr, &SubDeps); + if (VI.status != serialization::Status::Valid) { + diags.diagnose(diagnosticLoc, + diag::error_extracting_dependencies_from_cached_module, + DepName); + return true; + } + for (auto const &SubDep : SubDeps) { + if (AllDepNames.insert(SubDep.getPath()).second) { + Deps.push_back(SubDep); + if (dependencyTracker) + dependencyTracker->addDependency(SubDep.getPath(), + /*IsSystem=*/false); + } + } + } + } + return false; + } + +public: + ParseableInterfaceBuilder(ASTContext &ctx, + StringRef interfacePath, + StringRef moduleName, + StringRef moduleCachePath, + StringRef prebuiltCachePath, + bool serializeDependencyHashes = false, + SourceLoc diagnosticLoc = SourceLoc(), + DependencyTracker *tracker = nullptr) + : ctx(ctx), fs(*ctx.SourceMgr.getFileSystem()), diags(ctx.Diags), + interfacePath(interfacePath), moduleName(moduleName), + moduleCachePath(moduleCachePath), prebuiltCachePath(prebuiltCachePath), + serializeDependencyHashes(serializeDependencyHashes), + diagnosticLoc(diagnosticLoc), dependencyTracker(tracker) { + configureSubInvocation(); + } + + const CompilerInvocation &getSubInvocation() const { + return subInvocation; + } + + bool buildSwiftModule(StringRef OutPath, bool ShouldSerializeDeps) { + bool SubError = false; + bool RunSuccess = llvm::CrashRecoveryContext().RunSafelyOnThread([&] { + // Note that we don't assume cachePath is the same as the Clang + // module cache path at this point. + if (!moduleCachePath.empty()) + (void)llvm::sys::fs::create_directory(moduleCachePath); + + configureSubInvocationInputsAndOutputs(OutPath); + + FrontendOptions &FEOpts = subInvocation.getFrontendOptions(); + const auto &InputInfo = FEOpts.InputsAndOutputs.firstInput(); + StringRef InPath = InputInfo.file(); + const auto &OutputInfo = + InputInfo.getPrimarySpecificPaths().SupplementaryOutputs; + StringRef OutPath = OutputInfo.ModuleOutputPath; + + llvm::BumpPtrAllocator SubArgsAlloc; + llvm::StringSaver SubArgSaver(SubArgsAlloc); + SmallVector SubArgs; + swift::version::Version Vers; + if (extractSwiftInterfaceVersionAndArgs(Vers, SubArgSaver, SubArgs)) { + SubError = true; + return; + } + + // For now: we support anything with the same "major version" and assume + // minor versions might be interesting for debugging, or special-casing a + // compatible field variant. + if (Vers.asMajorVersion() != InterfaceFormatVersion.asMajorVersion()) { + diags.diagnose(diagnosticLoc, + diag::unsupported_version_of_parseable_interface, + interfacePath, Vers); + SubError = true; + return; + } + + SmallString<32> ExpectedModuleName = subInvocation.getModuleName(); + if (subInvocation.parseArgs(SubArgs, diags)) { + SubError = true; + return; + } + + if (subInvocation.getModuleName() != ExpectedModuleName) { + auto DiagKind = diag::serialization_name_mismatch; + if (subInvocation.getLangOptions().DebuggerSupport) + DiagKind = diag::serialization_name_mismatch_repl; + diags.diagnose(diagnosticLoc, DiagKind, subInvocation.getModuleName(), + ExpectedModuleName); + SubError = true; + return; + } + + // Optimize emitted modules. This has to happen after we parse arguments, + // because parseSILOpts would override the current optimization mode. + subInvocation.getSILOptions().OptMode = OptimizationMode::ForSpeed; + + // Build the .swiftmodule; this is a _very_ abridged version of the logic + // in performCompile in libFrontendTool, specialized, to just the one + // module-serialization task we're trying to do here. + LLVM_DEBUG(llvm::dbgs() << "Setting up instance to compile " + << InPath << " to " << OutPath << "\n"); + CompilerInstance SubInstance; + SubInstance.getSourceMgr().setFileSystem(&fs); + + ForwardingDiagnosticConsumer FDC(diags); + SubInstance.addDiagnosticConsumer(&FDC); + + SubInstance.createDependencyTracker(/*TrackSystemDeps=*/true); + if (SubInstance.setup(subInvocation)) { + SubError = true; + return; + } + + LLVM_DEBUG(llvm::dbgs() << "Performing sema\n"); + SubInstance.performSema(); + if (SubInstance.getASTContext().hadError()) { + LLVM_DEBUG(llvm::dbgs() << "encountered errors\n"); + SubError = true; + return; + } + + SILOptions &SILOpts = subInvocation.getSILOptions(); + auto Mod = SubInstance.getMainModule(); + auto SILMod = performSILGeneration(Mod, SILOpts); + if (!SILMod) { + LLVM_DEBUG(llvm::dbgs() << "SILGen did not produce a module\n"); + SubError = true; + return; + } + + // Setup the callbacks for serialization, which can occur during the + // optimization pipeline. + SerializationOptions SerializationOpts; + std::string OutPathStr = OutPath; + SerializationOpts.OutputPath = OutPathStr.c_str(); + SerializationOpts.ModuleLinkName = FEOpts.ModuleLinkName; + SmallVector Deps; + if (collectDepsForSerialization(SubInstance, Deps, + FEOpts.SerializeParseableModuleInterfaceDependencyHashes)) { + SubError = true; + return; + } + if (ShouldSerializeDeps) + SerializationOpts.Dependencies = Deps; + SILMod->setSerializeSILAction([&]() { + serialize(Mod, SerializationOpts, SILMod.get()); + }); + + LLVM_DEBUG(llvm::dbgs() << "Running SIL processing passes\n"); + if (SubInstance.performSILProcessing(SILMod.get())) { + LLVM_DEBUG(llvm::dbgs() << "encountered errors\n"); + SubError = true; + return; + } + + SubError = SubInstance.getDiags().hadAnyError(); + }); + return !RunSuccess || SubError; + } +}; + +#pragma mark - Module Loading + +namespace { + +/// Handles the details of loading parseable interfaces as modules, and will +/// do the necessary lookup to determine if we should be loading from the +/// normal cache, the prebuilt cache, a module adjacent to the interface, or +/// a module that we'll build from a parseable interface. +class ParseableInterfaceModuleLoaderImpl { + using AccessPathElem = std::pair; + friend class swift::ParseableInterfaceModuleLoader; + ASTContext &ctx; + llvm::vfs::FileSystem &fs; + DiagnosticEngine &diags; + const StringRef modulePath; + const std::string interfacePath; + const StringRef moduleName; + const StringRef prebuiltCacheDir; + const StringRef cacheDir; + const SourceLoc diagnosticLoc; + DependencyTracker *const dependencyTracker; + const ModuleLoadingMode loadMode; + + ParseableInterfaceModuleLoaderImpl( + ASTContext &ctx, StringRef modulePath, StringRef interfacePath, + StringRef moduleName, StringRef cacheDir, StringRef prebuiltCacheDir, + SourceLoc diagLoc, DependencyTracker *dependencyTracker = nullptr, + ModuleLoadingMode loadMode = ModuleLoadingMode::PreferSerialized) + : ctx(ctx), fs(*ctx.SourceMgr.getFileSystem()), diags(ctx.Diags), + modulePath(modulePath), interfacePath(interfacePath), + moduleName(moduleName), prebuiltCacheDir(prebuiltCacheDir), + cacheDir(cacheDir), diagnosticLoc(diagLoc), + dependencyTracker(dependencyTracker), loadMode(loadMode) {} + + /// Construct a cache key for the .swiftmodule being generated. There is a + /// balance to be struck here between things that go in the cache key and + /// things that go in the "up to date" check of the cache entry. We want to + /// avoid fighting over a single cache entry too much when (say) running + /// different compiler versions on the same machine or different inputs + /// that happen to have the same short module name, so we will disambiguate + /// those in the key. But we want to invalidate and rebuild a cache entry + /// -- rather than making a new one and potentially filling up the cache + /// with dead entries -- when other factors change, such as the contents of + /// the .swiftinterface input or its dependencies. + std::string getCacheHash(const CompilerInvocation &SubInvocation) { + // Start with the compiler version (which will be either tag names or revs). + // Explicitly don't pass in the "effective" language version -- this would + // mean modules built in different -swift-version modes would rebuild their + // dependencies. + llvm::hash_code H = hash_value(swift::version::getSwiftFullVersion()); + + // Simplest representation of input "identity" (not content) is just a + // pathname, and probably all we can get from the VFS in this regard + // anyways. + H = hash_combine(H, interfacePath); + + // Include the target CPU architecture. In practice, .swiftinterface files + // will be in architecture-specific subdirectories and would have + // architecture-specific pieces #if'd out. However, it doesn't hurt to + // include it, and it guards against mistakenly reusing cached modules + // across architectures. + H = hash_combine(H, SubInvocation.getLangOptions().Target.getArchName()); + + // The SDK path is going to affect how this module is imported, so include + // it. + H = hash_combine(H, SubInvocation.getSDKPath()); + + return llvm::APInt(64, H).toString(36, /*Signed=*/false); + } + + /// Calculate an output filename in \p SubInvocation's cache path that + /// includes a hash of relevant key data. + void computeCachedOutputPath(const CompilerInvocation &SubInvocation, + llvm::SmallString<256> &OutPath) { + OutPath = SubInvocation.getClangModuleCachePath(); + llvm::sys::path::append(OutPath, SubInvocation.getModuleName()); + OutPath.append("-"); + OutPath.append(getCacheHash(SubInvocation)); + OutPath.append("."); + auto OutExt = file_types::getExtension(file_types::TY_SwiftModuleFile); + OutPath.append(OutExt); + } + + // Checks that a dependency read from the cached module is up to date compared + // to the interface file it represents. + bool dependencyIsUpToDate(const FileDependency &dep) { + auto status = getStatusOfDependency(fs, dep.getPath(), interfacePath, + diags, diagnosticLoc); + if (!status) return false; + + // If the sizes differ, then we know the file has changed. + if (status->getSize() != dep.getSize()) return false; + + // Otherwise, if this dependency is verified by modification time, check + // it vs. the modification time of the file. + if (dep.isModificationTimeBased()) { + uint64_t mtime = + status->getLastModificationTime().time_since_epoch().count(); + return mtime == dep.getModificationTime(); + } + + // Slow path: if the dependency is verified by content hash, check it vs. + // the hash of the file. + auto buf = getBufferOfDependency(fs, dep.getPath(), interfacePath, + diags, diagnosticLoc); + if (!buf) return false; + + return xxHash64(buf->getBuffer()) == dep.getContentHash(); + } + + // Check if all the provided file dependencies are up-to-date compared to + // what's currently on disk. + bool dependenciesAreUpToDate(ArrayRef deps) { + for (auto &in : deps) { + if (dependencyTracker) + dependencyTracker->addDependency(in.getPath(), /*isSystem*/false); + if (!dependencyIsUpToDate(in)) { + LLVM_DEBUG(llvm::dbgs() << "Dep " << in.getPath() + << " is directly out of date\n"); + return false; + } + LLVM_DEBUG(llvm::dbgs() << "Dep " << in.getPath() << " is up to date\n"); + } + return true; + } + + // Check that the output .swiftmodule file is at least as new as all the + // dependencies it read when it was built last time. + bool serializedASTBufferIsUpToDate( + const llvm::MemoryBuffer &buf, SmallVectorImpl &allDeps) { + LLVM_DEBUG(llvm::dbgs() << "Validating deps of " << modulePath << "\n"); + auto validationInfo = serialization::validateSerializedAST( + buf.getBuffer(), /*ExtendedValidationInfo=*/nullptr, &allDeps); + + if (validationInfo.status != serialization::Status::Valid) + return false; + + assert(validationInfo.name == moduleName && + "we built a module at this path with a different name?"); + + return dependenciesAreUpToDate(allDeps); + } + + // Check that the output .swiftmodule file is at least as new as all the + // dependencies it read when it was built last time. + bool swiftModuleIsUpToDate(StringRef modulePath, + SmallVectorImpl &AllDeps) { + auto OutBuf = fs.getBufferForFile(modulePath); + if (!OutBuf) + return false; + return serializedASTBufferIsUpToDate(*OutBuf.get(), AllDeps); + } + + // Check that a "forwarding" .swiftmodule file is at least as new as all the + // dependencies it read when it was built last time. Requires that the + // forwarding module has been loaded from disk. + bool forwardingModuleIsUpToDate(const ForwardingModule &fwd, + SmallVectorImpl &deps) { + // First, make sure the underlying module path exists and is valid. + // FIXME: We should preserve this buffer, rather than opening it again + // when loading the module. + auto modBuf = fs.getBufferForFile(fwd.underlyingModulePath); + if (!modBuf || !serializedASTLooksValid(*modBuf.get())) + return false; + + // Next, check the dependencies in the forwarding file. + for (auto &dep : fwd.dependencies) { + deps.push_back( + FileDependency::modTimeBased( + dep.path, dep.size, dep.lastModificationTime)); + } + return dependenciesAreUpToDate(deps); + } + + Optional + computePrebuiltModulePath(llvm::SmallString<256> &scratch) { + namespace path = llvm::sys::path; + StringRef sdkPath = ctx.SearchPathOpts.SDKPath; + + // Check if the interface file comes from the SDK + if (sdkPath.empty() || !hasPrefix(path::begin(interfacePath), + path::end(interfacePath), + path::begin(sdkPath), + path::end(sdkPath))) + return None; + + // Assemble the expected path: $PREBUILT_CACHE/Foo.swiftmodule or + // $PREBUILT_CACHE/Foo.swiftmodule/arch.swiftmodule. Note that there's no + // cache key here. + scratch.append(prebuiltCacheDir); + + // FIXME: Would it be possible to only have architecture-specific names + // here? Then we could skip this check. + StringRef inParentDirName = + path::filename(path::parent_path(interfacePath)); + if (path::extension(inParentDirName) == ".swiftmodule") { + assert(path::stem(inParentDirName) == moduleName); + path::append(scratch, inParentDirName); + } + path::append(scratch, path::filename(modulePath)); + + return scratch.str(); + } + + /// Finds the most appropriate .swiftmodule, whose dependencies are up to + /// date, that we can load for the provided .swiftinterface file. + llvm::ErrorOr discoverUpToDateModuleForInterface( + StringRef modulePath, StringRef cachedOutputPath, + SmallVectorImpl &deps) { + auto notFoundError = + std::make_error_code(std::errc::no_such_file_or_directory); + + // Keep track of whether we should attempt to load a .swiftmodule adjacent + // to the .swiftinterface. + bool shouldLoadAdjacentModule = true; + + switch (loadMode) { + case ModuleLoadingMode::OnlyParseable: + // Always skip both the caches and adjacent modules, and always build the + // parseable interface. + return notFoundError; + case ModuleLoadingMode::PreferParseable: + // If we're in the load mode that prefers .swiftinterfaces, specifically + // skip the module adjacent to the interface, but use the caches if + // they're present. + shouldLoadAdjacentModule = false; + break; + case ModuleLoadingMode::PreferSerialized: + // The rest of the function should be covered by this. + break; + case ModuleLoadingMode::OnlySerialized: + llvm_unreachable("parseable module loader should not have been created"); + } + + // First, check the cached module path. Whatever's in this cache represents + // the most up-to-date knowledge we have about the module. + if (auto cachedBufOrError = fs.getBufferForFile(cachedOutputPath)) { + auto &buf = *cachedBufOrError.get(); + + // Check to see if the module is a serialized AST. If it's not, then we're + // probably dealing with a Forwarding Module, which is a YAML file. + bool isForwardingModule = + !serialization::isSerializedAST(buf.getBuffer()); + + // If it's a forwarding module, load the YAML file from disk and check + // if it's up-to-date. + if (isForwardingModule) { + auto modOrErr = ForwardingModule::load(buf); + if (modOrErr && forwardingModuleIsUpToDate(*modOrErr, deps)) + return DiscoveredModule::forwarded(modOrErr->underlyingModulePath); + // Otherwise, check if the AST buffer itself is up to date. + } else if (serializedASTBufferIsUpToDate(buf, deps)) { + return DiscoveredModule::normal(cachedOutputPath); + } + } + + // If we weren't able to open the file for any reason, including it not + // existing, keep going. + + // If we have a prebuilt cache path, check that too if the interface comes + // from the SDK. + if (!prebuiltCacheDir.empty()) { + llvm::SmallString<256> scratch; + auto path = computePrebuiltModulePath(scratch); + if (path && swiftModuleIsUpToDate(*path, deps)) + return DiscoveredModule::prebuilt(*path); + } + + // Finally, if there's a module adjacent to the .swiftinterface that we can + // _likely_ load (it validates OK and is up to date), bail early with + // errc::not_supported, so the next (serialized) loader in the chain will + // load it. Alternately, if there's a .swiftmodule present but we can't even + // read it (for whatever reason), we should let the other module loader + // diagnose it. + if (!shouldLoadAdjacentModule) + return notFoundError; + + auto adjacentModuleBuffer = fs.getBufferForFile(modulePath); + if (adjacentModuleBuffer) { + if (serializedASTBufferIsUpToDate(*adjacentModuleBuffer.get(), deps)) + return std::make_error_code(std::errc::not_supported); + } else if (adjacentModuleBuffer.getError() != notFoundError) { + return std::make_error_code(std::errc::not_supported); + } + + // Couldn't find an up-to-date .swiftmodule, will need to build module from + // interface. + return notFoundError; + } + + /// Writes the "forwarding module" that will forward to a module in the + /// prebuilt cache. + /// Since forwarding modules track dependencies separately from the module + /// they point to, we'll need to grab the up-to-date file status while doing + /// this. + bool writeForwardingModule(const DiscoveredModule &mod, + StringRef outputPath, + ArrayRef deps) { + assert(mod.isPrebuilt() && + "cannot write forwarding file for non-prebuilt module"); + ForwardingModule fwd(mod.path); + + // FIXME: We need to avoid re-statting all these dependencies, otherwise + // we may record out-of-date information. + auto addDependency = [&](StringRef path) { + auto status = fs.status(path); + uint64_t mtime = + status->getLastModificationTime().time_since_epoch().count(); + fwd.addDependency(path, status->getSize(), mtime); + }; + + // Add the prebuilt module as a dependency of the forwarding module. + addDependency(fwd.underlyingModulePath); + + // Add all the dependencies from the prebuilt module. + for (auto dep : deps) { + addDependency(dep.getPath()); + } + + return withOutputFile(diags, outputPath, + [&](llvm::raw_pwrite_stream &out) { + llvm::yaml::Output yamlWriter(out); + yamlWriter << fwd; + return false; + }); + } + + /// Looks up the best module to load for a given interface. See the main + /// comment in \c ParseableInterfaceModuleLoader.h for an explanation of + /// the module loading strategy. + llvm::ErrorOr findOrBuildLoadableModule() { + + // Set up a builder if we need to build the module. It'll also set up + // the subinvocation we'll need to use to compute the cache paths. + ParseableInterfaceBuilder builder( + ctx, interfacePath, moduleName, cacheDir, prebuiltCacheDir, + /*serializeDependencyHashes*/false, diagnosticLoc, dependencyTracker); + auto &subInvocation = builder.getSubInvocation(); + + // Compute the output path if we're loading or emitting a cached module. + llvm::SmallString<256> cachedOutputPath; + computeCachedOutputPath(subInvocation, cachedOutputPath); + + // Try to find the right module for this interface, either alongside it, + // in the cache, or in the prebuilt cache. + SmallVector allDeps; + auto moduleOrErr = + discoverUpToDateModuleForInterface(modulePath, cachedOutputPath, allDeps); + + // If we errored with anything other than 'no such file or directory', + // fail this load and let the other module loader diagnose it. + if (!moduleOrErr && + moduleOrErr.getError() != std::errc::no_such_file_or_directory) + return moduleOrErr.getError(); + + // We discovered a module! Return the module's path so we know what to load. + if (moduleOrErr) { + auto &module = *moduleOrErr; + // If it's prebuilt, use this time to generate a forwarding module. + if (module.isPrebuilt()) + if (writeForwardingModule(module, cachedOutputPath, allDeps)) + return std::make_error_code(std::errc::not_supported); + + // FIXME: return and load module buffer directly + return module.path; + } + + // We didn't discover a module corresponding to this interface. Build one. + if (builder.buildSwiftModule(cachedOutputPath, /*shouldSerializeDeps*/true)) + return std::make_error_code(std::errc::invalid_argument); + + // FIXME: return and load module buffer directly + return cachedOutputPath.str(); + } +}; + +} // end anonymous namespace + +/// Load a .swiftmodule associated with a .swiftinterface either from a +/// cache or by converting it in a subordinate \c CompilerInstance, caching +/// the results. +std::error_code ParseableInterfaceModuleLoader::findModuleFilesInDirectory( + AccessPathElem ModuleID, StringRef DirPath, StringRef ModuleFilename, + StringRef ModuleDocFilename, + std::unique_ptr *ModuleBuffer, + std::unique_ptr *ModuleDocBuffer) { + + // If running in OnlySerialized mode, ParseableInterfaceModuleLoader + // should not have been constructed at all. + assert(LoadMode != ModuleLoadingMode::OnlySerialized); + + auto &fs = *Ctx.SourceMgr.getFileSystem(); + llvm::SmallString<256> ModPath, InPath; + + // First check to see if the .swiftinterface exists at all. Bail if not. + ModPath = DirPath; + path::append(ModPath, ModuleFilename); + + auto Ext = file_types::getExtension(file_types::TY_SwiftParseableInterfaceFile); + InPath = ModPath; + path::replace_extension(InPath, Ext); + if (!fs.exists(InPath)) + return std::make_error_code(std::errc::no_such_file_or_directory); + + // Create an instance of the Impl to do the heavy lifting. + ParseableInterfaceModuleLoaderImpl Impl( + Ctx, ModPath, InPath, ModuleID.first.str(), + CacheDir, PrebuiltCacheDir, ModuleID.second, dependencyTracker, + LoadMode); + + // Ask the impl to find us a module that we can load or give us an error + // telling us that we couldn't load it. + auto PathOrErr = Impl.findOrBuildLoadableModule(); + if (!PathOrErr) + return PathOrErr.getError(); + std::string FinalPath = std::move(*PathOrErr); + + // Finish off by delegating back up to the SerializedModuleLoaderBase + // routine that can load the recently-manufactured serialized module. + LLVM_DEBUG(llvm::dbgs() << "Loading " << FinalPath + << " via normal module loader\n"); + llvm::SmallString<256> DocPath{DirPath}; + path::append(DocPath, ModuleDocFilename); + auto ErrorCode = SerializedModuleLoaderBase::openModuleFiles( + ModuleID, FinalPath, DocPath, ModuleBuffer, ModuleDocBuffer); + LLVM_DEBUG(llvm::dbgs() << "Loaded " << FinalPath + << " via normal module loader"); + if (ErrorCode) { + LLVM_DEBUG(llvm::dbgs() << " with error: " << ErrorCode.message()); + } + LLVM_DEBUG(llvm::dbgs() << "\n"); + return ErrorCode; +} + + +bool ParseableInterfaceModuleLoader::buildSwiftModuleFromSwiftInterface( + ASTContext &Ctx, StringRef CacheDir, StringRef PrebuiltCacheDir, + StringRef ModuleName, StringRef InPath, StringRef OutPath, + bool SerializeDependencyHashes) { + ParseableInterfaceBuilder builder(Ctx, InPath, ModuleName, + CacheDir, PrebuiltCacheDir, + SerializeDependencyHashes); + // FIXME: We really only want to serialize 'important' dependencies here, and + // make them relocatable (SDK-relative) if we want to ship the built + // swiftmodules to another machine. Just track them as absolute paths + // for now, so we can test the dependency tracking locally. + return builder.buildSwiftModule(OutPath, /*shouldSerializeDeps*/true); +} diff --git a/lib/Frontend/ParseableInterfaceSupport.cpp b/lib/Frontend/ParseableInterfaceSupport.cpp index ba0c3599e5e17..ce440f967785b 100644 --- a/lib/Frontend/ParseableInterfaceSupport.cpp +++ b/lib/Frontend/ParseableInterfaceSupport.cpp @@ -2,7 +2,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2018 Apple Inc. and the Swift project authors +// Copyright (c) 2019 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -10,7 +10,6 @@ // //===----------------------------------------------------------------------===// -#define DEBUG_TYPE "textual-module-interface" #include "swift/AST/ASTContext.h" #include "swift/AST/Decl.h" #include "swift/AST/DiagnosticsFrontend.h" @@ -24,627 +23,19 @@ #include "swift/Frontend/ParseableInterfaceSupport.h" #include "swift/Frontend/PrintingDiagnosticConsumer.h" #include "swift/SILOptimizer/PassManager/Passes.h" +#include "swift/Serialization/ModuleFormat.h" #include "swift/Serialization/SerializationOptions.h" +#include "swift/Serialization/Validation.h" #include "clang/Basic/Module.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Lex/Preprocessor.h" -#include "clang/Lex/PreprocessorOptions.h" -#include "clang/Lex/HeaderSearch.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/StringSet.h" -#include "llvm/Support/xxhash.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/CrashRecoveryContext.h" #include "llvm/Support/Path.h" #include "llvm/Support/Regex.h" #include "llvm/Support/StringSaver.h" using namespace swift; -using FileDependency = SerializationOptions::FileDependency; -#define SWIFT_INTERFACE_FORMAT_VERSION_KEY "swift-interface-format-version" -#define SWIFT_TOOLS_VERSION_KEY "swift-tools-version" -#define SWIFT_MODULE_FLAGS_KEY "swift-module-flags" - -static swift::version::Version InterfaceFormatVersion({1, 0}); - -static bool -extractSwiftInterfaceVersionAndArgs(DiagnosticEngine &Diags, SourceLoc DiagLoc, - llvm::vfs::FileSystem &FS, - StringRef SwiftInterfacePathIn, - swift::version::Version &Vers, - llvm::StringSaver &SubArgSaver, - SmallVectorImpl &SubArgs) { - auto FileOrError = swift::vfs::getFileOrSTDIN(FS, SwiftInterfacePathIn); - if (!FileOrError) { - Diags.diagnose(DiagLoc, diag::error_open_input_file, - SwiftInterfacePathIn, FileOrError.getError().message()); - return true; - } - auto SB = FileOrError.get()->getBuffer(); - auto VersRe = getSwiftInterfaceFormatVersionRegex(); - auto FlagRe = getSwiftInterfaceModuleFlagsRegex(); - SmallVector VersMatches, FlagMatches; - if (!VersRe.match(SB, &VersMatches)) { - Diags.diagnose(DiagLoc, - diag::error_extracting_version_from_parseable_interface); - return true; - } - if (!FlagRe.match(SB, &FlagMatches)) { - Diags.diagnose(DiagLoc, - diag::error_extracting_flags_from_parseable_interface); - return true; - } - assert(VersMatches.size() == 2); - assert(FlagMatches.size() == 2); - Vers = swift::version::Version(VersMatches[1], SourceLoc(), &Diags); - llvm::cl::TokenizeGNUCommandLine(FlagMatches[1], SubArgSaver, SubArgs); - return false; -} - -static std::unique_ptr -getBufferOfDependency(llvm::vfs::FileSystem &FS, - StringRef ModulePath, StringRef DepPath, - DiagnosticEngine &Diags, SourceLoc DiagLoc) { - auto DepBuf = FS.getBufferForFile(DepPath, /*FileSize=*/-1, - /*RequiresNullTerminator=*/false); - if (!DepBuf) { - Diags.diagnose(DiagLoc, - diag::missing_dependency_of_parseable_module_interface, - DepPath, ModulePath, DepBuf.getError().message()); - return nullptr; - } - return std::move(DepBuf.get()); -} - -static Optional -getStatusOfDependency(llvm::vfs::FileSystem &FS, - StringRef ModulePath, StringRef DepPath, - DiagnosticEngine &Diags, SourceLoc DiagLoc) { - auto Status = FS.status(DepPath); - if (!Status) { - Diags.diagnose(DiagLoc, - diag::missing_dependency_of_parseable_module_interface, - DepPath, ModulePath, Status.getError().message()); - return None; - } - return Status.get(); -} - -/// Construct a cache key for the .swiftmodule being generated. There is a -/// balance to be struck here between things that go in the cache key and -/// things that go in the "up to date" check of the cache entry. We want to -/// avoid fighting over a single cache entry too much when (say) running -/// different compiler versions on the same machine or different inputs -/// that happen to have the same short module name, so we will disambiguate -/// those in the key. But we want to invalidate and rebuild a cache entry -/// -- rather than making a new one and potentially filling up the cache -/// with dead entries -- when other factors change, such as the contents of -/// the .swiftinterface input or its dependencies. -static std::string getCacheHash(ASTContext &Ctx, - const CompilerInvocation &SubInvocation, - StringRef InPath) { - // Start with the compiler version (which will be either tag names or revs). - // Explicitly don't pass in the "effective" language version -- this would - // mean modules built in different -swift-version modes would rebuild their - // dependencies. - llvm::hash_code H = hash_value(swift::version::getSwiftFullVersion()); - - // Simplest representation of input "identity" (not content) is just a - // pathname, and probably all we can get from the VFS in this regard anyways. - H = hash_combine(H, InPath); - - // Include the target CPU architecture. In practice, .swiftinterface files - // will be in architecture-specific subdirectories and would have - // architecture-specific pieces #if'd out. However, it doesn't hurt to - // include it, and it guards against mistakenly reusing cached modules across - // architectures. - H = hash_combine(H, SubInvocation.getLangOptions().Target.getArchName()); - - // The SDK path is going to affect how this module is imported, so include it. - H = hash_combine(H, SubInvocation.getSDKPath()); - - return llvm::APInt(64, H).toString(36, /*Signed=*/false); -} - -static CompilerInvocation -createInvocationForBuildingFromInterface(ASTContext &Ctx, StringRef ModuleName, - StringRef CacheDir, - StringRef PrebuiltCacheDir) { - auto &SearchPathOpts = Ctx.SearchPathOpts; - auto &LangOpts = Ctx.LangOpts; - - CompilerInvocation SubInvocation; - - // Start with a SubInvocation that copies various state from our - // invoking ASTContext. - SubInvocation.setImportSearchPaths(SearchPathOpts.ImportSearchPaths); - SubInvocation.setFrameworkSearchPaths(SearchPathOpts.FrameworkSearchPaths); - SubInvocation.setSDKPath(SearchPathOpts.SDKPath); - SubInvocation.setInputKind(InputFileKind::SwiftModuleInterface); - SubInvocation.setRuntimeResourcePath(SearchPathOpts.RuntimeResourcePath); - SubInvocation.setTargetTriple(LangOpts.Target); - - SubInvocation.setModuleName(ModuleName); - SubInvocation.setClangModuleCachePath(CacheDir); - SubInvocation.getFrontendOptions().PrebuiltModuleCachePath = PrebuiltCacheDir; - - // Respect the detailed-record preprocessor setting of the parent context. - // This, and the "raw" clang module format it implicitly enables, are required - // by sourcekitd. - if (auto *ClangLoader = Ctx.getClangModuleLoader()) { - auto &Opts = ClangLoader->getClangInstance().getPreprocessorOpts(); - if (Opts.DetailedRecord) { - SubInvocation.getClangImporterOptions().DetailedPreprocessingRecord = true; - } - } - - // Inhibit warnings from the SubInvocation since we are assuming the user - // is not in a position to fix them. - SubInvocation.getDiagnosticOptions().SuppressWarnings = true; - - // Inherit this setting down so that it can affect error diagnostics (mostly - // by making them non-fatal). - SubInvocation.getLangOptions().DebuggerSupport = LangOpts.DebuggerSupport; - - // Disable this; deinitializers always get printed with `@objc` even in - // modules that don't import Foundation. - SubInvocation.getLangOptions().EnableObjCAttrRequiresFoundation = false; - - return SubInvocation; -} - -/// Calculate an output filename in \p SubInvocation's cache path that includes -/// a hash of relevant key data. -static void computeCachedOutputPath(ASTContext &Ctx, - const CompilerInvocation &SubInvocation, - StringRef InPath, - llvm::SmallString<256> &OutPath) { - OutPath = SubInvocation.getClangModuleCachePath(); - llvm::sys::path::append(OutPath, SubInvocation.getModuleName()); - OutPath.append("-"); - OutPath.append(getCacheHash(Ctx, SubInvocation, InPath)); - OutPath.append("."); - auto OutExt = file_types::getExtension(file_types::TY_SwiftModuleFile); - OutPath.append(OutExt); -} - -void ParseableInterfaceModuleLoader::configureSubInvocationInputsAndOutputs( - CompilerInvocation &SubInvocation, StringRef InPath, StringRef OutPath) { - auto &SubFEOpts = SubInvocation.getFrontendOptions(); - SubFEOpts.RequestedAction = FrontendOptions::ActionType::EmitModuleOnly; - SubFEOpts.EnableParseableModuleInterface = true; - SubFEOpts.InputsAndOutputs.addPrimaryInputFile(InPath); - SupplementaryOutputPaths SOPs; - SOPs.ModuleOutputPath = OutPath.str(); - - // Pick a primary output path that will cause problems to use. - StringRef MainOut = "/"; - SubFEOpts.InputsAndOutputs.setMainAndSupplementaryOutputs({MainOut}, {SOPs}); -} - -// Checks that a dependency read from the cached module is up to date compared -// to the interface file it represents. If it's up-to-date, return its status, -// so we can switch hash-based dependencies to mtimes after we've validated -// the hashes. -static Optional -dependencyIsUpToDate(llvm::vfs::FileSystem &FS, FileDependency In, - StringRef ModulePath, DiagnosticEngine &Diags, - SourceLoc DiagLoc) { - auto Status = getStatusOfDependency(FS, ModulePath, In.getPath(), - Diags, DiagLoc); - if (!Status) return None; - - // If the sizes differ, then we know the file has changed. - if (Status->getSize() != In.getSize()) return None; - - // Otherwise, if this dependency is verified by modification time, check - // it vs. the modification time of the file. - uint64_t mtime = - Status->getLastModificationTime().time_since_epoch().count(); - - if (In.isModificationTimeBased()) - return mtime == In.getModificationTime() ? Status : None; - - // Slow path: if the dependency is verified by content hash, check it vs. the - // hash of the file. - auto buf = getBufferOfDependency(FS, ModulePath, In.getPath(), - Diags, DiagLoc); - if (!buf) return None; - - if (xxHash64(buf->getBuffer()) == In.getContentHash()) - return Status; - - return None; -} - -} - -// Check that the output .swiftmodule file is at least as new as all the -// dependencies it read when it was built last time. -static bool -swiftModuleIsUpToDate(llvm::vfs::FileSystem &FS, - std::pair ModuleID, - StringRef OutPath, - DiagnosticEngine &Diags, - DependencyTracker *OuterTracker) { - - auto OutBuf = FS.getBufferForFile(OutPath); - if (!OutBuf) - return false; - - LLVM_DEBUG(llvm::dbgs() << "Validating deps of " << OutPath << "\n"); - SmallVector AllDeps; - auto VI = serialization::validateSerializedAST( - OutBuf.get()->getBuffer(), - /*ExtendedValidationInfo=*/nullptr, &AllDeps); - - if (VI.status != serialization::Status::Valid) - return false; - - assert(VI.name == ModuleID.first.str() && - "we built a module at this path with a different name?"); - - for (auto In : AllDeps) { - if (OuterTracker) - OuterTracker->addDependency(In.Path, /*IsSystem=*/false); - if (!dependencyIsUpToDate(FS, In, OutPath, Diags, ModuleID.second)) { - LLVM_DEBUG(llvm::dbgs() << "Dep " << In.Path - << " is directly out of date\n"); - return false; - } - LLVM_DEBUG(llvm::dbgs() << "Dep " << In.Path << " is up to date\n"); - } - return true; -} - -/// Populate the provided \p Deps with \c FileDependency entries including: -/// -/// - \p InPath - The .swiftinterface input file -/// -/// - All the dependencies mentioned by \p SubInstance's DependencyTracker, -/// that were read while compiling the module. -/// -/// - For any file in the latter set that is itself a .swiftmodule -/// living in \p ModuleCachePath, all of _its_ dependencies, copied -/// out to avoid having to do recursive scanning when rechecking this -/// dependency in the future. -static bool -collectDepsForSerialization(llvm::vfs::FileSystem &FS, - CompilerInstance &SubInstance, - StringRef InPath, StringRef ModuleCachePath, - SmallVectorImpl &Deps, - bool IsHashBased, - DiagnosticEngine &Diags, SourceLoc DiagLoc, - DependencyTracker *OuterTracker) { - auto DTDeps = SubInstance.getDependencyTracker()->getDependencies(); - SmallVector InitialDepNames(DTDeps.begin(), DTDeps.end()); - InitialDepNames.push_back(InPath); - llvm::StringSet<> AllDepNames; - for (auto const &DepName : InitialDepNames) { - if (AllDepNames.insert(DepName).second && OuterTracker) { - OuterTracker->addDependency(DepName, /*IsSystem=*/false); - } - auto DepBuf = getBufferOfDependency(FS, InPath, DepName, Diags, DiagLoc); - if (!DepBuf) - return true; - auto Status = getStatusOfDependency(FS, InPath, DepName, Diags, DiagLoc); - if (!Status) - return true; - - if (IsHashBased) { - uint64_t hash = xxHash64(DepBuf->getBuffer()); - Deps.push_back( - FileDependency::hashBased(DepName, Status->getSize(), hash)); - } else { - uint64_t mtime = - Status->getLastModificationTime().time_since_epoch().count(); - Deps.push_back( - FileDependency::modTimeBased(DepName, Status->getSize(), mtime)); - } - - if (ModuleCachePath.empty()) - continue; - - // If Dep is itself a .swiftmodule in the cache dir, pull out its deps - // and include them in our own, so we have a single-file view of - // transitive deps: removes redundancies, and avoids opening and reading - // multiple swiftmodules during future loads. - auto Ext = llvm::sys::path::extension(DepName); - auto Ty = file_types::lookupTypeForExtension(Ext); - if (Ty == file_types::TY_SwiftModuleFile && - DepName.startswith(ModuleCachePath)) { - SmallVector SubDeps; - auto VI = serialization::validateSerializedAST( - DepBuf->getBuffer(), - /*ExtendedValidationInfo=*/nullptr, &SubDeps); - if (VI.status != serialization::Status::Valid) { - Diags.diagnose(DiagLoc, - diag::error_extracting_dependencies_from_cached_module, - DepName); - return true; - } - for (auto const &SubDep : SubDeps) { - if (AllDepNames.insert(SubDep.getPath()).second) { - Deps.push_back(SubDep); - if (OuterTracker) - OuterTracker->addDependency(SubDep.getPath(), /*IsSystem=*/false); - } - } - } - } - return false; -} - -bool ParseableInterfaceModuleLoader::buildSwiftModuleFromSwiftInterface( - llvm::vfs::FileSystem &FS, DiagnosticEngine &Diags, SourceLoc DiagLoc, - CompilerInvocation &SubInvocation, StringRef InPath, StringRef OutPath, - StringRef ModuleCachePath, DependencyTracker *OuterTracker, - bool ShouldSerializeDeps) { - bool SubError = false; - bool RunSuccess = llvm::CrashRecoveryContext().RunSafelyOnThread([&] { - // Note that we don't assume ModuleCachePath is the same as the Clang - // module cache path at this point. - if (!ModuleCachePath.empty()) - (void)llvm::sys::fs::create_directory(ModuleCachePath); - - configureSubInvocationInputsAndOutputs(SubInvocation, InPath, OutPath); - - FrontendOptions &FEOpts = SubInvocation.getFrontendOptions(); - const auto &InputInfo = FEOpts.InputsAndOutputs.firstInput(); - StringRef InPath = InputInfo.file(); - const auto &OutputInfo = - InputInfo.getPrimarySpecificPaths().SupplementaryOutputs; - StringRef OutPath = OutputInfo.ModuleOutputPath; - - llvm::BumpPtrAllocator SubArgsAlloc; - llvm::StringSaver SubArgSaver(SubArgsAlloc); - SmallVector SubArgs; - swift::version::Version Vers; - if (extractSwiftInterfaceVersionAndArgs(Diags, DiagLoc, FS, InPath, Vers, - SubArgSaver, SubArgs)) { - SubError = true; - return; - } - - // For now: we support anything with the same "major version" and assume - // minor versions might be interesting for debugging, or special-casing a - // compatible field variant. - if (Vers.asMajorVersion() != InterfaceFormatVersion.asMajorVersion()) { - Diags.diagnose(DiagLoc, - diag::unsupported_version_of_parseable_interface, - InPath, Vers); - SubError = true; - return; - } - - SmallString<32> ExpectedModuleName = SubInvocation.getModuleName(); - if (SubInvocation.parseArgs(SubArgs, Diags)) { - SubError = true; - return; - } - - if (SubInvocation.getModuleName() != ExpectedModuleName) { - auto DiagKind = diag::serialization_name_mismatch; - if (SubInvocation.getLangOptions().DebuggerSupport) - DiagKind = diag::serialization_name_mismatch_repl; - Diags.diagnose(DiagLoc, DiagKind, SubInvocation.getModuleName(), - ExpectedModuleName); - SubError = true; - return; - } - - // Optimize emitted modules. This has to happen after we parse arguments, - // because parseSILOpts would override the current optimization mode. - SubInvocation.getSILOptions().OptMode = OptimizationMode::ForSpeed; - - // Build the .swiftmodule; this is a _very_ abridged version of the logic in - // performCompile in libFrontendTool, specialized, to just the one - // module-serialization task we're trying to do here. - LLVM_DEBUG(llvm::dbgs() << "Setting up instance to compile " - << InPath << " to " << OutPath << "\n"); - CompilerInstance SubInstance; - SubInstance.getSourceMgr().setFileSystem(&FS); - - ForwardingDiagnosticConsumer FDC(Diags); - SubInstance.addDiagnosticConsumer(&FDC); - - SubInstance.createDependencyTracker(/*TrackSystemDeps=*/false); - if (SubInstance.setup(SubInvocation)) { - SubError = true; - return; - } - - LLVM_DEBUG(llvm::dbgs() << "Performing sema\n"); - SubInstance.performSema(); - if (SubInstance.getASTContext().hadError()) { - LLVM_DEBUG(llvm::dbgs() << "encountered errors\n"); - SubError = true; - return; - } - - SILOptions &SILOpts = SubInvocation.getSILOptions(); - auto Mod = SubInstance.getMainModule(); - auto SILMod = performSILGeneration(Mod, SILOpts); - if (!SILMod) { - LLVM_DEBUG(llvm::dbgs() << "SILGen did not produce a module\n"); - SubError = true; - return; - } - - // Setup the callbacks for serialization, which can occur during the - // optimization pipeline. - SerializationOptions SerializationOpts; - std::string OutPathStr = OutPath; - SerializationOpts.OutputPath = OutPathStr.c_str(); - SerializationOpts.ModuleLinkName = FEOpts.ModuleLinkName; - SmallVector Deps; - if (collectDepsForSerialization( - FS, SubInstance, InPath, ModuleCachePath, Deps, - FEOpts.SerializeParseableModuleInterfaceDependencyHashes, - Diags, DiagLoc, OuterTracker)) { - SubError = true; - return; - } - if (ShouldSerializeDeps) - SerializationOpts.Dependencies = Deps; - SILMod->setSerializeSILAction([&]() { - serialize(Mod, SerializationOpts, SILMod.get()); - }); - - LLVM_DEBUG(llvm::dbgs() << "Running SIL processing passes\n"); - if (SubInstance.performSILProcessing(SILMod.get())) { - LLVM_DEBUG(llvm::dbgs() << "encountered errors\n"); - SubError = true; - return; - } - - SubError = SubInstance.getDiags().hadAnyError(); - }); - return !RunSuccess || SubError; -} - -static bool serializedASTLooksValidOrCannotBeRead(llvm::vfs::FileSystem &FS, - StringRef ModPath) { - auto ModBuf = FS.getBufferForFile(ModPath, /*FileSize=*/-1, - /*RequiresNullTerminator=*/false); - if (!ModBuf) - return ModBuf.getError() != std::errc::no_such_file_or_directory; - - auto VI = serialization::validateSerializedAST(ModBuf.get()->getBuffer()); - return VI.status == serialization::Status::Valid; -} - -/// Load a .swiftmodule associated with a .swiftinterface either from a -/// cache or by converting it in a subordinate \c CompilerInstance, caching -/// the results. -std::error_code ParseableInterfaceModuleLoader::findModuleFilesInDirectory( - AccessPathElem ModuleID, StringRef DirPath, StringRef ModuleFilename, - StringRef ModuleDocFilename, - std::unique_ptr *ModuleBuffer, - std::unique_ptr *ModuleDocBuffer) { - - namespace path = llvm::sys::path; - - // If running in OnlySerialized mode, ParseableInterfaceModuleLoader - // should not have been constructed at all. - assert(LoadMode != ModuleLoadingMode::OnlySerialized); - - auto &FS = *Ctx.SourceMgr.getFileSystem(); - auto &Diags = Ctx.Diags; - llvm::SmallString<256> ModPath, InPath, OutPath; - - // First check to see if the .swiftinterface exists at all. Bail if not. - ModPath = DirPath; - path::append(ModPath, ModuleFilename); - - auto Ext = file_types::getExtension(file_types::TY_SwiftParseableInterfaceFile); - InPath = ModPath; - path::replace_extension(InPath, Ext); - if (!FS.exists(InPath)) - return std::make_error_code(std::errc::no_such_file_or_directory); - - // Next, if we're in the load mode that prefers .swiftmodules, see if there's - // one here we can _likely_ load (validates OK). If so, bail early with - // errc::not_supported, so the next (serialized) loader in the chain will load - // it. Alternately, if there's a .swiftmodule present but we can't even read - // it (for whatever reason), we should let the other module loader diagnose - // it. - if (LoadMode == ModuleLoadingMode::PreferSerialized && - serializedASTLooksValidOrCannotBeRead(FS, ModPath)) { - return std::make_error_code(std::errc::not_supported); - } - - // If we have a prebuilt cache path, check that too if the interface comes - // from the SDK. - if (!PrebuiltCacheDir.empty()) { - StringRef SDKPath = Ctx.SearchPathOpts.SDKPath; - if (!SDKPath.empty() && hasPrefix(path::begin(InPath), - path::end(InPath), - path::begin(SDKPath), - path::end(SDKPath))) { - // Assemble the expected path: $PREBUILT_CACHE/Foo.swiftmodule or - // $PREBUILT_CACHE/Foo.swiftmodule/arch.swiftmodule. Note that there's no - // cache key here. - OutPath = PrebuiltCacheDir; - - // FIXME: Would it be possible to only have architecture-specific names - // here? Then we could skip this check. - StringRef InParentDirName = path::filename(path::parent_path(InPath)); - if (path::extension(InParentDirName) == ".swiftmodule") { - assert(path::stem(InParentDirName) == ModuleID.first.str()); - path::append(OutPath, InParentDirName); - } - path::append(OutPath, ModuleFilename); - - if (!swiftModuleIsUpToDate(FS, ModuleID, OutPath, Diags, - dependencyTracker)) { - OutPath.clear(); - } - } - } - - if (OutPath.empty()) { - // At this point we're either in PreferParseable mode or there's no credible - // adjacent .swiftmodule so we'll go ahead and start trying to convert the - // .swiftinterface. - - // Set up a _potential_ sub-invocation to consume the .swiftinterface and - // emit the .swiftmodule. - CompilerInvocation SubInvocation = - createInvocationForBuildingFromInterface(Ctx, ModuleID.first.str(), - CacheDir, PrebuiltCacheDir); - computeCachedOutputPath(Ctx, SubInvocation, InPath, OutPath); - - // Evaluate if we need to run this sub-invocation, and if so run it. - if (!swiftModuleIsUpToDate(FS, ModuleID, OutPath, Diags, - dependencyTracker)) { - if (buildSwiftModuleFromSwiftInterface(FS, Diags, ModuleID.second, - SubInvocation, InPath, OutPath, - CacheDir, dependencyTracker, - /*ShouldSerializeDeps*/true)) - return std::make_error_code(std::errc::invalid_argument); - } - } - - // Finish off by delegating back up to the SerializedModuleLoaderBase - // routine that can load the recently-manufactured serialized module. - LLVM_DEBUG(llvm::dbgs() << "Loading " << OutPath - << " via normal module loader\n"); - llvm::SmallString<256> DocPath{DirPath}; - path::append(DocPath, ModuleDocFilename); - auto ErrorCode = SerializedModuleLoaderBase::openModuleFiles( - ModuleID, OutPath, DocPath, ModuleBuffer, ModuleDocBuffer); - LLVM_DEBUG(llvm::dbgs() << "Loaded " << OutPath - << " via normal module loader"); - if (ErrorCode) { - LLVM_DEBUG(llvm::dbgs() << " with error: " << ErrorCode.message()); - } - LLVM_DEBUG(llvm::dbgs() << "\n"); - return ErrorCode; -} - -bool -ParseableInterfaceModuleLoader::buildSwiftModuleFromSwiftInterface( - ASTContext &Ctx, StringRef CacheDir, StringRef PrebuiltCacheDir, - StringRef ModuleName, StringRef InPath, StringRef OutPath) { - CompilerInvocation SubInvocation = - createInvocationForBuildingFromInterface(Ctx, ModuleName, CacheDir, - PrebuiltCacheDir); - - auto &FS = *Ctx.SourceMgr.getFileSystem(); - auto &Diags = Ctx.Diags; - // FIXME: We don't really want to ignore dependencies here, but we have to - // identify which ones are important, and make them relocatable - // (SDK-relative) if we want to ship the built swiftmodules to another - // machine. Just leave them out for now. - return buildSwiftModuleFromSwiftInterface(FS, Diags, /*DiagLoc*/SourceLoc(), - SubInvocation, InPath, OutPath, - /*CachePath*/"", - /*OuterTracker*/nullptr, - /*ShouldSerializeDeps*/false); -} +version::Version swift::InterfaceFormatVersion({1, 0}); /// Diagnose any scoped imports in \p imports, i.e. those with a non-empty /// access path. These are not yet supported by parseable interfaces, since the @@ -690,23 +81,6 @@ llvm::Regex swift::getSwiftInterfaceModuleFlagsRegex() { llvm::Regex::Newline); } -/// Extract the specified-or-defaulted -module-cache-path that winds up in -/// the clang importer, for reuse as the .swiftmodule cache path when -/// building a ParseableInterfaceModuleLoader. -std::string -swift::getModuleCachePathFromClang(const clang::CompilerInstance &Clang) { - if (!Clang.hasPreprocessor()) - return ""; - std::string SpecificModuleCachePath = Clang.getPreprocessor() - .getHeaderSearchInfo() - .getModuleCachePath(); - - // The returned-from-clang module cache path includes a suffix directory - // that is specific to the clang version and invocation; we want the - // directory above that. - return llvm::sys::path::parent_path(SpecificModuleCachePath); -} - /// Prints the imported modules in \p M to \p out in the form of \c import /// source declarations. static void printImports(raw_ostream &out, ModuleDecl *M) { diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index 579a8b0dfbd86..dcf98b8fa9661 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -51,6 +51,7 @@ #include "swift/Frontend/Frontend.h" #include "swift/Frontend/PrintingDiagnosticConsumer.h" #include "swift/Frontend/SerializedDiagnosticConsumer.h" +#include "swift/Frontend/ParseableInterfaceModuleLoader.h" #include "swift/Frontend/ParseableInterfaceSupport.h" #include "swift/Immediate/Immediate.h" #include "swift/Index/IndexRecord.h" @@ -585,7 +586,8 @@ static bool buildModuleFromParseableInterface(CompilerInvocation &Invocation, return ParseableInterfaceModuleLoader::buildSwiftModuleFromSwiftInterface( Instance.getASTContext(), Invocation.getClangModuleCachePath(), PrebuiltCachePath, Invocation.getModuleName(), InputPath, - Invocation.getOutputFilename()); + Invocation.getOutputFilename(), + FEOpts.SerializeParseableModuleInterfaceDependencyHashes); } static bool compileLLVMIR(CompilerInvocation &Invocation, diff --git a/test/ParseableInterface/ModuleCache/Inputs/check-is-forwarding-module.py b/test/ParseableInterface/ModuleCache/Inputs/check-is-forwarding-module.py new file mode 100644 index 0000000000000..d2509e0d1669b --- /dev/null +++ b/test/ParseableInterface/ModuleCache/Inputs/check-is-forwarding-module.py @@ -0,0 +1,7 @@ +import sys + +with open(sys.argv[1], 'r') as yaml_file: + # Forwarding files are YAML files that start with '---' + if yaml_file.read(3) != '---': + print("swiftmodule '%s' is not a forwarding module!") + sys.exit(1) diff --git a/test/ParseableInterface/ModuleCache/force-module-loading-mode-archs.swift b/test/ParseableInterface/ModuleCache/force-module-loading-mode-archs.swift index e6b7eba790b0f..62e425941ccf9 100644 --- a/test/ParseableInterface/ModuleCache/force-module-loading-mode-archs.swift +++ b/test/ParseableInterface/ModuleCache/force-module-loading-mode-archs.swift @@ -22,20 +22,29 @@ // RUN: %empty-directory(%t/Lib.swiftmodule) // RUN: sed -e 's/FromInterface/FromSerialized/g' %S/Inputs/force-module-loading-mode/Lib.swiftinterface | %target-swift-frontend -parse-stdlib -module-cache-path %t/MCP -emit-module-path %t/Lib.swiftmodule/%target-swiftmodule-name - -module-name Lib // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-SERIALIZED %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-SERIALIZED %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=NO-SUCH-MODULE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-SERIALIZED %s +// RUN: %empty-directory(%t/MCP) // (default) // RUN: not %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP -I %t %s 2>&1 | %FileCheck -check-prefix=FROM-SERIALIZED %s +// RUN: %empty-directory(%t/MCP) // 4. Both are present. // RUN: cp %S/Inputs/force-module-loading-mode/Lib.swiftinterface %t/Lib.swiftmodule/%target-cpu.swiftinterface // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-SERIALIZED %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-SERIALIZED %s // (default) // RUN: not %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP -I %t %s 2>&1 | %FileCheck -check-prefix=FROM-SERIALIZED %s +// RUN: %empty-directory(%t/MCP) // 5. Both are present but the module is invalid. // RUN: rm %t/Lib.swiftmodule/%target-swiftmodule-name && touch %t/Lib.swiftmodule/%target-swiftmodule-name @@ -49,11 +58,16 @@ // 6. Both are present but the module can't be opened. // RUN: chmod a-r %t/Lib.swiftmodule/%target-swiftmodule-name // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=NO-SUCH-MODULE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=NO-SUCH-MODULE %s +// RUN: %empty-directory(%t/MCP) // (default) // RUN: not %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP -I %t %s 2>&1 | %FileCheck -check-prefix=NO-SUCH-MODULE %s +// RUN: %empty-directory(%t/MCP) // 7. Both are present but for the wrong architecture. // RUN: %empty-directory(%t/Lib.swiftmodule) diff --git a/test/ParseableInterface/ModuleCache/force-module-loading-mode-framework.swift b/test/ParseableInterface/ModuleCache/force-module-loading-mode-framework.swift index 4b93db42aa2e0..5905fa4d68f19 100644 --- a/test/ParseableInterface/ModuleCache/force-module-loading-mode-framework.swift +++ b/test/ParseableInterface/ModuleCache/force-module-loading-mode-framework.swift @@ -31,29 +31,44 @@ // 4. Both are present. // RUN: cp %S/Inputs/force-module-loading-mode/Lib.swiftinterface %t/Lib.framework/Modules/Lib.swiftmodule/%target-cpu.swiftinterface // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -F %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -F %t 2>&1 | %FileCheck -check-prefix=FROM-SERIALIZED %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -F %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -F %t 2>&1 | %FileCheck -check-prefix=FROM-SERIALIZED %s +// RUN: %empty-directory(%t/MCP) // (default) // RUN: not %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP -F %t %s 2>&1 | %FileCheck -check-prefix=FROM-SERIALIZED %s +// RUN: %empty-directory(%t/MCP) // 5. Both are present but the module is invalid. // RUN: rm %t/Lib.framework/Modules/Lib.swiftmodule/%target-swiftmodule-name && touch %t/Lib.framework/Modules/Lib.swiftmodule/%target-swiftmodule-name // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -F %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -F %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -F %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -F %t 2>&1 | %FileCheck -check-prefix=BAD-MODULE %s +// RUN: %empty-directory(%t/MCP) // (default) // RUN: not %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP -F %t %s 2>&1 | %FileCheck -check-prefix=BAD-MODULE %s +// RUN: %empty-directory(%t/MCP) // 6. Both are present but the module can't be opened. // RUN: chmod a-r %t/Lib.framework/Modules/Lib.swiftmodule/%target-swiftmodule-name // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -F %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -F %t 2>&1 | %FileCheck -check-prefix=NO-SUCH-MODULE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -F %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -F %t 2>&1 | %FileCheck -check-prefix=NO-SUCH-MODULE %s +// RUN: %empty-directory(%t/MCP) // (default) // RUN: not %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP -F %t %s 2>&1 | %FileCheck -check-prefix=NO-SUCH-MODULE %s +// RUN: %empty-directory(%t/MCP) // 7. Both are present but for the wrong architecture. // RUN: %empty-directory(%t/Lib.framework/Modules/Lib.swiftmodule) diff --git a/test/ParseableInterface/ModuleCache/force-module-loading-mode.swift b/test/ParseableInterface/ModuleCache/force-module-loading-mode.swift index 2f934bce47b2f..311136423bf03 100644 --- a/test/ParseableInterface/ModuleCache/force-module-loading-mode.swift +++ b/test/ParseableInterface/ModuleCache/force-module-loading-mode.swift @@ -28,29 +28,44 @@ // 4. Both are present. // RUN: cp %S/Inputs/force-module-loading-mode/Lib.swiftinterface %t // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-SERIALIZED %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-SERIALIZED %s +// RUN: %empty-directory(%t/MCP) // (default) // RUN: not %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP -I %t %s 2>&1 | %FileCheck -check-prefix=FROM-SERIALIZED %s +// RUN: %empty-directory(%t/MCP) // 5. Both are present but the module is invalid. // RUN: rm %t/Lib.swiftmodule && touch %t/Lib.swiftmodule // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=BAD-MODULE %s +// RUN: %empty-directory(%t/MCP) // (default) // RUN: not %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP -I %t %s 2>&1 | %FileCheck -check-prefix=BAD-MODULE %s +// RUN: %empty-directory(%t/MCP) // 6. Both are present but the module can't be opened. // RUN: chmod a-r %t/Lib.swiftmodule // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=prefer-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=NO-SUCH-MODULE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-parseable %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s +// RUN: %empty-directory(%t/MCP) // RUN: not env SWIFT_FORCE_MODULE_LOADING=only-serialized %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP %s -I %t 2>&1 | %FileCheck -check-prefix=NO-SUCH-MODULE %s +// RUN: %empty-directory(%t/MCP) // (default) // RUN: not %target-swift-frontend -typecheck -parse-stdlib -module-cache-path %t/MCP -I %t %s 2>&1 | %FileCheck -check-prefix=NO-SUCH-MODULE %s +// RUN: %empty-directory(%t/MCP) import Lib // NO-SUCH-MODULE: [[@LINE-1]]:8: error: no such module 'Lib' diff --git a/test/ParseableInterface/ModuleCache/prebuilt-module-cache-archs.swift b/test/ParseableInterface/ModuleCache/prebuilt-module-cache-archs.swift index 06b5aa78e2f74..30728316ed3ad 100644 --- a/test/ParseableInterface/ModuleCache/prebuilt-module-cache-archs.swift +++ b/test/ParseableInterface/ModuleCache/prebuilt-module-cache-archs.swift @@ -16,7 +16,9 @@ // RUN: %empty-directory(%t/prebuilt-cache/Lib.swiftmodule) // RUN: sed -e 's/FromInterface/FromPrebuilt/g' %S/Inputs/prebuilt-module-cache/Lib.swiftinterface | %target-swift-frontend -parse-stdlib -module-cache-path %t/MCP -emit-module-path %t/prebuilt-cache/Lib.swiftmodule/%target-swiftmodule-name - -module-name Lib // RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %t/include -I %t/include/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-PREBUILT %s -// RUN: ls %t/MCP | not grep swiftmodule + +// Make sure we installed a forwarding module. +// RUN: %{python} %S/Inputs/check-is-forwarding-module.py %t/MCP/Lib-*.swiftmodule // What if the module is invalid? // RUN: rm %t/prebuilt-cache/Lib.swiftmodule/%target-swiftmodule-name && touch %t/prebuilt-cache/Lib.swiftmodule/%target-swiftmodule-name diff --git a/test/ParseableInterface/ModuleCache/prebuilt-module-cache-forwarding.swift b/test/ParseableInterface/ModuleCache/prebuilt-module-cache-forwarding.swift new file mode 100644 index 0000000000000..06ffcfd96c534 --- /dev/null +++ b/test/ParseableInterface/ModuleCache/prebuilt-module-cache-forwarding.swift @@ -0,0 +1,46 @@ +// RUN: %empty-directory(%t) +// RUN: %empty-directory(%t/MCP) +// RUN: %empty-directory(%t/prebuilt-cache) + +// First, prebuild a module and put it in the prebuilt cache. +// RUN: sed -e 's/FromInterface/FromPrebuilt/g' %S/Inputs/prebuilt-module-cache/Lib.swiftinterface > %t/Lib.swiftinterface +// RUN: %target-swift-frontend -build-module-from-parseable-interface -module-cache-path %t/MCP -serialize-parseable-module-interface-dependency-hashes -o %t/prebuilt-cache/Lib.swiftmodule %t/Lib.swiftinterface + +// Next, use the module and check if the forwarding module is in place. +// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-PREBUILT %s + +// Make sure we installed a forwarding module. +// RUN: %{python} %S/Inputs/check-is-forwarding-module.py %t/MCP/Lib-*.swiftmodule + +// Now invalidate a dependency of the prebuilt module, and make sure the forwarding file is replaced with a real module. +// RUN: touch %t/Lib.swiftinterface +// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s + +// Delete the cached module we just created, and create the forwarding module again +// RUN: %empty-directory(%t/MCP) +// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-PREBUILT %s + +// Move the prebuilt module out of the way, so the forwarding module points to nothing. +// RUN: mv %t/prebuilt-cache/Lib.swiftmodule %t/prebuilt-cache/NotLib.swiftmodule + +// Make sure we delete the existing forwarding module and rebuild from an interface. +// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s + +// Move the prebuilt module back to its original path +// RUN: mv %t/prebuilt-cache/NotLib.swiftmodule %t/prebuilt-cache/Lib.swiftmodule + +// If the forwarding module is corrupted, we shouldn't rebuild the module in the cache, +// we should delete it and generate a new forwarding module. +// RUN: %empty-directory(%t/MCP) +// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 +// RUN: %{python} %S/Inputs/check-is-forwarding-module.py %t/MCP/Lib-*.swiftmodule +// RUN: chmod a-r %t/MCP/Lib-*.swiftmodule +// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-PREBUILT %s +// RUN: %{python} %S/Inputs/check-is-forwarding-module.py %t/MCP/Lib-*.swiftmodule + +import Lib + +struct X {} +let _: X = Lib.testValue +// FROM-INTERFACE: [[@LINE-1]]:16: error: cannot convert value of type 'FromInterface' to specified type 'X' +// FROM-PREBUILT: [[@LINE-2]]:16: error: cannot convert value of type 'FromPrebuilt' to specified type 'X' diff --git a/test/ParseableInterface/ModuleCache/prebuilt-module-cache-recursive.swift b/test/ParseableInterface/ModuleCache/prebuilt-module-cache-recursive.swift index b33fdd86bb176..17a303eb6687a 100644 --- a/test/ParseableInterface/ModuleCache/prebuilt-module-cache-recursive.swift +++ b/test/ParseableInterface/ModuleCache/prebuilt-module-cache-recursive.swift @@ -9,7 +9,9 @@ // RUN: %empty-directory(%t/MCP) // RUN: sed -e 's/FromInterface/FromPrebuilt/g' %S/Inputs/prebuilt-module-cache/Lib.swiftinterface | %target-swift-frontend -parse-stdlib -module-cache-path %t/MCP -emit-module-path %t/prebuilt-cache/Lib.swiftmodule - -module-name Lib // RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-PREBUILT %s -// RUN: ls %t/MCP | grep -v LibExporter | not grep swiftmodule + +// Make sure we installed a forwarding module. +// RUN: %{python} %S/Inputs/check-is-forwarding-module.py %t/MCP/Lib-*.swiftmodule // What if the module is invalid? // RUN: rm %t/prebuilt-cache/Lib.swiftmodule && touch %t/prebuilt-cache/Lib.swiftmodule diff --git a/test/ParseableInterface/ModuleCache/prebuilt-module-cache.swift b/test/ParseableInterface/ModuleCache/prebuilt-module-cache.swift index e0e6f4ae52223..dd23403e1ef8c 100644 --- a/test/ParseableInterface/ModuleCache/prebuilt-module-cache.swift +++ b/test/ParseableInterface/ModuleCache/prebuilt-module-cache.swift @@ -13,7 +13,9 @@ // RUN: %empty-directory(%t/MCP) // RUN: sed -e 's/FromInterface/FromPrebuilt/g' %S/Inputs/prebuilt-module-cache/Lib.swiftinterface | %target-swift-frontend -parse-stdlib -module-cache-path %t/MCP -emit-module-path %t/prebuilt-cache/Lib.swiftmodule - -module-name Lib // RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-PREBUILT %s -// RUN: ls %t/MCP | not grep swiftmodule + +// Make sure we installed a forwarding module. +// RUN: %{python} %S/Inputs/check-is-forwarding-module.py %t/MCP/Lib-*.swiftmodule // Try some variations on the detection that the search path is in the SDK: // RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-PREBUILT %s diff --git a/test/ParseableInterface/SmokeTest.swiftinterface b/test/ParseableInterface/SmokeTest.swiftinterface index e615f5ea0813f..de15e5ee78c9b 100644 --- a/test/ParseableInterface/SmokeTest.swiftinterface +++ b/test/ParseableInterface/SmokeTest.swiftinterface @@ -14,7 +14,7 @@ // RUN: %target-swift-ide-test -print-module -module-to-print SmokeTest -I %t -source-filename x -print-interface > %t/SmokeTest.txt // RUN: %FileCheck %s < %t/SmokeTest.txt // RUN: %FileCheck -check-prefix NEGATIVE %s < %t/SmokeTest.txt -// RUN: llvm-bcanalyzer -dump %t/SmokeTest.swiftmodule | not grep FILE_DEPENDENCY +// RUN: llvm-bcanalyzer -dump %t/SmokeTest.swiftmodule | grep FILE_DEPENDENCY // CHECK-LABEL: public class TestClass public class TestClass {