[5.1][ParseableInterface] Flesh out parseable module loading

include/swift/Frontend/FrontendInputsAndOutputs.h

@@ -173,7 +173,7 @@ class FrontendInputsAndOutputs {
 
 private:
   friend class ArgsToFrontendOptionsConverter;
-  friend class ParseableInterfaceModuleLoader;
+  friend class ParseableInterfaceBuilder;
   void setMainAndSupplementaryOutputs(
       ArrayRef<std::string> outputFiles,
       ArrayRef<SupplementaryOutputPaths> supplementaryOutputs);

include/swift/Frontend/FrontendOptions.h

@@ -267,6 +267,10 @@ class FrontendOptions {
   /// dependencies as well.
   bool TrackSystemDeps = false;
 
+  /// Should we serialize the hashes of dependencies (vs. the modification
+  /// times) when compiling a parseable module interface?
+  bool SerializeParseableModuleInterfaceDependencyHashes = false;
+
   /// The different modes for validating TBD against the LLVM IR.
   enum class TBDValidationMode {
     Default,        ///< Do the default validation for the current platform.

include/swift/Frontend/ParseableInterfaceModuleLoader.h

@@ -0,0 +1,164 @@
+//===--- ParseableInterfaceModuleLoader.h - Loads .swiftinterface files ---===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+/// \file This implements the logic for loading and building parseable module
+/// interfaces.
+///
+/// === Loading Parseable Modules ===
+///
+/// If there is a .swiftinterface file corresponding to a given module name
+/// present in the frontend's search paths, then this module loader will look in
+/// the following places for a module:
+///
+/// - First, look in the module cache (specified by -module-cache-path)
+///   - We check here first because an existing .swiftmodule might be
+///     out-of-date, necessitating a rebuild. If a cached module is out-of-date,
+///     it's simply rebuilt.
+/// - Next, look adjacent to the .swiftinterface. If we find a module that's
+///   either loadable by this compiler, valid, and up-to-date, or totally
+///   unreadable, then delegate to the serialized module loader to either load
+///   or diagnose it.
+/// - Finally, look in the prebuilt module cache (specified
+///   by -prebuilt-module-cache-path)
+///
+/// If we can't find an appropriate module to load, we can always fall back and
+/// recompile the .swiftinterface file.
+///
+/// === Dependency Checking ===
+///
+/// Cached modules keep track of their dependencies' last modification time and
+/// file size. This means that checking if a module is up-to-date requires
+/// `stat`ing the dependencies and comparing the results from the filesystem
+/// with the results in the module.
+///
+/// Prebuilt modules, on the other hand, won't have a reliable modification
+/// time, as their dependencies live in the SDK. Prebuilt modules will instead
+/// keep track of the size and content hash of their dependencies.
+/// In order to avoid constantly re-hashing the dependencies, however, we will
+/// install a "forwarding module" in the regular cache.
+/// This "forwarding module"
+///  - Points to the prebuilt module on disk, and
+///  - Lists modification times from the last time we verified the content
+///
+/// So, to recap, there are 4 kinds of modules:
+/// ┌───────────────────────────────┐
+/// │     ┌───┐           ┌───┐     │
+/// │     │ I │           │ M │     │
+/// │     └───┘           └───┘     │
+/// │ .swiftinterface  .swiftmodule │
+/// │     ┌───┐           ┌───┐     │
+/// │     │ P │           │ F │     │
+/// │     └───┘           └───┘     │
+/// │   Prebuilt        Forwarding  │
+/// │ .swiftmodule     .swiftmodule │
+/// └───────────────────────────────┘
+///
+/// - Prebuilt modules have hash-based dependencies, cached modules have
+///   mod-time-based dependencies
+/// - Forwarding modules point to prebuilt modules and augment them with
+///   modification times
+///
+/// === Example Cache ===
+///
+/// Here's an example of what's in a prebuilt cache or module cache.
+///
+/// Say there are 4 frameworks, each exposing a .swiftinterface file.
+/// Then, we pre-build 2 of those frameworks and put them in the prebuilt cache.
+/// Finally, we import all 4 of those frameworks while building a project.
+///
+/// For the 2 frameworks with modules in the prebuilt cache, we'll have
+/// installed 2 forwarding modules. For the other 2 frameworks, we'll have
+/// compiled the interfaces and put them in the module cache.
+///
+///                                ┌─────┐
+///  ┌─────────────────────────────┤ SDK ├─────────────────────────┐
+///  │          ┌────────────────┐ └─────┘     ┌────────────────┐  │
+///  │  ┌───────┤ Framework Dirs ├────────┐   ┌┤ Prebuilt Cache ├┐ │
+///  │  │       └────────────────┘        │   │└────────────────┘│ │
+///  │  │  ┌───┐   ┌───┐   ┌───┐   ┌───┐  │   │   ┌───┐  ┌───┐   │ │
+///  │  │  │ I │   │ I │   │ I │   │ I │◀─┼───┼───│ P │  │ P │◀═╗│ │
+///  │  │  └───┘   └───┘   └───┘   └───┘  │   │   └───┘  └───┘  ║│ │
+///  │  │    ▲       ▲       ▲            │   │     ▲      │    ║│ │
+///  │  └────┼───────┼───────┼────────────┘   └─────╫──────┼────╫┘ │
+///  │       │       │       └──────────────────────╫──────┘    ║  │
+///  └───────┼───────┼──────────────────────────────╫───────────╫──┘
+///          │       │   ┌───────────────┐          ║           ║
+///          │  ┌────┼───┤ Module Cache  ├────────┐ ║           ║
+///          │  │    │   └───────────────┘        │ ║           ║
+///          │  │  ┌───┐   ┌───┐   ┌───┐   ┌───┐  │ ║           ║
+///          │  │  │ M │   │ M │   │ F │   │ F │  │ ║           ║
+///          │  │  └───┘   └───┘   └───┘   └───┘  │ ║           ║
+///          │  │            │       ║       ╚════╪═╝           ║
+///          │  └────────────┼───────╫────────────┘             ║
+///          └───────────────┘       ╚══════════════════════════╝
+///
+//===----------------------------------------------------------------------===//
+
+#include "swift/Basic/LLVM.h"
+#include "swift/Frontend/ParseableInterfaceSupport.h"
+#include "swift/Serialization/SerializedModuleLoader.h"
+
+namespace clang {
+  class CompilerInstance;
+}
+
+namespace swift {
+
+/// A ModuleLoader that runs a subordinate \c CompilerInvocation and
+/// \c CompilerInstance to convert .swiftinterface files to .swiftmodule
+/// files on the fly, caching the resulting .swiftmodules in the module cache
+/// directory, and loading the serialized .swiftmodules from there.
+class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase {
+  explicit ParseableInterfaceModuleLoader(ASTContext &ctx, StringRef cacheDir,
+                                          StringRef prebuiltCacheDir,
+                                          DependencyTracker *tracker,
+                                          ModuleLoadingMode loadMode)
+  : SerializedModuleLoaderBase(ctx, tracker, loadMode),
+  CacheDir(cacheDir), PrebuiltCacheDir(prebuiltCacheDir)
+  {}
+
+  std::string CacheDir;
+  std::string PrebuiltCacheDir;
+
+  std::error_code findModuleFilesInDirectory(
+    AccessPathElem ModuleID, StringRef DirPath, StringRef ModuleFilename,
+    StringRef ModuleDocFilename,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer) override;
+
+
+public:
+  static std::unique_ptr<ParseableInterfaceModuleLoader>
+  create(ASTContext &ctx, StringRef cacheDir, StringRef prebuiltCacheDir,
+         DependencyTracker *tracker, ModuleLoadingMode loadMode) {
+    return std::unique_ptr<ParseableInterfaceModuleLoader>(
+      new ParseableInterfaceModuleLoader(ctx, cacheDir, prebuiltCacheDir,
+                                         tracker, loadMode));
+  }
+
+  /// Unconditionally build \p InPath (a swiftinterface file) to \p OutPath (as
+  /// a swiftmodule file).
+  ///
+  /// A simplified version of the core logic in #openModuleFiles, mostly for
+  /// testing purposes.
+  static bool buildSwiftModuleFromSwiftInterface(
+    ASTContext &Ctx, StringRef CacheDir, StringRef PrebuiltCacheDir,
+    StringRef ModuleName, StringRef InPath, StringRef OutPath,
+    bool SerializeDependencyHashes);
+};
+
+/// Extract the specified-or-defaulted -module-cache-path that winds up in
+/// the clang importer, for reuse as the .swiftmodule cache path when
+/// building a ParseableInterfaceModuleLoader.
+std::string
+getModuleCachePathFromClang(const clang::CompilerInstance &Instance);
+
+}

include/swift/Frontend/ParseableInterfaceSupport.h

@@ -2,7 +2,7 @@
 //
 // This source file is part of the Swift.org open source project
 //
-// Copyright (c) 2018 Apple Inc. and the Swift project authors
+// Copyright (c) 2019 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See https://swift.org/LICENSE.txt for license information
@@ -14,9 +14,13 @@
 #define SWIFT_FRONTEND_PARSEABLEINTERFACESUPPORT_H
 
 #include "swift/Basic/LLVM.h"
-#include "swift/Serialization/SerializedModuleLoader.h"
+#include "swift/Basic/Version.h"
 #include "llvm/Support/Regex.h"
 
+#define SWIFT_INTERFACE_FORMAT_VERSION_KEY "swift-interface-format-version"
+#define SWIFT_TOOLS_VERSION_KEY "swift-tools-version"
+#define SWIFT_MODULE_FLAGS_KEY "swift-module-flags"
+
 namespace swift {
 
 class ModuleDecl;
@@ -29,6 +33,8 @@ struct ParseableInterfaceOptions {
   std::string ParseableInterfaceFlags;
 };
 
+extern version::Version InterfaceFormatVersion;
+
 llvm::Regex getSwiftInterfaceFormatVersionRegex();
 llvm::Regex getSwiftInterfaceModuleFlagsRegex();
 
@@ -49,71 +55,6 @@ bool emitParseableInterface(raw_ostream &out,
                             ParseableInterfaceOptions const &Opts,
                             ModuleDecl *M);
 
-/// Extract the specified-or-defaulted -module-cache-path that winds up in
-/// the clang importer, for reuse as the .swiftmodule cache path when
-/// building a ParseableInterfaceModuleLoader.
-std::string
-getModuleCachePathFromClang(const clang::CompilerInstance &Instance);
-
-/// A ModuleLoader that runs a subordinate \c CompilerInvocation and \c
-/// CompilerInstance to convert .swiftinterface files to .swiftmodule
-/// files on the fly, caching the resulting .swiftmodules in the module cache
-/// directory, and loading the serialized .swiftmodules from there.
-class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase {
-  explicit ParseableInterfaceModuleLoader(ASTContext &ctx, StringRef cacheDir,
-                                          StringRef prebuiltCacheDir,
-                                          DependencyTracker *tracker,
-                                          ModuleLoadingMode loadMode)
-    : SerializedModuleLoaderBase(ctx, tracker, loadMode),
-      CacheDir(cacheDir), PrebuiltCacheDir(prebuiltCacheDir)
-  {}
-
-  std::string CacheDir;
-  std::string PrebuiltCacheDir;
-
-  /// Wire up the SubInvocation's InputsAndOutputs to contain both input and
-  /// output filenames.
-  ///
-  /// This is a method rather than a helper function in the implementation file
-  /// because it accesses non-public bits of FrontendInputsAndOutputs.
-  static void configureSubInvocationInputsAndOutputs(
-    CompilerInvocation &SubInvocation, StringRef InPath, StringRef OutPath);
-
-  static bool buildSwiftModuleFromSwiftInterface(
-    llvm::vfs::FileSystem &FS, DiagnosticEngine &Diags, SourceLoc DiagLoc,
-    CompilerInvocation &SubInvocation, StringRef InPath, StringRef OutPath,
-    StringRef ModuleCachePath, DependencyTracker *OuterTracker,
-    bool ShouldSerializeDeps);
-
-  std::error_code findModuleFilesInDirectory(
-      AccessPathElem ModuleID, StringRef DirPath, StringRef ModuleFilename,
-      StringRef ModuleDocFilename,
-      std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
-      std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer) override;
-
-public:
-  static std::unique_ptr<ParseableInterfaceModuleLoader>
-  create(ASTContext &ctx, StringRef cacheDir, StringRef prebuiltCacheDir,
-         DependencyTracker *tracker, ModuleLoadingMode loadMode) {
-    return std::unique_ptr<ParseableInterfaceModuleLoader>(
-        new ParseableInterfaceModuleLoader(ctx, cacheDir, prebuiltCacheDir,
-                                           tracker, loadMode));
-  }
-
-  /// Unconditionally build \p InPath (a swiftinterface file) to \p OutPath (as
-  /// a swiftmodule file).
-  ///
-  /// A simplified version of the core logic in #openModuleFiles, mostly for
-  /// testing purposes.
-  static bool buildSwiftModuleFromSwiftInterface(ASTContext &Ctx,
-                                                 StringRef CacheDir,
-                                                 StringRef PrebuiltCacheDir,
-                                                 StringRef ModuleName,
-                                                 StringRef InPath,
-                                                 StringRef OutPath);
-};
-
-
 } // end namespace swift
 
 #endif

include/swift/Option/FrontendOptions.td

@@ -64,6 +64,10 @@ def emit_interface_path
   : Separate<["-"], "emit-interface-path">,
     Alias<emit_parseable_module_interface_path>;
 
+def serialize_parseable_module_interface_dependency_hashes
+  : Flag<["-"], "serialize-parseable-module-interface-dependency-hashes">,
+    Flags<[HelpHidden]>;
+
 def tbd_install_name
   : Separate<["-"], "tbd-install_name">, MetaVarName<"<path>">,
     HelpText<"The install_name to use in an emitted TBD file">;

include/swift/Serialization/ModuleFormat.h

@@ -52,7 +52,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0;
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
 /// Don't worry about adhering to the 80-column limit for this line.
-const uint16_t SWIFTMODULE_VERSION_MINOR = 476; // Last change: Opaque archetypes
+const uint16_t SWIFTMODULE_VERSION_MINOR = 477; // Last change: prebuilt module cache
 
 using DeclIDField = BCFixed<31>;
 
@@ -107,7 +107,7 @@ using CharOffset = BitOffset;
 using CharOffsetField = BitOffsetField;
 
 using FileSizeField = BCVBR<16>;
-using FileModTimeField = BCVBR<16>;
+using FileModTimeOrContentHashField = BCVBR<16>;
 using FileHashField = BCVBR<16>;
 
 // These IDs must \em not be renumbered or reordered without incrementing
@@ -671,9 +671,10 @@ namespace input_block {
 
   using FileDependencyLayout = BCRecordLayout<
     FILE_DEPENDENCY,
-    FileSizeField,    // file size (for validation)
-    FileModTimeField, // file mtime (for validation)
-    BCBlob            // path
+    FileSizeField,                 // file size (for validation)
+    FileModTimeOrContentHashField, // mtime or content hash (for validation)
+    BCFixed<1>,                    // are we reading mtime (0) or hash (1)?
+    BCBlob                         // path
   >;
 }