DependenciesScanner: include compiled module candidates for textual module interface in JSON output

Instead of replacing an interface file with its up-to-date compile module,
the dep-scanner should report potentially up-to-date module candidates either adjacent to
the interface file or in the prebuilt module cache. swift-driver should later pass down
these candidates to -compile-module-from-interface invocation and the front-end job
will check if one of the candidates is ready to use. The front-end job then either emits a forwarding
module to an up-to-date candidate or a binary module.

Author: nkcsgexi

include/swift/AST/ModuleDependencies.h

@@ -69,6 +69,9 @@ class SwiftModuleDependenciesStorage : public ModuleDependenciesStorageBase {
   /// The Swift interface file, if it can be used to generate the module file.
   const Optional<std::string> swiftInterfaceFile;
 
+  /// Potentially ready-to-use compiled modules for the interface file.
+  const std::vector<std::string> compiledModuleCandidates;
+
   /// The Swift frontend invocation arguments to build the Swift module from the
   /// interface.
   const std::vector<std::string> buildCommandLine;
@@ -96,11 +99,14 @@ class SwiftModuleDependenciesStorage : public ModuleDependenciesStorageBase {
   SwiftModuleDependenciesStorage(
       const std::string &compiledModulePath,
       const Optional<std::string> &swiftInterfaceFile,
+      ArrayRef<std::string> compiledModuleCandidates,
       ArrayRef<StringRef> buildCommandLine,
       ArrayRef<StringRef> extraPCMArgs,
       StringRef contextHash
   ) : ModuleDependenciesStorageBase(/*isSwiftModule=*/true, compiledModulePath),
       swiftInterfaceFile(swiftInterfaceFile),
+      compiledModuleCandidates(compiledModuleCandidates.begin(),
+                               compiledModuleCandidates.end()),
       buildCommandLine(buildCommandLine.begin(), buildCommandLine.end()),
       extraPCMArgs(extraPCMArgs.begin(), extraPCMArgs.end()),
       contextHash(contextHash) { }
@@ -181,13 +187,14 @@ class ModuleDependencies {
   /// built from a Swift interface file (\c .swiftinterface).
   static ModuleDependencies forSwiftInterface(
       const std::string &swiftInterfaceFile,
+      ArrayRef<std::string> compiledCandidates,
       ArrayRef<StringRef> buildCommands,
       ArrayRef<StringRef> extraPCMArgs,
       StringRef contextHash) {
     std::string compiledModulePath;
     return ModuleDependencies(
         std::make_unique<SwiftModuleDependenciesStorage>(
-          compiledModulePath, swiftInterfaceFile, buildCommands,
+          compiledModulePath, swiftInterfaceFile, compiledCandidates, buildCommands,
           extraPCMArgs, contextHash));
   }
 
@@ -196,7 +203,7 @@ class ModuleDependencies {
       const std::string &compiledModulePath) {
     return ModuleDependencies(
         std::make_unique<SwiftModuleDependenciesStorage>(
-          compiledModulePath, None, ArrayRef<StringRef>(),
+          compiledModulePath, None, ArrayRef<std::string>(), ArrayRef<StringRef>(),
           ArrayRef<StringRef>(), StringRef()));
   }
 
@@ -205,8 +212,8 @@ class ModuleDependencies {
     std::string compiledModulePath;
     return ModuleDependencies(
         std::make_unique<SwiftModuleDependenciesStorage>(
-          compiledModulePath, None, ArrayRef<StringRef>(), extraPCMArgs,
-          StringRef()));
+          compiledModulePath, None, ArrayRef<std::string>(),
+          ArrayRef<StringRef>(), extraPCMArgs, StringRef()));
   }
 
   /// Describe the module dependencies for a Clang module that can be

include/swift/Frontend/ModuleInterfaceLoader.h

@@ -237,8 +237,9 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase {
     bool SerializeDependencyHashes, bool TrackSystemDependencies,
     ModuleInterfaceLoaderOptions Opts);
 
-  std::string getUpToDateCompiledModuleForInterface(StringRef moduleName,
-                                                    StringRef interfacePath) override;
+  std::vector<std::string>
+  getCompiledModuleCandidatesForInterface(StringRef moduleName,
+                                          StringRef interfacePath) override;
 };
 
 struct InterfaceSubContextDelegateImpl: InterfaceSubContextDelegate {

include/swift/Serialization/SerializedModuleLoader.h

@@ -201,9 +201,10 @@ class SerializedModuleLoaderBase : public ModuleLoader {
       StringRef moduleName, ModuleDependenciesCache &cache,
       InterfaceSubContextDelegate &delegate) override;
 
-  virtual std::string getUpToDateCompiledModuleForInterface(StringRef moduleName,
-                                                      StringRef interfacePath) {
-    return std::string();
+  virtual std::vector<std::string>
+  getCompiledModuleCandidatesForInterface(StringRef moduleName,
+                                          StringRef interfacePath) {
+    return std::vector<std::string>();
   }
 };
 

lib/Frontend/ModuleInterfaceLoader.cpp

@@ -1015,9 +1015,9 @@ std::error_code ModuleInterfaceLoader::findModuleFilesInDirectory(
   return std::error_code();
 }
 
-std::string
-ModuleInterfaceLoader::getUpToDateCompiledModuleForInterface(StringRef moduleName,
-                                                      StringRef interfacePath) {
+std::vector<std::string>
+ModuleInterfaceLoader::getCompiledModuleCandidatesForInterface(StringRef moduleName,
+                                                               StringRef interfacePath) {
   // Derive .swiftmodule path from the .swiftinterface path.
   auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile);
   llvm::SmallString<32> modulePath = interfacePath;
@@ -1030,9 +1030,14 @@ ModuleInterfaceLoader::getUpToDateCompiledModuleForInterface(StringRef moduleNam
                 llvm::is_contained(PreferInterfaceForModules, moduleName) ?
                   ModuleLoadingMode::PreferInterface : LoadMode);
   SmallVector<FileDependency, 16> allDeps;
-  std::string usableModulePath;
-  Impl.discoverUpToDateCompiledModuleForInterface(allDeps, usableModulePath);
-  return usableModulePath;
+  std::vector<std::string> results;
+  auto pair = Impl.getCompiledModuleCandidates();
+  // Add compiled module candidates only when they are non-empty.
+  if (!pair.first.empty())
+    results.push_back(pair.first);
+  if (!pair.second.empty())
+    results.push_back(pair.second);
+  return results;
 }
 
 bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(

lib/FrontendTool/ScanDependencies.cpp

@@ -339,6 +339,17 @@ static void writeJSON(llvm::raw_ostream &out,
         }
         out.indent(5 * 2);
         out << "],\n";
+        out.indent(5 * 2);
+        out << "\"compiledModuleCandidates\": [\n";
+        for (auto &candidate: swiftDeps->compiledModuleCandidates) {
+          out.indent(6 * 2);
+          out << "\"" << candidate << "\"";
+          if (&candidate != &swiftDeps->compiledModuleCandidates.back())
+            out << ",";
+          out << "\n";
+        }
+        out.indent(5 * 2);
+        out << "],\n";
       } else if (!swiftDeps->compiledModulePath.empty()) {
         writeJSONSingleField(
             out, "compiledModulePath",

lib/Serialization/ModuleDependencyScanner.cpp

@@ -44,16 +44,6 @@ class ModuleDependencyScanner : public SerializedModuleLoaderBase {
                                    /*IgnoreSwiftSourceInfoFile=*/true),
         moduleName(moduleName), astDelegate(astDelegate) { }
 
-  std::string getCompiledModulePath(const SerializedModuleBaseName &BaseName) {
-    if (LoadMode == ModuleLoadingMode::OnlySerialized) {
-      return BaseName.getName(file_types::TY_SwiftModuleFile);
-    }
-    return static_cast<SerializedModuleLoaderBase*>(Ctx
-      .getModuleInterfaceLoader())->getUpToDateCompiledModuleForInterface(
-         moduleName.str(),
-         BaseName.getName(file_types::TY_SwiftModuleInterfaceFile));
-  }
-
   virtual std::error_code findModuleFilesInDirectory(
       AccessPathElem ModuleID,
       const SerializedModuleBaseName &BaseName,
@@ -65,26 +55,23 @@ class ModuleDependencyScanner : public SerializedModuleLoaderBase {
 
     auto &fs = *Ctx.SourceMgr.getFileSystem();
 
-    // Compute the full path of the module we're looking for.
-    auto ModPath = getCompiledModulePath(BaseName);
+    auto ModPath = BaseName.getName(file_types::TY_SwiftModuleFile);
+    auto InPath = BaseName.getName(file_types::TY_SwiftModuleInterfaceFile);
 
-    if (fs.exists(ModPath)) {
-      // The module file will be loaded directly.
-      auto dependencies = scanModuleFile(ModPath);
-      if (dependencies) {
-        this->dependencies = std::move(dependencies.get());
-        return std::error_code();
+    if (LoadMode == ModuleLoadingMode::OnlySerialized || !fs.exists(InPath)) {
+      if (fs.exists(ModPath)) {
+        // The module file will be loaded directly.
+        auto dependencies = scanModuleFile(ModPath);
+        if (dependencies) {
+          this->dependencies = std::move(dependencies.get());
+          return std::error_code();
+        }
+        return dependencies.getError();
+      } else {
+        return std::make_error_code(std::errc::no_such_file_or_directory);
       }
-
-      return dependencies.getError();
     }
-
-    // Check whether the .swiftinterface exists.
-    auto InPath = BaseName.getName(file_types::TY_SwiftModuleInterfaceFile);
-
-    if (!fs.exists(InPath))
-      return std::make_error_code(std::errc::no_such_file_or_directory);
-
+    assert(fs.exists(InPath));
     auto dependencies = scanInterfaceFile(InPath);
     if (dependencies) {
       this->dependencies = std::move(dependencies.get());
@@ -101,6 +88,14 @@ class ModuleDependencyScanner : public SerializedModuleLoaderBase {
 };
 }
 
+static std::vector<std::string> getCompiledCandidates(ASTContext &ctx,
+                                                      StringRef moduleName,
+                                                      StringRef interfacePath) {
+  return static_cast<SerializedModuleLoaderBase*>(ctx
+    .getModuleInterfaceLoader())->getCompiledModuleCandidatesForInterface(
+      moduleName.str(), interfacePath);
+}
+
 ErrorOr<ModuleDependencies> ModuleDependencyScanner::scanInterfaceFile(
     Twine moduleInterfacePath) {
   // Create a module filename.
@@ -117,7 +112,11 @@ ErrorOr<ModuleDependencies> ModuleDependencyScanner::scanInterfaceFile(
                                               SourceLoc(),
                 [&](ASTContext &Ctx, ArrayRef<StringRef> Args,
                     ArrayRef<StringRef> PCMArgs, StringRef Hash) {
-    Result = ModuleDependencies::forSwiftInterface(moduleInterfacePath.str(),
+    std::string InPath = moduleInterfacePath.str();
+    auto compiledCandidates = getCompiledCandidates(Ctx, moduleName.str(),
+                                                    InPath);
+    Result = ModuleDependencies::forSwiftInterface(InPath,
+                                                   compiledCandidates,
                                                    Args,
                                                    PCMArgs,
                                                    Hash);

test/ScanDependencies/Inputs/ModuleDependencyGraph.swift

@@ -63,6 +63,9 @@ struct SwiftModuleDetails: Codable {
   /// The module interface from which this module was built, if any.
   var moduleInterfacePath: String?
 
+  /// The paths of potentially ready-to-use compiled modules for the interface.
+  var compiledModuleCandidates: [String]?
+
   /// The compiled Swift module to use.
   var compiledModulePath: String?
 

test/ScanDependencies/compiled_swift_modules.swift

@@ -5,8 +5,10 @@
 
 import Foo
 
-// HAS_COMPILED:        "compiledModulePath": "{{.*}}Foo.swiftmodule{{.*}}.swiftmodule"
-// HAS_NO_COMPILED-NOT: "compiledModulePath": "{{.*}}Foo.swiftmodule{{.*}}.swiftmodule"
+// HAS_COMPILED: "compiledModuleCandidates": [
+// HAS_COMPILED-NEXT: "{{.*}}Foo.swiftmodule{{.*}}.swiftmodule"
+
+// HAS_NO_COMPILED-NOT: "{{.*}}Foo.swiftmodule{{.*}}.swiftmodule"
 
 // Step 1: build swift interface and swift module side by side
 // RUN: %target-swift-frontend -emit-module %t/Foo.swift -emit-module-path %t/Foo.swiftmodule/%target-swiftmodule-name -module-name Foo -emit-module-interface-path %t/Foo.swiftmodule/%target-swiftinterface-name
@@ -32,6 +34,6 @@ import Foo
 // Step 6: update the interface file from where the prebuilt module cache was built.
 // RUN: touch %t/Foo.swiftmodule/%target-swiftinterface-name
 
-// Step 4: scan dependency should give us the interface file.
-// RUN: %target-swift-frontend -scan-dependencies %s -o %t/deps.json -I %t -emit-dependencies -emit-dependencies-path %t/deps.d
-// RUN: %FileCheck %s -check-prefix=HAS_NO_COMPILED < %t/deps.json
+// Step 7: scan dependency should give us the prebuilt module file even though it's out-of-date.
+// RUN: %target-swift-frontend -scan-dependencies %s -o %t/deps.json -I %t -emit-dependencies -emit-dependencies-path %t/deps.d -sdk %t -prebuilt-module-cache-path %t/ResourceDir/%target-sdk-name/prebuilt-modules
+// RUN: %FileCheck %s -check-prefix=HAS_COMPILED < %t/deps.json