[LLDB] Set up an explicit Swift module loader

and populate it with explicitly specified Swift modules.

This change allows LLDB to precisely load the explicitly built main
module belonging to the current CU from disk. The mere presence of an
ESML also recursively enables the same behavior for all of its
dependencies, since that functionality is already implemented in the
Swift compiler.

This fixes a performance cliff encountered when debugging with dSYMs
which contain all user modules, but none from the SDK, thus triggering
an implicit build of any SDK dependencies.

During the transition period this behavior can be disabled with a setting.

rdar://164274588

Author: adrian-prantl

lldb/include/lldb/Core/ModuleList.h

@@ -83,6 +83,8 @@ class ModuleListProperties : public Properties {
   bool GetUseSwiftClangImporter() const;
   bool GetUseSwiftDWARFImporter() const;
   bool SetUseSwiftDWARFImporter(bool new_value);
+  bool GetUseSwiftExplicitModuleLoader() const;
+  bool SetUseSwiftExplicitModuleLoader(bool new_value);
   bool GetSwiftValidateTypeSystem() const;
   bool GetSwiftTypeSystemFallback() const;
   bool GetSwiftLoadConformances() const;

lldb/source/Core/CoreProperties.td

@@ -25,6 +25,10 @@ let Definition = "modulelist" in {
   def UseSwiftDWARFImporter: Property<"use-swift-dwarfimporter", "Boolean">,
     DefaultTrue,
     Desc<"Reconstruct Clang module dependencies from DWARF when debugging Swift code">;
+  def UseSwiftExplicitModuleLoader
+      : Property<"use-swift-explicit-module-loader", "Boolean">,
+        DefaultTrue,
+        Desc<"Prefer explicitly specified modules over ones found in dSYMs">;
   def SwiftValidateTypeSystem: Property<"swift-validate-typesystem", "Boolean">,
     DefaultFalse,
     Desc<"Validate all Swift typesystem queries. Used for testing an asserts-enabled LLDB only.">;

lldb/source/Core/ModuleList.cpp

@@ -199,6 +199,17 @@ bool ModuleListProperties::SetUseSwiftDWARFImporter(bool new_value) {
   return SetPropertyAtIndex(idx, new_value);
 }
 
+bool ModuleListProperties::GetUseSwiftExplicitModuleLoader() const {
+  const uint32_t idx = ePropertyUseSwiftExplicitModuleLoader;
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_modulelist_properties[idx].default_uint_value != 0);
+}
+
+bool ModuleListProperties::SetUseSwiftExplicitModuleLoader(bool new_value) {
+  const uint32_t idx = ePropertyUseSwiftExplicitModuleLoader;
+  return SetPropertyAtIndex(idx, new_value);
+}
+
 bool ModuleListProperties::GetSwiftValidateTypeSystem() const {
   const uint32_t idx = ePropertySwiftValidateTypeSystem;
   return GetPropertyAtIndexAs<bool>(

lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp

@@ -1899,6 +1899,22 @@ void SwiftASTContext::AddExtraClangArgs(
     RemoveExplicitModules(importer_options.ExtraArgs);
 }
 
+bool SwiftASTContext::IsModuleAvailableInCAS(const std::string &key) {
+  auto id = m_cas->parseID(key);
+  if (!id) {
+    HEALTH_LOG_PRINTF("failed to parse CASID when loading module: %s",
+                      toString(id.takeError()).c_str());
+    return false;
+  }
+  auto lookup = m_action_cache->get(*id);
+  if (!lookup) {
+    HEALTH_LOG_PRINTF("module lookup failure through action cache: %s",
+                      toString(lookup.takeError()).c_str());
+    return false;
+  }
+  return (bool)*lookup;
+};
+
 void SwiftASTContext::AddExtraClangCC1Args(
     const std::vector<std::string> &source,
     const std::vector<std::pair<std::string, bool>> module_search_paths,
@@ -1970,26 +1986,9 @@ void SwiftASTContext::AddExtraClangCC1Args(
     invocation.getCASOpts().PluginOptions =
         GetCASOptions().CASOpts.PluginOptions;
 
-    // Check the module availability in CAS, if not, fallback to regular load.
-    auto CheckModuleInCAS = [&](const std::string &key) {
-      auto id = m_cas->parseID(key);
-      if (!id) {
-        HEALTH_LOG_PRINTF("failed to parse CASID when loading module: %s",
-                          toString(id.takeError()).c_str());
-        return false;
-      }
-      auto lookup = m_action_cache->get(*id);
-      if (!lookup) {
-        HEALTH_LOG_PRINTF("module lookup failure through action cache: %s",
-                          toString(lookup.takeError()).c_str());
-        return false;
-      }
-      return (bool)*lookup;
-    };
-
     use_cas_module = llvm::all_of(
         invocation.getFrontendOpts().ModuleCacheKeys, [&](const auto &entry) {
-          auto exist = CheckModuleInCAS(entry.second);
+          bool exist = IsModuleAvailableInCAS(entry.second);
           if (!exist)
             HEALTH_LOG_PRINTF("module '%s' cannot be load "
                               "from CAS using key: %s, fallback to "
@@ -3748,7 +3747,6 @@ ThreadSafeASTContext SwiftASTContext::GetASTContext() {
   std::string moduleCachePath =
       GetCompilerInvocation().getClangModuleCachePath().str();
   std::unique_ptr<swift::ClangImporter> clang_importer_up;
-  auto &clang_importer_options = GetClangImporterOptions();
   if (!m_ast_context_up->SearchPathOpts.getSDKPath().empty() ||
       TargetHasNoSDK()) {
     // Create the DWARFImporterDelegate.
@@ -3848,15 +3846,34 @@ ThreadSafeASTContext SwiftASTContext::GetASTContext() {
     m_ast_context_up->addModuleLoader(std::move(memory_buffer_loader_up));
   }
 
+  // 2. Create the explicit swift module loader.
+  if (props.GetUseSwiftExplicitModuleLoader()) {
+    auto &search_path_opts = GetCompilerInvocation().getSearchPathOptions();
+    std::unique_ptr<swift::ModuleLoader> esml_up =
+        swift::ExplicitSwiftModuleLoader::create(
+            *m_ast_context_up, m_dependency_tracker.get(), loading_mode,
+            search_path_opts.ExplicitSwiftModuleMapPath,
+            search_path_opts.ExplicitSwiftModuleInputs,
+            /*IgnoreSwiftSourceInfo*/ false);
+    if (esml_up) {
+      m_explicit_swift_module_loader =
+          static_cast<swift::ExplicitSwiftModuleLoader *>(esml_up.get());
+      m_ast_context_up->addModuleLoader(std::move(esml_up), /*isClang=*/false,
+                                        /*isDwarf=*/false,
+                                        /*isInterface=*/false,
+                                        /*isExplicit=*/true);
+    }
+  }
+
   // Add a module interface checker.
   m_ast_context_up->addModuleInterfaceChecker(
-    std::make_unique<swift::ModuleInterfaceCheckerImpl>(*m_ast_context_up,
-      moduleCachePath, prebuiltModuleCachePath,
-      swift::ModuleInterfaceLoaderOptions()));
+      std::make_unique<swift::ModuleInterfaceCheckerImpl>(
+          *m_ast_context_up, moduleCachePath, prebuiltModuleCachePath,
+          swift::ModuleInterfaceLoaderOptions()));
 
-  // 2. Create and install the module interface loader.
+  // 3. Create and install the module interface loader.
   //
-  // The ordering of 2-4 is the same as the Swift compiler's 1-3,
+  // The ordering of 2-4 is the same as the Swift compiler's 2-4,
   // where unintuitively the serialized module loader comes before the
   // module interface loader. The reason for this is that the module
   // interface loader is actually 2-in-1 and secretly attempts to load
@@ -3868,21 +3885,22 @@ ThreadSafeASTContext SwiftASTContext::GetASTContext() {
   if (loading_mode != swift::ModuleLoadingMode::OnlySerialized) {
     std::unique_ptr<swift::ModuleLoader> module_interface_loader_up(
         swift::ModuleInterfaceLoader::create(
-          *m_ast_context_up, *static_cast<swift::ModuleInterfaceCheckerImpl*>(
-            m_ast_context_up->getModuleInterfaceChecker()), m_dependency_tracker.get(),
-          loading_mode));
+            *m_ast_context_up,
+            *static_cast<swift::ModuleInterfaceCheckerImpl *>(
+                m_ast_context_up->getModuleInterfaceChecker()),
+            m_dependency_tracker.get(), loading_mode));
     if (module_interface_loader_up)
       m_ast_context_up->addModuleLoader(std::move(module_interface_loader_up));
   }
 
-  // 3. Create and install the serialized module loader.
+  // 4. Create and install the serialized module loader.
   std::unique_ptr<swift::ModuleLoader> serialized_module_loader_up(
       swift::ImplicitSerializedModuleLoader::create(
           *m_ast_context_up, m_dependency_tracker.get(), loading_mode));
   if (serialized_module_loader_up)
     m_ast_context_up->addModuleLoader(std::move(serialized_module_loader_up));
 
-  // 4. Install the clang importer.
+  // 5. Install the clang importer.
   if (clang_importer_up) {
     m_clangimporter = (swift::ClangImporter *)clang_importer_up.get();
     m_ast_context_up->addModuleLoader(std::move(clang_importer_up),
@@ -4080,6 +4098,23 @@ SwiftASTContext::GetModule(const SourceModule &module, bool *cached) {
   // Create a diagnostic consumer for the diagnostics produced by the import.
   auto import_diags = getScopedDiagnosticConsumer();
 
+  // Is this an explicitly specified explicit Swift module?
+  StringRef module_path = module.search_path.GetStringRef();
+  bool is_esml_module = (module_path.ends_with(".swiftmodule") &&
+                         llvm::sys::fs::exists(module_path)) ||
+                        (m_cas && IsModuleAvailableInCAS(module_path.str()));
+  if (is_esml_module) {
+    std::string path = module_path.str();
+    bool unloaded = false;
+    if (m_explicit_swift_module_loader) {
+      ast->addExplicitModulePath(module_name, path);
+      if (auto *memory_loader = GetMemoryBufferModuleLoader())
+        unloaded = memory_loader->unregisterMemoryBuffer(module_name);
+    }
+    HEALTH_LOG_PRINTF("found explicit module \"%s\"%s", path.c_str(),
+                      unloaded ? "; replacing AST section module" : "");
+  }
+
   swift::ModuleDecl *module_decl = ast->getModuleByName(module_name);
 
   // Error handling.
@@ -4102,6 +4137,16 @@ SwiftASTContext::GetModule(const SourceModule &module, bool *cached) {
   LOG_PRINTF(GetLog(LLDBLog::Types), "(\"%s\") -- found %s",
              module_name.c_str(), module_decl->getName().str().str().c_str());
 
+  if (is_esml_module) {
+    // Simulate the effect of the BypassResilience flag in the
+    // MemoryBufferSerializedModuleLoader.  Explicitly specified
+    // modules are not typically produced from textual interfaces. By
+    // disabling resilience, the debugger can directly access private
+    // members.
+    //if (!module_decl->isBuiltFromInterface())
+    //  module_decl->setBypassResilience();
+  }
+
   m_swift_module_cache.insert({module_name, *module_decl});
   return *module_decl;
 }

lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.h

@@ -278,6 +278,9 @@ class SwiftASTContext : public TypeSystemSwift {
 
   void ConfigureModuleValidation(std::vector<std::string> &extra_args);
 
+  /// Check whether a module with key \c key is available in CAS.
+  bool IsModuleAvailableInCAS(const std::string &key);
+
   /// Add a list of Clang arguments to the ClangImporter options and
   /// apply the working directory to any relative paths.
   void AddExtraClangArgs(
@@ -972,6 +975,7 @@ class SwiftASTContext : public TypeSystemSwift {
   /// Owned by the AST.
   swift::MemoryBufferSerializedModuleLoader *m_memory_buffer_module_loader =
       nullptr;
+  swift::ModuleLoader *m_explicit_swift_module_loader = nullptr;
   swift::ClangImporter *m_clangimporter = nullptr;
   /// Wraps the clang::ASTContext owned by ClangImporter.
   std::shared_ptr<TypeSystemClang> m_clangimporter_typesystem;

lldb/test/API/lang/swift/explicit_modules/simple/TestSwiftExplicitModules.py

@@ -13,8 +13,30 @@ def test(self):
         target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(
             self, 'Set breakpoint here', lldb.SBFileSpec('main.swift'))
 
+        log = self.getBuildArtifact("types.log")
+        self.expect('log enable lldb types -f "%s"' % log)
         self.expect("expression c", substrs=['hello explicit'])
+        self.filecheck('platform shell cat "%s"' % log, __file__)
+        # CHECK: SwiftASTContextForExpressions(module: "a", cu: "main.swift"){{.*}} found explicit module {{.*}}a.swiftmodule
+        # CHECK: SwiftASTContextForExpressions(module: "a", cu: "main.swift"){{.*}} Module import remark: loaded module 'a'; source: '{{.*}}a.swiftmodule', loaded: '{{.*}}a.swiftmodule'
 
+    @swiftTest
+    def test_disable_esml(self):
+        """Test disabling the explicit Swift module loader"""
+        self.build()
+        self.expect("settings set symbols.use-swift-explicit-module-loader false")
+
+        target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(
+            self, 'Set breakpoint here', lldb.SBFileSpec('main.swift'))
+
+        log = self.getBuildArtifact("types.log")
+        self.expect('log enable lldb types -f "%s"' % log)
+        self.expect("expression c", substrs=['hello explicit'])
+        self.filecheck('platform shell cat "%s"' % log, __file__, '--check-prefix=DISABLED')
+        # DISABLED: SwiftASTContextForExpressions(module: "a", cu: "main.swift"){{.*}} found explicit module {{.*}}a.swiftmodule
+        # DISABLED: SwiftASTContextForExpressions(module: "a", cu: "main.swift"){{.*}} Module import remark: loaded module 'a'; source: 'a', loaded: 'a'
+
+        
     @swiftTest
     @skipUnlessDarwin
     def test_import(self):