Embed GDB pretty printers in rlibs and dylibs

Instead of collecting pretty printers transitively when building
executables/staticlibs/cdylibs, let the debugger find each crate's
pretty printers via its .debug_gdb_scripts section. This covers the case
where libraries defining custom pretty printers are loaded dynamically.

Author: sebastianpoeplau

compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs

@@ -1,13 +1,13 @@
 // .debug_gdb_scripts binary section.
 
+use std::collections::BTreeSet;
 use std::ffi::CString;
 
-use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive;
 use rustc_codegen_ssa::traits::*;
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::bug;
 use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerType;
-use rustc_session::config::{CrateType, DebugInfo};
+use rustc_session::config::DebugInfo;
 
 use crate::builder::Builder;
 use crate::common::CodegenCx;
@@ -51,10 +51,14 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>(
 
         // Next, add the pretty printers that were specified via the `#[debugger_visualizer]`
         // attribute.
-        let visualizers = collect_debugger_visualizers_transitive(
-            cx.tcx,
-            DebuggerVisualizerType::GdbPrettyPrinter,
-        );
+        let visualizers = cx
+            .tcx
+            .debugger_visualizers(LOCAL_CRATE)
+            .iter()
+            .filter(|visualizer| {
+                visualizer.visualizer_type == DebuggerVisualizerType::GdbPrettyPrinter
+            })
+            .collect::<BTreeSet<_>>();
         let crate_name = cx.tcx.crate_name(LOCAL_CRATE);
         for (index, visualizer) in visualizers.iter().enumerate() {
             // The initial byte `4` instructs GDB that the following pretty printer
@@ -91,30 +95,5 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>(
 }
 
 pub(crate) fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool {
-    // We collect pretty printers transitively for all crates, so we make sure
-    // that the section is only emitted for leaf crates.
-    let embed_visualizers = cx.tcx.crate_types().iter().any(|&crate_type| match crate_type {
-        CrateType::Executable | CrateType::Cdylib | CrateType::Staticlib | CrateType::Sdylib => {
-            // These are crate types for which we will embed pretty printers since they
-            // are treated as leaf crates.
-            true
-        }
-        CrateType::ProcMacro => {
-            // We could embed pretty printers for proc macro crates too but it does not
-            // seem like a good default, since this is a rare use case and we don't
-            // want to slow down the common case.
-            false
-        }
-        CrateType::Rlib | CrateType::Dylib => {
-            // Don't embed pretty printers for these crate types; the compiler
-            // can see the `#[debug_visualizer]` attributes when using the
-            // library, and emitting `.debug_gdb_scripts` regardless would
-            // break `#![omit_gdb_pretty_printer_section]`.
-            false
-        }
-    });
-
-    cx.sess().opts.debuginfo != DebugInfo::None
-        && cx.sess().target.emit_debug_gdb_scripts
-        && embed_visualizers
+    cx.sess().opts.debuginfo != DebugInfo::None && cx.sess().target.emit_debug_gdb_scripts
 }

compiler/rustc_codegen_ssa/src/base.rs

@@ -609,7 +609,15 @@ pub fn collect_debugger_visualizers_transitive(
 ) -> BTreeSet<DebuggerVisualizerFile> {
     tcx.debugger_visualizers(LOCAL_CRATE)
         .iter()
-        .chain(tcx.crates(()).iter().flat_map(|&cnum| tcx.debugger_visualizers(cnum)))
+        .chain(
+            tcx.crates(())
+                .iter()
+                .filter(|&cnum| {
+                    let used_crate_source = tcx.used_crate_source(*cnum);
+                    used_crate_source.rlib.is_some() || used_crate_source.rmeta.is_some()
+                })
+                .flat_map(|&cnum| tcx.debugger_visualizers(cnum)),
+        )
         .filter(|visualizer| visualizer.visualizer_type == visualizer_type)
         .cloned()
         .collect::<BTreeSet<_>>()

src/tools/compiletest/src/directives/directive_names.rs

@@ -68,6 +68,7 @@ pub(crate) const KNOWN_DIRECTIVE_NAMES: &[&str] = &[
     "ignore-gnu",
     "ignore-haiku",
     "ignore-horizon",
+    "ignore-i586-unknown-linux-gnu",
     "ignore-i686-pc-windows-gnu",
     "ignore-i686-pc-windows-msvc",
     "ignore-illumos",

tests/debuginfo/embedded-visualizer.rs

@@ -1,6 +1,8 @@
 //@ compile-flags:-g
 //@ ignore-lldb
 //@ ignore-windows-gnu: #128981
+//@ ignore-musl: linker too old in CI
+//@ ignore-i586-unknown-linux-gnu: linker too old in CI
 
 // === CDB TESTS ==================================================================================