Add -Zannotate-moves for profiler visibility of move/copy operations (codegen)

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1069,7 +1069,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
             OperandValue::Ref(place.val)
         };
 
-        OperandRef { val, layout: place.layout }
+        OperandRef { val, layout: place.layout, move_annotation: None }
     }
 
     fn write_operand_repeatedly(

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -19,7 +19,7 @@ use crate::context::CodegenCx;
 pub(super) const UNKNOWN_LINE_NUMBER: u32 = 0;
 pub(super) const UNKNOWN_COLUMN_NUMBER: u32 = 0;
 
-impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'gcc, 'tcx> {
+impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
     // FIXME(eddyb) find a common convention for all of the debuginfo-related
     // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
     fn dbg_var_addr(

compiler/rustc_codegen_llvm/src/abi.rs

@@ -247,7 +247,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 );
                 bx.lifetime_end(llscratch, scratch_size);
             }
-            _ => {
+            PassMode::Pair(..) | PassMode::Direct { .. } => {
                 OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst);
             }
         }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -752,7 +752,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             OperandValue::Ref(place.val)
         };
 
-        OperandRef { val, layout: place.layout }
+        OperandRef { val, layout: place.layout, move_annotation: None }
     }
 
     fn write_operand_repeatedly(

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -146,7 +146,7 @@ impl<'ll> Builder<'_, 'll, '_> {
     }
 }
 
-impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
+impl<'ll, 'tcx> DebugInfoBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     // FIXME(eddyb) find a common convention for all of the debuginfo-related
     // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
     fn dbg_var_addr(
@@ -284,6 +284,57 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
             llvm::set_value_name(value, name.as_bytes());
         }
     }
+
+    /// Annotate move/copy operations with debug info for profiling.
+    ///
+    /// This creates a temporary debug scope that makes the move/copy appear as an inlined call to
+    /// `compiler_move<T, SIZE>()` or `compiler_copy<T, SIZE>()`. The provided closure is executed
+    /// with this temporary debug location active.
+    ///
+    /// The `instance` parameter should be the monomorphized instance of the `compiler_move` or
+    /// `compiler_copy` function with the actual type and size.
+    fn with_move_annotation<R>(
+        &mut self,
+        instance: ty::Instance<'tcx>,
+        f: impl FnOnce(&mut Self) -> R,
+    ) -> R {
+        // Save the current debug location
+        let saved_loc = self.get_dbg_loc();
+
+        // Create a DIScope for the compiler_move/compiler_copy function
+        // We use the function's FnAbi for debug info generation
+        let fn_abi = self
+            .cx()
+            .tcx
+            .fn_abi_of_instance(
+                self.cx().typing_env().as_query_input((instance, ty::List::empty())),
+            )
+            .unwrap();
+
+        let di_scope = self.cx().dbg_scope_fn(instance, fn_abi, None);
+
+        // Create an inlined debug location:
+        // - scope: the compiler_move/compiler_copy function
+        // - inlined_at: the current location (where the move/copy actually occurs)
+        // - span: use the function's definition span
+        let fn_span = self.cx().tcx.def_span(instance.def_id());
+        let inlined_loc = self.cx().dbg_loc(di_scope, saved_loc, fn_span);
+
+        // Set the temporary debug location
+        self.set_dbg_loc(inlined_loc);
+
+        // Execute the closure (which will generate the memcpy)
+        let result = f(self);
+
+        // Restore the original debug location
+        if let Some(loc) = saved_loc {
+            self.set_dbg_loc(loc);
+        } else {
+            self.clear_dbg_loc();
+        }
+
+        result
+    }
 }
 
 /// A source code location used to generate debug information.

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -557,9 +557,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let op = match self.locals[mir::RETURN_PLACE] {
                     LocalRef::Operand(op) => op,
                     LocalRef::PendingOperand => bug!("use of return before def"),
-                    LocalRef::Place(cg_place) => {
-                        OperandRef { val: Ref(cg_place.val), layout: cg_place.layout }
-                    }
+                    LocalRef::Place(cg_place) => OperandRef {
+                        val: Ref(cg_place.val),
+                        layout: cg_place.layout,
+                        move_annotation: None,
+                    },
                     LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
                 };
                 let llslot = match op.val {
@@ -1145,7 +1147,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 | (&mir::Operand::Constant(_), Ref(PlaceValue { llextra: None, .. })) => {
                     let tmp = PlaceRef::alloca(bx, op.layout);
                     bx.lifetime_start(tmp.val.llval, tmp.layout.size);
-                    op.val.store(bx, tmp);
+                    op.store_with_annotation(bx, tmp);
                     op.val = Ref(tmp.val);
                     lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
                 }
@@ -1553,13 +1555,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     };
                     let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
                     bx.lifetime_start(scratch.llval, arg.layout.size);
-                    op.val.store(bx, scratch.with_type(arg.layout));
+                    op.store_with_annotation(bx, scratch.with_type(arg.layout));
                     lifetime_ends_after_call.push((scratch.llval, arg.layout.size));
                     (scratch.llval, scratch.align, true)
                 }
                 PassMode::Cast { .. } => {
                     let scratch = PlaceRef::alloca(bx, arg.layout);
-                    op.val.store(bx, scratch);
+                    op.store_with_annotation(bx, scratch);
                     (scratch.val.llval, scratch.val.align, true)
                 }
                 _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi, false),

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -480,6 +480,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                         return local(OperandRef {
                             val: OperandValue::Pair(a, b),
                             layout: arg.layout,
+                            move_annotation: None,
                         });
                     }
                     _ => {}
@@ -552,6 +553,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         fx.caller_location = Some(OperandRef {
             val: OperandValue::Immediate(bx.get_param(llarg_idx)),
             layout: arg.layout,
+            move_annotation: None,
         });
     }
 

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -5,12 +5,13 @@ use rustc_abi as abi;
 use rustc_abi::{
     Align, BackendRepr, FIRST_VARIANT, FieldIdx, Primitive, Size, TagEncoding, VariantIdx, Variants,
 };
+use rustc_hir::LangItem;
 use rustc_middle::mir::interpret::{Pointer, Scalar, alloc_range};
 use rustc_middle::mir::{self, ConstValue};
-use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
+use rustc_middle::ty::{self, Ty};
 use rustc_middle::{bug, span_bug};
-use rustc_session::config::OptLevel;
+use rustc_session::config::{AnnotateMoves, DebugInfo, OptLevel};
 use tracing::{debug, instrument};
 
 use super::place::{PlaceRef, PlaceValue};
@@ -131,6 +132,10 @@ pub struct OperandRef<'tcx, V> {
 
     /// The layout of value, based on its Rust type.
     pub layout: TyAndLayout<'tcx>,
+
+    /// Annotation for profiler visibility of move/copy operations.
+    /// When set, the store operation should appear as an inlined call to this function.
+    pub move_annotation: Option<ty::Instance<'tcx>>,
 }
 
 impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
@@ -142,7 +147,7 @@ impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
 impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
     pub fn zero_sized(layout: TyAndLayout<'tcx>) -> OperandRef<'tcx, V> {
         assert!(layout.is_zst());
-        OperandRef { val: OperandValue::ZeroSized, layout }
+        OperandRef { val: OperandValue::ZeroSized, layout, move_annotation: None }
     }
 
     pub(crate) fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
@@ -180,7 +185,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
             }
         };
 
-        OperandRef { val, layout }
+        OperandRef { val, layout, move_annotation: None }
     }
 
     fn from_const_alloc<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
@@ -214,7 +219,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                 let size = s.size(bx);
                 assert_eq!(size, layout.size, "abi::Scalar size does not match layout size");
                 let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
-                OperandRef { val: OperandValue::Immediate(val), layout }
+                OperandRef { val: OperandValue::Immediate(val), layout, move_annotation: None }
             }
             BackendRepr::ScalarPair(
                 a @ abi::Scalar::Initialized { .. },
@@ -235,7 +240,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                     b,
                     bx.scalar_pair_element_backend_type(layout, 1, true),
                 );
-                OperandRef { val: OperandValue::Pair(a_val, b_val), layout }
+                OperandRef { val: OperandValue::Pair(a_val, b_val), layout, move_annotation: None }
             }
             _ if layout.is_zst() => OperandRef::zero_sized(layout),
             _ => {
@@ -285,6 +290,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
         self.val.deref(layout.align.abi).with_type(layout)
     }
 
+    /// Store this operand into a place, applying move/copy annotation if present.
+    ///
+    /// This is the preferred method for storing operands, as it automatically
+    /// applies profiler annotations for tracked move/copy operations.
+    pub fn store_with_annotation<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+        self,
+        bx: &mut Bx,
+        dest: PlaceRef<'tcx, V>,
+    ) {
+        if let Some(instance) = self.move_annotation {
+            bx.with_move_annotation(instance, |bx| self.val.store(bx, dest))
+        } else {
+            self.val.store(bx, dest)
+        }
+    }
+
     /// If this operand is a `Pair`, we return an aggregate with the two values.
     /// For other cases, see `immediate`.
     pub fn immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
@@ -320,7 +341,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
         } else {
             OperandValue::Immediate(llval)
         };
-        OperandRef { val, layout }
+        OperandRef { val, layout, move_annotation: None }
     }
 
     pub(crate) fn extract_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
@@ -388,7 +409,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
             })
         };
 
-        OperandRef { val, layout: field }
+        OperandRef { val, layout: field, move_annotation: None }
     }
 
     /// Obtain the actual discriminant of a value.
@@ -828,10 +849,15 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> {
                 }
             },
         };
-        OperandRef { val, layout }
+        OperandRef { val, layout, move_annotation: None }
     }
 }
 
+/// Default size limit for move/copy annotations (in bytes). 64 bytes is a common size of a cache
+/// line, and the assumption is that anything this size or below is very cheap to move/copy, so only
+/// annotate copies larger than this.
+const MOVE_ANNOTATION_DEFAULT_LIMIT: u64 = 65;
+
 impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
     /// Returns an `OperandValue` that's generally UB to use in any way.
     ///
@@ -961,7 +987,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 abi::Variants::Single { index: vidx },
                             );
                             let layout = o.layout.for_variant(bx.cx(), vidx);
-                            o = OperandRef { val: o.val, layout }
+                            o = OperandRef { layout, ..o }
                         }
                         _ => return None,
                     }
@@ -1014,7 +1040,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         match *operand {
             mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
-                self.codegen_consume(bx, place.as_ref())
+                let kind = match operand {
+                    mir::Operand::Move(_) => LangItem::CompilerMove,
+                    mir::Operand::Copy(_) => LangItem::CompilerCopy,
+                    _ => unreachable!(),
+                };
+
+                // Check if we should annotate this move/copy for profiling
+                let move_annotation = self.move_copy_annotation_instance(bx, place.as_ref(), kind);
+
+                OperandRef { move_annotation, ..self.codegen_consume(bx, place.as_ref()) }
             }
 
             mir::Operand::Constant(ref constant) => {
@@ -1030,11 +1065,76 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         return OperandRef {
                             val: OperandValue::Immediate(llval),
                             layout: bx.layout_of(ty),
+                            move_annotation: None,
                         };
                     }
                 }
                 self.eval_mir_constant_to_operand(bx, constant)
             }
         }
     }
+
+    /// Creates an `Instance` for annotating a move/copy operation at codegen time.
+    ///
+    /// Returns `Some(instance)` if the operation should be annotated with debug info, `None`
+    /// otherwise. The instance represents a monomorphized `compiler_move<T, SIZE>` or
+    /// `compiler_copy<T, SIZE>` function that can be used to create debug scopes.
+    ///
+    /// There are a number of conditions that must be met for an annotation to be created, but aside
+    /// from the basics (annotation is enabled, we're generating debuginfo), the primary concern is
+    /// moves/copies which could result in a real `memcpy`. So we check for the size limit, but also
+    /// that the underlying representation of the type is in memory.
+    fn move_copy_annotation_instance(
+        &self,
+        bx: &Bx,
+        place: mir::PlaceRef<'tcx>,
+        kind: LangItem,
+    ) -> Option<ty::Instance<'tcx>> {
+        let tcx = bx.tcx();
+        let sess = tcx.sess;
+
+        // Skip if we're not generating debuginfo
+        if sess.opts.debuginfo == DebugInfo::None {
+            return None;
+        }
+
+        // Check if annotation is enabled and get size limit (otherwise skip)
+        let size_limit = match sess.opts.unstable_opts.annotate_moves {
+            AnnotateMoves::Disabled => return None,
+            AnnotateMoves::Enabled(None) => MOVE_ANNOTATION_DEFAULT_LIMIT,
+            AnnotateMoves::Enabled(Some(limit)) => limit,
+        };
+
+        let ty = self.monomorphized_place_ty(place);
+        let layout = bx.cx().layout_of(ty);
+        let ty_size = layout.size.bytes();
+
+        // Only annotate if type has a memory representation and exceeds size limit (and has a
+        // non-zero size)
+        if layout.is_zst()
+            || ty_size < size_limit
+            || !matches!(layout.backend_repr, BackendRepr::Memory { .. })
+        {
+            return None;
+        }
+
+        // Look up the DefId for compiler_move or compiler_copy lang item
+        let def_id = tcx.lang_items().get(kind)?;
+
+        // Create generic args: compiler_move<T, SIZE> or compiler_copy<T, SIZE>
+        let size_const = ty::Const::from_target_usize(tcx, ty_size);
+        let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]);
+
+        // Create the Instance
+        let typing_env = self.mir.typing_env(tcx);
+        let instance = ty::Instance::expect_resolve(
+            tcx,
+            typing_env,
+            def_id,
+            generic_args,
+            rustc_span::DUMMY_SP, // span only used for error messages
+        );
+
+        Some(instance)
+    }
 }