Add a fast path for lowering trivial consts

compiler/rustc_const_eval/src/const_eval/eval_queries.rs

@@ -7,7 +7,7 @@ use rustc_hir::def::DefKind;
 use rustc_middle::mir::interpret::{AllocId, ErrorHandled, InterpErrorInfo, ReportedErrorInfo};
 use rustc_middle::mir::{self, ConstAlloc, ConstValue};
 use rustc_middle::query::TyCtxtAt;
-use rustc_middle::ty::layout::HasTypingEnv;
+use rustc_middle::ty::layout::{HasTypingEnv, TyAndLayout};
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::{bug, throw_inval};
@@ -24,13 +24,11 @@ use crate::interpret::{
 };
 use crate::{CTRL_C_RECEIVED, errors};
 
-// Returns a pointer to where the result lives
-#[instrument(level = "trace", skip(ecx, body))]
-fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
+fn setup_for_eval<'tcx>(
     ecx: &mut CompileTimeInterpCx<'tcx>,
     cid: GlobalId<'tcx>,
-    body: &'tcx mir::Body<'tcx>,
-) -> InterpResult<'tcx, R> {
+    layout: TyAndLayout<'tcx>,
+) -> InterpResult<'tcx, (InternKind, MPlaceTy<'tcx>)> {
     let tcx = *ecx.tcx;
     assert!(
         cid.promoted.is_some()
@@ -46,7 +44,6 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
         "Unexpected DefKind: {:?}",
         ecx.tcx.def_kind(cid.instance.def_id())
     );
-    let layout = ecx.layout_of(body.bound_return_ty().instantiate(tcx, cid.instance.args))?;
     assert!(layout.is_sized());
 
     let intern_kind = if cid.promoted.is_some() {
@@ -58,12 +55,25 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
         }
     };
 
-    let ret = if let InternKind::Static(_) = intern_kind {
-        create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout)?
+    let return_place = if let InternKind::Static(_) = intern_kind {
+        create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout)
     } else {
-        ecx.allocate(layout, MemoryKind::Stack)?
+        ecx.allocate(layout, MemoryKind::Stack)
     };
 
+    return_place.map(|ret| (intern_kind, ret))
+}
+
+#[instrument(level = "trace", skip(ecx, body))]
+fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
+    ecx: &mut CompileTimeInterpCx<'tcx>,
+    cid: GlobalId<'tcx>,
+    body: &'tcx mir::Body<'tcx>,
+) -> InterpResult<'tcx, R> {
+    let tcx = *ecx.tcx;
+    let layout = ecx.layout_of(body.bound_return_ty().instantiate(tcx, cid.instance.args))?;
+    let (intern_kind, ret) = setup_for_eval(ecx, cid, layout)?;
+
     trace!(
         "eval_body_using_ecx: pushing stack frame for global: {}{}",
         with_no_trimmed_paths!(ecx.tcx.def_path_str(cid.instance.def_id())),
@@ -87,6 +97,31 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
         }
     }
 
+    intern_and_validate(ecx, cid, intern_kind, ret)
+}
+
+#[instrument(level = "trace", skip(ecx))]
+fn eval_trivial_const_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
+    ecx: &mut CompileTimeInterpCx<'tcx>,
+    cid: GlobalId<'tcx>,
+    val: ConstValue,
+    ty: Ty<'tcx>,
+) -> InterpResult<'tcx, R> {
+    let layout = ecx.layout_of(ty)?;
+    let (intern_kind, return_place) = setup_for_eval(ecx, cid, layout)?;
+
+    let opty = ecx.const_val_to_op(val, ty, Some(layout))?;
+    ecx.copy_op(&opty, &return_place)?;
+
+    intern_and_validate(ecx, cid, intern_kind, return_place)
+}
+
+fn intern_and_validate<'tcx, R: InterpretationResult<'tcx>>(
+    ecx: &mut CompileTimeInterpCx<'tcx>,
+    cid: GlobalId<'tcx>,
+    intern_kind: InternKind,
+    ret: MPlaceTy<'tcx>,
+) -> InterpResult<'tcx, R> {
     // Intern the result
     let intern_result = intern_const_alloc_recursive(ecx, intern_kind, &ret);
 
@@ -292,6 +327,9 @@ pub fn eval_to_const_value_raw_provider<'tcx>(
     tcx: TyCtxt<'tcx>,
     key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
 ) -> ::rustc_middle::mir::interpret::EvalToConstValueResult<'tcx> {
+    if let Some((value, _ty)) = tcx.trivial_const(key.value.instance.def_id()) {
+        return Ok(value);
+    }
     tcx.eval_to_allocation_raw(key).map(|val| turn_into_const_value(tcx, val, key))
 }
 
@@ -368,10 +406,14 @@ fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>(
         // so we have to reject reading mutable global memory.
         CompileTimeMachine::new(CanAccessMutGlobal::from(is_static), CheckAlignment::Error),
     );
-    let res = ecx.load_mir(cid.instance.def, cid.promoted);
-    res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, body))
-        .report_err()
-        .map_err(|error| report_eval_error(&ecx, cid, error))
+
+    let result = if let Some((value, ty)) = tcx.trivial_const(def) {
+        eval_trivial_const_using_ecx(&mut ecx, cid, value, ty)
+    } else {
+        ecx.load_mir(cid.instance.def, cid.promoted)
+            .and_then(|body| eval_body_using_ecx(&mut ecx, cid, body))
+    };
+    result.report_err().map_err(|error| report_eval_error(&ecx, cid, error))
 }
 
 #[inline(always)]

compiler/rustc_interface/src/passes.rs

@@ -1088,9 +1088,15 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
 
     sess.time("MIR_borrow_checking", || {
         tcx.par_hir_body_owners(|def_id| {
-            if !tcx.is_typeck_child(def_id.to_def_id()) {
+            let not_typeck_child = !tcx.is_typeck_child(def_id.to_def_id());
+            if not_typeck_child {
                 // Child unsafety and borrowck happens together with the parent
                 tcx.ensure_ok().check_unsafety(def_id);
+            }
+            if tcx.is_trivial_const(def_id) {
+                return;
+            }
+            if not_typeck_child {
                 tcx.ensure_ok().mir_borrowck(def_id);
                 tcx.ensure_ok().check_transmutes(def_id);
             }
@@ -1198,7 +1204,9 @@ fn analysis(tcx: TyCtxt<'_>, (): ()) {
     if tcx.sess.opts.unstable_opts.validate_mir {
         sess.time("ensuring_final_MIR_is_computable", || {
             tcx.par_hir_body_owners(|def_id| {
-                tcx.instance_mir(ty::InstanceKind::Item(def_id.into()));
+                if !tcx.is_trivial_const(def_id) {
+                    tcx.instance_mir(ty::InstanceKind::Item(def_id.into()));
+                }
             });
         });
     }

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -240,6 +240,7 @@ provide! { tcx, def_id, other, cdata,
     thir_abstract_const => { table }
     optimized_mir => { table }
     mir_for_ctfe => { table }
+    trivial_const => { table }
     closure_saved_names_of_captured_variables => { table }
     mir_coroutine_witnesses => { table }
     promoted_mir => { table }

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -1791,8 +1791,15 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                     record!(self.tables.mir_coroutine_witnesses[def_id.to_def_id()] <- witnesses);
                 }
             }
+            let mut is_trivial = false;
             if encode_const {
-                record!(self.tables.mir_for_ctfe[def_id.to_def_id()] <- tcx.mir_for_ctfe(def_id));
+                if let Some((val, ty)) = tcx.trivial_const(def_id) {
+                    is_trivial = true;
+                    record!(self.tables.trivial_const[def_id.to_def_id()] <- (val, ty));
+                } else {
+                    is_trivial = false;
+                    record!(self.tables.mir_for_ctfe[def_id.to_def_id()] <- tcx.mir_for_ctfe(def_id));
+                }
 
                 // FIXME(generic_const_exprs): this feels wrong to have in `encode_mir`
                 let abstract_const = tcx.thir_abstract_const(def_id);
@@ -1810,7 +1817,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                     }
                 }
             }
-            record!(self.tables.promoted_mir[def_id.to_def_id()] <- tcx.promoted_mir(def_id));
+            if !is_trivial {
+                record!(self.tables.promoted_mir[def_id.to_def_id()] <- tcx.promoted_mir(def_id));
+            }
 
             if self.tcx.is_coroutine(def_id.to_def_id())
                 && let Some(witnesses) = tcx.mir_coroutine_witnesses(def_id)
@@ -2234,6 +2243,9 @@ fn prefetch_mir(tcx: TyCtxt<'_>) {
 
     let reachable_set = tcx.reachable_set(());
     par_for_each_in(tcx.mir_keys(()), |&&def_id| {
+        if tcx.is_trivial_const(def_id) {
+            return;
+        }
         let (encode_const, encode_opt) = should_encode_mir(tcx, reachable_set, def_id);
 
         if encode_const {

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -29,6 +29,7 @@ use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo};
 use rustc_middle::middle::lib_features::FeatureStability;
 use rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault;
 use rustc_middle::mir;
+use rustc_middle::mir::ConstValue;
 use rustc_middle::ty::fast_reject::SimplifiedType;
 use rustc_middle::ty::{self, Ty, TyCtxt, UnusedGenericParams};
 use rustc_middle::util::Providers;
@@ -426,6 +427,7 @@ define_tables! {
     object_lifetime_default: Table<DefIndex, LazyValue<ObjectLifetimeDefault>>,
     optimized_mir: Table<DefIndex, LazyValue<mir::Body<'static>>>,
     mir_for_ctfe: Table<DefIndex, LazyValue<mir::Body<'static>>>,
+    trivial_const: Table<DefIndex, LazyValue<(ConstValue, Ty<'static>)>>,
     closure_saved_names_of_captured_variables: Table<DefIndex, LazyValue<IndexVec<FieldIdx, Symbol>>>,
     mir_coroutine_witnesses: Table<DefIndex, LazyValue<mir::CoroutineLayout<'static>>>,
     promoted_mir: Table<DefIndex, LazyValue<IndexVec<mir::Promoted, mir::Body<'static>>>>,

compiler/rustc_metadata/src/rmeta/parameterized.rs

@@ -102,6 +102,7 @@ trivially_parameterized_over_tcx! {
     rustc_middle::middle::lib_features::FeatureStability,
     rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault,
     rustc_middle::mir::ConstQualifs,
+    rustc_middle::mir::ConstValue,
     rustc_middle::ty::AnonConstKind,
     rustc_middle::ty::AssocContainer,
     rustc_middle::ty::AsyncDestructor,

compiler/rustc_middle/src/mir/graphviz.rs

@@ -16,6 +16,7 @@ where
 
     let mirs = def_ids
         .iter()
+        .filter(|def_id| !tcx.is_trivial_const(*def_id))
         .flat_map(|def_id| {
             if tcx.is_const_fn(*def_id) {
                 vec![tcx.optimized_mir(*def_id), tcx.mir_for_ctfe(*def_id)]

compiler/rustc_middle/src/mir/pretty.rs

@@ -353,8 +353,16 @@ pub fn write_mir_pretty<'tcx>(
             // are shared between mir_for_ctfe and optimized_mir
             writer.write_mir_fn(tcx.mir_for_ctfe(def_id), w)?;
         } else {
-            let instance_mir = tcx.instance_mir(ty::InstanceKind::Item(def_id));
-            render_body(w, instance_mir)?;
+            if let Some((val, ty)) = tcx.trivial_const(def_id) {
+                ty::print::with_forced_impl_filename_line! {
+                    // see notes on #41697 elsewhere
+                    write!(w, "const {}", tcx.def_path_str(def_id))?
+                }
+                writeln!(w, ": {} = const {};", ty, Const::Val(val, ty))?;
+            } else {
+                let instance_mir = tcx.instance_mir(ty::InstanceKind::Item(def_id));
+                render_body(w, instance_mir)?;
+            }
         }
     }
     Ok(())

compiler/rustc_middle/src/query/erase.rs

@@ -160,6 +160,10 @@ impl EraseType for Result<mir::ConstValue, mir::interpret::ErrorHandled> {
     type Result = [u8; size_of::<Result<mir::ConstValue, mir::interpret::ErrorHandled>>()];
 }
 
+impl EraseType for Option<(mir::ConstValue, Ty<'_>)> {
+    type Result = [u8; size_of::<Option<(mir::ConstValue, Ty<'_>)>>()];
+}
+
 impl EraseType for EvalToValTreeResult<'_> {
     type Result = [u8; size_of::<EvalToValTreeResult<'static>>()];
 }

compiler/rustc_middle/src/query/mod.rs

@@ -2719,6 +2719,12 @@ rustc_queries! {
         separate_provide_extern
     }
 
+    query trivial_const(def_id: DefId) -> Option<(mir::ConstValue, Ty<'tcx>)> {
+        desc { |tcx| "checking if `{}` is a trivial const", tcx.def_path_str(def_id) }
+        cache_on_disk_if { def_id.is_local() }
+        separate_provide_extern
+    }
+
     /// Checks for the nearest `#[sanitize(xyz = "off")]` or
     /// `#[sanitize(xyz = "on")]` on this def and any enclosing defs, up to the
     /// crate root.

compiler/rustc_middle/src/ty/context.rs

@@ -3546,6 +3546,13 @@ impl<'tcx> TyCtxt<'tcx> {
         self.get_diagnostic_attr(def_id, sym::do_not_recommend).is_some()
     }
 
+    pub fn is_trivial_const<P>(self, def_id: P) -> bool
+    where
+        P: IntoQueryParam<DefId>,
+    {
+        self.trivial_const(def_id).is_some()
+    }
+
     /// Whether this def is one of the special bin crate entrypoint functions that must have a
     /// monomorphization and also not be internalized in the bin crate.
     pub fn is_entrypoint(self, def_id: DefId) -> bool {

compiler/rustc_mir_transform/src/lib.rs

@@ -55,6 +55,7 @@ mod liveness;
 mod patch;
 mod shim;
 mod ssa;
+mod trivial_const;
 
 /// We import passes via this macro so that we can have a static list of pass names
 /// (used to verify CLI arguments). It takes a list of modules, followed by the passes
@@ -226,6 +227,7 @@ pub fn provide(providers: &mut Providers) {
         promoted_mir,
         deduced_param_attrs: deduce_param_attrs::deduced_param_attrs,
         coroutine_by_move_body_def_id: coroutine::coroutine_by_move_body_def_id,
+        trivial_const: trivial_const::trivial_const_provider,
         ..providers.queries
     };
 }
@@ -379,6 +381,16 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def: LocalDefId) -> ConstQualifs {
 fn mir_built(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> {
     let mut body = build_mir(tcx, def);
 
+    // Identifying trivial consts based on their mir_built is easy, but a little wasteful.
+    // Trying to push this logic earlier in the compiler and never even produce the Body would
+    // probably improve compile time.
+    if trivial_const::trivial_const(tcx, def, || &body).is_some() {
+        // Skip all the passes below for trivial consts.
+        let body = tcx.alloc_steal_mir(body);
+        pass_manager::dump_mir_for_phase_change(tcx, &body.borrow());
+        return body;
+    }
+
     pass_manager::dump_mir_for_phase_change(tcx, &body);
 
     pm::run_passes(
@@ -409,6 +421,8 @@ fn mir_promoted(
     tcx: TyCtxt<'_>,
     def: LocalDefId,
 ) -> (&Steal<Body<'_>>, &Steal<IndexVec<Promoted, Body<'_>>>) {
+    debug_assert!(!tcx.is_trivial_const(def), "Tried to get mir_promoted of a trivial const");
+
     // Ensure that we compute the `mir_const_qualif` for constants at
     // this point, before we steal the mir-const result.
     // Also this means promotion can rely on all const checks having been done.
@@ -436,6 +450,9 @@ fn mir_promoted(
         tcx.ensure_done().coroutine_by_move_body_def_id(def);
     }
 
+    // the `trivial_const` query uses mir_built, so make sure it is run.
+    tcx.ensure_done().trivial_const(def);
+
     let mut body = tcx.mir_built(def).steal();
     if let Some(error_reported) = const_qualifs.tainted_by_errors {
         body.tainted_by_errors = Some(error_reported);
@@ -463,6 +480,7 @@ fn mir_promoted(
 
 /// Compute the MIR that is used during CTFE (and thus has no optimizations run on it)
 fn mir_for_ctfe(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &Body<'_> {
+    debug_assert!(!tcx.is_trivial_const(def_id), "Tried to get mir_for_ctfe of a trivial const");
     tcx.arena.alloc(inner_mir_for_ctfe(tcx, def_id))
 }