Skip to content

Commit b652d9a

Browse files
borsbors
committed
Auto merge of #108062 - Zoxc:spec-incr, r=cjgillot
Specialize query execution for incremental and non-incremental This specializes query execution for incremental and non-incremental by passing in a separate `dyn QueryEngine` types, taking advantage of the virtual dispatch to avoid a branch. This ends up duplicating `try_execute_query`, hopefully the compile time cost of that is relatively low. This is a performance improvement for the non-incremental path: <table><tr><td rowspan="2">Benchmark</td><td colspan="1"><b>Before</b></th><td colspan="2"><b>After</b></th></tr><tr><td align="right">Time</td><td align="right">Time</td><td align="right">%</th></tr><tr><td>🟣 <b>clap</b>:check</td><td align="right">1.8420s</td><td align="right">1.8331s</td><td align="right"> -0.48%</td></tr><tr><td>🟣 <b>hyper</b>:check</td><td align="right">0.2652s</td><td align="right">0.2631s</td><td align="right"> -0.78%</td></tr><tr><td>🟣 <b>regex</b>:check</td><td align="right">1.0161s</td><td align="right">1.0062s</td><td align="right"> -0.98%</td></tr><tr><td>🟣 <b>syn</b>:check</td><td align="right">1.6408s</td><td align="right">1.6197s</td><td align="right">💚 -1.28%</td></tr><tr><td>🟣 <b>syntex_syntax</b>:check</td><td align="right">6.3939s</td><td align="right">6.3558s</td><td align="right"> -0.60%</td></tr><tr><td>Total</td><td align="right">11.1580s</td><td align="right">11.0780s</td><td align="right"> -0.72%</td></tr><tr><td>Summary</td><td align="right">1.0000s</td><td align="right">0.9918s</td><td align="right"> -0.82%</td></tr></table> The incremental path is more neutral: <table><tr><td rowspan="2">Benchmark</td><td colspan="1"><b>Before</b></th><td colspan="2"><b>After</b></th></tr><tr><td align="right">Time</td><td align="right">Time</td><td align="right">%</th></tr><tr><td>🟣 <b>clap</b>:check:initial</td><td align="right">2.2210s</td><td align="right">2.2227s</td><td align="right"> 0.08%</td></tr><tr><td>🟣 <b>hyper</b>:check:initial</td><td align="right">0.3441s</td><td align="right">0.3443s</td><td align="right"> 0.05%</td></tr><tr><td>🟣 <b>regex</b>:check:initial</td><td align="right">1.2919s</td><td align="right">1.2877s</td><td align="right"> -0.33%</td></tr><tr><td>🟣 <b>syn</b>:check:initial</td><td align="right">2.0749s</td><td align="right">2.0721s</td><td align="right"> -0.14%</td></tr><tr><td>🟣 <b>syntex_syntax</b>:check:initial</td><td align="right">7.9266s</td><td align="right">7.9206s</td><td align="right"> -0.07%</td></tr><tr><td>Total</td><td align="right">13.8585s</td><td align="right">13.8474s</td><td align="right"> -0.08%</td></tr><tr><td>Summary</td><td align="right">1.0000s</td><td align="right">0.9992s</td><td align="right"> -0.08%</td></tr></table> r? `@cjgillot`
2 parents 9239760 + 882a968 commit b652d9a

File tree

4 files changed

+76
-17
lines changed

4 files changed

+76
-17
lines changed

‎compiler/rustc_interface/src/passes.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -691,6 +691,8 @@ pub fn create_global_ctxt<'tcx>(
691691
callback(sess, &mut local_providers, &mut extern_providers);
692692
}
693693

694+
let incremental = dep_graph.is_fully_enabled();
695+
694696
sess.time("setup_global_ctxt", || {
695697
gcx_cell.get_or_init(move || {
696698
TyCtxt::create_global_ctxt(
@@ -705,6 +707,7 @@ pub fn create_global_ctxt<'tcx>(
705707
local_providers,
706708
extern_providers,
707709
query_result_on_disk_cache,
710+
incremental,
708711
),
709712
)
710713
})

‎compiler/rustc_query_impl/src/lib.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ use rustc_middle::ty::TyCtxt;
3434
use rustc_query_system::dep_graph::SerializedDepNodeIndex;
3535
use rustc_query_system::ich::StableHashingContext;
3636
use rustc_query_system::query::{
37-
get_query, HashResult, QueryCache, QueryConfig, QueryInfo, QueryMap, QueryMode, QueryState,
37+
get_query_incr, get_query_non_incr, HashResult, QueryCache, QueryConfig, QueryInfo, QueryMap,
38+
QueryMode, QueryState,
3839
};
3940
use rustc_query_system::HandleCycleError;
4041
use rustc_query_system::Value;
@@ -203,6 +204,7 @@ pub fn query_system<'tcx>(
203204
local_providers: Providers,
204205
extern_providers: ExternProviders,
205206
on_disk_cache: Option<OnDiskCache<'tcx>>,
207+
incremental: bool,
206208
) -> QuerySystem<'tcx> {
207209
QuerySystem {
208210
states: Default::default(),
@@ -211,7 +213,7 @@ pub fn query_system<'tcx>(
211213
dynamic_queries: dynamic_queries(),
212214
on_disk_cache,
213215
fns: QuerySystemFns {
214-
engine: engine(),
216+
engine: engine(incremental),
215217
local_providers,
216218
extern_providers,
217219
query_structs: make_dep_kind_array!(query_structs).to_vec(),

‎compiler/rustc_query_impl/src/plumbing.rs‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -494,7 +494,7 @@ macro_rules! define_queries {
494494
(
495495
$($(#[$attr:meta])*
496496
[$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
497-
mod get_query {
497+
mod get_query_incr {
498498
use super::*;
499499

500500
$(
@@ -506,7 +506,7 @@ macro_rules! define_queries {
506506
key: query_keys::$name<'tcx>,
507507
mode: QueryMode,
508508
) -> Option<Erase<query_values::$name<'tcx>>> {
509-
get_query(
509+
get_query_incr(
510510
queries::$name::config(tcx),
511511
QueryCtxt::new(tcx),
512512
span,
@@ -517,9 +517,37 @@ macro_rules! define_queries {
517517
)*
518518
}
519519

520-
pub(crate) fn engine() -> QueryEngine {
521-
QueryEngine {
522-
$($name: get_query::$name,)*
520+
mod get_query_non_incr {
521+
use super::*;
522+
523+
$(
524+
#[inline(always)]
525+
#[tracing::instrument(level = "trace", skip(tcx))]
526+
pub(super) fn $name<'tcx>(
527+
tcx: TyCtxt<'tcx>,
528+
span: Span,
529+
key: query_keys::$name<'tcx>,
530+
__mode: QueryMode,
531+
) -> Option<Erase<query_values::$name<'tcx>>> {
532+
Some(get_query_non_incr(
533+
queries::$name::config(tcx),
534+
QueryCtxt::new(tcx),
535+
span,
536+
key,
537+
))
538+
}
539+
)*
540+
}
541+
542+
pub(crate) fn engine(incremental: bool) -> QueryEngine {
543+
if incremental {
544+
QueryEngine {
545+
$($name: get_query_incr::$name,)*
546+
}
547+
} else {
548+
QueryEngine {
549+
$($name: get_query_non_incr::$name,)*
550+
}
523551
}
524552
}
525553

‎compiler/rustc_query_system/src/query/plumbing.rs‎

Lines changed: 36 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,7 @@ where
312312
}
313313

314314
#[inline(never)]
315-
fn try_execute_query<Q, Qcx>(
315+
fn try_execute_query<Q, Qcx, const INCR: bool>(
316316
query: Q,
317317
qcx: Qcx,
318318
span: Span,
@@ -355,7 +355,7 @@ where
355355
// Drop the lock before we start executing the query
356356
drop(state_lock);
357357

358-
execute_job(query, qcx, state, key, id, dep_node)
358+
execute_job::<_, _, INCR>(query, qcx, state, key, id, dep_node)
359359
}
360360
Entry::Occupied(mut entry) => {
361361
match entry.get_mut() {
@@ -383,7 +383,7 @@ where
383383
}
384384

385385
#[inline(always)]
386-
fn execute_job<Q, Qcx>(
386+
fn execute_job<Q, Qcx, const INCR: bool>(
387387
query: Q,
388388
qcx: Qcx,
389389
state: &QueryState<Q::Key, Qcx::DepKind>,
@@ -398,9 +398,19 @@ where
398398
// Use `JobOwner` so the query will be poisoned if executing it panics.
399399
let job_owner = JobOwner { state, key };
400400

401-
let (result, dep_node_index) = match qcx.dep_context().dep_graph().data() {
402-
None => execute_job_non_incr(query, qcx, key, id),
403-
Some(data) => execute_job_incr(query, qcx, data, key, dep_node, id),
401+
debug_assert_eq!(qcx.dep_context().dep_graph().is_fully_enabled(), INCR);
402+
403+
let (result, dep_node_index) = if INCR {
404+
execute_job_incr(
405+
query,
406+
qcx,
407+
qcx.dep_context().dep_graph().data().unwrap(),
408+
key,
409+
dep_node,
410+
id,
411+
)
412+
} else {
413+
execute_job_non_incr(query, qcx, key, id)
404414
};
405415

406416
let cache = query.query_cache(qcx);
@@ -784,7 +794,18 @@ pub enum QueryMode {
784794
}
785795

786796
#[inline(always)]
787-
pub fn get_query<Q, Qcx>(
797+
pub fn get_query_non_incr<Q, Qcx>(query: Q, qcx: Qcx, span: Span, key: Q::Key) -> Q::Value
798+
where
799+
Q: QueryConfig<Qcx>,
800+
Qcx: QueryContext,
801+
{
802+
debug_assert!(!qcx.dep_context().dep_graph().is_fully_enabled());
803+
804+
ensure_sufficient_stack(|| try_execute_query::<Q, Qcx, false>(query, qcx, span, key, None).0)
805+
}
806+
807+
#[inline(always)]
808+
pub fn get_query_incr<Q, Qcx>(
788809
query: Q,
789810
qcx: Qcx,
790811
span: Span,
@@ -795,6 +816,8 @@ where
795816
Q: QueryConfig<Qcx>,
796817
Qcx: QueryContext,
797818
{
819+
debug_assert!(qcx.dep_context().dep_graph().is_fully_enabled());
820+
798821
let dep_node = if let QueryMode::Ensure { check_cache } = mode {
799822
let (must_run, dep_node) = ensure_must_run(query, qcx, &key, check_cache);
800823
if !must_run {
@@ -805,8 +828,9 @@ where
805828
None
806829
};
807830

808-
let (result, dep_node_index) =
809-
ensure_sufficient_stack(|| try_execute_query(query, qcx, span, key, dep_node));
831+
let (result, dep_node_index) = ensure_sufficient_stack(|| {
832+
try_execute_query::<_, _, true>(query, qcx, span, key, dep_node)
833+
});
810834
if let Some(dep_node_index) = dep_node_index {
811835
qcx.dep_context().dep_graph().read_index(dep_node_index)
812836
}
@@ -831,5 +855,7 @@ pub fn force_query<Q, Qcx>(
831855

832856
debug_assert!(!query.anon());
833857

834-
ensure_sufficient_stack(|| try_execute_query(query, qcx, DUMMY_SP, key, Some(dep_node)));
858+
ensure_sufficient_stack(|| {
859+
try_execute_query::<_, _, true>(query, qcx, DUMMY_SP, key, Some(dep_node))
860+
});
835861
}

0 commit comments

Comments
 (0)