From 33aaa94f68e0bc71b88cb0292ea43e184079e197 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Thu, 9 Apr 2026 18:14:36 +0300 Subject: [PATCH 01/15] Make try_load_from_disk_fn a static function --- compiler/rustc_middle/src/query/inner.rs | 22 +++-- compiler/rustc_middle/src/query/mod.rs | 4 +- compiler/rustc_middle/src/query/plumbing.rs | 62 +++++++++++-- compiler/rustc_query_impl/src/execution.rs | 93 ++++++++++++------- compiler/rustc_query_impl/src/lib.rs | 5 +- compiler/rustc_query_impl/src/plumbing.rs | 38 +++----- .../rustc_query_impl/src/profiling_support.rs | 7 +- compiler/rustc_query_impl/src/query_impl.rs | 22 ++--- 8 files changed, 151 insertions(+), 102 deletions(-) diff --git a/compiler/rustc_middle/src/query/inner.rs b/compiler/rustc_middle/src/query/inner.rs index 402c448a1fa3a..f6b5cd7d47304 100644 --- a/compiler/rustc_middle/src/query/inner.rs +++ b/compiler/rustc_middle/src/query/inner.rs @@ -6,7 +6,7 @@ use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span}; use crate::dep_graph; use crate::dep_graph::DepNodeKey; use crate::query::erase::{self, Erasable, Erased}; -use crate::query::{EnsureMode, QueryCache, QueryMode, QueryVTable}; +use crate::query::{EnsureMode, QueryCache, QueryHelper, QueryMode, QueryVTable}; use crate::ty::TyCtxt; /// Checks whether there is already a value for this key in the in-memory @@ -31,14 +31,15 @@ where /// Shared implementation of `tcx.$query(..)` and `tcx.at(span).$query(..)` /// for all queries. #[inline(always)] -pub(crate) fn query_get_at<'tcx, C>( +pub(crate) fn query_get_at<'tcx, C, H>( tcx: TyCtxt<'tcx>, span: Span, - query: &'tcx QueryVTable<'tcx, C>, + query: &'tcx QueryVTable<'tcx, C, H>, key: C::Key, ) -> C::Value where C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, { match try_get_cached(tcx, &query.cache, key) { Some(value) => value, @@ -49,13 +50,14 @@ where /// Shared implementation of `tcx.ensure_ok().$query(..)` and /// `tcx.ensure_done().$query(..)` for all queries. #[inline] -pub(crate) fn query_ensure_ok_or_done<'tcx, C>( +pub(crate) fn query_ensure_ok_or_done<'tcx, C, H>( tcx: TyCtxt<'tcx>, - query: &'tcx QueryVTable<'tcx, C>, + query: &'tcx QueryVTable<'tcx, C, H>, key: C::Key, ensure_mode: EnsureMode, ) where C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, { match try_get_cached(tcx, &query.cache, key) { Some(_value) => {} @@ -68,14 +70,15 @@ pub(crate) fn query_ensure_ok_or_done<'tcx, C>( /// Implementation of `tcx.ensure_result().$query(..)` for queries that /// return `Result<_, ErrorGuaranteed>`. #[inline] -pub(crate) fn query_ensure_result<'tcx, C, T>( +pub(crate) fn query_ensure_result<'tcx, C, T, H>( tcx: TyCtxt<'tcx>, - query: &'tcx QueryVTable<'tcx, C>, + query: &'tcx QueryVTable<'tcx, C, H>, key: C::Key, ) -> Result<(), ErrorGuaranteed> where C: QueryCache>>, Result: Erasable, + H: QueryHelper<'tcx, C::Key, C::Value>, { let convert = |value: Erased>| -> Result<(), ErrorGuaranteed> { match erase::restore_val(value) { @@ -110,14 +113,15 @@ where /// "Feeds" a feedable query by adding a given key/value pair to its in-memory cache. /// Called by macro-generated methods of [`rustc_middle::ty::TyCtxtFeed`]. -pub(crate) fn query_feed<'tcx, C>( +pub(crate) fn query_feed<'tcx, C, H>( tcx: TyCtxt<'tcx>, - query: &'tcx QueryVTable<'tcx, C>, + query: &'tcx QueryVTable<'tcx, C, H>, key: C::Key, value: C::Value, ) where C: QueryCache, C::Key: DepNodeKey<'tcx>, + H: QueryHelper<'tcx, C::Key, C::Value>, { let format_value = query.format_value; diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index b7e5e9bcb5e32..b780c8d57726f 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -5,8 +5,8 @@ pub use self::into_query_key::IntoQueryKey; pub use self::job::{QueryJob, QueryJobId, QueryLatch, QueryWaiter}; pub use self::keys::{AsLocalQueryKey, LocalCrate, QueryKey}; pub use self::plumbing::{ - ActiveKeyStatus, Cycle, EnsureMode, QueryMode, QueryState, QuerySystem, QueryVTable, TyCtxtAt, - TyCtxtEnsureDone, TyCtxtEnsureOk, TyCtxtEnsureResult, + ActiveKeyStatus, Cycle, EnsureMode, QueryHelper, QueryMode, QueryState, QuerySystem, + QueryVTable, TyCtxtAt, TyCtxtEnsureDone, TyCtxtEnsureOk, TyCtxtEnsureResult, }; pub use self::stack::QueryStackFrame; pub use crate::queries::Providers; diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 87fea2fc6aa94..108e1adc182c2 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -8,12 +8,13 @@ use rustc_data_structures::sharded::Sharded; use rustc_data_structures::sync::{AtomicU64, Lock, WorkerLocal}; use rustc_errors::Diag; use rustc_hir::def_id::LocalDefId; +use rustc_serialize::Decodable; use rustc_span::Span; use crate::dep_graph::{DepKind, DepNodeIndex, QuerySideEffect, SerializedDepNodeIndex}; use crate::ich::StableHashingContext; use crate::queries::{ExternProviders, Providers, QueryArenas, QueryVTables, TaggedQueryKey}; -use crate::query::on_disk_cache::OnDiskCache; +use crate::query::on_disk_cache::{CacheDecoder, OnDiskCache}; use crate::query::{IntoQueryKey, QueryCache, QueryJob, QueryStackFrame}; use crate::ty::{self, TyCtxt}; @@ -76,7 +77,7 @@ pub enum EnsureMode { } /// Stores data and metadata (e.g. function pointers) for a particular query. -pub struct QueryVTable<'tcx, C: QueryCache> { +pub struct QueryVTable<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>> { pub name: &'static str, /// True if this query has the `eval_always` modifier. @@ -99,11 +100,7 @@ pub struct QueryVTable<'tcx, C: QueryCache> { pub will_cache_on_disk_for_key_fn: fn(key: C::Key) -> bool, - /// Function pointer that tries to load a query value from disk. - /// - /// This should only be called after a successful check of `will_cache_on_disk_for_key_fn`. - pub try_load_from_disk_fn: - fn(tcx: TyCtxt<'tcx>, prev_index: SerializedDepNodeIndex) -> Option, + pub helper: H, /// Function pointer that hashes this query's result values. /// @@ -133,7 +130,13 @@ pub struct QueryVTable<'tcx, C: QueryCache> { pub execute_query_fn: fn(TyCtxt<'tcx>, Span, C::Key, QueryMode) -> Option, } -impl<'tcx, C: QueryCache> fmt::Debug for QueryVTable<'tcx, C> { +pub trait QueryHelper<'tcx, K, V>: Default + 'static { + fn try_load_from_disk_fn(tcx: TyCtxt<'tcx>, prev_index: SerializedDepNodeIndex) -> Option; +} + +impl<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>> fmt::Debug + for QueryVTable<'tcx, C, H> +{ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // When debug-printing a query vtable (e.g. for ICE or tracing), // just print the query name to know what query we're dealing with. @@ -339,6 +342,32 @@ macro_rules! define_callbacks { #[cfg(not($arena_cache))] pub type ProvidedValue<'tcx> = Value<'tcx>; + #[derive(Default)] + pub struct Helper; + + impl<'tcx> crate::query::QueryHelper<'tcx, Key<'tcx>, Erased>> for Helper { + #[cfg($cache_on_disk)] + fn try_load_from_disk_fn( + tcx: TyCtxt<'tcx>, + prev_index: crate::dep_graph::SerializedDepNodeIndex, + ) -> Option>> { + use rustc_middle::queries::$name::{ProvidedValue, provided_to_erased}; + + let loaded_value: ProvidedValue<'tcx> = + rustc_middle::query::plumbing::try_load_from_disk(tcx, prev_index)?; + + // Arena-alloc the value if appropriate, and erase it. + Some(provided_to_erased(tcx, loaded_value)) + } + #[cfg(not($cache_on_disk))] + fn try_load_from_disk_fn( + _tcx: TyCtxt<'tcx>, + _prev_index: crate::dep_graph::SerializedDepNodeIndex, + ) -> Option>> { + None + } + } + pub type Cache<'tcx> = as $crate::query::QueryKey>::Cache>>; @@ -468,7 +497,7 @@ macro_rules! define_callbacks { /// Holds a `QueryVTable` for each query. pub struct QueryVTables<'tcx> { $( - pub $name: $crate::query::QueryVTable<'tcx, $name::Cache<'tcx>>, + pub $name: $crate::query::QueryVTable<'tcx, $name::Cache<'tcx>, $name::Helper>, )* } @@ -660,3 +689,18 @@ pub(crate) fn default_extern_query(name: &str, key: &dyn std::fmt::Debug) -> ! { perhaps the `{name}` query was never assigned a provider function", ) } + +pub(crate) fn try_load_from_disk<'tcx, V>( + tcx: TyCtxt<'tcx>, + prev_index: SerializedDepNodeIndex, +) -> Option +where + V: for<'a> Decodable>, +{ + let on_disk_cache = tcx.query_system.on_disk_cache.as_ref()?; + + // The call to `with_query_deserialization` enforces that no new `DepNodes` + // are created during deserialization. See the docs of that method for more + // details. + tcx.dep_graph.with_query_deserialization(|| on_disk_cache.try_load_query_value(tcx, prev_index)) +} diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index 45be65b02964e..c1e1615fd590f 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -8,8 +8,8 @@ use rustc_data_structures::{outline, sharded, sync}; use rustc_errors::FatalError; use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex}; use rustc_middle::query::{ - ActiveKeyStatus, Cycle, EnsureMode, QueryCache, QueryJob, QueryJobId, QueryKey, QueryLatch, - QueryMode, QueryState, QueryVTable, + ActiveKeyStatus, Cycle, EnsureMode, QueryCache, QueryHelper, QueryJob, QueryJobId, QueryKey, + QueryLatch, QueryMode, QueryState, QueryVTable, }; use rustc_middle::ty::TyCtxt; use rustc_middle::verify_ich::incremental_verify_ich; @@ -62,13 +62,14 @@ pub fn collect_active_query_jobs<'tcx>( /// Internal plumbing for collecting the set of active jobs for this query. /// /// Aborts if jobs can't be gathered as specified by `collect_kind`. -fn collect_active_query_jobs_inner<'tcx, C>( - query: &'tcx QueryVTable<'tcx, C>, +fn collect_active_query_jobs_inner<'tcx, C, H>( + query: &'tcx QueryVTable<'tcx, C, H>, collect_kind: CollectActiveJobsKind, job_map: &mut QueryJobMap<'tcx>, ) where C: QueryCache, - QueryVTable<'tcx, C>: DynSync, + H: QueryHelper<'tcx, C::Key, C::Value>, + QueryVTable<'tcx, C, H>: DynSync, { let mut collect_shard_jobs = |shard: &HashTable<(C::Key, ActiveKeyStatus<'tcx>)>| { for (key, status) in shard.iter() { @@ -108,8 +109,8 @@ fn collect_active_query_jobs_inner<'tcx, C>( #[cold] #[inline(never)] -fn handle_cycle<'tcx, C: QueryCache>( - query: &'tcx QueryVTable<'tcx, C>, +fn handle_cycle<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( + query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, key: C::Key, cycle: Cycle<'tcx>, @@ -194,8 +195,8 @@ where #[cold] #[inline(never)] -fn find_and_handle_cycle<'tcx, C: QueryCache>( - query: &'tcx QueryVTable<'tcx, C>, +fn find_and_handle_cycle<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( + query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, key: C::Key, try_execute: QueryJobId, @@ -210,8 +211,8 @@ fn find_and_handle_cycle<'tcx, C: QueryCache>( } #[inline(always)] -fn wait_for_query<'tcx, C: QueryCache>( - query: &'tcx QueryVTable<'tcx, C>, +fn wait_for_query<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( + query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, span: Span, key: C::Key, @@ -256,13 +257,17 @@ fn wait_for_query<'tcx, C: QueryCache>( /// Shared main part of both [`execute_query_incr_inner`] and [`execute_query_non_incr_inner`]. #[inline(never)] -fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>( - query: &'tcx QueryVTable<'tcx, C>, +fn try_execute_query<'tcx, C, H, const INCR: bool>( + query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, span: Span, key: C::Key, dep_node: Option, // `None` for non-incremental, `Some` for incremental -) -> (C::Value, Option) { +) -> (C::Value, Option) +where + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +{ let key_hash = sharded::make_hash(&key); let mut state_lock = query.state.active.lock_shard_by_hash(key_hash); @@ -340,9 +345,9 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>( } #[inline(always)] -fn check_feedable_consistency<'tcx, C: QueryCache>( +fn check_feedable_consistency<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( tcx: TyCtxt<'tcx>, - query: &'tcx QueryVTable<'tcx, C>, + query: &'tcx QueryVTable<'tcx, C, H>, key: C::Key, value: &C::Value, ) { @@ -382,8 +387,8 @@ fn check_feedable_consistency<'tcx, C: QueryCache>( // Fast path for when incr. comp. is off. #[inline(always)] -fn execute_job_non_incr<'tcx, C: QueryCache>( - query: &'tcx QueryVTable<'tcx, C>, +fn execute_job_non_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( + query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, key: C::Key, job_id: QueryJobId, @@ -410,8 +415,8 @@ fn execute_job_non_incr<'tcx, C: QueryCache>( } #[inline(always)] -fn execute_job_incr<'tcx, C: QueryCache>( - query: &'tcx QueryVTable<'tcx, C>, +fn execute_job_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( + query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, key: C::Key, dep_node: DepNode, @@ -462,10 +467,14 @@ fn execute_job_incr<'tcx, C: QueryCache>( /// by loading one from disk if possible, or by invoking its query provider if /// necessary. #[inline(always)] -fn load_from_disk_or_invoke_provider_green<'tcx, C: QueryCache>( +fn load_from_disk_or_invoke_provider_green< + 'tcx, + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +>( tcx: TyCtxt<'tcx>, dep_graph_data: &DepGraphData, - query: &'tcx QueryVTable<'tcx, C>, + query: &'tcx QueryVTable<'tcx, C, H>, key: C::Key, dep_node: &DepNode, prev_index: SerializedDepNodeIndex, @@ -479,7 +488,7 @@ fn load_from_disk_or_invoke_provider_green<'tcx, C: QueryCache>( // First try to load the result from the on-disk cache. Some things are never cached on disk. let try_value = if (query.will_cache_on_disk_for_key_fn)(key) { let prof_timer = tcx.prof.incr_cache_loading(); - let value = (query.try_load_from_disk_fn)(tcx, prev_index); + let value = H::try_load_from_disk_fn(tcx, prev_index); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); value } else { @@ -545,8 +554,8 @@ fn load_from_disk_or_invoke_provider_green<'tcx, C: QueryCache>( /// on having the dependency graph (and in some cases a disk-cached value) /// from the previous incr-comp session. #[inline(never)] -fn ensure_can_skip_execution<'tcx, C: QueryCache>( - query: &'tcx QueryVTable<'tcx, C>, +fn ensure_can_skip_execution<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( + query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, key: C::Key, dep_node: DepNode, @@ -593,25 +602,33 @@ fn ensure_can_skip_execution<'tcx, C: QueryCache>( /// Called by a macro-generated impl of [`QueryVTable::execute_query_fn`], /// in non-incremental mode. #[inline(always)] -pub(super) fn execute_query_non_incr_inner<'tcx, C: QueryCache>( - query: &'tcx QueryVTable<'tcx, C>, +pub(super) fn execute_query_non_incr_inner< + 'tcx, + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +>( + query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, span: Span, key: C::Key, ) -> C::Value { - ensure_sufficient_stack(|| try_execute_query::(query, tcx, span, key, None).0) + ensure_sufficient_stack(|| try_execute_query::(query, tcx, span, key, None).0) } /// Called by a macro-generated impl of [`QueryVTable::execute_query_fn`], /// in incremental mode. #[inline(always)] -pub(super) fn execute_query_incr_inner<'tcx, C: QueryCache>( - query: &'tcx QueryVTable<'tcx, C>, +pub(super) fn execute_query_incr_inner<'tcx, C, H>( + query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, span: Span, key: C::Key, mode: QueryMode, -) -> Option { +) -> Option +where + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +{ let dep_node = DepNode::construct(tcx, query.dep_kind, &key); // Check if query execution can be skipped, for `ensure_ok` or `ensure_done`. @@ -622,7 +639,7 @@ pub(super) fn execute_query_incr_inner<'tcx, C: QueryCache>( } let (result, dep_node_index) = ensure_sufficient_stack(|| { - try_execute_query::(query, tcx, span, key, Some(dep_node)) + try_execute_query::(query, tcx, span, key, Some(dep_node)) }); if let Some(dep_node_index) = dep_node_index { tcx.dep_graph.read_index(dep_node_index) @@ -634,11 +651,15 @@ pub(super) fn execute_query_incr_inner<'tcx, C: QueryCache>( /// for query nodes. /// /// [force_fn]: rustc_middle::dep_graph::DepKindVTable::force_from_dep_node_fn -pub(crate) fn force_query_dep_node<'tcx, C: QueryCache>( +pub(crate) fn force_query_dep_node<'tcx, C, H>( tcx: TyCtxt<'tcx>, - query: &'tcx QueryVTable<'tcx, C>, + query: &'tcx QueryVTable<'tcx, C, H>, dep_node: DepNode, -) -> bool { +) -> bool +where + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +{ let Some(key) = C::Key::try_recover_key(tcx, &dep_node) else { // We couldn't recover a key from the node's key fingerprint. // Tell the caller that we couldn't force the node. @@ -646,7 +667,7 @@ pub(crate) fn force_query_dep_node<'tcx, C: QueryCache>( }; ensure_sufficient_stack(|| { - try_execute_query::(query, tcx, DUMMY_SP, key, Some(dep_node)) + try_execute_query::(query, tcx, DUMMY_SP, key, Some(dep_node)) }); // We did manage to recover a key and force the node, though it's up to diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index d13ecaa2dee5c..0b429dee69627 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -12,7 +12,7 @@ use rustc_data_structures::sync::AtomicU64; use rustc_middle::dep_graph; use rustc_middle::queries::{ExternProviders, Providers}; use rustc_middle::query::on_disk_cache::OnDiskCache; -use rustc_middle::query::{QueryCache, QuerySystem, QueryVTable}; +use rustc_middle::query::{QueryCache, QueryHelper, QuerySystem, QueryVTable}; use rustc_middle::ty::TyCtxt; pub use crate::dep_kind_vtables::make_dep_kind_vtables; @@ -38,8 +38,9 @@ mod query_impl; /// on the type `rustc_query_impl::query_impl::$name::VTableGetter`. trait GetQueryVTable<'tcx> { type Cache: QueryCache + 'tcx; + type Helper: QueryHelper<'tcx, ::Key, ::Value>; - fn query_vtable(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, Self::Cache>; + fn query_vtable(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, Self::Cache, Self::Helper>; } pub fn query_system<'tcx>( diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 11f960598c387..1b1439dc114e4 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -7,11 +7,11 @@ use rustc_middle::bug; use rustc_middle::dep_graph::DepKindVTable; use rustc_middle::dep_graph::{DepNode, DepNodeKey, SerializedDepNodeIndex}; use rustc_middle::query::erase::{Erasable, Erased}; -use rustc_middle::query::on_disk_cache::{CacheDecoder, CacheEncoder}; -use rustc_middle::query::{QueryCache, QueryJobId, QueryMode, QueryVTable, erase}; +use rustc_middle::query::on_disk_cache::CacheEncoder; +use rustc_middle::query::{QueryCache, QueryHelper, QueryJobId, QueryMode, QueryVTable, erase}; use rustc_middle::ty::TyCtxt; use rustc_middle::ty::tls::{self, ImplicitCtxt}; -use rustc_serialize::{Decodable, Encodable}; +use rustc_serialize::Encodable; use rustc_span::DUMMY_SP; use rustc_span::def_id::LOCAL_CRATE; @@ -81,12 +81,13 @@ pub(crate) fn encode_query_values<'tcx>(tcx: TyCtxt<'tcx>, encoder: &mut CacheEn }); } -fn encode_query_values_inner<'a, 'tcx, C, V>( +fn encode_query_values_inner<'a, 'tcx, C, V, H>( tcx: TyCtxt<'tcx>, - query: &'tcx QueryVTable<'tcx, C>, + query: &'tcx QueryVTable<'tcx, C, H>, encoder: &mut CacheEncoder<'a, 'tcx>, ) where C: QueryCache>, + H: QueryHelper<'tcx, C::Key, C::Value>, V: Erasable + Encodable>, { let _timer = tcx.prof.generic_activity_with_arg("encode_query_results_for", query.name); @@ -109,8 +110,8 @@ pub(crate) fn verify_query_key_hashes<'tcx>(tcx: TyCtxt<'tcx>) { } } -fn verify_query_key_hashes_inner<'tcx, C: QueryCache>( - query: &'tcx QueryVTable<'tcx, C>, +fn verify_query_key_hashes_inner<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( + query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, ) { let _timer = tcx.prof.generic_activity_with_arg("query_key_hash_verify_for", query.name); @@ -136,11 +137,13 @@ fn verify_query_key_hashes_inner<'tcx, C: QueryCache>( } /// Inner implementation of [`DepKindVTable::promote_from_disk_fn`] for queries. -pub(crate) fn promote_from_disk_inner<'tcx, C: QueryCache>( +pub(crate) fn promote_from_disk_inner<'tcx, C: QueryCache, H>( tcx: TyCtxt<'tcx>, - query: &'tcx QueryVTable<'tcx, C>, + query: &'tcx QueryVTable<'tcx, C, H>, dep_node: DepNode, -) { +) where + H: QueryHelper<'tcx, C::Key, C::Value>, +{ debug_assert!(tcx.dep_graph.is_green(&dep_node)); let key = C::Key::try_recover_key(tcx, &dep_node).unwrap_or_else(|| { @@ -180,18 +183,3 @@ pub(crate) fn loadable_from_disk<'tcx>(tcx: TyCtxt<'tcx>, id: SerializedDepNodeI false } } - -pub(crate) fn try_load_from_disk<'tcx, V>( - tcx: TyCtxt<'tcx>, - prev_index: SerializedDepNodeIndex, -) -> Option -where - V: for<'a> Decodable>, -{ - let on_disk_cache = tcx.query_system.on_disk_cache.as_ref()?; - - // The call to `with_query_deserialization` enforces that no new `DepNodes` - // are created during deserialization. See the docs of that method for more - // details. - tcx.dep_graph.with_query_deserialization(|| on_disk_cache.try_load_query_value(tcx, prev_index)) -} diff --git a/compiler/rustc_query_impl/src/profiling_support.rs b/compiler/rustc_query_impl/src/profiling_support.rs index 980e2b1305245..817155f16a8cb 100644 --- a/compiler/rustc_query_impl/src/profiling_support.rs +++ b/compiler/rustc_query_impl/src/profiling_support.rs @@ -6,7 +6,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::profiling::SelfProfiler; use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LOCAL_CRATE, LocalDefId}; use rustc_hir::definitions::DefPathData; -use rustc_middle::query::{QueryCache, QueryVTable}; +use rustc_middle::query::{QueryCache, QueryHelper, QueryVTable}; use rustc_middle::ty::TyCtxt; use crate::query_impl::for_each_query_vtable; @@ -200,12 +200,13 @@ pub(crate) fn alloc_self_profile_query_strings(tcx: TyCtxt<'_>) { /// Allocate the self-profiling query strings for a single query cache. This /// method is called from `alloc_self_profile_query_strings` which knows all /// the queries via macro magic. -fn alloc_self_profile_query_strings_inner<'tcx, C>( +fn alloc_self_profile_query_strings_inner<'tcx, C, H>( tcx: TyCtxt<'tcx>, - query: &'tcx QueryVTable<'tcx, C>, + query: &'tcx QueryVTable<'tcx, C, H>, string_cache: &mut QueryKeyStringCache, ) where C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, C::Key: Debug + Clone, { tcx.prof.with_profiler(|profiler| { diff --git a/compiler/rustc_query_impl/src/query_impl.rs b/compiler/rustc_query_impl/src/query_impl.rs index 4425acc6b86b8..487492286f57c 100644 --- a/compiler/rustc_query_impl/src/query_impl.rs +++ b/compiler/rustc_query_impl/src/query_impl.rs @@ -142,7 +142,7 @@ macro_rules! define_queries { } pub(crate) fn make_query_vtable<'tcx>(incremental: bool) - -> QueryVTable<'tcx, rustc_middle::queries::$name::Cache<'tcx>> + -> QueryVTable<'tcx, rustc_middle::queries::$name::Cache<'tcx>, rustc_middle::queries::$name::Helper> { use rustc_middle::queries::$name::Value; QueryVTable { @@ -159,18 +159,7 @@ macro_rules! define_queries { will_cache_on_disk_for_key_fn: $crate::query_impl::$name::will_cache_on_disk_for_key, - #[cfg($cache_on_disk)] - try_load_from_disk_fn: |tcx, prev_index| { - use rustc_middle::queries::$name::{ProvidedValue, provided_to_erased}; - - let loaded_value: ProvidedValue<'tcx> = - $crate::plumbing::try_load_from_disk(tcx, prev_index)?; - - // Arena-alloc the value if appropriate, and erase it. - Some(provided_to_erased(tcx, loaded_value)) - }, - #[cfg(not($cache_on_disk))] - try_load_from_disk_fn: |_tcx, _prev_index| None, + helper: Default::default(), #[cfg($handle_cycle_error)] handle_cycle_error_fn: |tcx, key, cycle, err| { @@ -208,9 +197,10 @@ macro_rules! define_queries { impl<'tcx> GetQueryVTable<'tcx> for VTableGetter { type Cache = rustc_middle::queries::$name::Cache<'tcx>; + type Helper = rustc_middle::queries::$name::Helper; #[inline(always)] - fn query_vtable(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, Self::Cache> { + fn query_vtable(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, Self::Cache, Self::Helper> { &tcx.query_system.query_vtables.$name } } @@ -246,7 +236,7 @@ macro_rules! define_queries { (ALL, $tcx:expr, $closure:expr) => {{ let tcx: rustc_middle::ty::TyCtxt<'_> = $tcx; $( - let query: &rustc_middle::query::QueryVTable<'_, _> = + let query: &rustc_middle::query::QueryVTable<'_, _, _> = &tcx.query_system.query_vtables.$name; $closure(query); )* @@ -261,7 +251,7 @@ macro_rules! define_queries { $( #[cfg($cache_on_disk)] { - let query: &rustc_middle::query::QueryVTable<'_, _> = + let query: &rustc_middle::query::QueryVTable<'_, _, _> = &tcx.query_system.query_vtables.$name; $closure(query); } From cd2adf86888ff23ef9e044c01a844fc1acc6cb45 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 13 Apr 2026 13:37:34 +0300 Subject: [PATCH 02/15] Move rustc_query_impl code to rustc_middle --- Cargo.lock | 19 +----------------- compiler/rustc_interface/Cargo.toml | 1 - compiler/rustc_interface/src/interface.rs | 2 +- compiler/rustc_interface/src/passes.rs | 6 +++--- compiler/rustc_interface/src/util.rs | 4 ++-- compiler/rustc_middle/Cargo.toml | 1 + .../src/query/impl_}/README.md | 0 .../src/query/impl_}/dep_kind_vtables.rs | 11 +++++----- .../src/query/impl_}/error.rs | 0 .../src/query/impl_}/execution.rs | 12 +++++++---- .../src/query/impl_}/handle_cycle_error.rs | 3 +-- .../src/query/impl_}/job.rs | 18 +++++++++-------- .../src/query/impl_/mod.rs} | 16 ++++----------- .../src/query/impl_}/plumbing.rs | 11 +++++----- .../src/query/impl_}/profiling_support.rs | 2 +- .../src/query/impl_}/query_impl.rs | 20 +++++++++---------- compiler/rustc_middle/src/query/mod.rs | 1 + compiler/rustc_middle/src/query/modifiers.rs | 2 +- compiler/rustc_middle/src/ty/context.rs | 2 +- compiler/rustc_query_impl/Cargo.toml | 19 ------------------ .../builder/cli_paths/snapshots/x_bench.snap | 1 - .../cli_paths/snapshots/x_build_compiler.snap | 1 - .../builder/cli_paths/snapshots/x_check.snap | 1 - .../cli_paths/snapshots/x_check_compiler.snap | 1 - ...eck_compiletest_include_default_paths.snap | 1 - .../builder/cli_paths/snapshots/x_clippy.snap | 1 - .../builder/cli_paths/snapshots/x_fix.snap | 1 - .../builder/cli_paths/snapshots/x_test.snap | 1 - .../snapshots/x_test_skip_coverage.snap | 1 - .../snapshots/x_test_skip_tests.snap | 1 - .../snapshots/x_test_skip_tests_etc.snap | 1 - triagebot.toml | 2 -- 32 files changed, 55 insertions(+), 108 deletions(-) rename compiler/{rustc_query_impl/src => rustc_middle/src/query/impl_}/README.md (100%) rename compiler/{rustc_query_impl/src => rustc_middle/src/query/impl_}/dep_kind_vtables.rs (94%) rename compiler/{rustc_query_impl/src => rustc_middle/src/query/impl_}/error.rs (100%) rename compiler/{rustc_query_impl/src => rustc_middle/src/query/impl_}/execution.rs (98%) rename compiler/{rustc_query_impl/src => rustc_middle/src/query/impl_}/handle_cycle_error.rs (99%) rename compiler/{rustc_query_impl/src => rustc_middle/src/query/impl_}/job.rs (96%) rename compiler/{rustc_query_impl/src/lib.rs => rustc_middle/src/query/impl_/mod.rs} (80%) rename compiler/{rustc_query_impl/src => rustc_middle/src/query/impl_}/plumbing.rs (95%) rename compiler/{rustc_query_impl/src => rustc_middle/src/query/impl_}/profiling_support.rs (99%) rename compiler/{rustc_query_impl/src => rustc_middle/src/query/impl_}/query_impl.rs (92%) delete mode 100644 compiler/rustc_query_impl/Cargo.toml diff --git a/Cargo.lock b/Cargo.lock index 563d99d5475c4..c28d524bda2db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4169,7 +4169,6 @@ dependencies = [ "rustc_parse", "rustc_passes", "rustc_privacy", - "rustc_query_impl", "rustc_resolve", "rustc_session", "rustc_span", @@ -4303,6 +4302,7 @@ dependencies = [ "bitflags", "either", "gsgdt", + "measureme", "parking_lot", "polonius-engine", "rustc_abi", @@ -4564,23 +4564,6 @@ dependencies = [ "rustc_target", ] -[[package]] -name = "rustc_query_impl" -version = "0.0.0" -dependencies = [ - "measureme", - "rustc_abi", - "rustc_data_structures", - "rustc_errors", - "rustc_hir", - "rustc_macros", - "rustc_middle", - "rustc_serialize", - "rustc_span", - "rustc_thread_pool", - "tracing", -] - [[package]] name = "rustc_resolve" version = "0.0.0" diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml index c234e21b92541..e07eafb81eb5f 100644 --- a/compiler/rustc_interface/Cargo.toml +++ b/compiler/rustc_interface/Cargo.toml @@ -34,7 +34,6 @@ rustc_monomorphize = { path = "../rustc_monomorphize" } rustc_parse = { path = "../rustc_parse" } rustc_passes = { path = "../rustc_passes" } rustc_privacy = { path = "../rustc_privacy" } -rustc_query_impl = { path = "../rustc_query_impl" } rustc_resolve = { path = "../rustc_resolve" } rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index fa4d4588caab4..44537fdf78016 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -9,6 +9,7 @@ use rustc_data_structures::jobserver::{self, Proxy}; use rustc_data_structures::stable_hasher::StableHasher; use rustc_errors::{DiagCtxtHandle, ErrorGuaranteed}; use rustc_lint::LintStore; +use rustc_middle::query::impl_::print_query_stack; use rustc_middle::ty; use rustc_middle::ty::CurrentGcx; use rustc_middle::util::Providers; @@ -16,7 +17,6 @@ use rustc_parse::lexer::StripTokens; use rustc_parse::new_parser_from_source_str; use rustc_parse::parser::Recovery; use rustc_parse::parser::attr::AllowLeadingUnsafe; -use rustc_query_impl::print_query_stack; use rustc_session::config::{self, Cfg, CheckCfg, ExpectedValues, Input, OutFileName}; use rustc_session::parse::ParseSess; use rustc_session::{CompilerIO, EarlyDiagCtxt, Session, lint}; diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index ab61ba635720b..c914d58c42f4d 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -901,7 +901,7 @@ pub static DEFAULT_QUERY_PROVIDERS: LazyLock = LazyLock::new(|| { rustc_mir_transform::provide(providers); rustc_monomorphize::provide(providers); rustc_privacy::provide(&mut providers.queries); - rustc_query_impl::provide(providers); + rustc_middle::query::impl_::provide(providers); rustc_resolve::provide(&mut providers.queries); rustc_hir_analysis::provide(&mut providers.queries); rustc_hir_typeck::provide(&mut providers.queries); @@ -995,8 +995,8 @@ pub fn create_and_enter_global_ctxt FnOnce(TyCtxt<'tcx>) -> T>( &hir_arena, untracked, dep_graph, - rustc_query_impl::make_dep_kind_vtables(&arena), - rustc_query_impl::query_system( + rustc_middle::query::impl_::make_dep_kind_vtables(&arena), + rustc_middle::query::impl_::query_system( providers.queries, providers.extern_queries, query_result_on_disk_cache, diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index 24b23cc4199e9..13bee4b1acc3c 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -17,8 +17,8 @@ use rustc_data_structures::jobserver::Proxy; use rustc_data_structures::sync; use rustc_metadata::{DylibError, EncodedMetadata, load_symbol_from_dylib}; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; +use rustc_middle::query::impl_::{CollectActiveJobsKind, collect_active_query_jobs}; use rustc_middle::ty::{CurrentGcx, TyCtxt}; -use rustc_query_impl::{CollectActiveJobsKind, collect_active_query_jobs}; use rustc_session::config::{ Cfg, CrateType, OutFileName, OutputFilenames, OutputTypes, Sysroot, host_tuple, }; @@ -183,8 +183,8 @@ pub(crate) fn run_in_thread_pool_with_globals< use std::process; use rustc_data_structures::defer; + use rustc_middle::query::impl_::break_query_cycle; use rustc_middle::ty::tls; - use rustc_query_impl::break_query_cycle; let thread_stack_size = init_stack_size(thread_builder_diag); diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml index 8bad0e291bf8b..dc10d2e542474 100644 --- a/compiler/rustc_middle/Cargo.toml +++ b/compiler/rustc_middle/Cargo.toml @@ -8,6 +8,7 @@ edition = "2024" bitflags = "2.4.1" either = "1.5.0" gsgdt = "0.1.2" +measureme = "12.0.1" parking_lot = "0.12" polonius-engine = "0.13.0" rustc_abi = { path = "../rustc_abi" } diff --git a/compiler/rustc_query_impl/src/README.md b/compiler/rustc_middle/src/query/impl_/README.md similarity index 100% rename from compiler/rustc_query_impl/src/README.md rename to compiler/rustc_middle/src/query/impl_/README.md diff --git a/compiler/rustc_query_impl/src/dep_kind_vtables.rs b/compiler/rustc_middle/src/query/impl_/dep_kind_vtables.rs similarity index 94% rename from compiler/rustc_query_impl/src/dep_kind_vtables.rs rename to compiler/rustc_middle/src/query/impl_/dep_kind_vtables.rs index d12db3784f711..fd06e2c42a7b6 100644 --- a/compiler/rustc_query_impl/src/dep_kind_vtables.rs +++ b/compiler/rustc_middle/src/query/impl_/dep_kind_vtables.rs @@ -1,10 +1,9 @@ use rustc_middle::arena::Arena; -use rustc_middle::bug; use rustc_middle::dep_graph::{DepKindVTable, DepNodeKey, KeyFingerprintStyle}; use rustc_middle::query::QueryCache; -use crate::GetQueryVTable; -use crate::plumbing::promote_from_disk_inner; +use crate::query::impl_::GetQueryVTable; +use crate::query::impl_::plumbing::promote_from_disk_inner; /// [`DepKindVTable`] constructors for special dep kinds that aren't queries. #[expect(non_snake_case, reason = "use non-snake case to avoid collision with query names")] @@ -114,7 +113,7 @@ where force_from_dep_node_fn: (can_recover && !is_no_force).then_some( |tcx, dep_node, _prev_index| { let query = Q::query_vtable(tcx); - crate::execution::force_query_dep_node(tcx, query, dep_node) + crate::query::impl_::execution::force_query_dep_node(tcx, query, dep_node) }, ), promote_from_disk_fn: (can_recover && is_cache_on_disk).then_some(|tcx, dep_node| { @@ -163,8 +162,8 @@ macro_rules! define_dep_kind_vtables { // The large number of query vtables. let q_vtables: [DepKindVTable<'tcx>; _] = [ $( - $crate::dep_kind_vtables::make_dep_kind_vtable_for_query::< - $crate::query_impl::$name::VTableGetter, + $crate::query::impl_::dep_kind_vtables::make_dep_kind_vtable_for_query::< + $crate::query::impl_::query_impl::$name::VTableGetter, >( $cache_on_disk, $eval_always, diff --git a/compiler/rustc_query_impl/src/error.rs b/compiler/rustc_middle/src/query/impl_/error.rs similarity index 100% rename from compiler/rustc_query_impl/src/error.rs rename to compiler/rustc_middle/src/query/impl_/error.rs diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_middle/src/query/impl_/execution.rs similarity index 98% rename from compiler/rustc_query_impl/src/execution.rs rename to compiler/rustc_middle/src/query/impl_/execution.rs index c1e1615fd590f..e21a64dc5aac0 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_middle/src/query/impl_/execution.rs @@ -16,10 +16,14 @@ use rustc_middle::verify_ich::incremental_verify_ich; use rustc_span::{DUMMY_SP, Span}; use tracing::warn; -use crate::dep_graph::{DepNode, DepNodeIndex}; -use crate::job::{QueryJobInfo, QueryJobMap, create_cycle_error, find_cycle_in_stack}; -use crate::plumbing::{current_query_job, loadable_from_disk, next_job_id, start_query}; -use crate::query_impl::for_each_query_vtable; +use crate::query::impl_::dep_graph::{DepNode, DepNodeIndex}; +use crate::query::impl_::job::{ + QueryJobInfo, QueryJobMap, create_cycle_error, find_cycle_in_stack, +}; +use crate::query::impl_::plumbing::{ + current_query_job, loadable_from_disk, next_job_id, start_query, +}; +use crate::query::impl_::query_impl::for_each_query_vtable; #[inline] fn equivalent_key(k: K) -> impl Fn(&(K, V)) -> bool { diff --git a/compiler/rustc_query_impl/src/handle_cycle_error.rs b/compiler/rustc_middle/src/query/impl_/handle_cycle_error.rs similarity index 99% rename from compiler/rustc_query_impl/src/handle_cycle_error.rs rename to compiler/rustc_middle/src/query/impl_/handle_cycle_error.rs index 07565254969c8..3fbf8f06cadab 100644 --- a/compiler/rustc_query_impl/src/handle_cycle_error.rs +++ b/compiler/rustc_middle/src/query/impl_/handle_cycle_error.rs @@ -8,14 +8,13 @@ use rustc_errors::codes::*; use rustc_errors::{Applicability, Diag, MultiSpan, pluralize, struct_span_code_err}; use rustc_hir as hir; use rustc_hir::def::{DefKind, Res}; -use rustc_middle::bug; use rustc_middle::queries::TaggedQueryKey; use rustc_middle::query::Cycle; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::def_id::{DefId, LocalDefId}; use rustc_span::{ErrorGuaranteed, Span}; -use crate::job::create_cycle_error; +use crate::query::impl_::job::create_cycle_error; // Default cycle handler used for all queries that don't use the `handle_cycle_error` query // modifier. diff --git a/compiler/rustc_query_impl/src/job.rs b/compiler/rustc_middle/src/query/impl_/job.rs similarity index 96% rename from compiler/rustc_query_impl/src/job.rs rename to compiler/rustc_middle/src/query/impl_/job.rs index 2486c0abfde80..9abf37ffb94f3 100644 --- a/compiler/rustc_query_impl/src/job.rs +++ b/compiler/rustc_middle/src/query/impl_/job.rs @@ -11,7 +11,7 @@ use rustc_middle::query::{Cycle, QueryJob, QueryJobId, QueryLatch, QueryStackFra use rustc_middle::ty::TyCtxt; use rustc_span::{DUMMY_SP, Span}; -use crate::{CollectActiveJobsKind, collect_active_query_jobs}; +use crate::query::impl_::{CollectActiveJobsKind, collect_active_query_jobs}; /// Map from query job IDs to job information collected by /// `collect_active_query_jobs`. @@ -420,7 +420,7 @@ pub(crate) fn create_cycle_error<'tcx>( let mut cycle_stack = Vec::new(); - use crate::error::StackCount; + use crate::query::impl_::error::StackCount; let stack_bottom = frames[0].tagged_key.description(tcx); let stack_count = if frames.len() == 1 { StackCount::Single { stack_bottom: stack_bottom.clone() } @@ -431,11 +431,13 @@ pub(crate) fn create_cycle_error<'tcx>( for i in 1..frames.len() { let frame = &frames[i]; let span = frame.tagged_key.default_span(tcx, frames[(i + 1) % frames.len()].span); - cycle_stack - .push(crate::error::CycleStack { span, desc: frame.tagged_key.description(tcx) }); + cycle_stack.push(crate::query::impl_::error::CycleStack { + span, + desc: frame.tagged_key.description(tcx), + }); } - let cycle_usage = usage.as_ref().map(|usage| crate::error::CycleUsage { + let cycle_usage = usage.as_ref().map(|usage| crate::query::impl_::error::CycleUsage { span: usage.tagged_key.default_span(tcx, usage.span), usage: usage.tagged_key.description(tcx), }); @@ -455,14 +457,14 @@ pub(crate) fn create_cycle_error<'tcx>( }; let alias = if is_all_def_kind(DefKind::TyAlias) { - Some(crate::error::Alias::Ty) + Some(crate::query::impl_::error::Alias::Ty) } else if is_all_def_kind(DefKind::TraitAlias) { - Some(crate::error::Alias::Trait) + Some(crate::query::impl_::error::Alias::Trait) } else { None }; - let cycle_diag = crate::error::Cycle { + let cycle_diag = crate::query::impl_::error::Cycle { span, cycle_stack, stack_bottom, diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_middle/src/query/impl_/mod.rs similarity index 80% rename from compiler/rustc_query_impl/src/lib.rs rename to compiler/rustc_middle/src/query/impl_/mod.rs index 0b429dee69627..395270ca7575a 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_middle/src/query/impl_/mod.rs @@ -1,13 +1,5 @@ //! Support for serializing the dep-graph and reloading it. -// tidy-alphabetical-start -#![allow(internal_features)] -#![feature(core_intrinsics)] -#![feature(min_specialization)] -#![feature(rustc_attrs)] -#![feature(try_blocks)] -// tidy-alphabetical-end - use rustc_data_structures::sync::AtomicU64; use rustc_middle::dep_graph; use rustc_middle::queries::{ExternProviders, Providers}; @@ -15,9 +7,9 @@ use rustc_middle::query::on_disk_cache::OnDiskCache; use rustc_middle::query::{QueryCache, QueryHelper, QuerySystem, QueryVTable}; use rustc_middle::ty::TyCtxt; -pub use crate::dep_kind_vtables::make_dep_kind_vtables; -pub use crate::execution::{CollectActiveJobsKind, collect_active_query_jobs}; -pub use crate::job::{QueryJobMap, break_query_cycle, print_query_stack}; +pub use crate::query::impl_::dep_kind_vtables::make_dep_kind_vtables; +pub use crate::query::impl_::execution::{CollectActiveJobsKind, collect_active_query_jobs}; +pub use crate::query::impl_::job::{QueryJobMap, break_query_cycle, print_query_stack}; mod dep_kind_vtables; mod error; @@ -35,7 +27,7 @@ mod query_impl; /// expansion. /// /// There is one macro-generated implementation of this trait for each query, -/// on the type `rustc_query_impl::query_impl::$name::VTableGetter`. +/// on the type `rustc_middle::query::impl_::query_impl::$name::VTableGetter`. trait GetQueryVTable<'tcx> { type Cache: QueryCache + 'tcx; type Helper: QueryHelper<'tcx, ::Key, ::Value>; diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_middle/src/query/impl_/plumbing.rs similarity index 95% rename from compiler/rustc_query_impl/src/plumbing.rs rename to compiler/rustc_middle/src/query/impl_/plumbing.rs index 1b1439dc114e4..3498fbc67aaed 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_middle/src/query/impl_/plumbing.rs @@ -2,7 +2,6 @@ use std::num::NonZero; use rustc_data_structures::unord::UnordMap; use rustc_hir::limit::Limit; -use rustc_middle::bug; #[expect(unused_imports, reason = "used by doc comments")] use rustc_middle::dep_graph::DepKindVTable; use rustc_middle::dep_graph::{DepNode, DepNodeKey, SerializedDepNodeIndex}; @@ -15,11 +14,11 @@ use rustc_serialize::Encodable; use rustc_span::DUMMY_SP; use rustc_span::def_id::LOCAL_CRATE; -use crate::error::{QueryOverflow, QueryOverflowNote}; -use crate::execution::all_inactive; -use crate::job::find_dep_kind_root; -use crate::query_impl::for_each_query_vtable; -use crate::{CollectActiveJobsKind, collect_active_query_jobs}; +use crate::query::impl_::error::{QueryOverflow, QueryOverflowNote}; +use crate::query::impl_::execution::all_inactive; +use crate::query::impl_::job::find_dep_kind_root; +use crate::query::impl_::query_impl::for_each_query_vtable; +use crate::query::impl_::{CollectActiveJobsKind, collect_active_query_jobs}; fn depth_limit_error<'tcx>(tcx: TyCtxt<'tcx>, job: QueryJobId) { let job_map = collect_active_query_jobs(tcx, CollectActiveJobsKind::Full); diff --git a/compiler/rustc_query_impl/src/profiling_support.rs b/compiler/rustc_middle/src/query/impl_/profiling_support.rs similarity index 99% rename from compiler/rustc_query_impl/src/profiling_support.rs rename to compiler/rustc_middle/src/query/impl_/profiling_support.rs index 817155f16a8cb..a24e11c748441 100644 --- a/compiler/rustc_query_impl/src/profiling_support.rs +++ b/compiler/rustc_middle/src/query/impl_/profiling_support.rs @@ -9,7 +9,7 @@ use rustc_hir::definitions::DefPathData; use rustc_middle::query::{QueryCache, QueryHelper, QueryVTable}; use rustc_middle::ty::TyCtxt; -use crate::query_impl::for_each_query_vtable; +use crate::query::impl_::query_impl::for_each_query_vtable; pub(crate) struct QueryKeyStringCache { def_id_cache: FxHashMap, diff --git a/compiler/rustc_query_impl/src/query_impl.rs b/compiler/rustc_middle/src/query/impl_/query_impl.rs similarity index 92% rename from compiler/rustc_query_impl/src/query_impl.rs rename to compiler/rustc_middle/src/query/impl_/query_impl.rs index 487492286f57c..de07c26e07796 100644 --- a/compiler/rustc_query_impl/src/query_impl.rs +++ b/compiler/rustc_middle/src/query/impl_/query_impl.rs @@ -4,7 +4,7 @@ use rustc_middle::query::{AsLocalQueryKey, QueryMode, QueryVTable}; use rustc_middle::ty::TyCtxt; use rustc_span::Span; -use crate::GetQueryVTable; +use crate::query::impl_::GetQueryVTable; macro_rules! define_queries { ( @@ -33,7 +33,7 @@ macro_rules! define_queries { // Non-queries are unused here. non_queries { $($_:tt)* } ) => { - // This macro expects to be expanded into `crate::query_impl`, which is this file. + // This macro expects to be expanded into `crate::query::impl_::query_impl`, which is this file. $( pub(crate) mod $name { use super::*; @@ -58,7 +58,7 @@ macro_rules! define_queries { ) -> Option>> { #[cfg(debug_assertions)] let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); - crate::execution::execute_query_incr_inner( + crate::query::impl_::execution::execute_query_incr_inner( &tcx.query_system.query_vtables.$name, tcx, span, @@ -79,7 +79,7 @@ macro_rules! define_queries { key: Key<'tcx>, __mode: QueryMode, ) -> Option>> { - Some(crate::execution::execute_query_non_incr_inner( + Some(crate::query::impl_::execution::execute_query_non_incr_inner( &tcx.query_system.query_vtables.$name, tcx, span, @@ -157,7 +157,7 @@ macro_rules! define_queries { invoke_provider_fn: self::invoke_provider_fn::__rust_begin_short_backtrace, will_cache_on_disk_for_key_fn: - $crate::query_impl::$name::will_cache_on_disk_for_key, + $crate::query::impl_::query_impl::$name::will_cache_on_disk_for_key, helper: Default::default(), @@ -165,11 +165,11 @@ macro_rules! define_queries { handle_cycle_error_fn: |tcx, key, cycle, err| { use rustc_middle::query::erase::erase_val; - erase_val($crate::handle_cycle_error::$name(tcx, key, cycle, err)) + erase_val($crate::query::impl_::handle_cycle_error::$name(tcx, key, cycle, err)) }, #[cfg(not($handle_cycle_error))] handle_cycle_error_fn: |_tcx, _key, _cycle, err| { - $crate::handle_cycle_error::default(err) + $crate::query::impl_::handle_cycle_error::default(err) }, #[cfg($no_hash)] @@ -185,9 +185,9 @@ macro_rules! define_queries { }, create_tagged_key: TaggedQueryKey::$name, execute_query_fn: if incremental { - crate::query_impl::$name::execute_query_incr::__rust_end_short_backtrace + crate::query::impl_::query_impl::$name::execute_query_incr::__rust_end_short_backtrace } else { - crate::query_impl::$name::execute_query_non_incr::__rust_end_short_backtrace + crate::query::impl_::query_impl::$name::execute_query_non_incr::__rust_end_short_backtrace }, } } @@ -212,7 +212,7 @@ macro_rules! define_queries { { rustc_middle::queries::QueryVTables { $( - $name: crate::query_impl::$name::make_query_vtable(incremental), + $name: crate::query::impl_::query_impl::$name::make_query_vtable(incremental), )* } } diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index b780c8d57726f..904ce429a1d4e 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -15,6 +15,7 @@ use crate::ty::TyCtxt; pub(crate) mod arena_cached; mod caches; pub mod erase; +pub mod impl_; pub(crate) mod inner; mod into_query_key; mod job; diff --git a/compiler/rustc_middle/src/query/modifiers.rs b/compiler/rustc_middle/src/query/modifiers.rs index 4fd91caa94cd7..a77b9590f7507 100644 --- a/compiler/rustc_middle/src/query/modifiers.rs +++ b/compiler/rustc_middle/src/query/modifiers.rs @@ -62,7 +62,7 @@ pub(crate) struct feedable; /// /// The default behaviour for a query cycle is to emit a cycle error and halt /// compilation. Queries with this modifier will instead use a custom handler, -/// which must be provided at `rustc_query_impl::handle_cycle_error::$name`, +/// which must be provided at `rustc_middle::query::impl_::handle_cycle_error::$name`, /// where `$name` is the query name. pub(crate) struct handle_cycle_error; diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index f0707cfbbb40c..32fb506fa4e51 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1220,7 +1220,7 @@ impl<'tcx> TyCtxt<'tcx> { pub fn needs_crate_hash(self) -> bool { // Why is the crate hash needed for these configurations? // - debug_assertions: for the "fingerprint the result" check in - // `rustc_query_impl::execution::execute_job`. + // `rustc_middle::query::impl_::execution::execute_job`. // - incremental: for query lookups. // - needs_metadata: for putting into crate metadata. // - instrument_coverage: for putting into coverage data (see diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml deleted file mode 100644 index 02d3b0110cb8c..0000000000000 --- a/compiler/rustc_query_impl/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "rustc_query_impl" -version = "0.0.0" -edition = "2024" - -[dependencies] -# tidy-alphabetical-start -measureme = "12.0.1" -rustc_abi = { path = "../rustc_abi" } -rustc_data_structures = { path = "../rustc_data_structures" } -rustc_errors = { path = "../rustc_errors" } -rustc_hir = { path = "../rustc_hir" } -rustc_macros = { path = "../rustc_macros" } -rustc_middle = { path = "../rustc_middle" } -rustc_serialize = { path = "../rustc_serialize" } -rustc_span = { path = "../rustc_span" } -rustc_thread_pool = { path = "../rustc_thread_pool" } -tracing = "0.1" -# tidy-alphabetical-end diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_bench.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_bench.snap index 294623f073864..7c94ba32ab52f 100644 --- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_bench.snap +++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_bench.snap @@ -78,7 +78,6 @@ expression: bench - Set({bench::compiler/rustc_proc_macro}) - Set({bench::compiler/rustc_public}) - Set({bench::compiler/rustc_public_bridge}) - - Set({bench::compiler/rustc_query_impl}) - Set({bench::compiler/rustc_resolve}) - Set({bench::compiler/rustc_sanitizers}) - Set({bench::compiler/rustc_serialize}) diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_build_compiler.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_build_compiler.snap index d5da908c8a443..a18d35a9c7f5f 100644 --- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_build_compiler.snap +++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_build_compiler.snap @@ -60,7 +60,6 @@ expression: build compiler - Set({build::compiler/rustc_proc_macro}) - Set({build::compiler/rustc_public}) - Set({build::compiler/rustc_public_bridge}) - - Set({build::compiler/rustc_query_impl}) - Set({build::compiler/rustc_resolve}) - Set({build::compiler/rustc_sanitizers}) - Set({build::compiler/rustc_serialize}) diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check.snap index 242a2272b4d16..89fdd165a447a 100644 --- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check.snap +++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check.snap @@ -62,7 +62,6 @@ expression: check - Set({check::compiler/rustc_proc_macro}) - Set({check::compiler/rustc_public}) - Set({check::compiler/rustc_public_bridge}) - - Set({check::compiler/rustc_query_impl}) - Set({check::compiler/rustc_resolve}) - Set({check::compiler/rustc_sanitizers}) - Set({check::compiler/rustc_serialize}) diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiler.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiler.snap index dab86b792127f..17242935ef8a6 100644 --- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiler.snap +++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiler.snap @@ -62,7 +62,6 @@ expression: check compiler - Set({check::compiler/rustc_proc_macro}) - Set({check::compiler/rustc_public}) - Set({check::compiler/rustc_public_bridge}) - - Set({check::compiler/rustc_query_impl}) - Set({check::compiler/rustc_resolve}) - Set({check::compiler/rustc_sanitizers}) - Set({check::compiler/rustc_serialize}) diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiletest_include_default_paths.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiletest_include_default_paths.snap index e43d5380a398d..b3f6e9f99a99f 100644 --- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiletest_include_default_paths.snap +++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiletest_include_default_paths.snap @@ -62,7 +62,6 @@ expression: check compiletest --include-default-paths - Set({check::compiler/rustc_proc_macro}) - Set({check::compiler/rustc_public}) - Set({check::compiler/rustc_public_bridge}) - - Set({check::compiler/rustc_query_impl}) - Set({check::compiler/rustc_resolve}) - Set({check::compiler/rustc_sanitizers}) - Set({check::compiler/rustc_serialize}) diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_clippy.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_clippy.snap index 827f2f8b60acb..30848e6362494 100644 --- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_clippy.snap +++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_clippy.snap @@ -77,7 +77,6 @@ expression: clippy - Set({clippy::compiler/rustc_proc_macro}) - Set({clippy::compiler/rustc_public}) - Set({clippy::compiler/rustc_public_bridge}) - - Set({clippy::compiler/rustc_query_impl}) - Set({clippy::compiler/rustc_resolve}) - Set({clippy::compiler/rustc_sanitizers}) - Set({clippy::compiler/rustc_serialize}) diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_fix.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_fix.snap index d380cb416acf8..c293a1fbb798c 100644 --- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_fix.snap +++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_fix.snap @@ -62,7 +62,6 @@ expression: fix - Set({fix::compiler/rustc_proc_macro}) - Set({fix::compiler/rustc_public}) - Set({fix::compiler/rustc_public_bridge}) - - Set({fix::compiler/rustc_query_impl}) - Set({fix::compiler/rustc_resolve}) - Set({fix::compiler/rustc_sanitizers}) - Set({fix::compiler/rustc_serialize}) diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test.snap index ac2f315d39d96..4f5e04b7d6a22 100644 --- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test.snap +++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test.snap @@ -128,7 +128,6 @@ expression: test - Set({test::compiler/rustc_proc_macro}) - Set({test::compiler/rustc_public}) - Set({test::compiler/rustc_public_bridge}) - - Set({test::compiler/rustc_query_impl}) - Set({test::compiler/rustc_resolve}) - Set({test::compiler/rustc_sanitizers}) - Set({test::compiler/rustc_serialize}) diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_coverage.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_coverage.snap index 09adbb0041ae6..de6a0645916e7 100644 --- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_coverage.snap +++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_coverage.snap @@ -127,7 +127,6 @@ expression: test --skip=coverage - Set({test::compiler/rustc_proc_macro}) - Set({test::compiler/rustc_public}) - Set({test::compiler/rustc_public_bridge}) - - Set({test::compiler/rustc_query_impl}) - Set({test::compiler/rustc_resolve}) - Set({test::compiler/rustc_sanitizers}) - Set({test::compiler/rustc_serialize}) diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests.snap index b5fccfcb966bb..e5003bdf5606e 100644 --- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests.snap +++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests.snap @@ -91,7 +91,6 @@ expression: test --skip=tests - Set({test::compiler/rustc_proc_macro}) - Set({test::compiler/rustc_public}) - Set({test::compiler/rustc_public_bridge}) - - Set({test::compiler/rustc_query_impl}) - Set({test::compiler/rustc_resolve}) - Set({test::compiler/rustc_sanitizers}) - Set({test::compiler/rustc_serialize}) diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests_etc.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests_etc.snap index 9ad8914f58e30..0c3209d037e2f 100644 --- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests_etc.snap +++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests_etc.snap @@ -71,7 +71,6 @@ expression: test --skip=tests --skip=coverage-map --skip=coverage-run --skip=lib - Set({test::compiler/rustc_proc_macro}) - Set({test::compiler/rustc_public}) - Set({test::compiler/rustc_public_bridge}) - - Set({test::compiler/rustc_query_impl}) - Set({test::compiler/rustc_resolve}) - Set({test::compiler/rustc_sanitizers}) - Set({test::compiler/rustc_serialize}) diff --git a/triagebot.toml b/triagebot.toml index 7708bdbceffcb..8f90a1c58b014 100644 --- a/triagebot.toml +++ b/triagebot.toml @@ -545,7 +545,6 @@ trigger_files = [ [autolabel."A-query-system"] trigger_files = [ - "compiler/rustc_query_impl", "compiler/rustc_macros/src/query.rs" ] @@ -1592,7 +1591,6 @@ dep-bumps = [ "/compiler/rustc_public_bridge" = ["project-stable-mir"] "/compiler/rustc_parse" = ["compiler", "parser"] "/compiler/rustc_parse/src/lexer" = ["compiler", "lexer"] -"/compiler/rustc_query_impl" = ["compiler", "query-system"] "/compiler/rustc_trait_selection" = ["compiler", "types"] "/compiler/rustc_traits" = ["compiler", "types"] "/compiler/rustc_type_ir" = ["compiler", "types"] From a7fbe0c2533576b0ca3749a2002772de2accc960 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 13 Apr 2026 13:49:36 +0300 Subject: [PATCH 03/15] make will_cache_on_disk_for_key a static function --- .../rustc_middle/src/query/impl_/execution.rs | 12 +++++++---- .../rustc_middle/src/query/impl_/plumbing.rs | 4 ++-- .../src/query/impl_/query_impl.rs | 17 ---------------- compiler/rustc_middle/src/query/plumbing.rs | 20 +++++++++++++++++-- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/compiler/rustc_middle/src/query/impl_/execution.rs b/compiler/rustc_middle/src/query/impl_/execution.rs index e21a64dc5aac0..a29e995394a73 100644 --- a/compiler/rustc_middle/src/query/impl_/execution.rs +++ b/compiler/rustc_middle/src/query/impl_/execution.rs @@ -490,7 +490,7 @@ fn load_from_disk_or_invoke_provider_green< debug_assert!(dep_graph_data.is_index_green(prev_index)); // First try to load the result from the on-disk cache. Some things are never cached on disk. - let try_value = if (query.will_cache_on_disk_for_key_fn)(key) { + let try_value = if H::will_cache_on_disk_for_key(key) { let prof_timer = tcx.prof.incr_cache_loading(); let value = H::try_load_from_disk_fn(tcx, prev_index); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -558,13 +558,17 @@ fn load_from_disk_or_invoke_provider_green< /// on having the dependency graph (and in some cases a disk-cached value) /// from the previous incr-comp session. #[inline(never)] -fn ensure_can_skip_execution<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( +fn ensure_can_skip_execution<'tcx, C, H>( query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, key: C::Key, dep_node: DepNode, ensure_mode: EnsureMode, -) -> bool { +) -> bool +where + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +{ // Queries with `eval_always` should never skip execution. if query.eval_always { return false; @@ -595,7 +599,7 @@ fn ensure_can_skip_execution<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C // needed, which guarantees the query provider will never run // for this key. EnsureMode::Done => { - (query.will_cache_on_disk_for_key_fn)(key) + H::will_cache_on_disk_for_key(key) && loadable_from_disk(tcx, serialized_dep_node_index) } } diff --git a/compiler/rustc_middle/src/query/impl_/plumbing.rs b/compiler/rustc_middle/src/query/impl_/plumbing.rs index 3498fbc67aaed..22d6485ec9fb0 100644 --- a/compiler/rustc_middle/src/query/impl_/plumbing.rs +++ b/compiler/rustc_middle/src/query/impl_/plumbing.rs @@ -93,7 +93,7 @@ fn encode_query_values_inner<'a, 'tcx, C, V, H>( assert!(all_inactive(&query.state)); query.cache.for_each(&mut |key, value, dep_node| { - if (query.will_cache_on_disk_for_key_fn)(*key) { + if H::will_cache_on_disk_for_key(*key) { encoder.encode_query_value::(dep_node, &erase::restore_val::(*value)); } }); @@ -154,7 +154,7 @@ pub(crate) fn promote_from_disk_inner<'tcx, C: QueryCache, H>( // If the recovered key isn't eligible for cache-on-disk, then there's no // value on disk to promote. - if !(query.will_cache_on_disk_for_key_fn)(key) { + if !H::will_cache_on_disk_for_key(key) { return; } diff --git a/compiler/rustc_middle/src/query/impl_/query_impl.rs b/compiler/rustc_middle/src/query/impl_/query_impl.rs index de07c26e07796..8e83430581c1e 100644 --- a/compiler/rustc_middle/src/query/impl_/query_impl.rs +++ b/compiler/rustc_middle/src/query/impl_/query_impl.rs @@ -127,20 +127,6 @@ macro_rules! define_queries { } } - fn will_cache_on_disk_for_key<'tcx>( - _key: rustc_middle::queries::$name::Key<'tcx>, - ) -> bool { - cfg_select! { - // If a query has both `cache_on_disk` and `separate_provide_extern`, only - // disk-cache values for "local" keys, i.e. things in the current crate. - all($cache_on_disk, $separate_provide_extern) => { - AsLocalQueryKey::as_local_key(&_key).is_some() - } - all($cache_on_disk, not($separate_provide_extern)) => true, - not($cache_on_disk) => false, - } - } - pub(crate) fn make_query_vtable<'tcx>(incremental: bool) -> QueryVTable<'tcx, rustc_middle::queries::$name::Cache<'tcx>, rustc_middle::queries::$name::Helper> { @@ -156,9 +142,6 @@ macro_rules! define_queries { invoke_provider_fn: self::invoke_provider_fn::__rust_begin_short_backtrace, - will_cache_on_disk_for_key_fn: - $crate::query::impl_::query_impl::$name::will_cache_on_disk_for_key, - helper: Default::default(), #[cfg($handle_cycle_error)] diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 108e1adc182c2..827e593736920 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -98,8 +98,6 @@ pub struct QueryVTable<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Valu /// This should be the only code that calls the provider function. pub invoke_provider_fn: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value, - pub will_cache_on_disk_for_key_fn: fn(key: C::Key) -> bool, - pub helper: H, /// Function pointer that hashes this query's result values. @@ -132,6 +130,8 @@ pub struct QueryVTable<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Valu pub trait QueryHelper<'tcx, K, V>: Default + 'static { fn try_load_from_disk_fn(tcx: TyCtxt<'tcx>, prev_index: SerializedDepNodeIndex) -> Option; + + fn will_cache_on_disk_for_key(key: K) -> bool; } impl<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>> fmt::Debug @@ -366,6 +366,22 @@ macro_rules! define_callbacks { ) -> Option>> { None } + #[cfg_attr(all($cache_on_disk, $separate_provide_extern), inline)] + #[cfg_attr(not(all($cache_on_disk, $separate_provide_extern)), inline(always))] + fn will_cache_on_disk_for_key( + _key: rustc_middle::queries::$name::Key<'tcx>, + ) -> bool { + cfg_select! { + // If a query has both `cache_on_disk` and `separate_provide_extern`, only + // disk-cache values for "local" keys, i.e. things in the current crate. + all($cache_on_disk, $separate_provide_extern) => { + crate::query::AsLocalQueryKey::as_local_key(&_key).is_some() + } + all($cache_on_disk, not($separate_provide_extern)) => true, + not($cache_on_disk) => false, + } + } + } pub type Cache<'tcx> = From cbe4c61360931fe1c27a34179120c183eb23d9d9 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 13 Apr 2026 14:07:51 +0300 Subject: [PATCH 04/15] make invoke_provider_fn into a static function --- .../rustc_middle/src/query/impl_/execution.rs | 8 +-- .../src/query/impl_/query_impl.rs | 43 +------------- compiler/rustc_middle/src/query/plumbing.rs | 57 ++++++++++++++++--- 3 files changed, 53 insertions(+), 55 deletions(-) diff --git a/compiler/rustc_middle/src/query/impl_/execution.rs b/compiler/rustc_middle/src/query/impl_/execution.rs index a29e995394a73..5ad641754f22f 100644 --- a/compiler/rustc_middle/src/query/impl_/execution.rs +++ b/compiler/rustc_middle/src/query/impl_/execution.rs @@ -401,7 +401,7 @@ fn execute_job_non_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Val let prof_timer = tcx.prof.query_provider(); // Call the query provider. - let value = start_query(job_id, query.depth_limit, || (query.invoke_provider_fn)(tcx, key)); + let value = start_query(job_id, query.depth_limit, || H::invoke_provider_fn(tcx, key)); let dep_node_index = tcx.dep_graph.next_virtual_depnode_index(); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -456,8 +456,8 @@ fn execute_job_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>> dep_graph_data.with_task( dep_node, tcx, - (query, key), - |tcx, (query, key)| (query.invoke_provider_fn)(tcx, key), + key, + |tcx, key| H::invoke_provider_fn(tcx, key), query.hash_value_fn, ) }); @@ -521,7 +521,7 @@ fn load_from_disk_or_invoke_provider_green< // We could not load a result from the on-disk cache, so recompute. The dep-graph for // this computation is already in-place, so we can just call the query provider. let prof_timer = tcx.prof.query_provider(); - let value = tcx.dep_graph.with_ignore(|| (query.invoke_provider_fn)(tcx, key)); + let value = tcx.dep_graph.with_ignore(|| H::invoke_provider_fn(tcx, key)); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); (value, true) diff --git a/compiler/rustc_middle/src/query/impl_/query_impl.rs b/compiler/rustc_middle/src/query/impl_/query_impl.rs index 8e83430581c1e..30775fc0b06a4 100644 --- a/compiler/rustc_middle/src/query/impl_/query_impl.rs +++ b/compiler/rustc_middle/src/query/impl_/query_impl.rs @@ -1,6 +1,6 @@ use rustc_middle::queries::TaggedQueryKey; use rustc_middle::query::erase::{self, Erased}; -use rustc_middle::query::{AsLocalQueryKey, QueryMode, QueryVTable}; +use rustc_middle::query::{QueryMode, QueryVTable}; use rustc_middle::ty::TyCtxt; use rustc_span::Span; @@ -88,45 +88,6 @@ macro_rules! define_queries { } } - /// Defines an `invoke_provider` function that calls the query's provider, - /// to be used as a function pointer in the query's vtable. - /// - /// To mark a short-backtrace boundary, the function's actual name - /// (after demangling) must be `__rust_begin_short_backtrace`. - mod invoke_provider_fn { - use super::*; - use rustc_middle::queries::$name::{Key, Value, provided_to_erased}; - - #[inline(never)] - pub(crate) fn __rust_begin_short_backtrace<'tcx>( - tcx: TyCtxt<'tcx>, - key: Key<'tcx>, - ) -> Erased> { - #[cfg(debug_assertions)] - let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); - - // Call the actual provider function for this query. - - #[cfg($separate_provide_extern)] - let provided_value = if let Some(local_key) = key.as_local_key() { - (tcx.query_system.local_providers.$name)(tcx, local_key) - } else { - (tcx.query_system.extern_providers.$name)(tcx, key) - }; - - #[cfg(not($separate_provide_extern))] - let provided_value = (tcx.query_system.local_providers.$name)(tcx, key); - - rustc_middle::ty::print::with_reduced_queries!({ - tracing::trace!(?provided_value); - }); - - // Erase the returned value, because `QueryVTable` uses erased values. - // For queries with `arena_cache`, this also arena-allocates the value. - provided_to_erased(tcx, provided_value) - } - } - pub(crate) fn make_query_vtable<'tcx>(incremental: bool) -> QueryVTable<'tcx, rustc_middle::queries::$name::Cache<'tcx>, rustc_middle::queries::$name::Helper> { @@ -140,8 +101,6 @@ macro_rules! define_queries { state: Default::default(), cache: Default::default(), - invoke_provider_fn: self::invoke_provider_fn::__rust_begin_short_backtrace, - helper: Default::default(), #[cfg($handle_cycle_error)] diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 827e593736920..1690921ae9254 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -91,13 +91,6 @@ pub struct QueryVTable<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Valu pub state: QueryState<'tcx, C::Key>, pub cache: C, - /// Function pointer that actually calls this query's provider. - /// Also performs some associated secondary tasks; see the macro-defined - /// implementation in `mod invoke_provider_fn` for more details. - /// - /// This should be the only code that calls the provider function. - pub invoke_provider_fn: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value, - pub helper: H, /// Function pointer that hashes this query's result values. @@ -132,6 +125,13 @@ pub trait QueryHelper<'tcx, K, V>: Default + 'static { fn try_load_from_disk_fn(tcx: TyCtxt<'tcx>, prev_index: SerializedDepNodeIndex) -> Option; fn will_cache_on_disk_for_key(key: K) -> bool; + + /// Function pointer that actually calls this query's provider. + /// Also performs some associated secondary tasks; see the macro-defined + /// implementation in `mod invoke_provider_fn` for more details. + /// + /// This should be the only code that calls the provider function. + fn invoke_provider_fn(tcx: TyCtxt<'tcx>, key: K) -> V; } impl<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>> fmt::Debug @@ -351,8 +351,6 @@ macro_rules! define_callbacks { tcx: TyCtxt<'tcx>, prev_index: crate::dep_graph::SerializedDepNodeIndex, ) -> Option>> { - use rustc_middle::queries::$name::{ProvidedValue, provided_to_erased}; - let loaded_value: ProvidedValue<'tcx> = rustc_middle::query::plumbing::try_load_from_disk(tcx, prev_index)?; @@ -382,6 +380,47 @@ macro_rules! define_callbacks { } } + fn invoke_provider_fn(tcx: TyCtxt<'tcx>, key: Key<'tcx>) -> Erased> { + invoke_provider_fn::__rust_begin_short_backtrace(tcx, key) + } + } + + /// Defines an `invoke_provider` function that calls the query's provider, + /// to be used as a function pointer in the query's vtable. + /// + /// To mark a short-backtrace boundary, the function's actual name + /// (after demangling) must be `__rust_begin_short_backtrace`. + mod invoke_provider_fn { + use super::*; + + #[inline(never)] + pub(crate) fn __rust_begin_short_backtrace<'tcx>( + tcx: TyCtxt<'tcx>, + key: Key<'tcx>, + ) -> Erased> { + #[cfg(debug_assertions)] + let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); + + // Call the actual provider function for this query. + + #[cfg($separate_provide_extern)] + let provided_value = if let Some(local_key) = crate::query::AsLocalQueryKey::as_local_key(&key) { + (tcx.query_system.local_providers.$name)(tcx, local_key) + } else { + (tcx.query_system.extern_providers.$name)(tcx, key) + }; + + #[cfg(not($separate_provide_extern))] + let provided_value = (tcx.query_system.local_providers.$name)(tcx, key); + + rustc_middle::ty::print::with_reduced_queries!({ + tracing::trace!(?provided_value); + }); + + // Erase the returned value, because `QueryVTable` uses erased values. + // For queries with `arena_cache`, this also arena-allocates the value. + provided_to_erased(tcx, provided_value) + } } pub type Cache<'tcx> = From 94defaa2889d8d049eca0dec86da24684a550adc Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 13 Apr 2026 14:32:52 +0300 Subject: [PATCH 05/15] make execute_query_fn into a static function --- compiler/rustc_interface/src/passes.rs | 3 --- .../rustc_middle/src/query/impl_/execution.rs | 14 +++++----- compiler/rustc_middle/src/query/impl_/mod.rs | 5 ++-- .../rustc_middle/src/query/impl_/plumbing.rs | 2 +- .../src/query/impl_/query_impl.rs | 11 +++----- compiler/rustc_middle/src/query/inner.rs | 6 ++--- compiler/rustc_middle/src/query/plumbing.rs | 27 +++++++++++++++++-- 7 files changed, 41 insertions(+), 27 deletions(-) diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index c914d58c42f4d..9a8ffbc0d114f 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -969,8 +969,6 @@ pub fn create_and_enter_global_ctxt FnOnce(TyCtxt<'tcx>) -> T>( callback(sess, &mut providers); } - let incremental = dep_graph.is_fully_enabled(); - // Note: this function body is the origin point of the widely-used 'tcx lifetime. // // `gcx_cell` is defined here and `&gcx_cell` is passed to `create_global_ctxt`, which then @@ -1000,7 +998,6 @@ pub fn create_and_enter_global_ctxt FnOnce(TyCtxt<'tcx>) -> T>( providers.queries, providers.extern_queries, query_result_on_disk_cache, - incremental, ), providers.hooks, compiler.current_gcx.clone(), diff --git a/compiler/rustc_middle/src/query/impl_/execution.rs b/compiler/rustc_middle/src/query/impl_/execution.rs index 5ad641754f22f..99fc9d7830019 100644 --- a/compiler/rustc_middle/src/query/impl_/execution.rs +++ b/compiler/rustc_middle/src/query/impl_/execution.rs @@ -610,23 +610,23 @@ where /// Called by a macro-generated impl of [`QueryVTable::execute_query_fn`], /// in non-incremental mode. #[inline(always)] -pub(super) fn execute_query_non_incr_inner< - 'tcx, - C: QueryCache, - H: QueryHelper<'tcx, C::Key, C::Value>, ->( +pub(crate) fn execute_query_non_incr_inner<'tcx, C, H>( query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, span: Span, key: C::Key, -) -> C::Value { +) -> C::Value +where + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +{ ensure_sufficient_stack(|| try_execute_query::(query, tcx, span, key, None).0) } /// Called by a macro-generated impl of [`QueryVTable::execute_query_fn`], /// in incremental mode. #[inline(always)] -pub(super) fn execute_query_incr_inner<'tcx, C, H>( +pub(crate) fn execute_query_incr_inner<'tcx, C, H>( query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, span: Span, diff --git a/compiler/rustc_middle/src/query/impl_/mod.rs b/compiler/rustc_middle/src/query/impl_/mod.rs index 395270ca7575a..ce065baae0a0b 100644 --- a/compiler/rustc_middle/src/query/impl_/mod.rs +++ b/compiler/rustc_middle/src/query/impl_/mod.rs @@ -13,7 +13,7 @@ pub use crate::query::impl_::job::{QueryJobMap, break_query_cycle, print_query_s mod dep_kind_vtables; mod error; -mod execution; +pub(crate) mod execution; mod handle_cycle_error; mod job; mod plumbing; @@ -39,11 +39,10 @@ pub fn query_system<'tcx>( local_providers: Providers, extern_providers: ExternProviders, on_disk_cache: Option, - incremental: bool, ) -> QuerySystem<'tcx> { QuerySystem { arenas: Default::default(), - query_vtables: query_impl::make_query_vtables(incremental), + query_vtables: query_impl::make_query_vtables(), side_effects: Default::default(), on_disk_cache, local_providers, diff --git a/compiler/rustc_middle/src/query/impl_/plumbing.rs b/compiler/rustc_middle/src/query/impl_/plumbing.rs index 22d6485ec9fb0..6de613adfa0dd 100644 --- a/compiler/rustc_middle/src/query/impl_/plumbing.rs +++ b/compiler/rustc_middle/src/query/impl_/plumbing.rs @@ -170,7 +170,7 @@ pub(crate) fn promote_from_disk_inner<'tcx, C: QueryCache, H>( // FIXME(Zalathar): Is there a reasonable way to skip more of the // query bookkeeping when doing this? None => { - (query.execute_query_fn)(tcx, DUMMY_SP, key, QueryMode::Get); + query.execute_query_fn(tcx, DUMMY_SP, key, QueryMode::Get); } } } diff --git a/compiler/rustc_middle/src/query/impl_/query_impl.rs b/compiler/rustc_middle/src/query/impl_/query_impl.rs index 30775fc0b06a4..8bfeb5ad792db 100644 --- a/compiler/rustc_middle/src/query/impl_/query_impl.rs +++ b/compiler/rustc_middle/src/query/impl_/query_impl.rs @@ -88,7 +88,7 @@ macro_rules! define_queries { } } - pub(crate) fn make_query_vtable<'tcx>(incremental: bool) + pub(crate) fn make_query_vtable<'tcx>() -> QueryVTable<'tcx, rustc_middle::queries::$name::Cache<'tcx>, rustc_middle::queries::$name::Helper> { use rustc_middle::queries::$name::Value; @@ -126,11 +126,6 @@ macro_rules! define_queries { format!("{:?}", erase::restore_val(*erased_value)) }, create_tagged_key: TaggedQueryKey::$name, - execute_query_fn: if incremental { - crate::query::impl_::query_impl::$name::execute_query_incr::__rust_end_short_backtrace - } else { - crate::query::impl_::query_impl::$name::execute_query_non_incr::__rust_end_short_backtrace - }, } } @@ -149,12 +144,12 @@ macro_rules! define_queries { } )* - pub(crate) fn make_query_vtables<'tcx>(incremental: bool) + pub(crate) fn make_query_vtables<'tcx>() -> rustc_middle::queries::QueryVTables<'tcx> { rustc_middle::queries::QueryVTables { $( - $name: crate::query::impl_::query_impl::$name::make_query_vtable(incremental), + $name: crate::query::impl_::query_impl::$name::make_query_vtable(), )* } } diff --git a/compiler/rustc_middle/src/query/inner.rs b/compiler/rustc_middle/src/query/inner.rs index f6b5cd7d47304..ae201d01b3031 100644 --- a/compiler/rustc_middle/src/query/inner.rs +++ b/compiler/rustc_middle/src/query/inner.rs @@ -43,7 +43,7 @@ where { match try_get_cached(tcx, &query.cache, key) { Some(value) => value, - None => (query.execute_query_fn)(tcx, span, key, QueryMode::Get).unwrap(), + None => query.execute_query_fn(tcx, span, key, QueryMode::Get).unwrap(), } } @@ -62,7 +62,7 @@ pub(crate) fn query_ensure_ok_or_done<'tcx, C, H>( match try_get_cached(tcx, &query.cache, key) { Some(_value) => {} None => { - (query.execute_query_fn)(tcx, DUMMY_SP, key, QueryMode::Ensure { ensure_mode }); + query.execute_query_fn(tcx, DUMMY_SP, key, QueryMode::Ensure { ensure_mode }); } } } @@ -90,7 +90,7 @@ where match try_get_cached(tcx, &query.cache, key) { Some(value) => convert(value), None => { - match (query.execute_query_fn)( + match query.execute_query_fn( tcx, DUMMY_SP, key, diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 1690921ae9254..cce9340ba977a 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -77,7 +77,11 @@ pub enum EnsureMode { } /// Stores data and metadata (e.g. function pointers) for a particular query. -pub struct QueryVTable<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>> { +pub struct QueryVTable<'tcx, C, H> +where + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +{ pub name: &'static str, /// True if this query has the `eval_always` modifier. @@ -108,7 +112,13 @@ pub struct QueryVTable<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Valu pub format_value: fn(&C::Value) -> String, pub create_tagged_key: fn(C::Key) -> TaggedQueryKey<'tcx>, +} +impl<'tcx, C, H> QueryVTable<'tcx, C, H> +where + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +{ /// Function pointer that is called by the query methods on [`TyCtxt`] and /// friends[^1], after they have checked the in-memory cache and found no /// existing value for this key. @@ -118,7 +128,20 @@ pub struct QueryVTable<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Valu /// and putting the obtained value into the in-memory cache. /// /// [^1]: [`TyCtxt`], [`TyCtxtAt`], [`TyCtxtEnsureOk`], [`TyCtxtEnsureDone`] - pub execute_query_fn: fn(TyCtxt<'tcx>, Span, C::Key, QueryMode) -> Option, + pub(crate) fn execute_query_fn( + &'tcx self, + tcx: TyCtxt<'tcx>, + span: Span, + key: C::Key, + mode: QueryMode, + ) -> Option { + // FIXME: Figure out likely or unlikely + if tcx.dep_graph.is_fully_enabled() { + crate::query::impl_::execution::execute_query_incr_inner(self, tcx, span, key, mode) + } else { + Some(crate::query::impl_::execution::execute_query_non_incr_inner(self, tcx, span, key)) + } + } } pub trait QueryHelper<'tcx, K, V>: Default + 'static { From d210989374d786ea704077a687ceecf5bb826a4c Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 13 Apr 2026 15:43:03 +0300 Subject: [PATCH 06/15] Make hash_value_fn into static function --- compiler/rustc_interface/src/passes.rs | 4 +-- compiler/rustc_middle/src/arena.rs | 1 - compiler/rustc_middle/src/dep_graph/graph.rs | 18 +++++----- compiler/rustc_middle/src/queries.rs | 7 +--- .../rustc_middle/src/query/impl_/execution.rs | 19 ++++++----- .../src/query/impl_/query_impl.rs | 8 ----- compiler/rustc_middle/src/query/inner.rs | 34 +++++++++---------- compiler/rustc_middle/src/query/mod.rs | 4 +-- compiler/rustc_middle/src/query/plumbing.rs | 31 +++++++++++++---- compiler/rustc_middle/src/ty/context.rs | 3 ++ compiler/rustc_resolve/src/macros.rs | 9 +++-- 11 files changed, 76 insertions(+), 62 deletions(-) diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 9a8ffbc0d114f..30562b62925db 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -787,7 +787,7 @@ fn resolver_for_lowering_raw<'tcx>( ) -> (&'tcx Steal<(ty::ResolverAstLowering<'tcx>, Arc)>, &'tcx ty::ResolverGlobalCtxt) { let arenas = Resolver::arenas(); let _ = tcx.registered_tools(()); // Uses `crate_for_resolver`. - let (krate, pre_configured_attrs) = tcx.crate_for_resolver(()).steal(); + let (krate, pre_configured_attrs) = tcx.crate_for_resolver.borrow_mut().take().unwrap(); let mut resolver = Resolver::new( tcx, &pre_configured_attrs, @@ -1013,8 +1013,8 @@ pub fn create_and_enter_global_ctxt FnOnce(TyCtxt<'tcx>) -> T>( &pre_configured_attrs, crate_name, ))); - feed.crate_for_resolver(tcx.arena.alloc(Steal::new((krate, pre_configured_attrs)))); feed.output_filenames(Arc::new(outputs)); + *tcx.crate_for_resolver.borrow_mut() = Some((krate, pre_configured_attrs)); let res = f(tcx); // FIXME maybe run finish even when a fatal error occurred? or at least diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs index de6a105ee2b7b..87f6bf0733152 100644 --- a/compiler/rustc_middle/src/arena.rs +++ b/compiler/rustc_middle/src/arena.rs @@ -35,7 +35,6 @@ macro_rules! arena_types { rustc_middle::ty::ResolverAstLowering<'tcx>, std::sync::Arc, )>, - [] crate_for_resolver: rustc_data_structures::steal::Steal<(rustc_ast::Crate, rustc_ast::AttrVec)>, [] resolutions: rustc_middle::ty::ResolverGlobalCtxt, [] const_allocs: rustc_middle::mir::interpret::Allocation, [] region_scope_tree: rustc_middle::middle::region::ScopeTree, diff --git a/compiler/rustc_middle/src/dep_graph/graph.rs b/compiler/rustc_middle/src/dep_graph/graph.rs index abd37f2488009..89edbd4318e04 100644 --- a/compiler/rustc_middle/src/dep_graph/graph.rs +++ b/compiler/rustc_middle/src/dep_graph/graph.rs @@ -574,7 +574,7 @@ impl DepGraph { node: DepNode, tcx: TyCtxt<'tcx>, result: &R, - hash_result: Option, &R) -> Fingerprint>, + hash_result: fn(&mut StableHashingContext<'_>, &R) -> Fingerprint, format_value_fn: fn(&R) -> String, ) -> DepNodeIndex { if let Some(data) = self.data.as_ref() { @@ -592,18 +592,16 @@ impl DepGraph { data, result, prev_index, - hash_result, + Some(hash_result), format_value_fn, ); #[cfg(debug_assertions)] - if hash_result.is_some() { - data.current.record_edge( - dep_node_index, - node, - data.prev_value_fingerprint_of(prev_index), - ); - } + data.current.record_edge( + dep_node_index, + node, + data.prev_value_fingerprint_of(prev_index), + ); return dep_node_index; } @@ -621,7 +619,7 @@ impl DepGraph { } }); - data.hash_result_and_alloc_node(tcx, node, edges, result, hash_result) + data.hash_result_and_alloc_node(tcx, node, edges, result, Some(hash_result)) } else { // Incremental compilation is turned off. We just execute the task // without tracking. We still provide a dep-node index that uniquely diff --git a/compiler/rustc_middle/src/queries.rs b/compiler/rustc_middle/src/queries.rs index 1017ccffb0b2a..a76887ab6cd1d 100644 --- a/compiler/rustc_middle/src/queries.rs +++ b/compiler/rustc_middle/src/queries.rs @@ -154,6 +154,7 @@ rustc_queries! { /// Collects the list of all tools registered using `#![register_tool]`. query registered_tools(_: ()) -> &'tcx ty::RegisteredTools { + eval_always arena_cache desc { "compute registered tools for crate" } } @@ -2602,12 +2603,6 @@ rustc_queries! { desc { "looking up enabled feature gates" } } - query crate_for_resolver((): ()) -> &'tcx Steal<(rustc_ast::Crate, rustc_ast::AttrVec)> { - feedable - no_hash - desc { "the ast before macro expansion and name resolution" } - } - /// Attempt to resolve the given `DefId` to an `Instance`, for the /// given generics args (`GenericArgsRef`), returning one of: /// * `Ok(Some(instance))` on success diff --git a/compiler/rustc_middle/src/query/impl_/execution.rs b/compiler/rustc_middle/src/query/impl_/execution.rs index 99fc9d7830019..7074307bb5835 100644 --- a/compiler/rustc_middle/src/query/impl_/execution.rs +++ b/compiler/rustc_middle/src/query/impl_/execution.rs @@ -349,19 +349,22 @@ where } #[inline(always)] -fn check_feedable_consistency<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( +fn check_feedable_consistency<'tcx, C, H>( tcx: TyCtxt<'tcx>, query: &'tcx QueryVTable<'tcx, C, H>, key: C::Key, value: &C::Value, -) { +) where + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +{ // We should not compute queries that also got a value via feeding. // This can't happen, as query feeding adds the very dependencies to the fed query // as its feeding query had. So if the fed query is red, so is its feeder, which will // get evaluated first, and re-feed the query. let Some((cached_value, _)) = query.cache.lookup(&key) else { return }; - let Some(hash_value_fn) = query.hash_value_fn else { + if H::NO_HASH { panic!( "no_hash fed query later has its value computed.\n\ Remove `no_hash` modifier to allow recomputation.\n\ @@ -371,7 +374,7 @@ fn check_feedable_consistency<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, }; let (old_hash, new_hash) = tcx.with_stable_hashing_context(|mut hcx| { - (hash_value_fn(&mut hcx, &cached_value), hash_value_fn(&mut hcx, value)) + (H::hash_value_fn(&mut hcx, &cached_value), H::hash_value_fn(&mut hcx, value)) }); let formatter = query.format_value; if old_hash != new_hash { @@ -408,9 +411,9 @@ fn execute_job_non_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Val // Sanity: Fingerprint the key and the result to assert they don't contain anything unhashable. if cfg!(debug_assertions) { let _ = key.to_fingerprint(tcx); - if let Some(hash_value_fn) = query.hash_value_fn { + if !H::NO_HASH { tcx.with_stable_hashing_context(|mut hcx| { - hash_value_fn(&mut hcx, &value); + H::hash_value_fn(&mut hcx, &value); }); } } @@ -458,7 +461,7 @@ fn execute_job_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>> tcx, key, |tcx, key| H::invoke_provider_fn(tcx, key), - query.hash_value_fn, + (!H::NO_HASH).then_some(H::hash_value_fn), ) }); @@ -543,7 +546,7 @@ fn load_from_disk_or_invoke_provider_green< dep_graph_data, &value, prev_index, - query.hash_value_fn, + (!H::NO_HASH).then_some(H::hash_value_fn), query.format_value, ); } diff --git a/compiler/rustc_middle/src/query/impl_/query_impl.rs b/compiler/rustc_middle/src/query/impl_/query_impl.rs index 8bfeb5ad792db..d8e736bf14ef5 100644 --- a/compiler/rustc_middle/src/query/impl_/query_impl.rs +++ b/compiler/rustc_middle/src/query/impl_/query_impl.rs @@ -114,14 +114,6 @@ macro_rules! define_queries { $crate::query::impl_::handle_cycle_error::default(err) }, - #[cfg($no_hash)] - hash_value_fn: None, - #[cfg(not($no_hash))] - hash_value_fn: Some(|hcx, erased_value: &erase::Erased>| { - let value = erase::restore_val(*erased_value); - rustc_middle::dep_graph::hash_result(hcx, &value) - }), - format_value: |erased_value: &erase::Erased>| { format!("{:?}", erase::restore_val(*erased_value)) }, diff --git a/compiler/rustc_middle/src/query/inner.rs b/compiler/rustc_middle/src/query/inner.rs index ae201d01b3031..758178f5fba98 100644 --- a/compiler/rustc_middle/src/query/inner.rs +++ b/compiler/rustc_middle/src/query/inner.rs @@ -131,22 +131,7 @@ pub(crate) fn query_feed<'tcx, C, H>( // The query already has a cached value for this key. // That's OK if both values are the same, i.e. they have the same hash, // so now we check their hashes. - if let Some(hash_value_fn) = query.hash_value_fn { - let (old_hash, value_hash) = tcx.with_stable_hashing_context(|ref mut hcx| { - (hash_value_fn(hcx, &old), hash_value_fn(hcx, &value)) - }); - if old_hash != value_hash { - // We have an inconsistency. This can happen if one of the two - // results is tainted by errors. In this case, delay a bug to - // ensure compilation is doomed, and keep the `old` value. - tcx.dcx().delayed_bug(format!( - "Trying to feed an already recorded value for query {query:?} key={key:?}:\n\ - old value: {old}\nnew value: {value}", - old = format_value(&old), - value = format_value(&value), - )); - } - } else { + if H::NO_HASH { // The query is `no_hash`, so we have no way to perform a sanity check. // If feeding the same value multiple times needs to be supported, // the query should not be marked `no_hash`. @@ -157,6 +142,21 @@ pub(crate) fn query_feed<'tcx, C, H>( value = format_value(&value), ) } + + let (old_hash, value_hash) = tcx.with_stable_hashing_context(|ref mut hcx| { + (H::hash_value_fn(hcx, &old), H::hash_value_fn(hcx, &value)) + }); + if old_hash != value_hash { + // We have an inconsistency. This can happen if one of the two + // results is tainted by errors. In this case, delay a bug to + // ensure compilation is doomed, and keep the `old` value. + tcx.dcx().delayed_bug(format!( + "Trying to feed an already recorded value for query {query:?} key={key:?}:\n\ + old value: {old}\nnew value: {value}", + old = format_value(&old), + value = format_value(&value), + )); + } } None => { // There is no cached value for this key, so feed the query by @@ -166,7 +166,7 @@ pub(crate) fn query_feed<'tcx, C, H>( dep_node, tcx, &value, - query.hash_value_fn, + H::hash_value_fn, query.format_value, ); query.cache.complete(key, value, dep_node_index); diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 904ce429a1d4e..831b7ac63bc87 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -5,8 +5,8 @@ pub use self::into_query_key::IntoQueryKey; pub use self::job::{QueryJob, QueryJobId, QueryLatch, QueryWaiter}; pub use self::keys::{AsLocalQueryKey, LocalCrate, QueryKey}; pub use self::plumbing::{ - ActiveKeyStatus, Cycle, EnsureMode, QueryHelper, QueryMode, QueryState, QuerySystem, - QueryVTable, TyCtxtAt, TyCtxtEnsureDone, TyCtxtEnsureOk, TyCtxtEnsureResult, + ActiveKeyStatus, Cycle, EnsureMode, QueryHashHelper, QueryHelper, QueryMode, QueryState, + QuerySystem, QueryVTable, TyCtxtAt, TyCtxtEnsureDone, TyCtxtEnsureOk, TyCtxtEnsureResult, }; pub use self::stack::QueryStackFrame; pub use crate::queries::Providers; diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index cce9340ba977a..fdfe922c1bc06 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -97,11 +97,6 @@ where pub helper: H, - /// Function pointer that hashes this query's result values. - /// - /// For `no_hash` queries, this function pointer is None. - pub hash_value_fn: Option, &C::Value) -> Fingerprint>, - /// Function pointer that handles a cycle error. `error` must be consumed, e.g. with `emit` (if /// it should be emitted) or `delay_as_bug` (if it need not be emitted because an alternative /// error is created and emitted). A value may be returned, or (more commonly) the function may @@ -144,7 +139,7 @@ where } } -pub trait QueryHelper<'tcx, K, V>: Default + 'static { +pub trait QueryHelper<'tcx, K, V>: QueryHashHelper + Default + 'static { fn try_load_from_disk_fn(tcx: TyCtxt<'tcx>, prev_index: SerializedDepNodeIndex) -> Option; fn will_cache_on_disk_for_key(key: K) -> bool; @@ -157,6 +152,15 @@ pub trait QueryHelper<'tcx, K, V>: Default + 'static { fn invoke_provider_fn(tcx: TyCtxt<'tcx>, key: K) -> V; } +pub trait QueryHashHelper: Default + 'static { + const NO_HASH: bool; + + /// Function that hashes query's result values. + /// + /// For `no_hash` queries, this function pointer is None. + fn hash_value_fn(hcx: &mut StableHashingContext<'_>, value: &V) -> Fingerprint; +} + impl<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>> fmt::Debug for QueryVTable<'tcx, C, H> { @@ -408,6 +412,21 @@ macro_rules! define_callbacks { } } + impl<'tcx> crate::query::QueryHashHelper>> for Helper { + const NO_HASH: bool = $no_hash; + + #[cfg($no_hash)] + fn hash_value_fn(_: &mut crate::ich::StableHashingContext<'_>, _: &Erased>) -> rustc_data_structures::fingerprint::Fingerprint { + panic!("Tried to hash value for no_hash query. `no_hash` query modifier is unsupported with `feedable` modifier enabled") + } + + #[cfg(not($no_hash))] + fn hash_value_fn(hcx: &mut crate::ich::StableHashingContext<'_>, value: &Erased>) -> rustc_data_structures::fingerprint::Fingerprint { + let value = erase::restore_val(*value); + rustc_middle::dep_graph::hash_result(hcx, &value) + } + } + /// Defines an `invoke_provider` function that calls the query's provider, /// to be used as a function pointer in the query's vtable. /// diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 32fb506fa4e51..bbe5af0bc1ba6 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -837,6 +837,8 @@ pub struct GlobalCtxt<'tcx> { /// A jobserver reference used to release then acquire a token while waiting on a query. pub jobserver_proxy: Arc, + + pub crate_for_resolver: Lock>, } impl<'tcx> GlobalCtxt<'tcx> { @@ -1062,6 +1064,7 @@ impl<'tcx> TyCtxt<'tcx> { alloc_map: interpret::AllocMap::new(), current_gcx, jobserver_proxy, + crate_for_resolver: Default::default(), }); // This is a separate function to work around a crash with parallel rustc (#135870) diff --git a/compiler/rustc_resolve/src/macros.rs b/compiler/rustc_resolve/src/macros.rs index 67a896bdd7557..81b7ac5b8282f 100644 --- a/compiler/rustc_resolve/src/macros.rs +++ b/compiler/rustc_resolve/src/macros.rs @@ -124,8 +124,13 @@ fn fast_print_path(path: &ast::Path) -> Symbol { } pub(crate) fn registered_tools(tcx: TyCtxt<'_>, (): ()) -> RegisteredTools { - let (_, pre_configured_attrs) = &*tcx.crate_for_resolver(()).borrow(); - registered_tools_ast(tcx.dcx(), pre_configured_attrs, tcx.sess, tcx.features()) + let crate_for_resolver = tcx.crate_for_resolver.borrow(); + registered_tools_ast( + tcx.dcx(), + &crate_for_resolver.as_ref().unwrap().1, + tcx.sess, + tcx.features(), + ) } pub fn registered_tools_ast( From 6470e62a12bc431c56c5c9451f7f241c05542623 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 13 Apr 2026 17:06:20 +0300 Subject: [PATCH 07/15] push with_task arguments to compile time --- compiler/rustc_codegen_llvm/src/base.rs | 219 ++++++++++-------- compiler/rustc_metadata/src/rmeta/encoder.rs | 47 +++- compiler/rustc_middle/src/dep_graph/graph.rs | 73 +++--- .../rustc_middle/src/query/impl_/execution.rs | 37 +-- compiler/rustc_middle/src/verify_ich.rs | 27 ++- 5 files changed, 236 insertions(+), 167 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index e1a6785942638..5118933df7edf 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -22,6 +22,7 @@ use rustc_hir::attrs::Linkage; use rustc_middle::dep_graph; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrs, SanitizerFnAttrs}; use rustc_middle::mono::Visibility; +use rustc_middle::query::{QueryHashHelper, QueryHelper}; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, Offload}; use rustc_span::Symbol; @@ -62,122 +63,146 @@ pub(crate) fn compile_codegen_unit( let start_time = Instant::now(); let dep_node = tcx.codegen_unit(cgu_name).codegen_dep_node(tcx); - let (module, _) = tcx.dep_graph.with_task( - dep_node, - tcx, - cgu_name, - module_codegen, - Some(dep_graph::hash_result), - ); - let time_to_codegen = start_time.elapsed(); - // We assume that the cost to run LLVM on a CGU is proportional to - // the time we needed for codegenning it. - let cost = time_to_codegen.as_nanos() as u64; + #[derive(Default)] + struct CompileCguHashHelper; - fn module_codegen(tcx: TyCtxt<'_>, cgu_name: Symbol) -> ModuleCodegen { - let cgu = tcx.codegen_unit(cgu_name); - let _prof_timer = - tcx.prof.generic_activity_with_arg_recorder("codegen_module", |recorder| { - recorder.record_arg(cgu_name.to_string()); - recorder.record_arg(cgu.size_estimate().to_string()); - }); - // Instantiate monomorphizations without filling out definitions yet... - let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str()); - { - let mut cx = CodegenCx::new(tcx, cgu, &llvm_module); - - // Declare and store globals shared by all offload kernels - // - // These globals are left in the LLVM-IR host module so all kernels can access them. - // They are necessary for correct offload execution. We do this here to simplify the - // `offload` intrinsic, avoiding the need for tracking whether it's the first - // intrinsic call or not. - let has_host_offload = cx - .sess() - .opts - .unstable_opts - .offload - .iter() - .any(|o| matches!(o, Offload::Host(_) | Offload::Test)); - if has_host_offload && !cx.sess().target.is_like_gpu { - cx.offload_globals.replace(Some(OffloadGlobals::declare(&cx))); - } + impl QueryHashHelper> for CompileCguHashHelper { + const NO_HASH: bool = false; - let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx); - for &(mono_item, data) in &mono_items { - mono_item.predefine::>( - &mut cx, - cgu_name.as_str(), - data.linkage, - data.visibility, - ); - } + fn hash_value_fn( + hcx: &mut rustc_middle::ich::StableHashingContext<'_>, + result: &ModuleCodegen, + ) -> rustc_data_structures::fingerprint::Fingerprint { + dep_graph::hash_result(hcx, result) + } + } - // ... and now that we have everything pre-defined, fill out those definitions. - for &(mono_item, item_data) in &mono_items { - mono_item.define::>(&mut cx, cgu_name.as_str(), item_data); - } + impl<'tcx> QueryHelper<'tcx, Symbol, ModuleCodegen> for CompileCguHashHelper { + fn try_load_from_disk_fn( + _: TyCtxt<'tcx>, + _: dep_graph::SerializedDepNodeIndex, + ) -> Option> { + unimplemented!() + } - // If this codegen unit contains the main function, also create the - // wrapper here - if let Some(entry) = - maybe_create_entry_wrapper::>(&cx, cx.codegen_unit) + fn will_cache_on_disk_for_key(_: Symbol) -> bool { + unimplemented!() + } + + fn invoke_provider_fn(tcx: TyCtxt<'tcx>, cgu_name: Symbol) -> ModuleCodegen { + // Module codegen + let cgu = tcx.codegen_unit(cgu_name); + let _prof_timer = + tcx.prof.generic_activity_with_arg_recorder("codegen_module", |recorder| { + recorder.record_arg(cgu_name.to_string()); + recorder.record_arg(cgu.size_estimate().to_string()); + }); + // Instantiate monomorphizations without filling out definitions yet... + let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str()); { - let attrs = attributes::sanitize_attrs(&cx, tcx, SanitizerFnAttrs::default()); - attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); - } + let mut cx = CodegenCx::new(tcx, cgu, &llvm_module); + + // Declare and store globals shared by all offload kernels + // + // These globals are left in the LLVM-IR host module so all kernels can access them. + // They are necessary for correct offload execution. We do this here to simplify the + // `offload` intrinsic, avoiding the need for tracking whether it's the first + // intrinsic call or not. + let has_host_offload = cx + .sess() + .opts + .unstable_opts + .offload + .iter() + .any(|o| matches!(o, Offload::Host(_) | Offload::Test)); + if has_host_offload && !cx.sess().target.is_like_gpu { + cx.offload_globals.replace(Some(OffloadGlobals::declare(&cx))); + } - // Define Objective-C module info and module flags. Note, the module info will - // also be added to the `llvm.compiler.used` variable, created later. - // - // These are only necessary when we need the linker to do its Objective-C-specific - // magic. We could theoretically do it unconditionally, but at a slight cost to linker - // performance in the common case where it's unnecessary. - if !cx.objc_classrefs.borrow().is_empty() || !cx.objc_selrefs.borrow().is_empty() { - if cx.objc_abi_version() == 1 { - cx.define_objc_module_info(); + let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx); + for &(mono_item, data) in &mono_items { + mono_item.predefine::>( + &mut cx, + cgu_name.as_str(), + data.linkage, + data.visibility, + ); } - cx.add_objc_module_flags(); - } - // Finalize code coverage by injecting the coverage map. Note, the coverage map will - // also be added to the `llvm.compiler.used` variable, created next. - if cx.sess().instrument_coverage() { - cx.coverageinfo_finalize(); - } + // ... and now that we have everything pre-defined, fill out those definitions. + for &(mono_item, item_data) in &mono_items { + mono_item.define::>(&mut cx, cgu_name.as_str(), item_data); + } - // Create the llvm.used variable. - if !cx.used_statics.is_empty() { - cx.create_used_variable_impl(c"llvm.used", &cx.used_statics); - } + // If this codegen unit contains the main function, also create the + // wrapper here + if let Some(entry) = + maybe_create_entry_wrapper::>(&cx, cx.codegen_unit) + { + let attrs = attributes::sanitize_attrs(&cx, tcx, SanitizerFnAttrs::default()); + attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); + } - // Create the llvm.compiler.used variable. - { - let compiler_used_statics = cx.compiler_used_statics.borrow(); - if !compiler_used_statics.is_empty() { - cx.create_used_variable_impl(c"llvm.compiler.used", &compiler_used_statics); + // Define Objective-C module info and module flags. Note, the module info will + // also be added to the `llvm.compiler.used` variable, created later. + // + // These are only necessary when we need the linker to do its Objective-C-specific + // magic. We could theoretically do it unconditionally, but at a slight cost to linker + // performance in the common case where it's unnecessary. + if !cx.objc_classrefs.borrow().is_empty() || !cx.objc_selrefs.borrow().is_empty() { + if cx.objc_abi_version() == 1 { + cx.define_objc_module_info(); + } + cx.add_objc_module_flags(); } - } - // Run replace-all-uses-with for statics that need it. This must - // happen after the llvm.used variables are created. - for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() { - unsafe { - llvm::LLVMReplaceAllUsesWith(old_g, new_g); - llvm::LLVMDeleteGlobal(old_g); + // Finalize code coverage by injecting the coverage map. Note, the coverage map will + // also be added to the `llvm.compiler.used` variable, created next. + if cx.sess().instrument_coverage() { + cx.coverageinfo_finalize(); + } + + // Create the llvm.used variable. + if !cx.used_statics.is_empty() { + cx.create_used_variable_impl(c"llvm.used", &cx.used_statics); + } + + // Create the llvm.compiler.used variable. + { + let compiler_used_statics = cx.compiler_used_statics.borrow(); + if !compiler_used_statics.is_empty() { + cx.create_used_variable_impl(c"llvm.compiler.used", &compiler_used_statics); + } + } + + // Run replace-all-uses-with for statics that need it. This must + // happen after the llvm.used variables are created. + for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() { + unsafe { + llvm::LLVMReplaceAllUsesWith(old_g, new_g); + llvm::LLVMDeleteGlobal(old_g); + } } - } - // Finalize debuginfo - if cx.sess().opts.debuginfo != DebugInfo::None { - cx.debuginfo_finalize(); + // Finalize debuginfo + if cx.sess().opts.debuginfo != DebugInfo::None { + cx.debuginfo_finalize(); + } } - } - ModuleCodegen::new_regular(cgu_name.to_string(), llvm_module) + ModuleCodegen::new_regular(cgu_name.to_string(), llvm_module) + } } + let (module, _) = + tcx.dep_graph.with_task::<_, _, CompileCguHashHelper>(dep_node, tcx, cgu_name); + let time_to_codegen = start_time.elapsed(); + + // We assume that the cost to run LLVM on a CGU is proportional to + // the time we needed for codegenning it. + let cost = time_to_codegen.as_nanos() as u64; + (module, cost) } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 72787c841204b..6bf5836080614 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -20,7 +20,7 @@ use rustc_hir_pretty::id_to_string; use rustc_middle::dep_graph::WorkProductId; use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::mir::interpret; -use rustc_middle::query::Providers; +use rustc_middle::query::{Providers, QueryHashHelper, QueryHelper}; use rustc_middle::traits::specialization_graph; use rustc_middle::ty::AssocContainer; use rustc_middle::ty::codec::TyEncoder; @@ -2472,14 +2472,34 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { ); } - // Perform metadata encoding inside a task, so the dep-graph can check if any encoded - // information changes, and maybe reuse the work product. - tcx.dep_graph.with_task( - dep_node, - tcx, - path, - |tcx, path| { - with_encode_metadata_header(tcx, path, |ecx| { + #[derive(Default)] + struct EncodeMetadataHelper; + + impl QueryHashHelper<()> for EncodeMetadataHelper { + const NO_HASH: bool = true; + + fn hash_value_fn( + _: &mut rustc_middle::ich::StableHashingContext<'_>, + _: &(), + ) -> rustc_data_structures::fingerprint::Fingerprint { + unimplemented!() + } + } + + impl<'tcx> QueryHelper<'tcx, &Path, ()> for EncodeMetadataHelper { + fn try_load_from_disk_fn( + _: TyCtxt<'tcx>, + _: rustc_middle::dep_graph::SerializedDepNodeIndex, + ) -> Option<()> { + unimplemented!() + } + + fn will_cache_on_disk_for_key(_: &Path) -> bool { + unimplemented!() + } + + fn invoke_provider_fn(tcx: TyCtxt<'tcx>, key: &Path) { + with_encode_metadata_header(tcx, key, |ecx| { // Encode all the entries and extra information in the crate, // culminating in the `CrateRoot` which points to all of it. let root = ecx.encode_crate_root(); @@ -2495,9 +2515,12 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { root.position.get() }) - }, - None, - ); + } + } + + // Perform metadata encoding inside a task, so the dep-graph can check if any encoded + // information changes, and maybe reuse the work product. + tcx.dep_graph.with_task::<_, _, EncodeMetadataHelper>(dep_node, tcx, path); } fn with_encode_metadata_header( diff --git a/compiler/rustc_middle/src/dep_graph/graph.rs b/compiler/rustc_middle/src/dep_graph/graph.rs index 89edbd4318e04..cc0390b4cbe6a 100644 --- a/compiler/rustc_middle/src/dep_graph/graph.rs +++ b/compiler/rustc_middle/src/dep_graph/graph.rs @@ -27,8 +27,9 @@ use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex use super::{DepKind, DepNode, WorkProductId, read_deps, with_deps}; use crate::dep_graph::edges::EdgesVec; use crate::ich::StableHashingContext; +use crate::query::QueryHelper; use crate::ty::TyCtxt; -use crate::verify_ich::incremental_verify_ich; +use crate::verify_ich::incremental_verify_ich_green; /// Tracks 'side effects' for a particular query. /// This struct is saved to disk along with the query result, @@ -274,17 +275,19 @@ impl DepGraph { } #[inline(always)] - pub fn with_task<'tcx, A: Debug, R>( + pub fn with_task<'tcx, A, R, H>( &self, dep_node: DepNode, tcx: TyCtxt<'tcx>, - task_arg: A, - task_fn: fn(tcx: TyCtxt<'tcx>, task_arg: A) -> R, - hash_result: Option, &R) -> Fingerprint>, - ) -> (R, DepNodeIndex) { + key: A, + ) -> (R, DepNodeIndex) + where + H: QueryHelper<'tcx, A, R>, + A: Debug, + { match self.data() { - Some(data) => data.with_task(dep_node, tcx, task_arg, task_fn, hash_result), - None => (task_fn(tcx, task_arg), self.next_virtual_depnode_index()), + Some(data) => data.with_task::<_, _, H>(dep_node, tcx, key), + None => (H::invoke_provider_fn(tcx, key), self.next_virtual_depnode_index()), } } @@ -325,14 +328,16 @@ impl DepGraphData { /// /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/queries/incremental-compilation.html #[inline(always)] - pub fn with_task<'tcx, A: Debug, R>( + pub fn with_task<'tcx, K, V, H>( &self, dep_node: DepNode, tcx: TyCtxt<'tcx>, - task_arg: A, - task_fn: fn(tcx: TyCtxt<'tcx>, task_arg: A) -> R, - hash_result: Option, &R) -> Fingerprint>, - ) -> (R, DepNodeIndex) { + key: K, + ) -> (V, DepNodeIndex) + where + K: Debug, + H: QueryHelper<'tcx, K, V>, + { // If the following assertion triggers, it can have two reasons: // 1. Something is wrong with DepNode creation, either here or // in `DepGraph::try_mark_green()`. @@ -341,12 +346,12 @@ impl DepGraphData { self.assert_dep_node_not_yet_allocated_in_current_session(tcx.sess, &dep_node, || { format!( "forcing query with already existing `DepNode`\n\ - - query-key: {task_arg:?}\n\ + - query-key: {key:?}\n\ - dep-node: {dep_node:?}" ) }); - let with_deps = |task_deps| with_deps(task_deps, || task_fn(tcx, task_arg)); + let with_deps = |task_deps| with_deps(task_deps, || H::invoke_provider_fn(tcx, key)); let (result, edges) = if tcx.is_eval_always(dep_node.kind) { (with_deps(TaskDepsRef::EvalAlways), EdgesVec::new()) } else { @@ -358,8 +363,11 @@ impl DepGraphData { (with_deps(TaskDepsRef::Allow(&task_deps)), task_deps.into_inner().reads) }; - let dep_node_index = - self.hash_result_and_alloc_node(tcx, dep_node, edges, &result, hash_result); + let dep_node_index = if H::NO_HASH { + self.alloc_and_color_node(dep_node, edges, None) + } else { + self.hash_result_and_alloc_node(tcx, dep_node, edges, &result, H::hash_value_fn) + }; (result, dep_node_index) } @@ -443,19 +451,21 @@ impl DepGraphData { } /// Intern the new `DepNode` with the dependencies up-to-now. - fn hash_result_and_alloc_node<'tcx, R>( + fn hash_result_and_alloc_node<'tcx, R, F>( &self, tcx: TyCtxt<'tcx>, node: DepNode, edges: EdgesVec, result: &R, - hash_result: Option, &R) -> Fingerprint>, - ) -> DepNodeIndex { + hash_result: F, + ) -> DepNodeIndex + where + F: Fn(&mut StableHashingContext<'_>, &R) -> Fingerprint, + { let hashing_timer = tcx.prof.incr_result_hashing(); - let current_fingerprint = hash_result.map(|hash_result| { - tcx.with_stable_hashing_context(|mut hcx| hash_result(&mut hcx, result)) - }); - let dep_node_index = self.alloc_and_color_node(node, edges, current_fingerprint); + let current_fingerprint = + tcx.with_stable_hashing_context(|mut hcx| hash_result(&mut hcx, result)); + let dep_node_index = self.alloc_and_color_node(node, edges, Some(current_fingerprint)); hashing_timer.finish_with_query_invocation_id(dep_node_index.into()); dep_node_index } @@ -569,14 +579,17 @@ impl DepGraph { /// FIXME: If the code is changed enough for this node to be marked before requiring the /// caller's node, we suppose that those changes will be enough to mark this node red and /// force a recomputation using the "normal" way. - pub fn with_feed_task<'tcx, R>( + pub fn with_feed_task<'tcx, R, F>( &self, node: DepNode, tcx: TyCtxt<'tcx>, result: &R, - hash_result: fn(&mut StableHashingContext<'_>, &R) -> Fingerprint, + hash_result: F, format_value_fn: fn(&R) -> String, - ) -> DepNodeIndex { + ) -> DepNodeIndex + where + F: Fn(&mut StableHashingContext<'_>, &R) -> Fingerprint, + { if let Some(data) = self.data.as_ref() { // The caller query has more dependencies than the node we are creating. We may // encounter a case where this created node is marked as green, but the caller query is @@ -587,12 +600,12 @@ impl DepGraph { if let Some(prev_index) = data.previous.node_to_index_opt(&node) { let dep_node_index = data.colors.current(prev_index); if let Some(dep_node_index) = dep_node_index { - incremental_verify_ich( + incremental_verify_ich_green( tcx, data, result, prev_index, - Some(hash_result), + hash_result, format_value_fn, ); @@ -619,7 +632,7 @@ impl DepGraph { } }); - data.hash_result_and_alloc_node(tcx, node, edges, result, Some(hash_result)) + data.hash_result_and_alloc_node(tcx, node, edges, result, hash_result) } else { // Incremental compilation is turned off. We just execute the task // without tracking. We still provide a dep-node index that uniquely diff --git a/compiler/rustc_middle/src/query/impl_/execution.rs b/compiler/rustc_middle/src/query/impl_/execution.rs index 7074307bb5835..1daf0df305469 100644 --- a/compiler/rustc_middle/src/query/impl_/execution.rs +++ b/compiler/rustc_middle/src/query/impl_/execution.rs @@ -12,7 +12,7 @@ use rustc_middle::query::{ QueryLatch, QueryMode, QueryState, QueryVTable, }; use rustc_middle::ty::TyCtxt; -use rustc_middle::verify_ich::incremental_verify_ich; +use rustc_middle::verify_ich::incremental_verify_ich_green; use rustc_span::{DUMMY_SP, Span}; use tracing::warn; @@ -24,6 +24,7 @@ use crate::query::impl_::plumbing::{ current_query_job, loadable_from_disk, next_job_id, start_query, }; use crate::query::impl_::query_impl::for_each_query_vtable; +use crate::verify_ich::assert_previous_green; #[inline] fn equivalent_key(k: K) -> impl Fn(&(K, V)) -> bool { @@ -422,13 +423,17 @@ fn execute_job_non_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Val } #[inline(always)] -fn execute_job_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( +fn execute_job_incr<'tcx, C, H>( query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, key: C::Key, dep_node: DepNode, job_id: QueryJobId, -) -> (C::Value, DepNodeIndex) { +) -> (C::Value, DepNodeIndex) +where + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +{ let dep_graph_data = tcx.dep_graph.data().expect("should always be present in incremental mode"); @@ -456,13 +461,7 @@ fn execute_job_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>> let (result, dep_node_index) = start_query(job_id, query.depth_limit, || { // Call the query provider. - dep_graph_data.with_task( - dep_node, - tcx, - key, - |tcx, key| H::invoke_provider_fn(tcx, key), - (!H::NO_HASH).then_some(H::hash_value_fn), - ) + dep_graph_data.with_task::<_, _, H>(dep_node, tcx, key) }); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -474,11 +473,7 @@ fn execute_job_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>> /// by loading one from disk if possible, or by invoking its query provider if /// necessary. #[inline(always)] -fn load_from_disk_or_invoke_provider_green< - 'tcx, - C: QueryCache, - H: QueryHelper<'tcx, C::Key, C::Value>, ->( +fn load_from_disk_or_invoke_provider_green<'tcx, C, H>( tcx: TyCtxt<'tcx>, dep_graph_data: &DepGraphData, query: &'tcx QueryVTable<'tcx, C, H>, @@ -486,7 +481,11 @@ fn load_from_disk_or_invoke_provider_green< dep_node: &DepNode, prev_index: SerializedDepNodeIndex, dep_node_index: DepNodeIndex, -) -> C::Value { +) -> C::Value +where + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +{ // Note this function can be called concurrently from the same query // We must ensure that this is handled correctly. @@ -532,6 +531,8 @@ fn load_from_disk_or_invoke_provider_green< }; if verify { + assert_previous_green(tcx, dep_graph_data, prev_index); + // Verify that re-running the query produced a result with the expected hash. // This catches bugs in query implementations, turning them into ICEs. // For example, a query might sort its result by `DefId` - since `DefId`s are @@ -541,12 +542,12 @@ fn load_from_disk_or_invoke_provider_green< // // See issue #82920 for an example of a miscompilation that would get turned into // an ICE by this check - incremental_verify_ich( + incremental_verify_ich_green( tcx, dep_graph_data, &value, prev_index, - (!H::NO_HASH).then_some(H::hash_value_fn), + H::hash_value_fn, query.format_value, ); } diff --git a/compiler/rustc_middle/src/verify_ich.rs b/compiler/rustc_middle/src/verify_ich.rs index a1ab4d8cc4d00..a5e2d77dc111a 100644 --- a/compiler/rustc_middle/src/verify_ich.rs +++ b/compiler/rustc_middle/src/verify_ich.rs @@ -9,21 +9,17 @@ use crate::ty::TyCtxt; #[inline] #[instrument(skip(tcx, dep_graph_data, result, hash_result, format_value), level = "debug")] -pub fn incremental_verify_ich<'tcx, V>( +pub fn incremental_verify_ich_green<'tcx, V, F>( tcx: TyCtxt<'tcx>, dep_graph_data: &DepGraphData, result: &V, prev_index: SerializedDepNodeIndex, - hash_result: Option, &V) -> Fingerprint>, + hash_result: F, format_value: fn(&V) -> String, -) { - if !dep_graph_data.is_index_green(prev_index) { - incremental_verify_ich_not_green(tcx, prev_index) - } - - let new_hash = hash_result.map_or(Fingerprint::ZERO, |f| { - tcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, result)) - }); +) where + F: Fn(&mut StableHashingContext<'_>, &V) -> Fingerprint, +{ + let new_hash = tcx.with_stable_hashing_context(|mut hcx| hash_result(&mut hcx, result)); let old_hash = dep_graph_data.prev_value_fingerprint_of(prev_index); @@ -32,6 +28,17 @@ pub fn incremental_verify_ich<'tcx, V>( } } +#[inline(always)] +pub fn assert_previous_green<'tcx>( + tcx: TyCtxt<'tcx>, + dep_graph_data: &DepGraphData, + prev_index: SerializedDepNodeIndex, +) { + if !dep_graph_data.is_index_green(prev_index) { + incremental_verify_ich_not_green(tcx, prev_index) + } +} + #[cold] #[inline(never)] fn incremental_verify_ich_not_green<'tcx>(tcx: TyCtxt<'tcx>, prev_index: SerializedDepNodeIndex) { From 4a2db865b7ceee7726da5100d03461d7285d6f70 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 13 Apr 2026 17:12:03 +0300 Subject: [PATCH 08/15] Make canonicalize a static function --- compiler/rustc_middle/src/infer/canonical.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs index 32c6b6e9c0ba1..d485df98832a9 100644 --- a/compiler/rustc_middle/src/infer/canonical.rs +++ b/compiler/rustc_middle/src/infer/canonical.rs @@ -153,17 +153,20 @@ impl<'tcx> CanonicalParamEnvCache<'tcx> { /// `canonicalize_op` is intentionally not allowed to be a closure to /// statically prevent it from capturing `InferCtxt` and resolving /// inference variables, which invalidates the cache. - pub fn get_or_insert( + pub fn get_or_insert( &self, tcx: TyCtxt<'tcx>, key: ty::ParamEnv<'tcx>, state: &mut OriginalQueryValues<'tcx>, - canonicalize_op: fn( + canonicalize_op: F, + ) -> Canonical<'tcx, ty::ParamEnv<'tcx>> + where + F: Fn( TyCtxt<'tcx>, ty::ParamEnv<'tcx>, &mut OriginalQueryValues<'tcx>, ) -> Canonical<'tcx, ty::ParamEnv<'tcx>>, - ) -> Canonical<'tcx, ty::ParamEnv<'tcx>> { + { if !key.has_type_flags( TypeFlags::HAS_INFER | TypeFlags::HAS_PLACEHOLDER | TypeFlags::HAS_FREE_REGIONS, ) { From fb7f00864fc4a5abba7ad6fe4a4a6fb87464e1b5 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 13 Apr 2026 17:20:20 +0300 Subject: [PATCH 09/15] Make format_value into a static function --- compiler/rustc_middle/src/dep_graph/graph.rs | 5 +++-- compiler/rustc_middle/src/query/impl_/execution.rs | 13 +++++-------- compiler/rustc_middle/src/query/impl_/query_impl.rs | 6 +----- compiler/rustc_middle/src/query/inner.rs | 12 +++++------- compiler/rustc_middle/src/query/plumbing.rs | 8 ++++++-- compiler/rustc_middle/src/verify_ich.rs | 5 +++-- 6 files changed, 23 insertions(+), 26 deletions(-) diff --git a/compiler/rustc_middle/src/dep_graph/graph.rs b/compiler/rustc_middle/src/dep_graph/graph.rs index cc0390b4cbe6a..cf85eca7f9e82 100644 --- a/compiler/rustc_middle/src/dep_graph/graph.rs +++ b/compiler/rustc_middle/src/dep_graph/graph.rs @@ -579,16 +579,17 @@ impl DepGraph { /// FIXME: If the code is changed enough for this node to be marked before requiring the /// caller's node, we suppose that those changes will be enough to mark this node red and /// force a recomputation using the "normal" way. - pub fn with_feed_task<'tcx, R, F>( + pub fn with_feed_task<'tcx, R, F, G>( &self, node: DepNode, tcx: TyCtxt<'tcx>, result: &R, hash_result: F, - format_value_fn: fn(&R) -> String, + format_value_fn: G, ) -> DepNodeIndex where F: Fn(&mut StableHashingContext<'_>, &R) -> Fingerprint, + G: Fn(&R) -> String, { if let Some(data) = self.data.as_ref() { // The caller query has more dependencies than the node we are creating. We may diff --git a/compiler/rustc_middle/src/query/impl_/execution.rs b/compiler/rustc_middle/src/query/impl_/execution.rs index 1daf0df305469..5450010ee587e 100644 --- a/compiler/rustc_middle/src/query/impl_/execution.rs +++ b/compiler/rustc_middle/src/query/impl_/execution.rs @@ -370,14 +370,13 @@ fn check_feedable_consistency<'tcx, C, H>( "no_hash fed query later has its value computed.\n\ Remove `no_hash` modifier to allow recomputation.\n\ The already cached value: {}", - (query.format_value)(&cached_value) + H::format_value(&cached_value) ); }; let (old_hash, new_hash) = tcx.with_stable_hashing_context(|mut hcx| { (H::hash_value_fn(&mut hcx, &cached_value), H::hash_value_fn(&mut hcx, value)) }); - let formatter = query.format_value; if old_hash != new_hash { // We have an inconsistency. This can happen if one of the two // results is tainted by errors. @@ -387,8 +386,8 @@ fn check_feedable_consistency<'tcx, C, H>( computed={:#?}\nfed={:#?}", query.dep_kind, key, - formatter(value), - formatter(&cached_value), + H::format_value(value), + H::format_value(&cached_value), ); } } @@ -442,10 +441,9 @@ where // `try_mark_green()`, so we can ignore them here. if let Some(ret) = start_query(job_id, false, || try { let (prev_index, dep_node_index) = dep_graph_data.try_mark_green(tcx, &dep_node)?; - let value = load_from_disk_or_invoke_provider_green( + let value = load_from_disk_or_invoke_provider_green::( tcx, dep_graph_data, - query, key, &dep_node, prev_index, @@ -476,7 +474,6 @@ where fn load_from_disk_or_invoke_provider_green<'tcx, C, H>( tcx: TyCtxt<'tcx>, dep_graph_data: &DepGraphData, - query: &'tcx QueryVTable<'tcx, C, H>, key: C::Key, dep_node: &DepNode, prev_index: SerializedDepNodeIndex, @@ -548,7 +545,7 @@ where &value, prev_index, H::hash_value_fn, - query.format_value, + H::format_value, ); } diff --git a/compiler/rustc_middle/src/query/impl_/query_impl.rs b/compiler/rustc_middle/src/query/impl_/query_impl.rs index d8e736bf14ef5..3efe2ca8163eb 100644 --- a/compiler/rustc_middle/src/query/impl_/query_impl.rs +++ b/compiler/rustc_middle/src/query/impl_/query_impl.rs @@ -1,5 +1,5 @@ use rustc_middle::queries::TaggedQueryKey; -use rustc_middle::query::erase::{self, Erased}; +use rustc_middle::query::erase::Erased; use rustc_middle::query::{QueryMode, QueryVTable}; use rustc_middle::ty::TyCtxt; use rustc_span::Span; @@ -91,7 +91,6 @@ macro_rules! define_queries { pub(crate) fn make_query_vtable<'tcx>() -> QueryVTable<'tcx, rustc_middle::queries::$name::Cache<'tcx>, rustc_middle::queries::$name::Helper> { - use rustc_middle::queries::$name::Value; QueryVTable { name: stringify!($name), eval_always: $eval_always, @@ -114,9 +113,6 @@ macro_rules! define_queries { $crate::query::impl_::handle_cycle_error::default(err) }, - format_value: |erased_value: &erase::Erased>| { - format!("{:?}", erase::restore_val(*erased_value)) - }, create_tagged_key: TaggedQueryKey::$name, } } diff --git a/compiler/rustc_middle/src/query/inner.rs b/compiler/rustc_middle/src/query/inner.rs index 758178f5fba98..f9954d73662cf 100644 --- a/compiler/rustc_middle/src/query/inner.rs +++ b/compiler/rustc_middle/src/query/inner.rs @@ -123,8 +123,6 @@ pub(crate) fn query_feed<'tcx, C, H>( C::Key: DepNodeKey<'tcx>, H: QueryHelper<'tcx, C::Key, C::Value>, { - let format_value = query.format_value; - // Check whether the in-memory cache already has a value for this key. match try_get_cached(tcx, &query.cache, key) { Some(old) => { @@ -138,8 +136,8 @@ pub(crate) fn query_feed<'tcx, C, H>( bug!( "Trying to feed an already recorded value for query {query:?} key={key:?}:\n\ old value: {old}\nnew value: {value}", - old = format_value(&old), - value = format_value(&value), + old = H::format_value(&old), + value = H::format_value(&value), ) } @@ -153,8 +151,8 @@ pub(crate) fn query_feed<'tcx, C, H>( tcx.dcx().delayed_bug(format!( "Trying to feed an already recorded value for query {query:?} key={key:?}:\n\ old value: {old}\nnew value: {value}", - old = format_value(&old), - value = format_value(&value), + old = H::format_value(&old), + value = H::format_value(&value), )); } } @@ -167,7 +165,7 @@ pub(crate) fn query_feed<'tcx, C, H>( tcx, &value, H::hash_value_fn, - query.format_value, + H::format_value, ); query.cache.complete(key, value, dep_node_index); } diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index fdfe922c1bc06..00b143248ba0a 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -104,8 +104,6 @@ where pub handle_cycle_error_fn: fn(tcx: TyCtxt<'tcx>, key: C::Key, cycle: Cycle<'tcx>, error: Diag<'_>) -> C::Value, - pub format_value: fn(&C::Value) -> String, - pub create_tagged_key: fn(C::Key) -> TaggedQueryKey<'tcx>, } @@ -159,6 +157,8 @@ pub trait QueryHashHelper: Default + 'static { /// /// For `no_hash` queries, this function pointer is None. fn hash_value_fn(hcx: &mut StableHashingContext<'_>, value: &V) -> Fingerprint; + + fn format_value(value: &V) -> String; } impl<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>> fmt::Debug @@ -425,6 +425,10 @@ macro_rules! define_callbacks { let value = erase::restore_val(*value); rustc_middle::dep_graph::hash_result(hcx, &value) } + + fn format_value(erased_value: &erase::Erased>) -> String { + format!("{:?}", erase::restore_val(*erased_value)) + } } /// Defines an `invoke_provider` function that calls the query's provider, diff --git a/compiler/rustc_middle/src/verify_ich.rs b/compiler/rustc_middle/src/verify_ich.rs index a5e2d77dc111a..dc9829de84812 100644 --- a/compiler/rustc_middle/src/verify_ich.rs +++ b/compiler/rustc_middle/src/verify_ich.rs @@ -9,15 +9,16 @@ use crate::ty::TyCtxt; #[inline] #[instrument(skip(tcx, dep_graph_data, result, hash_result, format_value), level = "debug")] -pub fn incremental_verify_ich_green<'tcx, V, F>( +pub fn incremental_verify_ich_green<'tcx, V, F, G>( tcx: TyCtxt<'tcx>, dep_graph_data: &DepGraphData, result: &V, prev_index: SerializedDepNodeIndex, hash_result: F, - format_value: fn(&V) -> String, + format_value: G, ) where F: Fn(&mut StableHashingContext<'_>, &V) -> Fingerprint, + G: Fn(&V) -> String, { let new_hash = tcx.with_stable_hashing_context(|mut hcx| hash_result(&mut hcx, result)); From f316b88d1eb08ed6cdd771d7f77641f95bcdb574 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 13 Apr 2026 17:26:58 +0300 Subject: [PATCH 10/15] Add inline attrs to query helper functions --- compiler/rustc_middle/src/query/plumbing.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 00b143248ba0a..d144d2e6e589c 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -374,6 +374,7 @@ macro_rules! define_callbacks { impl<'tcx> crate::query::QueryHelper<'tcx, Key<'tcx>, Erased>> for Helper { #[cfg($cache_on_disk)] + #[inline] fn try_load_from_disk_fn( tcx: TyCtxt<'tcx>, prev_index: crate::dep_graph::SerializedDepNodeIndex, @@ -385,6 +386,7 @@ macro_rules! define_callbacks { Some(provided_to_erased(tcx, loaded_value)) } #[cfg(not($cache_on_disk))] + #[inline(always)] fn try_load_from_disk_fn( _tcx: TyCtxt<'tcx>, _prev_index: crate::dep_graph::SerializedDepNodeIndex, @@ -407,6 +409,7 @@ macro_rules! define_callbacks { } } + #[inline(always)] fn invoke_provider_fn(tcx: TyCtxt<'tcx>, key: Key<'tcx>) -> Erased> { invoke_provider_fn::__rust_begin_short_backtrace(tcx, key) } @@ -416,16 +419,19 @@ macro_rules! define_callbacks { const NO_HASH: bool = $no_hash; #[cfg($no_hash)] + #[cold] fn hash_value_fn(_: &mut crate::ich::StableHashingContext<'_>, _: &Erased>) -> rustc_data_structures::fingerprint::Fingerprint { panic!("Tried to hash value for no_hash query. `no_hash` query modifier is unsupported with `feedable` modifier enabled") } #[cfg(not($no_hash))] + #[inline] fn hash_value_fn(hcx: &mut crate::ich::StableHashingContext<'_>, value: &Erased>) -> rustc_data_structures::fingerprint::Fingerprint { let value = erase::restore_val(*value); rustc_middle::dep_graph::hash_result(hcx, &value) } + #[inline] fn format_value(erased_value: &erase::Erased>) -> String { format!("{:?}", erase::restore_val(*erased_value)) } From d85470966faef03fecb2d4e437e00cf8feb9a155 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 13 Apr 2026 17:37:12 +0300 Subject: [PATCH 11/15] Fix impls --- compiler/rustc_codegen_llvm/src/base.rs | 4 ++++ compiler/rustc_metadata/src/rmeta/encoder.rs | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 5118933df7edf..a20c7ca018e29 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -76,6 +76,10 @@ pub(crate) fn compile_codegen_unit( ) -> rustc_data_structures::fingerprint::Fingerprint { dep_graph::hash_result(hcx, result) } + + fn format_value(value: &ModuleCodegen) -> String { + unimplemented!() + } } impl<'tcx> QueryHelper<'tcx, Symbol, ModuleCodegen> for CompileCguHashHelper { diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 6bf5836080614..367e072f7629c 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2484,6 +2484,10 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { ) -> rustc_data_structures::fingerprint::Fingerprint { unimplemented!() } + + fn format_value((): &()) -> String { + "()".to_string() + } } impl<'tcx> QueryHelper<'tcx, &Path, ()> for EncodeMetadataHelper { From dccb386e11e84b2fb638d6fb86a7618186048dc6 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 13 Apr 2026 17:39:06 +0300 Subject: [PATCH 12/15] Fix warning --- compiler/rustc_codegen_llvm/src/base.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index a20c7ca018e29..728aaf520ab15 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -77,7 +77,7 @@ pub(crate) fn compile_codegen_unit( dep_graph::hash_result(hcx, result) } - fn format_value(value: &ModuleCodegen) -> String { + fn format_value(_: &ModuleCodegen) -> String { unimplemented!() } } From dd7ab4604ec6a9a49e9052fc6dfdae349097b015 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 13 Apr 2026 17:44:18 +0300 Subject: [PATCH 13/15] Fix incremental tests --- .../rustc_middle/src/query/impl_/execution.rs | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/compiler/rustc_middle/src/query/impl_/execution.rs b/compiler/rustc_middle/src/query/impl_/execution.rs index 5450010ee587e..3a1d4bd7cbdd6 100644 --- a/compiler/rustc_middle/src/query/impl_/execution.rs +++ b/compiler/rustc_middle/src/query/impl_/execution.rs @@ -530,23 +530,25 @@ where if verify { assert_previous_green(tcx, dep_graph_data, prev_index); - // Verify that re-running the query produced a result with the expected hash. - // This catches bugs in query implementations, turning them into ICEs. - // For example, a query might sort its result by `DefId` - since `DefId`s are - // not stable across compilation sessions, the result could get up getting sorted - // in a different order when the query is re-run, even though all of the inputs - // (e.g. `DefPathHash` values) were green. - // - // See issue #82920 for an example of a miscompilation that would get turned into - // an ICE by this check - incremental_verify_ich_green( - tcx, - dep_graph_data, - &value, - prev_index, - H::hash_value_fn, - H::format_value, - ); + if !H::NO_HASH { + // Verify that re-running the query produced a result with the expected hash. + // This catches bugs in query implementations, turning them into ICEs. + // For example, a query might sort its result by `DefId` - since `DefId`s are + // not stable across compilation sessions, the result could get up getting sorted + // in a different order when the query is re-run, even though all of the inputs + // (e.g. `DefPathHash` values) were green. + // + // See issue #82920 for an example of a miscompilation that would get turned into + // an ICE by this check + incremental_verify_ich_green( + tcx, + dep_graph_data, + &value, + prev_index, + H::hash_value_fn, + H::format_value, + ); + } } value From dc9617599d4f02bda518ff74bdc19e9d1e1ca3a2 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Tue, 14 Apr 2026 14:52:22 +0300 Subject: [PATCH 14/15] Less inline control --- compiler/rustc_middle/src/query/caches.rs | 1 - .../rustc_middle/src/query/impl_/execution.rs | 10 --- .../rustc_middle/src/query/impl_/plumbing.rs | 1 - .../src/query/impl_/query_impl.rs | 54 +--------------- compiler/rustc_middle/src/query/inner.rs | 4 -- compiler/rustc_middle/src/query/job.rs | 3 +- compiler/rustc_middle/src/query/plumbing.rs | 64 +++++++------------ 7 files changed, 24 insertions(+), 113 deletions(-) diff --git a/compiler/rustc_middle/src/query/caches.rs b/compiler/rustc_middle/src/query/caches.rs index 0c71a98b7fb29..bd02ccb467a29 100644 --- a/compiler/rustc_middle/src/query/caches.rs +++ b/compiler/rustc_middle/src/query/caches.rs @@ -147,7 +147,6 @@ where type Key = DefId; type Value = V; - #[inline(always)] fn lookup(&self, key: &DefId) -> Option<(V, DepNodeIndex)> { if key.krate == LOCAL_CRATE { self.local.lookup(&key.index) diff --git a/compiler/rustc_middle/src/query/impl_/execution.rs b/compiler/rustc_middle/src/query/impl_/execution.rs index 3a1d4bd7cbdd6..eaa013c2d3980 100644 --- a/compiler/rustc_middle/src/query/impl_/execution.rs +++ b/compiler/rustc_middle/src/query/impl_/execution.rs @@ -190,7 +190,6 @@ impl<'tcx, K> Drop for ActiveJobGuard<'tcx, K> where K: Eq + Hash + Copy, { - #[inline(never)] #[cold] fn drop(&mut self) { // Poison the query so jobs waiting on it panic. @@ -215,7 +214,6 @@ fn find_and_handle_cycle<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Va (handle_cycle(query, tcx, key, cycle), None) } -#[inline(always)] fn wait_for_query<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, @@ -261,7 +259,6 @@ fn wait_for_query<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( } /// Shared main part of both [`execute_query_incr_inner`] and [`execute_query_non_incr_inner`]. -#[inline(never)] fn try_execute_query<'tcx, C, H, const INCR: bool>( query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, @@ -349,7 +346,6 @@ where } } -#[inline(always)] fn check_feedable_consistency<'tcx, C, H>( tcx: TyCtxt<'tcx>, query: &'tcx QueryVTable<'tcx, C, H>, @@ -393,7 +389,6 @@ fn check_feedable_consistency<'tcx, C, H>( } // Fast path for when incr. comp. is off. -#[inline(always)] fn execute_job_non_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, @@ -421,7 +416,6 @@ fn execute_job_non_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Val (value, dep_node_index) } -#[inline(always)] fn execute_job_incr<'tcx, C, H>( query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, @@ -470,7 +464,6 @@ where /// Given that the dep node for this query+key is green, obtain a value for it /// by loading one from disk if possible, or by invoking its query provider if /// necessary. -#[inline(always)] fn load_from_disk_or_invoke_provider_green<'tcx, C, H>( tcx: TyCtxt<'tcx>, dep_graph_data: &DepGraphData, @@ -560,7 +553,6 @@ where /// This only makes sense during incremental compilation, because it relies /// on having the dependency graph (and in some cases a disk-cached value) /// from the previous incr-comp session. -#[inline(never)] fn ensure_can_skip_execution<'tcx, C, H>( query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, @@ -612,7 +604,6 @@ where /// Called by a macro-generated impl of [`QueryVTable::execute_query_fn`], /// in non-incremental mode. -#[inline(always)] pub(crate) fn execute_query_non_incr_inner<'tcx, C, H>( query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, @@ -628,7 +619,6 @@ where /// Called by a macro-generated impl of [`QueryVTable::execute_query_fn`], /// in incremental mode. -#[inline(always)] pub(crate) fn execute_query_incr_inner<'tcx, C, H>( query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, diff --git a/compiler/rustc_middle/src/query/impl_/plumbing.rs b/compiler/rustc_middle/src/query/impl_/plumbing.rs index 6de613adfa0dd..239e8190674d1 100644 --- a/compiler/rustc_middle/src/query/impl_/plumbing.rs +++ b/compiler/rustc_middle/src/query/impl_/plumbing.rs @@ -51,7 +51,6 @@ pub(crate) fn current_query_job() -> Option { } /// Executes a job by changing the `ImplicitCtxt` to point to the new query job while it executes. -#[inline(always)] pub(crate) fn start_query( job_id: QueryJobId, depth_limit: bool, diff --git a/compiler/rustc_middle/src/query/impl_/query_impl.rs b/compiler/rustc_middle/src/query/impl_/query_impl.rs index 3efe2ca8163eb..6a3b3e680eafe 100644 --- a/compiler/rustc_middle/src/query/impl_/query_impl.rs +++ b/compiler/rustc_middle/src/query/impl_/query_impl.rs @@ -1,8 +1,6 @@ use rustc_middle::queries::TaggedQueryKey; -use rustc_middle::query::erase::Erased; -use rustc_middle::query::{QueryMode, QueryVTable}; +use rustc_middle::query::QueryVTable; use rustc_middle::ty::TyCtxt; -use rustc_span::Span; use crate::query::impl_::GetQueryVTable; @@ -38,56 +36,6 @@ macro_rules! define_queries { pub(crate) mod $name { use super::*; - // It seems to be important that every query has its own monomorphic - // copy of `execute_query_incr` and `execute_query_non_incr`. - // Trying to inline these wrapper functions into their generic - // "inner" helpers tends to break `tests/run-make/short-ice`. - - pub(crate) mod execute_query_incr { - use super::*; - use rustc_middle::queries::$name::{Key, Value}; - - // Adding `__rust_end_short_backtrace` marker to backtraces so that we emit the frames - // when `RUST_BACKTRACE=1`, add a new mod with `$name` here is to allow duplicate naming - #[inline(never)] - pub(crate) fn __rust_end_short_backtrace<'tcx>( - tcx: TyCtxt<'tcx>, - span: Span, - key: Key<'tcx>, - mode: QueryMode, - ) -> Option>> { - #[cfg(debug_assertions)] - let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); - crate::query::impl_::execution::execute_query_incr_inner( - &tcx.query_system.query_vtables.$name, - tcx, - span, - key, - mode - ) - } - } - - pub(crate) mod execute_query_non_incr { - use super::*; - use rustc_middle::queries::$name::{Key, Value}; - - #[inline(never)] - pub(crate) fn __rust_end_short_backtrace<'tcx>( - tcx: TyCtxt<'tcx>, - span: Span, - key: Key<'tcx>, - __mode: QueryMode, - ) -> Option>> { - Some(crate::query::impl_::execution::execute_query_non_incr_inner( - &tcx.query_system.query_vtables.$name, - tcx, - span, - key, - )) - } - } - pub(crate) fn make_query_vtable<'tcx>() -> QueryVTable<'tcx, rustc_middle::queries::$name::Cache<'tcx>, rustc_middle::queries::$name::Helper> { diff --git a/compiler/rustc_middle/src/query/inner.rs b/compiler/rustc_middle/src/query/inner.rs index f9954d73662cf..59a6551697259 100644 --- a/compiler/rustc_middle/src/query/inner.rs +++ b/compiler/rustc_middle/src/query/inner.rs @@ -13,7 +13,6 @@ use crate::ty::TyCtxt; /// query cache, returning that value if present. /// /// (Also performs some associated bookkeeping, if a value was found.) -#[inline(always)] fn try_get_cached<'tcx, C>(tcx: TyCtxt<'tcx>, cache: &C, key: C::Key) -> Option where C: QueryCache, @@ -30,7 +29,6 @@ where /// Shared implementation of `tcx.$query(..)` and `tcx.at(span).$query(..)` /// for all queries. -#[inline(always)] pub(crate) fn query_get_at<'tcx, C, H>( tcx: TyCtxt<'tcx>, span: Span, @@ -49,7 +47,6 @@ where /// Shared implementation of `tcx.ensure_ok().$query(..)` and /// `tcx.ensure_done().$query(..)` for all queries. -#[inline] pub(crate) fn query_ensure_ok_or_done<'tcx, C, H>( tcx: TyCtxt<'tcx>, query: &'tcx QueryVTable<'tcx, C, H>, @@ -69,7 +66,6 @@ pub(crate) fn query_ensure_ok_or_done<'tcx, C, H>( /// Implementation of `tcx.ensure_result().$query(..)` for queries that /// return `Result<_, ErrorGuaranteed>`. -#[inline] pub(crate) fn query_ensure_result<'tcx, C, T, H>( tcx: TyCtxt<'tcx>, query: &'tcx QueryVTable<'tcx, C, H>, diff --git a/compiler/rustc_middle/src/query/job.rs b/compiler/rustc_middle/src/query/job.rs index 24c4daf9855d2..accc56827a7b0 100644 --- a/compiler/rustc_middle/src/query/job.rs +++ b/compiler/rustc_middle/src/query/job.rs @@ -30,7 +30,7 @@ pub struct QueryJob<'tcx> { impl<'tcx> QueryJob<'tcx> { /// Creates a new query job. - #[inline] + #[inline(always)] pub fn new(id: QueryJobId, span: Span, parent: Option) -> Self { QueryJob { id, span, parent, latch: None } } @@ -46,7 +46,6 @@ impl<'tcx> QueryJob<'tcx> { /// /// This does nothing for single threaded rustc, /// as there are no concurrent jobs which could be waiting on us - #[inline] pub fn signal_complete(self) { if let Some(latch) = self.latch { latch.set(); diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index d144d2e6e589c..8d20e4244b331 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -374,7 +374,6 @@ macro_rules! define_callbacks { impl<'tcx> crate::query::QueryHelper<'tcx, Key<'tcx>, Erased>> for Helper { #[cfg($cache_on_disk)] - #[inline] fn try_load_from_disk_fn( tcx: TyCtxt<'tcx>, prev_index: crate::dep_graph::SerializedDepNodeIndex, @@ -393,7 +392,6 @@ macro_rules! define_callbacks { ) -> Option>> { None } - #[cfg_attr(all($cache_on_disk, $separate_provide_extern), inline)] #[cfg_attr(not(all($cache_on_disk, $separate_provide_extern)), inline(always))] fn will_cache_on_disk_for_key( _key: rustc_middle::queries::$name::Key<'tcx>, @@ -409,47 +407,7 @@ macro_rules! define_callbacks { } } - #[inline(always)] fn invoke_provider_fn(tcx: TyCtxt<'tcx>, key: Key<'tcx>) -> Erased> { - invoke_provider_fn::__rust_begin_short_backtrace(tcx, key) - } - } - - impl<'tcx> crate::query::QueryHashHelper>> for Helper { - const NO_HASH: bool = $no_hash; - - #[cfg($no_hash)] - #[cold] - fn hash_value_fn(_: &mut crate::ich::StableHashingContext<'_>, _: &Erased>) -> rustc_data_structures::fingerprint::Fingerprint { - panic!("Tried to hash value for no_hash query. `no_hash` query modifier is unsupported with `feedable` modifier enabled") - } - - #[cfg(not($no_hash))] - #[inline] - fn hash_value_fn(hcx: &mut crate::ich::StableHashingContext<'_>, value: &Erased>) -> rustc_data_structures::fingerprint::Fingerprint { - let value = erase::restore_val(*value); - rustc_middle::dep_graph::hash_result(hcx, &value) - } - - #[inline] - fn format_value(erased_value: &erase::Erased>) -> String { - format!("{:?}", erase::restore_val(*erased_value)) - } - } - - /// Defines an `invoke_provider` function that calls the query's provider, - /// to be used as a function pointer in the query's vtable. - /// - /// To mark a short-backtrace boundary, the function's actual name - /// (after demangling) must be `__rust_begin_short_backtrace`. - mod invoke_provider_fn { - use super::*; - - #[inline(never)] - pub(crate) fn __rust_begin_short_backtrace<'tcx>( - tcx: TyCtxt<'tcx>, - key: Key<'tcx>, - ) -> Erased> { #[cfg(debug_assertions)] let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); @@ -475,6 +433,28 @@ macro_rules! define_callbacks { } } + impl<'tcx> crate::query::QueryHashHelper>> for Helper { + const NO_HASH: bool = $no_hash; + + #[cfg($no_hash)] + #[cold] + fn hash_value_fn(_: &mut crate::ich::StableHashingContext<'_>, _: &Erased>) -> rustc_data_structures::fingerprint::Fingerprint { + panic!("Tried to hash value for no_hash query. `no_hash` query modifier is unsupported with `feedable` modifier enabled") + } + + #[cfg(not($no_hash))] + #[inline] + fn hash_value_fn(hcx: &mut crate::ich::StableHashingContext<'_>, value: &Erased>) -> rustc_data_structures::fingerprint::Fingerprint { + let value = erase::restore_val(*value); + rustc_middle::dep_graph::hash_result(hcx, &value) + } + + #[inline] + fn format_value(erased_value: &erase::Erased>) -> String { + format!("{:?}", erase::restore_val(*erased_value)) + } + } + pub type Cache<'tcx> = as $crate::query::QueryKey>::Cache>>; From 7e84d8e3821372c1b54c1e3930796cacb5697304 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Tue, 14 Apr 2026 16:08:12 +0300 Subject: [PATCH 15/15] Move the rest of QueryVTable into compile time --- compiler/rustc_codegen_llvm/src/base.rs | 25 ++++++ compiler/rustc_metadata/src/rmeta/encoder.rs | 27 ++++++- .../rustc_middle/src/query/impl_/execution.rs | 81 ++++++++++--------- compiler/rustc_middle/src/query/impl_/mod.rs | 4 +- .../rustc_middle/src/query/impl_/plumbing.rs | 32 ++++---- .../src/query/impl_/profiling_support.rs | 4 +- .../src/query/impl_/query_impl.rs | 40 --------- compiler/rustc_middle/src/query/inner.rs | 2 +- compiler/rustc_middle/src/query/plumbing.rs | 71 +++++++++++----- 9 files changed, 168 insertions(+), 118 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 728aaf520ab15..abedb4d2731ec 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -77,12 +77,21 @@ pub(crate) fn compile_codegen_unit( dep_graph::hash_result(hcx, result) } + #[cold] fn format_value(_: &ModuleCodegen) -> String { unimplemented!() } } + // FIXME: use a separate trait for this impl<'tcx> QueryHelper<'tcx, Symbol, ModuleCodegen> for CompileCguHashHelper { + const EVAL_ALWAYS: bool = false; + const DEPTH_LIMIT: bool = false; + const DEP_KIND: dep_graph::DepKind = dep_graph::DepKind::CompileCodegenUnit; + const FEEDABLE: bool = false; + const NAME: &'static str = ""; + + #[cold] fn try_load_from_disk_fn( _: TyCtxt<'tcx>, _: dep_graph::SerializedDepNodeIndex, @@ -90,6 +99,7 @@ pub(crate) fn compile_codegen_unit( unimplemented!() } + #[cold] fn will_cache_on_disk_for_key(_: Symbol) -> bool { unimplemented!() } @@ -197,6 +207,21 @@ pub(crate) fn compile_codegen_unit( ModuleCodegen::new_regular(cgu_name.to_string(), llvm_module) } + + #[cold] + fn create_tagged_key(_: Symbol) -> rustc_middle::queries::TaggedQueryKey<'tcx> { + unimplemented!() + } + + #[cold] + fn handle_cycle_error_fn( + _: TyCtxt<'tcx>, + _: Symbol, + _: rustc_middle::query::Cycle<'tcx>, + _: rustc_errors::Diag<'_>, + ) -> ModuleCodegen { + unimplemented!() + } } let (module, _) = diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 367e072f7629c..13b48c9dd6d57 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -17,7 +17,7 @@ use rustc_hir::def_id::{CRATE_DEF_ID, CRATE_DEF_INDEX, LOCAL_CRATE, LocalDefId, use rustc_hir::definitions::DefPathData; use rustc_hir::find_attr; use rustc_hir_pretty::id_to_string; -use rustc_middle::dep_graph::WorkProductId; +use rustc_middle::dep_graph::{DepKind, WorkProductId}; use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::mir::interpret; use rustc_middle::query::{Providers, QueryHashHelper, QueryHelper}; @@ -2478,6 +2478,7 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { impl QueryHashHelper<()> for EncodeMetadataHelper { const NO_HASH: bool = true; + #[cold] fn hash_value_fn( _: &mut rustc_middle::ich::StableHashingContext<'_>, _: &(), @@ -2485,12 +2486,20 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { unimplemented!() } + #[cold] fn format_value((): &()) -> String { "()".to_string() } } impl<'tcx> QueryHelper<'tcx, &Path, ()> for EncodeMetadataHelper { + const EVAL_ALWAYS: bool = false; + const DEPTH_LIMIT: bool = false; + const FEEDABLE: bool = false; + const NAME: &'static str = ""; + const DEP_KIND: rustc_middle::dep_graph::DepKind = DepKind::Metadata; + + #[cold] fn try_load_from_disk_fn( _: TyCtxt<'tcx>, _: rustc_middle::dep_graph::SerializedDepNodeIndex, @@ -2498,6 +2507,7 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { unimplemented!() } + #[cold] fn will_cache_on_disk_for_key(_: &Path) -> bool { unimplemented!() } @@ -2520,6 +2530,21 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { root.position.get() }) } + + #[cold] + fn create_tagged_key(_: &Path) -> rustc_middle::queries::TaggedQueryKey<'tcx> { + unimplemented!() + } + + #[cold] + fn handle_cycle_error_fn( + _: TyCtxt<'tcx>, + _: &Path, + _: rustc_middle::query::Cycle<'tcx>, + _: rustc_errors::Diag<'_>, + ) -> () { + unimplemented!() + } } // Perform metadata encoding inside a task, so the dep-graph can check if any encoded diff --git a/compiler/rustc_middle/src/query/impl_/execution.rs b/compiler/rustc_middle/src/query/impl_/execution.rs index eaa013c2d3980..de79ca291e646 100644 --- a/compiler/rustc_middle/src/query/impl_/execution.rs +++ b/compiler/rustc_middle/src/query/impl_/execution.rs @@ -21,7 +21,8 @@ use crate::query::impl_::job::{ QueryJobInfo, QueryJobMap, create_cycle_error, find_cycle_in_stack, }; use crate::query::impl_::plumbing::{ - current_query_job, loadable_from_disk, next_job_id, start_query, + current_query_job, loadable_from_disk, next_job_id, start_query_depth_limit, + start_query_no_depth_limit, }; use crate::query::impl_::query_impl::for_each_query_vtable; use crate::verify_ich::assert_previous_green; @@ -81,7 +82,7 @@ fn collect_active_query_jobs_inner<'tcx, C, H>( if let ActiveKeyStatus::Started(job) = status { // It's fine to call `create_tagged_key` with the shard locked, // because it's just a `TaggedQueryKey` variant constructor. - let tagged_key = (query.create_tagged_key)(*key); + let tagged_key = H::create_tagged_key(*key); job_map.insert(job.id, QueryJobInfo { tagged_key, job: job.clone() }); } } @@ -97,7 +98,7 @@ fn collect_active_query_jobs_inner<'tcx, C, H>( for shard in query.state.active.try_lock_shards() { match shard { Some(shard) => collect_shard_jobs(&shard), - None => panic!("Failed to collect active jobs for query `{}`!", query.name), + None => panic!("Failed to collect active jobs for query `{}`!", H::NAME), } } } @@ -105,7 +106,7 @@ fn collect_active_query_jobs_inner<'tcx, C, H>( for shard in query.state.active.try_lock_shards() { match shard { Some(shard) => collect_shard_jobs(&shard), - None => warn!("Failed to collect active jobs for query `{}`!", query.name), + None => warn!("Failed to collect active jobs for query `{}`!", H::NAME), } } } @@ -113,15 +114,13 @@ fn collect_active_query_jobs_inner<'tcx, C, H>( } #[cold] -#[inline(never)] -fn handle_cycle<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( - query: &'tcx QueryVTable<'tcx, C, H>, - tcx: TyCtxt<'tcx>, - key: C::Key, - cycle: Cycle<'tcx>, -) -> C::Value { +fn handle_cycle<'tcx, C, H>(tcx: TyCtxt<'tcx>, key: C::Key, cycle: Cycle<'tcx>) -> C::Value +where + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +{ let error = create_cycle_error(tcx, &cycle); - (query.handle_cycle_error_fn)(tcx, key, cycle, error) + H::handle_cycle_error_fn(tcx, key, cycle, error) } /// Guard object representing the responsibility to execute a query job and @@ -198,9 +197,7 @@ where } #[cold] -#[inline(never)] fn find_and_handle_cycle<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( - query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, key: C::Key, try_execute: QueryJobId, @@ -211,7 +208,7 @@ fn find_and_handle_cycle<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Va let job_map = collect_active_query_jobs(tcx, CollectActiveJobsKind::FullNoContention); let cycle = find_cycle_in_stack(try_execute, job_map, ¤t_query_job(), span); - (handle_cycle(query, tcx, key, cycle), None) + (handle_cycle::(tcx, key, cycle), None) } fn wait_for_query<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( @@ -243,7 +240,7 @@ fn wait_for_query<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( Some((_, ActiveKeyStatus::Poisoned)) => FatalError.raise(), _ => panic!( "query '{}' result must be in the cache or the query must be poisoned after a wait", - query.name + H::NAME ), } }) @@ -254,7 +251,7 @@ fn wait_for_query<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( (v, Some(index)) } - Err(cycle) => (handle_cycle(query, tcx, key, cycle), None), + Err(cycle) => (handle_cycle::(tcx, key, cycle), None), } } @@ -305,12 +302,12 @@ where // Delegate to another function to actually execute the query job. let (value, dep_node_index) = if INCR { - execute_job_incr(query, tcx, key, dep_node.unwrap(), id) + execute_job_incr::(tcx, key, dep_node.unwrap(), id) } else { - execute_job_non_incr(query, tcx, key, id) + execute_job_non_incr::(tcx, key, id) }; - if query.feedable { + if H::FEEDABLE { check_feedable_consistency(tcx, query, key, &value); } @@ -337,7 +334,7 @@ where // If we are single-threaded we know that we have cycle error, // so we just return the error. - find_and_handle_cycle(query, tcx, key, id, span) + find_and_handle_cycle::(tcx, key, id, span) } } ActiveKeyStatus::Poisoned => FatalError.raise(), @@ -380,7 +377,7 @@ fn check_feedable_consistency<'tcx, C, H>( tcx.dcx().has_errors().is_some(), "Computed query value for {:?}({:?}) is inconsistent with fed value,\n\ computed={:#?}\nfed={:#?}", - query.dep_kind, + H::DEP_KIND, key, H::format_value(value), H::format_value(&cached_value), @@ -389,17 +386,25 @@ fn check_feedable_consistency<'tcx, C, H>( } // Fast path for when incr. comp. is off. -fn execute_job_non_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>>( - query: &'tcx QueryVTable<'tcx, C, H>, +fn execute_job_non_incr<'tcx, C, H>( tcx: TyCtxt<'tcx>, key: C::Key, job_id: QueryJobId, -) -> (C::Value, DepNodeIndex) { +) -> (C::Value, DepNodeIndex) +where + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, +{ debug_assert!(!tcx.dep_graph.is_fully_enabled()); let prof_timer = tcx.prof.query_provider(); // Call the query provider. - let value = start_query(job_id, query.depth_limit, || H::invoke_provider_fn(tcx, key)); + let invoke = || H::invoke_provider_fn(tcx, key); + let value = if H::DEPTH_LIMIT { + start_query_depth_limit(job_id, invoke) + } else { + start_query_no_depth_limit(job_id, invoke) + }; let dep_node_index = tcx.dep_graph.next_virtual_depnode_index(); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -417,7 +422,6 @@ fn execute_job_non_incr<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Val } fn execute_job_incr<'tcx, C, H>( - query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, key: C::Key, dep_node: DepNode, @@ -430,10 +434,10 @@ where let dep_graph_data = tcx.dep_graph.data().expect("should always be present in incremental mode"); - if !query.eval_always { + if !H::EVAL_ALWAYS { // The diagnostics for this query will be promoted to the current session during // `try_mark_green()`, so we can ignore them here. - if let Some(ret) = start_query(job_id, false, || try { + if let Some(ret) = start_query_no_depth_limit(job_id, || try { let (prev_index, dep_node_index) = dep_graph_data.try_mark_green(tcx, &dep_node)?; let value = load_from_disk_or_invoke_provider_green::( tcx, @@ -451,10 +455,15 @@ where let prof_timer = tcx.prof.query_provider(); - let (result, dep_node_index) = start_query(job_id, query.depth_limit, || { + let compute = || { // Call the query provider. dep_graph_data.with_task::<_, _, H>(dep_node, tcx, key) - }); + }; + let (result, dep_node_index) = if H::DEPTH_LIMIT { + start_query_depth_limit(job_id, compute) + } else { + start_query_no_depth_limit(job_id, compute) + }; prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -554,7 +563,6 @@ where /// on having the dependency graph (and in some cases a disk-cached value) /// from the previous incr-comp session. fn ensure_can_skip_execution<'tcx, C, H>( - query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, key: C::Key, dep_node: DepNode, @@ -565,7 +573,7 @@ where H: QueryHelper<'tcx, C::Key, C::Value>, { // Queries with `eval_always` should never skip execution. - if query.eval_always { + if H::EVAL_ALWAYS { return false; } @@ -630,11 +638,12 @@ where C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>, { - let dep_node = DepNode::construct(tcx, query.dep_kind, &key); + let dep_node = DepNode::construct(tcx, H::DEP_KIND, &key); // Check if query execution can be skipped, for `ensure_ok` or `ensure_done`. - if let QueryMode::Ensure { ensure_mode } = mode - && ensure_can_skip_execution(query, tcx, key, dep_node, ensure_mode) + if !H::EVAL_ALWAYS + && let QueryMode::Ensure { ensure_mode } = mode + && ensure_can_skip_execution::(tcx, key, dep_node, ensure_mode) { return None; } diff --git a/compiler/rustc_middle/src/query/impl_/mod.rs b/compiler/rustc_middle/src/query/impl_/mod.rs index ce065baae0a0b..a6a402630da2e 100644 --- a/compiler/rustc_middle/src/query/impl_/mod.rs +++ b/compiler/rustc_middle/src/query/impl_/mod.rs @@ -14,7 +14,7 @@ pub use crate::query::impl_::job::{QueryJobMap, break_query_cycle, print_query_s mod dep_kind_vtables; mod error; pub(crate) mod execution; -mod handle_cycle_error; +pub(crate) mod handle_cycle_error; mod job; mod plumbing; mod profiling_support; @@ -42,7 +42,7 @@ pub fn query_system<'tcx>( ) -> QuerySystem<'tcx> { QuerySystem { arenas: Default::default(), - query_vtables: query_impl::make_query_vtables(), + query_vtables: Default::default(), side_effects: Default::default(), on_disk_cache, local_providers, diff --git a/compiler/rustc_middle/src/query/impl_/plumbing.rs b/compiler/rustc_middle/src/query/impl_/plumbing.rs index 239e8190674d1..19d75c744492e 100644 --- a/compiler/rustc_middle/src/query/impl_/plumbing.rs +++ b/compiler/rustc_middle/src/query/impl_/plumbing.rs @@ -20,6 +20,7 @@ use crate::query::impl_::job::find_dep_kind_root; use crate::query::impl_::query_impl::for_each_query_vtable; use crate::query::impl_::{CollectActiveJobsKind, collect_active_query_jobs}; +#[cold] fn depth_limit_error<'tcx>(tcx: TyCtxt<'tcx>, job: QueryJobId) { let job_map = collect_active_query_jobs(tcx, CollectActiveJobsKind::Full); let (span, desc, depth) = find_dep_kind_root(tcx, job, job_map); @@ -51,22 +52,25 @@ pub(crate) fn current_query_job() -> Option { } /// Executes a job by changing the `ImplicitCtxt` to point to the new query job while it executes. -pub(crate) fn start_query( - job_id: QueryJobId, - depth_limit: bool, - compute: impl FnOnce() -> R, -) -> R { +pub(crate) fn start_query_depth_limit(job_id: QueryJobId, compute: impl FnOnce() -> R) -> R { tls::with_context(move |icx| { - if depth_limit && !icx.tcx.recursion_limit().value_within_limit(icx.query_depth) { + if !icx.tcx.recursion_limit().value_within_limit(icx.query_depth) { depth_limit_error(icx.tcx, job_id); } // Update the `ImplicitCtxt` to point to our new query job. - let icx = ImplicitCtxt { - query: Some(job_id), - query_depth: icx.query_depth + if depth_limit { 1 } else { 0 }, - ..*icx - }; + let icx = ImplicitCtxt { query: Some(job_id), query_depth: icx.query_depth + 1, ..*icx }; + + // Use the `ImplicitCtxt` while we execute the query. + tls::enter_context(&icx, compute) + }) +} + +/// Executes a job by changing the `ImplicitCtxt` to point to the new query job while it executes. +pub(crate) fn start_query_no_depth_limit(job_id: QueryJobId, compute: impl FnOnce() -> R) -> R { + tls::with_context(move |icx| { + // Update the `ImplicitCtxt` to point to our new query job. + let icx = ImplicitCtxt { query: Some(job_id), ..*icx }; // Use the `ImplicitCtxt` while we execute the query. tls::enter_context(&icx, compute) @@ -88,7 +92,7 @@ fn encode_query_values_inner<'a, 'tcx, C, V, H>( H: QueryHelper<'tcx, C::Key, C::Value>, V: Erasable + Encodable>, { - let _timer = tcx.prof.generic_activity_with_arg("encode_query_results_for", query.name); + let _timer = tcx.prof.generic_activity_with_arg("encode_query_results_for", H::NAME); assert!(all_inactive(&query.state)); query.cache.for_each(&mut |key, value, dep_node| { @@ -112,12 +116,12 @@ fn verify_query_key_hashes_inner<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Ke query: &'tcx QueryVTable<'tcx, C, H>, tcx: TyCtxt<'tcx>, ) { - let _timer = tcx.prof.generic_activity_with_arg("query_key_hash_verify_for", query.name); + let _timer = tcx.prof.generic_activity_with_arg("query_key_hash_verify_for", H::NAME); let cache = &query.cache; let mut map = UnordMap::with_capacity(cache.len()); cache.for_each(&mut |key, _, _| { - let node = DepNode::construct(tcx, query.dep_kind, key); + let node = DepNode::construct(tcx, H::DEP_KIND, key); if let Some(other_key) = map.insert(node, *key) { bug!( "query key:\n\ diff --git a/compiler/rustc_middle/src/query/impl_/profiling_support.rs b/compiler/rustc_middle/src/query/impl_/profiling_support.rs index a24e11c748441..28e85fe89bdb7 100644 --- a/compiler/rustc_middle/src/query/impl_/profiling_support.rs +++ b/compiler/rustc_middle/src/query/impl_/profiling_support.rs @@ -218,7 +218,7 @@ fn alloc_self_profile_query_strings_inner<'tcx, C, H>( if profiler.query_key_recording_enabled() { let mut query_string_builder = QueryKeyStringBuilder::new(profiler, tcx, string_cache); - let query_name = profiler.get_or_alloc_cached_string(query.name); + let query_name = profiler.get_or_alloc_cached_string(H::NAME); // Since building the string representation of query keys might // need to invoke queries itself, we cannot keep the query caches @@ -246,7 +246,7 @@ fn alloc_self_profile_query_strings_inner<'tcx, C, H>( } } else { // In this branch we don't allocate query keys - let query_name = profiler.get_or_alloc_cached_string(query.name); + let query_name = profiler.get_or_alloc_cached_string(H::NAME); let event_id = event_id_builder.from_label(query_name).to_string_id(); // FIXME(eddyb) make this O(1) by using a pre-cached query name `EventId`, diff --git a/compiler/rustc_middle/src/query/impl_/query_impl.rs b/compiler/rustc_middle/src/query/impl_/query_impl.rs index 6a3b3e680eafe..599cb7e7cadba 100644 --- a/compiler/rustc_middle/src/query/impl_/query_impl.rs +++ b/compiler/rustc_middle/src/query/impl_/query_impl.rs @@ -1,4 +1,3 @@ -use rustc_middle::queries::TaggedQueryKey; use rustc_middle::query::QueryVTable; use rustc_middle::ty::TyCtxt; @@ -36,35 +35,6 @@ macro_rules! define_queries { pub(crate) mod $name { use super::*; - pub(crate) fn make_query_vtable<'tcx>() - -> QueryVTable<'tcx, rustc_middle::queries::$name::Cache<'tcx>, rustc_middle::queries::$name::Helper> - { - QueryVTable { - name: stringify!($name), - eval_always: $eval_always, - depth_limit: $depth_limit, - feedable: $feedable, - dep_kind: rustc_middle::dep_graph::DepKind::$name, - state: Default::default(), - cache: Default::default(), - - helper: Default::default(), - - #[cfg($handle_cycle_error)] - handle_cycle_error_fn: |tcx, key, cycle, err| { - use rustc_middle::query::erase::erase_val; - - erase_val($crate::query::impl_::handle_cycle_error::$name(tcx, key, cycle, err)) - }, - #[cfg(not($handle_cycle_error))] - handle_cycle_error_fn: |_tcx, _key, _cycle, err| { - $crate::query::impl_::handle_cycle_error::default(err) - }, - - create_tagged_key: TaggedQueryKey::$name, - } - } - /// Marker type that implements [`GetQueryVTable`] for this query. pub(crate) enum VTableGetter {} @@ -80,16 +50,6 @@ macro_rules! define_queries { } )* - pub(crate) fn make_query_vtables<'tcx>() - -> rustc_middle::queries::QueryVTables<'tcx> - { - rustc_middle::queries::QueryVTables { - $( - $name: crate::query::impl_::query_impl::$name::make_query_vtable(), - )* - } - } - /// Given a filter condition (e.g. `ALL` or `CACHE_ON_DISK`), a `tcx`, /// and a closure expression that accepts `&QueryVTable`, this macro /// calls that closure with each query vtable that satisfies the filter diff --git a/compiler/rustc_middle/src/query/inner.rs b/compiler/rustc_middle/src/query/inner.rs index 59a6551697259..cc734bda425d6 100644 --- a/compiler/rustc_middle/src/query/inner.rs +++ b/compiler/rustc_middle/src/query/inner.rs @@ -155,7 +155,7 @@ pub(crate) fn query_feed<'tcx, C, H>( None => { // There is no cached value for this key, so feed the query by // adding the provided value to the cache. - let dep_node = dep_graph::DepNode::construct(tcx, query.dep_kind, &key); + let dep_node = dep_graph::DepNode::construct(tcx, H::DEP_KIND, &key); let dep_node_index = tcx.dep_graph.with_feed_task( dep_node, tcx, diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 8d20e4244b331..5fc40cf590720 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -82,29 +82,19 @@ where C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>, { - pub name: &'static str, - - /// True if this query has the `eval_always` modifier. - pub eval_always: bool, - /// True if this query has the `depth_limit` modifier. - pub depth_limit: bool, - /// True if this query has the `feedable` modifier. - pub feedable: bool, - - pub dep_kind: DepKind, pub state: QueryState<'tcx, C::Key>, pub cache: C, - pub helper: H, +} - /// Function pointer that handles a cycle error. `error` must be consumed, e.g. with `emit` (if - /// it should be emitted) or `delay_as_bug` (if it need not be emitted because an alternative - /// error is created and emitted). A value may be returned, or (more commonly) the function may - /// just abort after emitting the error. - pub handle_cycle_error_fn: - fn(tcx: TyCtxt<'tcx>, key: C::Key, cycle: Cycle<'tcx>, error: Diag<'_>) -> C::Value, - - pub create_tagged_key: fn(C::Key) -> TaggedQueryKey<'tcx>, +impl<'tcx, C, H> Default for QueryVTable<'tcx, C, H> +where + C: QueryCache + Default, + H: QueryHelper<'tcx, C::Key, C::Value>, +{ + fn default() -> Self { + Self { state: Default::default(), cache: Default::default(), helper: Default::default() } + } } impl<'tcx, C, H> QueryVTable<'tcx, C, H> @@ -138,6 +128,12 @@ where } pub trait QueryHelper<'tcx, K, V>: QueryHashHelper + Default + 'static { + const EVAL_ALWAYS: bool; + const DEPTH_LIMIT: bool; + const FEEDABLE: bool; + const DEP_KIND: DepKind; + const NAME: &'static str; + fn try_load_from_disk_fn(tcx: TyCtxt<'tcx>, prev_index: SerializedDepNodeIndex) -> Option; fn will_cache_on_disk_for_key(key: K) -> bool; @@ -148,6 +144,14 @@ pub trait QueryHelper<'tcx, K, V>: QueryHashHelper + Default + 'static { /// /// This should be the only code that calls the provider function. fn invoke_provider_fn(tcx: TyCtxt<'tcx>, key: K) -> V; + + fn create_tagged_key(key: K) -> TaggedQueryKey<'tcx>; + + /// Function pointer that handles a cycle error. `error` must be consumed, e.g. with `emit` (if + /// it should be emitted) or `delay_as_bug` (if it need not be emitted because an alternative + /// error is created and emitted). A value may be returned, or (more commonly) the function may + /// just abort after emitting the error. + fn handle_cycle_error_fn(tcx: TyCtxt<'tcx>, key: K, cycle: Cycle<'tcx>, error: Diag<'_>) -> V; } pub trait QueryHashHelper: Default + 'static { @@ -161,8 +165,10 @@ pub trait QueryHashHelper: Default + 'static { fn format_value(value: &V) -> String; } -impl<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>> fmt::Debug - for QueryVTable<'tcx, C, H> +impl<'tcx, C, H> fmt::Debug for QueryVTable<'tcx, C, H> +where + C: QueryCache, + H: QueryHelper<'tcx, C::Key, C::Value>, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // When debug-printing a query vtable (e.g. for ICE or tracing), @@ -171,7 +177,7 @@ impl<'tcx, C: QueryCache, H: QueryHelper<'tcx, C::Key, C::Value>> fmt::Debug // // If there is need for a more detailed dump of all flags and fields, // consider writing a separate dump method and calling it explicitly. - f.write_str(self.name) + f.write_str(H::NAME) } } @@ -373,6 +379,12 @@ macro_rules! define_callbacks { pub struct Helper; impl<'tcx> crate::query::QueryHelper<'tcx, Key<'tcx>, Erased>> for Helper { + const EVAL_ALWAYS: bool = $eval_always; + const DEPTH_LIMIT: bool = $depth_limit; + const FEEDABLE: bool = $feedable; + const DEP_KIND: crate::dep_graph::DepKind = crate::dep_graph::DepKind::$name; + const NAME: &'static str = stringify!($name); + #[cfg($cache_on_disk)] fn try_load_from_disk_fn( tcx: TyCtxt<'tcx>, @@ -431,6 +443,20 @@ macro_rules! define_callbacks { // For queries with `arena_cache`, this also arena-allocates the value. provided_to_erased(tcx, provided_value) } + + #[inline(always)] + fn create_tagged_key(key: Key<'tcx>) -> TaggedQueryKey<'tcx> { + TaggedQueryKey::$name(key) + } + + #[cfg($handle_cycle_error)] + fn handle_cycle_error_fn(tcx: TyCtxt<'tcx>, key: Key<'tcx>, cycle: $crate::query::plumbing::Cycle<'tcx>, err: rustc_errors::Diag<'_>) -> Erased> { + rustc_middle::query::erase::erase_val($crate::query::impl_::handle_cycle_error::$name(tcx, key, cycle, err)) + } + #[cfg(not($handle_cycle_error))] + fn handle_cycle_error_fn(_: TyCtxt<'tcx>, _: Key<'tcx>, _: $crate::query::plumbing::Cycle<'tcx>, err: rustc_errors::Diag<'_>) -> Erased> { + $crate::query::impl_::handle_cycle_error::default(err) + } } impl<'tcx> crate::query::QueryHashHelper>> for Helper { @@ -582,6 +608,7 @@ macro_rules! define_callbacks { } /// Holds a `QueryVTable` for each query. + #[derive(Default)] pub struct QueryVTables<'tcx> { $( pub $name: $crate::query::QueryVTable<'tcx, $name::Cache<'tcx>, $name::Helper>,