Create definitions for promoted constants.

compiler/rustc_borrowck/src/consumers.rs

@@ -2,9 +2,8 @@
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def_id::LocalDefId;
-use rustc_index::IndexVec;
 use rustc_middle::bug;
-use rustc_middle::mir::{Body, Promoted};
+use rustc_middle::mir::Body;
 use rustc_middle::ty::TyCtxt;
 
 pub use super::borrow_set::{BorrowData, BorrowSet, TwoPhaseActivation};
@@ -85,8 +84,6 @@ pub enum ConsumerOptions {
 pub struct BodyWithBorrowckFacts<'tcx> {
     /// A mir body that contains region identifiers.
     pub body: Body<'tcx>,
-    /// The mir bodies of promoteds.
-    pub promoted: IndexVec<Promoted, Body<'tcx>>,
     /// The set of borrows occurring in `body` with data about them.
     pub borrow_set: BorrowSet<'tcx>,
     /// Context generated during borrowck, intended to be passed to

compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs

@@ -1223,7 +1223,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
 
     fn suggest_using_iter_mut(&self, err: &mut Diag<'_>) {
         let source = self.body.source;
-        if let InstanceKind::Item(def_id) = source.instance
+        if let InstanceKind::Item(def_id) = source
             && let Some(Node::Expr(hir::Expr { hir_id, kind, .. })) =
                 self.infcx.tcx.hir_get_if_local(def_id)
             && let ExprKind::Closure(hir::Closure { kind: hir::ClosureKind::Closure, .. }) = kind
@@ -1614,7 +1614,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         }
 
         // Make sure we are inside a closure.
-        let InstanceKind::Item(body_def_id) = self.body.source.instance else {
+        let InstanceKind::Item(body_def_id) = self.body.source else {
             return false;
         };
         let Some(Node::Expr(hir::Expr { hir_id: body_hir_id, kind, .. })) =

compiler/rustc_borrowck/src/lib.rs

@@ -28,8 +28,8 @@ use rustc_data_structures::graph::dominators::Dominators;
 use rustc_hir as hir;
 use rustc_hir::CRATE_HIR_ID;
 use rustc_hir::def_id::LocalDefId;
+use rustc_index::IndexVec;
 use rustc_index::bit_set::MixedBitSet;
-use rustc_index::{IndexSlice, IndexVec};
 use rustc_infer::infer::outlives::env::RegionBoundPairs;
 use rustc_infer::infer::{
     InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin, TyCtxtInferExt,
@@ -289,7 +289,6 @@ impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
 struct CollectRegionConstraintsResult<'tcx> {
     infcx: BorrowckInferCtxt<'tcx>,
     body_owned: Body<'tcx>,
-    promoted: IndexVec<Promoted, Body<'tcx>>,
     move_data: MoveData<'tcx>,
     borrow_set: BorrowSet<'tcx>,
     location_table: PoloniusLocationTable,
@@ -313,9 +312,8 @@ fn borrowck_collect_region_constraints<'tcx>(
 ) -> CollectRegionConstraintsResult<'tcx> {
     let tcx = root_cx.tcx;
     let infcx = BorrowckInferCtxt::new(tcx, def, root_cx.root_def_id());
-    let (input_body, promoted) = tcx.mir_promoted(def);
+    let (input_body, _) = tcx.mir_promoted(def);
     let input_body: &Body<'_> = &input_body.borrow();
-    let input_promoted: &IndexSlice<_, _> = &promoted.borrow();
     if let Some(e) = input_body.tainted_by_errors {
         infcx.set_tainted_by_errors(e);
         root_cx.set_tainted_by_errors(e);
@@ -326,8 +324,7 @@ fn borrowck_collect_region_constraints<'tcx>(
     // be modified (in place) to contain non-lexical lifetimes. It
     // will have a lifetime tied to the inference context.
     let mut body_owned = input_body.clone();
-    let mut promoted = input_promoted.to_owned();
-    let universal_regions = nll::replace_regions_in_mir(&infcx, &mut body_owned, &mut promoted);
+    let universal_regions = nll::replace_regions_in_mir(&infcx, &mut body_owned);
     let body = &body_owned; // no further changes
 
     let location_table = PoloniusLocationTable::new(body);
@@ -356,7 +353,6 @@ fn borrowck_collect_region_constraints<'tcx>(
         root_cx,
         &infcx,
         body,
-        &promoted,
         universal_regions,
         &location_table,
         &borrow_set,
@@ -368,7 +364,6 @@ fn borrowck_collect_region_constraints<'tcx>(
     CollectRegionConstraintsResult {
         infcx,
         body_owned,
-        promoted,
         move_data,
         borrow_set,
         location_table,
@@ -392,7 +387,6 @@ fn borrowck_check_region_constraints<'tcx>(
     CollectRegionConstraintsResult {
         infcx,
         body_owned,
-        promoted,
         move_data,
         borrow_set,
         location_table,
@@ -456,54 +450,6 @@ fn borrowck_check_region_constraints<'tcx>(
         && tcx.coroutine_movability(def.to_def_id()) == hir::Movability::Movable;
 
     let diags_buffer = &mut BorrowckDiagnosticsBuffer::default();
-    // While promoteds should mostly be correct by construction, we need to check them for
-    // invalid moves to detect moving out of arrays:`struct S; fn main() { &([S][0]); }`.
-    for promoted_body in &promoted {
-        use rustc_middle::mir::visit::Visitor;
-        // This assumes that we won't use some of the fields of the `promoted_mbcx`
-        // when detecting and reporting move errors. While it would be nice to move
-        // this check out of `MirBorrowckCtxt`, actually doing so is far from trivial.
-        let move_data = MoveData::gather_moves(promoted_body, tcx, |_| true);
-        let mut promoted_mbcx = MirBorrowckCtxt {
-            root_cx,
-            infcx: &infcx,
-            body: promoted_body,
-            move_data: &move_data,
-            // no need to create a real location table for the promoted, it is not used
-            location_table: &location_table,
-            movable_coroutine,
-            fn_self_span_reported: Default::default(),
-            access_place_error_reported: Default::default(),
-            reservation_error_reported: Default::default(),
-            uninitialized_error_reported: Default::default(),
-            regioncx: &regioncx,
-            used_mut: Default::default(),
-            used_mut_upvars: SmallVec::new(),
-            borrow_set: &borrow_set,
-            upvars: &[],
-            local_names: OnceCell::from(IndexVec::from_elem(None, &promoted_body.local_decls)),
-            region_names: RefCell::default(),
-            next_region_name: RefCell::new(1),
-            polonius_output: None,
-            move_errors: Vec::new(),
-            diags_buffer,
-            polonius_context: polonius_context.as_ref(),
-        };
-        struct MoveVisitor<'a, 'b, 'infcx, 'tcx> {
-            ctxt: &'a mut MirBorrowckCtxt<'b, 'infcx, 'tcx>,
-        }
-
-        impl<'tcx> Visitor<'tcx> for MoveVisitor<'_, '_, '_, 'tcx> {
-            fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) {
-                if let Operand::Move(place) = operand {
-                    self.ctxt.check_movable_place(location, *place);
-                }
-            }
-        }
-        MoveVisitor { ctxt: &mut promoted_mbcx }.visit_body(promoted_body);
-        promoted_mbcx.report_move_errors();
-    }
-
     let mut mbcx = MirBorrowckCtxt {
         root_cx,
         infcx: &infcx,
@@ -580,7 +526,6 @@ fn borrowck_check_region_constraints<'tcx>(
             def,
             BodyWithBorrowckFacts {
                 body: body_owned,
-                promoted,
                 borrow_set,
                 region_inference_context: regioncx,
                 location_table: polonius_input.as_ref().map(|_| location_table),

compiler/rustc_borrowck/src/nll.rs

@@ -8,9 +8,8 @@ use std::str::FromStr;
 use polonius_engine::{Algorithm, AllFacts, Output};
 use rustc_data_structures::frozen::Frozen;
 use rustc_hir::find_attr;
-use rustc_index::IndexSlice;
 use rustc_middle::mir::pretty::PrettyPrintMirOptions;
-use rustc_middle::mir::{Body, MirDumper, PassWhere, Promoted};
+use rustc_middle::mir::{Body, MirDumper, PassWhere};
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_mir_dataflow::move_paths::MoveData;
@@ -52,11 +51,10 @@ pub(crate) struct NllOutput<'tcx> {
 /// Rewrites the regions in the MIR to use NLL variables, also scraping out the set of universal
 /// regions (e.g., region parameters) declared on the function. That set will need to be given to
 /// `compute_regions`.
-#[instrument(skip(infcx, body, promoted), level = "debug")]
+#[instrument(skip(infcx, body), level = "debug")]
 pub(crate) fn replace_regions_in_mir<'tcx>(
     infcx: &BorrowckInferCtxt<'tcx>,
     body: &mut Body<'tcx>,
-    promoted: &mut IndexSlice<Promoted, Body<'tcx>>,
 ) -> UniversalRegions<'tcx> {
     let def = body.source.def_id().expect_local();
 
@@ -66,7 +64,7 @@ pub(crate) fn replace_regions_in_mir<'tcx>(
     let universal_regions = UniversalRegions::new(infcx, def);
 
     // Replace all remaining regions with fresh inference variables.
-    renumber::renumber_mir(infcx, body, promoted);
+    renumber::renumber_mir(infcx, body);
 
     if let Some(dumper) = MirDumper::new(infcx.tcx, "renumber", body) {
         dumper.dump_mir(body);

compiler/rustc_borrowck/src/region_infer/values.rs

@@ -1,7 +1,7 @@
 use std::fmt::Debug;
 use std::rc::Rc;
 
-use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_index::Idx;
 use rustc_index::bit_set::SparseBitMatrix;
 use rustc_index::interval::{IntervalSet, SparseIntervalMatrix};
@@ -42,15 +42,11 @@ pub(crate) struct LivenessValues {
     /// The map from locations to points.
     location_map: Rc<DenseLocationMap>,
 
-    /// Which regions are live. This is exclusive with the fine-grained tracking in `points`, and
-    /// currently only used for validating promoteds (which don't care about more precise tracking).
-    live_regions: Option<FxHashSet<RegionVid>>,
-
     /// For each region: the points where it is live.
     ///
     /// This is not initialized for promoteds, because we don't care *where* within a promoted a
     /// region is live, only that it is.
-    points: Option<SparseIntervalMatrix<RegionVid, PointIndex>>,
+    points: SparseIntervalMatrix<RegionVid, PointIndex>,
 
     /// When using `-Zpolonius=next`, the set of loans that are live at a given point in the CFG.
     live_loans: Option<LiveLoans>,
@@ -60,21 +56,7 @@ impl LivenessValues {
     /// Create an empty map of regions to locations where they're live.
     pub(crate) fn with_specific_points(location_map: Rc<DenseLocationMap>) -> Self {
         LivenessValues {
-            live_regions: None,
-            points: Some(SparseIntervalMatrix::new(location_map.num_points())),
-            location_map,
-            live_loans: None,
-        }
-    }
-
-    /// Create an empty map of regions to locations where they're live.
-    ///
-    /// Unlike `with_specific_points`, does not track exact locations where something is live, only
-    /// which regions are live.
-    pub(crate) fn without_specific_points(location_map: Rc<DenseLocationMap>) -> Self {
-        LivenessValues {
-            live_regions: Some(Default::default()),
-            points: None,
+            points: SparseIntervalMatrix::new(location_map.num_points()),
             location_map,
             live_loans: None,
         }
@@ -83,52 +65,30 @@ impl LivenessValues {
     /// Returns the liveness matrix of points where each region is live. Panics if the liveness
     /// values have been created without any per-point data (that is, for promoteds).
     pub(crate) fn points(&self) -> &SparseIntervalMatrix<RegionVid, PointIndex> {
-        self.points
-            .as_ref()
-            .expect("this `LivenessValues` wasn't created using `with_specific_points`")
+        &self.points
     }
 
     /// Iterate through each region that has a value in this set.
     pub(crate) fn regions(&self) -> impl Iterator<Item = RegionVid> {
-        self.points.as_ref().expect("use with_specific_points").rows()
-    }
-
-    /// Iterate through each region that has a value in this set.
-    // We are passing query instability implications to the caller.
-    #[rustc_lint_query_instability]
-    #[allow(rustc::potential_query_instability)]
-    pub(crate) fn live_regions_unordered(&self) -> impl Iterator<Item = RegionVid> {
-        self.live_regions.as_ref().unwrap().iter().copied()
+        self.points.rows()
     }
 
     /// Records `region` as being live at the given `location`.
     pub(crate) fn add_location(&mut self, region: RegionVid, location: Location) {
         let point = self.location_map.point_from_location(location);
         debug!("LivenessValues::add_location(region={:?}, location={:?})", region, location);
-        if let Some(points) = &mut self.points {
-            points.insert(region, point);
-        } else if self.location_map.point_in_range(point) {
-            self.live_regions.as_mut().unwrap().insert(region);
-        }
+        self.points.insert(region, point);
     }
 
     /// Records `region` as being live at all the given `points`.
     pub(crate) fn add_points(&mut self, region: RegionVid, points: &IntervalSet<PointIndex>) {
         debug!("LivenessValues::add_points(region={:?}, points={:?})", region, points);
-        if let Some(this) = &mut self.points {
-            this.union_row(region, points);
-        } else if points.iter().any(|point| self.location_map.point_in_range(point)) {
-            self.live_regions.as_mut().unwrap().insert(region);
-        }
+        self.points.union_row(region, points);
     }
 
     /// Records `region` as being live at all the control-flow points.
     pub(crate) fn add_all_points(&mut self, region: RegionVid) {
-        if let Some(points) = &mut self.points {
-            points.insert_all_into_row(region);
-        } else {
-            self.live_regions.as_mut().unwrap().insert(region);
-        }
+        self.points.insert_all_into_row(region);
     }
 
     /// Returns whether `region` is marked live at the given
@@ -142,23 +102,12 @@ impl LivenessValues {
     /// [`point`][rustc_mir_dataflow::points::PointIndex].
     #[inline]
     pub(crate) fn is_live_at_point(&self, region: RegionVid, point: PointIndex) -> bool {
-        if let Some(points) = &self.points {
-            points.row(region).is_some_and(|r| r.contains(point))
-        } else {
-            unreachable!(
-                "Should be using LivenessValues::with_specific_points to ask whether live at a location"
-            )
-        }
+        self.points.row(region).is_some_and(|r| r.contains(point))
     }
 
     /// Returns an iterator of all the points where `region` is live.
     fn live_points(&self, region: RegionVid) -> impl Iterator<Item = PointIndex> {
-        let Some(points) = &self.points else {
-            unreachable!(
-                "Should be using LivenessValues::with_specific_points to ask whether live at a location"
-            )
-        };
-        points
+        self.points
             .row(region)
             .into_iter()
             .flat_map(|set| set.iter())
@@ -328,10 +277,7 @@ impl<'tcx, N: Idx> RegionValues<'tcx, N> {
     /// elements for the region `from` from `values` and add them to
     /// the region `to` in `self`.
     pub(crate) fn merge_liveness(&mut self, to: N, from: RegionVid, values: &LivenessValues) {
-        let Some(value_points) = &values.points else {
-            panic!("LivenessValues must track specific points for use in merge_liveness");
-        };
-        if let Some(set) = value_points.row(from) {
+        if let Some(set) = values.points.row(from) {
             self.points.union_row(to, set);
         }
     }

compiler/rustc_borrowck/src/renumber.rs

@@ -1,7 +1,6 @@
-use rustc_index::IndexSlice;
 use rustc_infer::infer::NllRegionVariableOrigin;
 use rustc_middle::mir::visit::{MutVisitor, TyContext};
-use rustc_middle::mir::{Body, ConstOperand, Location, Promoted};
+use rustc_middle::mir::{Body, ConstOperand, Location};
 use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, TypeFoldable, fold_regions};
 use rustc_span::Symbol;
 use tracing::{debug, instrument};
@@ -10,21 +9,10 @@ use crate::BorrowckInferCtxt;
 
 /// Replaces all free regions appearing in the MIR with fresh
 /// inference variables, returning the number of variables created.
-#[instrument(skip(infcx, body, promoted), level = "debug")]
-pub(crate) fn renumber_mir<'tcx>(
-    infcx: &BorrowckInferCtxt<'tcx>,
-    body: &mut Body<'tcx>,
-    promoted: &mut IndexSlice<Promoted, Body<'tcx>>,
-) {
+#[instrument(skip(infcx, body), level = "debug")]
+pub(crate) fn renumber_mir<'tcx>(infcx: &BorrowckInferCtxt<'tcx>, body: &mut Body<'tcx>) {
     debug!(?body.arg_count);
-
-    let mut renumberer = RegionRenumberer { infcx };
-
-    for body in promoted.iter_mut() {
-        renumberer.visit_body_preserves_cfg(body);
-    }
-
-    renumberer.visit_body_preserves_cfg(body);
+    RegionRenumberer { infcx }.visit_body_preserves_cfg(body);
 }
 
 // The fields are used only for debugging output in `sccs_info`.

compiler/rustc_borrowck/src/root_cx.rs

@@ -266,6 +266,14 @@ impl<'tcx> BorrowCheckRootCtxt<'tcx> {
             .iter()
             .chain(std::iter::once(self.root_def_id));
         for def_id in all_bodies {
+            // The list of promoted from a given body is a flat list,
+            // in topological order from innermost to outermost.
+            let promoted = self.tcx.promoted_mir(def_id);
+            for &promoted_def_id in promoted {
+                let result = borrowck_collect_region_constraints(self, promoted_def_id);
+                self.collect_region_constraints_results.insert(promoted_def_id, result);
+            }
+
             let result = borrowck_collect_region_constraints(self, def_id);
             self.collect_region_constraints_results.insert(def_id, result);
         }