diff --git a/verifast-proofs/alloc/raw_vec/mod.rs/original/raw_vec.rs b/verifast-proofs/alloc/raw_vec/mod.rs/original/raw_vec.rs index b0027e964e467..bc9692f5b6c2f 100644 --- a/verifast-proofs/alloc/raw_vec/mod.rs/original/raw_vec.rs +++ b/verifast-proofs/alloc/raw_vec/mod.rs/original/raw_vec.rs @@ -23,8 +23,7 @@ mod tests; // ensure that the code generation related to these panics is minimal as there's // only one location which panics rather than a bunch throughout the module. #[cfg(not(no_global_oom_handling))] -#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))] -#[track_caller] +#[cfg_attr(not(panic = "immediate-abort"), inline(never))] fn capacity_overflow() -> ! { panic!("capacity overflow"); } @@ -123,7 +122,6 @@ impl RawVec { #[cfg(not(any(no_global_oom_handling, test)))] #[must_use] #[inline] - #[track_caller] pub(crate) fn with_capacity(capacity: usize) -> Self { Self { inner: RawVecInner::with_capacity(capacity, T::LAYOUT), _marker: PhantomData } } @@ -132,7 +130,6 @@ impl RawVec { #[cfg(not(any(no_global_oom_handling, test)))] #[must_use] #[inline] - #[track_caller] pub(crate) fn with_capacity_zeroed(capacity: usize) -> Self { Self { inner: RawVecInner::with_capacity_zeroed_in(capacity, Global, T::LAYOUT), @@ -145,7 +142,6 @@ impl RawVecInner { #[cfg(not(any(no_global_oom_handling, test)))] #[must_use] #[inline] - #[track_caller] fn with_capacity(capacity: usize, elem_layout: Layout) -> Self { match Self::try_allocate_in(capacity, AllocInit::Uninitialized, Global, elem_layout) { Ok(res) => res, @@ -177,6 +173,8 @@ impl RawVec { /// the returned `RawVec`. #[inline] pub(crate) const fn new_in(alloc: A) -> Self { + // Check assumption made in `current_memory` + const { assert!(T::LAYOUT.size() % T::LAYOUT.align() == 0) }; Self { inner: RawVecInner::new_in(alloc, Alignment::of::()), _marker: PhantomData } } @@ -184,7 +182,6 @@ impl RawVec { /// allocator for the returned `RawVec`. #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] pub(crate) fn with_capacity_in(capacity: usize, alloc: A) -> Self { Self { inner: RawVecInner::with_capacity_in(capacity, alloc, T::LAYOUT), @@ -206,7 +203,6 @@ impl RawVec { /// of allocator for the returned `RawVec`. #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] pub(crate) fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self { Self { inner: RawVecInner::with_capacity_zeroed_in(capacity, alloc, T::LAYOUT), @@ -326,18 +322,18 @@ impl RawVec { /// Aborts on OOM. #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] pub(crate) fn reserve(&mut self, len: usize, additional: usize) { - self.inner.reserve(len, additional, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.reserve(len, additional, T::LAYOUT) } } /// A specialized version of `self.reserve(len, 1)` which requires the /// caller to ensure `len == self.capacity()`. #[cfg(not(no_global_oom_handling))] #[inline(never)] - #[track_caller] pub(crate) fn grow_one(&mut self) { - self.inner.grow_one(T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.grow_one(T::LAYOUT) } } /// The same as `reserve`, but returns on errors instead of panicking or aborting. @@ -346,7 +342,8 @@ impl RawVec { len: usize, additional: usize, ) -> Result<(), TryReserveError> { - self.inner.try_reserve(len, additional, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.try_reserve(len, additional, T::LAYOUT) } } /// Ensures that the buffer contains at least enough space to hold `len + @@ -367,9 +364,9 @@ impl RawVec { /// /// Aborts on OOM. #[cfg(not(no_global_oom_handling))] - #[track_caller] pub(crate) fn reserve_exact(&mut self, len: usize, additional: usize) { - self.inner.reserve_exact(len, additional, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.reserve_exact(len, additional, T::LAYOUT) } } /// The same as `reserve_exact`, but returns on errors instead of panicking or aborting. @@ -378,7 +375,8 @@ impl RawVec { len: usize, additional: usize, ) -> Result<(), TryReserveError> { - self.inner.try_reserve_exact(len, additional, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.try_reserve_exact(len, additional, T::LAYOUT) } } /// Shrinks the buffer down to the specified capacity. If the given amount @@ -392,10 +390,10 @@ impl RawVec { /// /// Aborts on OOM. #[cfg(not(no_global_oom_handling))] - #[track_caller] #[inline] pub(crate) fn shrink_to_fit(&mut self, cap: usize) { - self.inner.shrink_to_fit(cap, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.shrink_to_fit(cap, T::LAYOUT) } } } @@ -417,7 +415,6 @@ impl RawVecInner { #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] fn with_capacity_in(capacity: usize, alloc: A, elem_layout: Layout) -> Self { match Self::try_allocate_in(capacity, AllocInit::Uninitialized, alloc, elem_layout) { Ok(this) => { @@ -442,7 +439,6 @@ impl RawVecInner { #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] fn with_capacity_zeroed_in(capacity: usize, alloc: A, elem_layout: Layout) -> Self { match Self::try_allocate_in(capacity, AllocInit::Zeroed, alloc, elem_layout) { Ok(res) => res, @@ -518,8 +514,12 @@ impl RawVecInner { &self.alloc } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[inline] - fn current_memory(&self, elem_layout: Layout) -> Option<(NonNull, Layout)> { + unsafe fn current_memory(&self, elem_layout: Layout) -> Option<(NonNull, Layout)> { if elem_layout.size() == 0 || self.cap.as_inner() == 0 { None } else { @@ -535,48 +535,65 @@ impl RawVecInner { } } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] - fn reserve(&mut self, len: usize, additional: usize, elem_layout: Layout) { + unsafe fn reserve(&mut self, len: usize, additional: usize, elem_layout: Layout) { // Callers expect this function to be very cheap when there is already sufficient capacity. // Therefore, we move all the resizing and error-handling logic from grow_amortized and // handle_reserve behind a call, while making sure that this function is likely to be // inlined as just a comparison and a call if the comparison fails. #[cold] - fn do_reserve_and_handle( + unsafe fn do_reserve_and_handle( slf: &mut RawVecInner, len: usize, additional: usize, elem_layout: Layout, ) { - if let Err(err) = slf.grow_amortized(len, additional, elem_layout) { + // SAFETY: Precondition passed to caller + if let Err(err) = unsafe { slf.grow_amortized(len, additional, elem_layout) } { handle_error(err); } } if self.needs_to_grow(len, additional, elem_layout) { - do_reserve_and_handle(self, len, additional, elem_layout); + unsafe { + do_reserve_and_handle(self, len, additional, elem_layout); + } } } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] - fn grow_one(&mut self, elem_layout: Layout) { - if let Err(err) = self.grow_amortized(self.cap.as_inner(), 1, elem_layout) { + unsafe fn grow_one(&mut self, elem_layout: Layout) { + // SAFETY: Precondition passed to caller + if let Err(err) = unsafe { self.grow_amortized(self.cap.as_inner(), 1, elem_layout) } { handle_error(err); } } - fn try_reserve( + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + unsafe fn try_reserve( &mut self, len: usize, additional: usize, elem_layout: Layout, ) -> Result<(), TryReserveError> { if self.needs_to_grow(len, additional, elem_layout) { - self.grow_amortized(len, additional, elem_layout)?; + // SAFETY: Precondition passed to caller + unsafe { + self.grow_amortized(len, additional, elem_layout)?; + } } unsafe { // Inform the optimizer that the reservation has succeeded or wasn't needed @@ -585,22 +602,33 @@ impl RawVecInner { Ok(()) } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] - #[track_caller] - fn reserve_exact(&mut self, len: usize, additional: usize, elem_layout: Layout) { - if let Err(err) = self.try_reserve_exact(len, additional, elem_layout) { + unsafe fn reserve_exact(&mut self, len: usize, additional: usize, elem_layout: Layout) { + // SAFETY: Precondition passed to caller + if let Err(err) = unsafe { self.try_reserve_exact(len, additional, elem_layout) } { handle_error(err); } } - fn try_reserve_exact( + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + unsafe fn try_reserve_exact( &mut self, len: usize, additional: usize, elem_layout: Layout, ) -> Result<(), TryReserveError> { if self.needs_to_grow(len, additional, elem_layout) { - self.grow_exact(len, additional, elem_layout)?; + // SAFETY: Precondition passed to caller + unsafe { + self.grow_exact(len, additional, elem_layout)?; + } } unsafe { // Inform the optimizer that the reservation has succeeded or wasn't needed @@ -609,11 +637,15 @@ impl RawVecInner { Ok(()) } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] - fn shrink_to_fit(&mut self, cap: usize, elem_layout: Layout) { - if let Err(err) = self.shrink(cap, elem_layout) { + unsafe fn shrink_to_fit(&mut self, cap: usize, elem_layout: Layout) { + if let Err(err) = unsafe { self.shrink(cap, elem_layout) } { handle_error(err); } } @@ -632,7 +664,13 @@ impl RawVecInner { self.cap = unsafe { Cap::new_unchecked(cap) }; } - fn grow_amortized( + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - The sum of `len` and `additional` must be greater than or equal to + /// `self.capacity(elem_layout.size())` + unsafe fn grow_amortized( &mut self, len: usize, additional: usize, @@ -657,14 +695,25 @@ impl RawVecInner { let new_layout = layout_array(cap, elem_layout)?; - let ptr = finish_grow(new_layout, self.current_memory(elem_layout), &mut self.alloc)?; - // SAFETY: layout_array would have resulted in a capacity overflow if we tried to allocate more than `isize::MAX` items + // SAFETY: + // - For the `current_memory` call: Precondition passed to caller + // - For the `finish_grow` call: Precondition passed to caller + // + `current_memory` does the right thing + let ptr = + unsafe { finish_grow(new_layout, self.current_memory(elem_layout), &mut self.alloc)? }; + // SAFETY: layout_array would have resulted in a capacity overflow if we tried to allocate more than `isize::MAX` items unsafe { self.set_ptr_and_cap(ptr, cap) }; Ok(()) } - fn grow_exact( + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - The sum of `len` and `additional` must be greater than or equal to + /// `self.capacity(elem_layout.size())` + unsafe fn grow_exact( &mut self, len: usize, additional: usize, @@ -679,7 +728,12 @@ impl RawVecInner { let cap = len.checked_add(additional).ok_or(CapacityOverflow)?; let new_layout = layout_array(cap, elem_layout)?; - let ptr = finish_grow(new_layout, self.current_memory(elem_layout), &mut self.alloc)?; + // SAFETY: + // - For the `current_memory` call: Precondition passed to caller + // - For the `finish_grow` call: Precondition passed to caller + // + `current_memory` does the right thing + let ptr = + unsafe { finish_grow(new_layout, self.current_memory(elem_layout), &mut self.alloc)? }; // SAFETY: layout_array would have resulted in a capacity overflow if we tried to allocate more than `isize::MAX` items unsafe { self.set_ptr_and_cap(ptr, cap); @@ -687,9 +741,14 @@ impl RawVecInner { Ok(()) } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` #[cfg(not(no_global_oom_handling))] #[inline] - fn shrink(&mut self, cap: usize, elem_layout: Layout) -> Result<(), TryReserveError> { + unsafe fn shrink(&mut self, cap: usize, elem_layout: Layout) -> Result<(), TryReserveError> { assert!(cap <= self.capacity(elem_layout.size()), "Tried to shrink to a larger capacity"); // SAFETY: Just checked this isn't trying to grow unsafe { self.shrink_unchecked(cap, elem_layout) } @@ -711,8 +770,12 @@ impl RawVecInner { cap: usize, elem_layout: Layout, ) -> Result<(), TryReserveError> { - let (ptr, layout) = - if let Some(mem) = self.current_memory(elem_layout) { mem } else { return Ok(()) }; + // SAFETY: Precondition passed to caller + let (ptr, layout) = if let Some(mem) = unsafe { self.current_memory(elem_layout) } { + mem + } else { + return Ok(()); + }; // If shrinking to 0, deallocate the buffer. We don't reach this point // for the T::IS_ZST case since current_memory() will have returned @@ -748,7 +811,8 @@ impl RawVecInner { /// Ideally this function would take `self` by move, but it cannot because it exists to be /// called from a `Drop` impl. unsafe fn deallocate(&mut self, elem_layout: Layout) { - if let Some((ptr, layout)) = self.current_memory(elem_layout) { + // SAFETY: Precondition passed to caller + if let Some((ptr, layout)) = unsafe { self.current_memory(elem_layout) } { unsafe { self.alloc.deallocate(ptr, layout); } @@ -756,10 +820,17 @@ impl RawVecInner { } } +/// # Safety +/// If `current_memory` matches `Some((ptr, old_layout))`: +/// - `ptr` must denote a block of memory *currently allocated* via `alloc` +/// - `old_layout` must *fit* that block of memory +/// - `new_layout` must have the same alignment as `old_layout` +/// - `new_layout.size()` must be greater than or equal to `old_layout.size()` +/// If `current_memory` is `None`, this function is safe. // not marked inline(never) since we want optimizers to be able to observe the specifics of this // function, see tests/codegen-llvm/vec-reserve-extend.rs. #[cold] -fn finish_grow( +unsafe fn finish_grow( new_layout: Layout, current_memory: Option<(NonNull, Layout)>, alloc: &mut A, @@ -785,7 +856,6 @@ where #[cfg(not(no_global_oom_handling))] #[cold] #[optimize(size)] -#[track_caller] fn handle_error(e: TryReserveError) -> ! { match e.kind() { CapacityOverflow => capacity_overflow(), diff --git a/verifast-proofs/alloc/raw_vec/mod.rs/verified/raw_vec.rs b/verifast-proofs/alloc/raw_vec/mod.rs/verified/raw_vec.rs index 12533c063dc90..bb53ee74a2fa8 100644 --- a/verifast-proofs/alloc/raw_vec/mod.rs/verified/raw_vec.rs +++ b/verifast-proofs/alloc/raw_vec/mod.rs/verified/raw_vec.rs @@ -483,8 +483,6 @@ impl RawVec { #[cfg(not(no_global_oom_handling))] pub(crate) const MIN_NON_ZERO_CAP: usize = min_non_zero_cap(size_of::()); - // Check assumption made in `current_memory` - const { assert!(T::LAYOUT.size() % T::LAYOUT.align() == 0) }; /// Like `new`, but parameterized over the choice of allocator for /// the returned `RawVec`. #[inline] @@ -503,6 +501,8 @@ impl RawVec { //@ std::alloc::Layout_inv(Layout::new_::()); //@ std::alloc::is_valid_layout_size_of_align_of::(); //@ std::ptr::Alignment_as_nonzero__new_(std::mem::align_of::()); + // Check assumption made in `current_memory` + const { assert!(T::LAYOUT.size() % T::LAYOUT.align() == 0) }; let r = Self { inner: RawVecInner::new_in(alloc, Alignment::of::()), _marker: PhantomData }; //@ RawVecInner_inv::(); //@ close RawVec::(t, r, alloc_id, ?ptr, ?capacity); @@ -846,7 +846,8 @@ impl RawVec { //@ close_points_to(&(*self).inner); //@ open RawVec(t, self0, alloc_id, ptr0, capacity0); //@ array_at_lft__to_u8s_at_lft_(ptr0, capacity0); - let r = self.inner.try_reserve(len, additional, T::LAYOUT); + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + let r = unsafe { self.inner.try_reserve(len, additional, T::LAYOUT) }; //@ open_points_to(&(*self).inner); //@ close_points_to(self); //@ assert *self |-> ?self1; @@ -934,7 +935,8 @@ impl RawVec { //@ close_points_to(&(*self).inner); //@ open RawVec(t, self0, alloc_id, ptr0, capacity0); //@ array_at_lft__to_u8s_at_lft_(ptr0, capacity0); - let r = self.inner.try_reserve_exact(len, additional, T::LAYOUT); + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + let r = unsafe { self.inner.try_reserve_exact(len, additional, T::LAYOUT) }; //@ open_points_to(&(*self).inner); //@ close_points_to(self); //@ assert *self |-> ?self1; @@ -1141,10 +1143,6 @@ impl RawVecInner { } //@ end_lifetime(k); //@ std::alloc::end_ref_Allocator_at_lifetime::(); - /// # Safety - /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to - /// initially construct `self` - /// - `elem_layout`'s size must be a multiple of its alignment r } #[cfg(not(no_global_oom_handling))] @@ -1162,10 +1160,6 @@ impl RawVecInner { //@ std::alloc::end_ref_Allocator_at_lifetime::(); r } - /// # Safety - /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to - /// initially construct `self` - /// - `elem_layout`'s size must be a multiple of its alignment }; let ptr = match result { Ok(ptr) => ptr, @@ -1302,8 +1296,12 @@ impl RawVecInner { &self.alloc } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[inline] - fn current_memory(&self, elem_layout: Layout) -> Option<(NonNull, Layout)> + unsafe fn current_memory(&self, elem_layout: Layout) -> Option<(NonNull, Layout)> //@ req [_]RawVecInner_share_(?k, ?t, self, elem_layout, ?alloc_id, ?ptr, ?capacity) &*& [?q]lifetime_token(k); /*@ ens [q]lifetime_token(k) &*& @@ -1314,7 +1312,6 @@ impl RawVecInner { }; @*/ //@ on_unwind_ens false; - //@ safety_proof { assume(false); } { //@ open RawVecInner_share_(k, t, self, elem_layout, alloc_id, ptr, capacity); //@ open_frac_borrow(k, RawVecInner_frac_borrow_content(self, elem_layout, alloc_id, ptr, capacity), q); @@ -1348,6 +1345,10 @@ impl RawVecInner { } } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] #[inline] unsafe fn reserve(&mut self, len: usize, additional: usize, elem_layout: Layout) { @@ -1523,11 +1524,6 @@ impl RawVecInner { //@ init_ref_RawVecInner(self_ref2); //@ open_frac_borrow(k2, ref_initialized_(self_ref2), 1/2); //@ open [?f2]ref_initialized_::>(self_ref2)(); - /// # Safety - /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to - /// initially construct `self` - /// - `elem_layout`'s size must be a multiple of its alignment - /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` let needs_to_grow2 = self.needs_to_grow(len, additional, elem_layout); //@ close [f2]ref_initialized_::>(self_ref2)(); //@ close_frac_borrow(f2, ref_initialized_(self_ref2)); @@ -1541,6 +1537,11 @@ impl RawVecInner { Ok(()) } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` #[cfg(not(no_global_oom_handling))] #[inline] unsafe fn shrink_to_fit(&mut self, cap: usize, elem_layout: Layout) { @@ -1605,7 +1606,6 @@ impl RawVecInner { //@ safety_proof { assume(false); } { // This is ensured by the calling contexts. - // SAFETY: layout_array would have resulted in a capacity overflow if we tried to allocate more than `isize::MAX` items if cfg!(debug_assertions) { //~allow_dead_code // FIXME: The source location associated with a dead `else` branch is the entire `if` statement :-( assert!(additional > 0); } @@ -1662,7 +1662,11 @@ impl RawVecInner { //@ open_points_to(self); - match core::ops::Try::branch(finish_grow(new_layout, current_memory, &mut self.alloc)) { + // SAFETY: + // - For the `current_memory` call: Precondition passed to caller + // - For the `finish_grow` call: Precondition passed to caller + // + `current_memory` does the right thing + match core::ops::Try::branch(unsafe { finish_grow(new_layout, current_memory, &mut self.alloc) }) { core::ops::ControlFlow::Break(residual) => { //@ let self1 = *self; //@ close RawVecInner0(alloc_id, ptr0_, cap0_, elem_layout, ptr0, capacity0); @@ -1719,11 +1723,6 @@ impl RawVecInner { Result::Err(e) => RawVecInner(t, self1, elem_layout, alloc_id, ptr0, capacity0) &*& array_at_lft_(alloc_id.lft, ptr0, capacity0 * Layout::size_(elem_layout), _) &*& .own(t, e) - /// # Safety - /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to - /// initially construct `self` - /// - `elem_layout`'s size must be a multiple of its alignment - /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` }; @*/ //@ safety_proof { assume(false); } @@ -1772,7 +1771,11 @@ impl RawVecInner { //@ mul_mono_l(1, Layout::size_(elem_layout), cap); //@ mul_mono_l(capacity0, cap, Layout::size_(elem_layout)); - match core::ops::Try::branch(finish_grow(new_layout, current_memory, &mut self.alloc)) { + // SAFETY: + // - For the `current_memory` call: Precondition passed to caller + // - For the `finish_grow` call: Precondition passed to caller + // + `current_memory` does the right thing + match core::ops::Try::branch(unsafe { finish_grow(new_layout, current_memory, &mut self.alloc) }) { core::ops::ControlFlow::Break(residual) => { //@ let self1 = *self; //@ close RawVecInner0(alloc_id, ptr0_, cap0_, elem_layout, ptr0, capacity0); @@ -1793,13 +1796,6 @@ impl RawVecInner { //@ assert 0 <= Cap_as_inner_(self0.cap); //@ assert 0 <= logical_capacity(self0.cap, Layout::size_(elem_layout)); //@ assert cap != 0; -/// # Safety -/// If `current_memory` matches `Some((ptr, old_layout))`: -/// - `ptr` must denote a block of memory *currently allocated* via `alloc` -/// - `old_layout` must *fit* that block of memory -/// - `new_layout` must have the same alignment as `old_layout` -/// - `new_layout.size()` must be greater than or equal to `old_layout.size()` -/// If `current_memory` is `None`, this function is safe. //@ std::alloc::Layout_inv(new_layout); //@ close RawVecInner0(alloc_id, Unique::from_non_null_(ptr.ptr), UsizeNoHighBit::new_(cap), elem_layout, NonNull_ptr(ptr.ptr), _); //@ close RawVecInner::(t, self1, elem_layout, alloc_id, _, _); @@ -1809,9 +1805,14 @@ impl RawVecInner { } } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` #[cfg(not(no_global_oom_handling))] #[inline] - fn shrink(&mut self, cap: usize, elem_layout: Layout) -> Result<(), TryReserveError> + unsafe fn shrink(&mut self, cap: usize, elem_layout: Layout) -> Result<(), TryReserveError> /*@ req thread_token(?t) &*& t == currentThread &*& Layout::size_(elem_layout) % Layout::align_(elem_layout) == 0 &*& @@ -1830,7 +1831,6 @@ impl RawVecInner { .own(t, e) }; @*/ - //@ safety_proof { assume(false); } { //@ let k = begin_lifetime(); //@ share_RawVecInner(k, self); @@ -1890,7 +1890,8 @@ impl RawVecInner { //@ init_ref_RawVecInner(self_ref); //@ open_frac_borrow(k, ref_initialized_(self_ref), 1/2); //@ open [?f]ref_initialized_::>(self_ref)(); - let current_memory = self.current_memory(elem_layout); + // SAFETY: Precondition passed to caller + let current_memory = unsafe { self.current_memory(elem_layout) }; //@ close [f]ref_initialized_::>(self_ref)(); //@ close_frac_borrow(f, ref_initialized_(self_ref)); //@ end_lifetime(k); @@ -1996,7 +1997,8 @@ impl RawVecInner { //@ init_ref_RawVecInner(self_ref); //@ open_frac_borrow(k, ref_initialized_(self_ref), 1/2); //@ open [?f]ref_initialized_::>(self_ref)(); - let current_memory = self.current_memory(elem_layout); + // SAFETY: Precondition passed to caller + let current_memory = unsafe { self.current_memory(elem_layout) }; //@ close [f]ref_initialized_::>(self_ref)(); //@ close_frac_borrow(f, ref_initialized_(self_ref)); //@ end_lifetime(k); @@ -2027,6 +2029,13 @@ impl RawVecInner { } } +/// # Safety +/// If `current_memory` matches `Some((ptr, old_layout))`: +/// - `ptr` must denote a block of memory *currently allocated* via `alloc` +/// - `old_layout` must *fit* that block of memory +/// - `new_layout` must have the same alignment as `old_layout` +/// - `new_layout.size()` must be greater than or equal to `old_layout.size()` +/// If `current_memory` is `None`, this function is safe. // not marked inline(never) since we want optimizers to be able to observe the specifics of this // function, see tests/codegen-llvm/vec-reserve-extend.rs. #[cold] diff --git a/verifast-proofs/alloc/raw_vec/mod.rs/with-directives/raw_vec.rs b/verifast-proofs/alloc/raw_vec/mod.rs/with-directives/raw_vec.rs index 1cd1dce6329f9..9fd200414f9d8 100644 --- a/verifast-proofs/alloc/raw_vec/mod.rs/with-directives/raw_vec.rs +++ b/verifast-proofs/alloc/raw_vec/mod.rs/with-directives/raw_vec.rs @@ -23,8 +23,7 @@ mod tests; // ensure that the code generation related to these panics is minimal as there's // only one location which panics rather than a bunch throughout the module. #[cfg(not(no_global_oom_handling))] -#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))] -#[track_caller] +#[cfg_attr(not(panic = "immediate-abort"), inline(never))] fn capacity_overflow() -> ! { panic!("capacity overflow"); } @@ -123,7 +122,6 @@ impl RawVec { #[cfg(not(any(no_global_oom_handling, test)))] #[must_use] #[inline] - #[track_caller] pub(crate) fn with_capacity(capacity: usize) -> Self { Self { inner: RawVecInner::with_capacity(capacity, T::LAYOUT), _marker: PhantomData } } @@ -132,7 +130,6 @@ impl RawVec { #[cfg(not(any(no_global_oom_handling, test)))] #[must_use] #[inline] - #[track_caller] pub(crate) fn with_capacity_zeroed(capacity: usize) -> Self { Self { inner: RawVecInner::with_capacity_zeroed_in(capacity, Global, T::LAYOUT), @@ -145,7 +142,6 @@ impl RawVecInner { #[cfg(not(any(no_global_oom_handling, test)))] #[must_use] #[inline] - #[track_caller] fn with_capacity(capacity: usize, elem_layout: Layout) -> Self { match Self::try_allocate_in(capacity, AllocInit::Uninitialized, Global, elem_layout) { Ok(res) => res, @@ -177,6 +173,8 @@ impl RawVec { /// the returned `RawVec`. #[inline] pub(crate) const fn new_in(alloc: A) -> Self { + // Check assumption made in `current_memory` + const { assert!(T::LAYOUT.size() % T::LAYOUT.align() == 0) }; Self { inner: RawVecInner::new_in(alloc, Alignment::of::()), _marker: PhantomData } } @@ -184,7 +182,6 @@ impl RawVec { /// allocator for the returned `RawVec`. #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] pub(crate) fn with_capacity_in(capacity: usize, alloc: A) -> Self { Self { inner: RawVecInner::with_capacity_in(capacity, alloc, T::LAYOUT), @@ -206,7 +203,6 @@ impl RawVec { /// of allocator for the returned `RawVec`. #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] pub(crate) fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self { Self { inner: RawVecInner::with_capacity_zeroed_in(capacity, alloc, T::LAYOUT), @@ -326,18 +322,18 @@ impl RawVec { /// Aborts on OOM. #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] pub(crate) fn reserve(&mut self, len: usize, additional: usize) { - self.inner.reserve(len, additional, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.reserve(len, additional, T::LAYOUT) } } /// A specialized version of `self.reserve(len, 1)` which requires the /// caller to ensure `len == self.capacity()`. #[cfg(not(no_global_oom_handling))] #[inline(never)] - #[track_caller] pub(crate) fn grow_one(&mut self) { - self.inner.grow_one(T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.grow_one(T::LAYOUT) } } /// The same as `reserve`, but returns on errors instead of panicking or aborting. @@ -346,7 +342,8 @@ impl RawVec { len: usize, additional: usize, ) -> Result<(), TryReserveError> { - self.inner.try_reserve(len, additional, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.try_reserve(len, additional, T::LAYOUT) } } /// Ensures that the buffer contains at least enough space to hold `len + @@ -367,9 +364,9 @@ impl RawVec { /// /// Aborts on OOM. #[cfg(not(no_global_oom_handling))] - #[track_caller] pub(crate) fn reserve_exact(&mut self, len: usize, additional: usize) { - self.inner.reserve_exact(len, additional, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.reserve_exact(len, additional, T::LAYOUT) } } /// The same as `reserve_exact`, but returns on errors instead of panicking or aborting. @@ -378,7 +375,8 @@ impl RawVec { len: usize, additional: usize, ) -> Result<(), TryReserveError> { - self.inner.try_reserve_exact(len, additional, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.try_reserve_exact(len, additional, T::LAYOUT) } } /// Shrinks the buffer down to the specified capacity. If the given amount @@ -392,10 +390,10 @@ impl RawVec { /// /// Aborts on OOM. #[cfg(not(no_global_oom_handling))] - #[track_caller] #[inline] pub(crate) fn shrink_to_fit(&mut self, cap: usize) { - self.inner.shrink_to_fit(cap, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.shrink_to_fit(cap, T::LAYOUT) } } } @@ -417,7 +415,6 @@ impl RawVecInner { #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] fn with_capacity_in(capacity: usize, alloc: A, elem_layout: Layout) -> Self { match Self::try_allocate_in(capacity, AllocInit::Uninitialized, alloc, elem_layout) { Ok(/*@~mut@*/ this) => { @@ -442,7 +439,6 @@ impl RawVecInner { #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] fn with_capacity_zeroed_in(capacity: usize, alloc: A, elem_layout: Layout) -> Self { match Self::try_allocate_in(capacity, AllocInit::Zeroed, alloc, elem_layout) { Ok(res) => res, @@ -518,8 +514,12 @@ impl RawVecInner { &self.alloc } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[inline] - fn current_memory(&self, elem_layout: Layout) -> Option<(NonNull, Layout)> { + unsafe fn current_memory(&self, elem_layout: Layout) -> Option<(NonNull, Layout)> { if elem_layout.size() == 0 || self.cap.as_inner() == 0 { None } else { @@ -535,48 +535,65 @@ impl RawVecInner { } } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] - fn reserve(&mut self, len: usize, additional: usize, elem_layout: Layout) { + unsafe fn reserve(&mut self, len: usize, additional: usize, elem_layout: Layout) { // Callers expect this function to be very cheap when there is already sufficient capacity. // Therefore, we move all the resizing and error-handling logic from grow_amortized and // handle_reserve behind a call, while making sure that this function is likely to be // inlined as just a comparison and a call if the comparison fails. #[cold] - fn do_reserve_and_handle( + unsafe fn do_reserve_and_handle( slf: &mut RawVecInner, len: usize, additional: usize, elem_layout: Layout, ) { - if let Err(err) = slf.grow_amortized(len, additional, elem_layout) { + // SAFETY: Precondition passed to caller + if let Err(err) = unsafe { slf.grow_amortized(len, additional, elem_layout) } { handle_error(err); } } if self.needs_to_grow(len, additional, elem_layout) { - do_reserve_and_handle(self, len, additional, elem_layout); + unsafe { + do_reserve_and_handle(self, len, additional, elem_layout); + } } } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] - fn grow_one(&mut self, elem_layout: Layout) { - if let Err(err) = self.grow_amortized(self.cap.as_inner(), 1, elem_layout) { + unsafe fn grow_one(&mut self, elem_layout: Layout) { + // SAFETY: Precondition passed to caller + if let Err(err) = unsafe { self.grow_amortized(self.cap.as_inner(), 1, elem_layout) } { handle_error(err); } } - fn try_reserve( + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + unsafe fn try_reserve( &mut self, len: usize, additional: usize, elem_layout: Layout, ) -> Result<(), TryReserveError> { if self.needs_to_grow(len, additional, elem_layout) { - self.grow_amortized(len, additional, elem_layout)?; + // SAFETY: Precondition passed to caller + unsafe { + self.grow_amortized(len, additional, elem_layout)?; + } } unsafe { // Inform the optimizer that the reservation has succeeded or wasn't needed @@ -585,22 +602,33 @@ impl RawVecInner { Ok(()) } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] - #[track_caller] - fn reserve_exact(&mut self, len: usize, additional: usize, elem_layout: Layout) { - if let Err(err) = self.try_reserve_exact(len, additional, elem_layout) { + unsafe fn reserve_exact(&mut self, len: usize, additional: usize, elem_layout: Layout) { + // SAFETY: Precondition passed to caller + if let Err(err) = unsafe { self.try_reserve_exact(len, additional, elem_layout) } { handle_error(err); } } - fn try_reserve_exact( + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + unsafe fn try_reserve_exact( &mut self, len: usize, additional: usize, elem_layout: Layout, ) -> Result<(), TryReserveError> { if self.needs_to_grow(len, additional, elem_layout) { - self.grow_exact(len, additional, elem_layout)?; + // SAFETY: Precondition passed to caller + unsafe { + self.grow_exact(len, additional, elem_layout)?; + } } unsafe { // Inform the optimizer that the reservation has succeeded or wasn't needed @@ -609,11 +637,15 @@ impl RawVecInner { Ok(()) } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] - fn shrink_to_fit(&mut self, cap: usize, elem_layout: Layout) { - if let Err(err) = self.shrink(cap, elem_layout) { + unsafe fn shrink_to_fit(&mut self, cap: usize, elem_layout: Layout) { + if let Err(err) = unsafe { self.shrink(cap, elem_layout) } { handle_error(err); } } @@ -632,7 +664,13 @@ impl RawVecInner { self.cap = unsafe { Cap::new_unchecked(cap) }; } - fn grow_amortized( + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - The sum of `len` and `additional` must be greater than or equal to + /// `self.capacity(elem_layout.size())` + unsafe fn grow_amortized( &mut self, len: usize, additional: usize, @@ -657,14 +695,25 @@ impl RawVecInner { let new_layout = layout_array(cap, elem_layout)?; - let ptr = finish_grow(new_layout, self.current_memory(elem_layout), &mut self.alloc)?; - // SAFETY: layout_array would have resulted in a capacity overflow if we tried to allocate more than `isize::MAX` items + // SAFETY: + // - For the `current_memory` call: Precondition passed to caller + // - For the `finish_grow` call: Precondition passed to caller + // + `current_memory` does the right thing + let ptr = + unsafe { finish_grow(new_layout, self.current_memory(elem_layout), &mut self.alloc)? }; + // SAFETY: layout_array would have resulted in a capacity overflow if we tried to allocate more than `isize::MAX` items unsafe { self.set_ptr_and_cap(ptr, cap) }; Ok(()) } - fn grow_exact( + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - The sum of `len` and `additional` must be greater than or equal to + /// `self.capacity(elem_layout.size())` + unsafe fn grow_exact( &mut self, len: usize, additional: usize, @@ -679,7 +728,12 @@ impl RawVecInner { let cap = len.checked_add(additional).ok_or(CapacityOverflow)?; let new_layout = layout_array(cap, elem_layout)?; - let ptr = finish_grow(new_layout, self.current_memory(elem_layout), &mut self.alloc)?; + // SAFETY: + // - For the `current_memory` call: Precondition passed to caller + // - For the `finish_grow` call: Precondition passed to caller + // + `current_memory` does the right thing + let ptr = + unsafe { finish_grow(new_layout, self.current_memory(elem_layout), &mut self.alloc)? }; // SAFETY: layout_array would have resulted in a capacity overflow if we tried to allocate more than `isize::MAX` items unsafe { self.set_ptr_and_cap(ptr, cap); @@ -687,9 +741,14 @@ impl RawVecInner { Ok(()) } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` #[cfg(not(no_global_oom_handling))] #[inline] - fn shrink(&mut self, cap: usize, elem_layout: Layout) -> Result<(), TryReserveError> { + unsafe fn shrink(&mut self, cap: usize, elem_layout: Layout) -> Result<(), TryReserveError> { assert!(cap <= self.capacity(elem_layout.size()), "Tried to shrink to a larger capacity"); // SAFETY: Just checked this isn't trying to grow unsafe { self.shrink_unchecked(cap, elem_layout) } @@ -711,8 +770,12 @@ impl RawVecInner { cap: usize, elem_layout: Layout, ) -> Result<(), TryReserveError> { - let (ptr, layout) = - if let Some(mem) = self.current_memory(elem_layout) { mem } else { return Ok(()) }; + // SAFETY: Precondition passed to caller + let (ptr, layout) = if let Some(mem) = unsafe { self.current_memory(elem_layout) } { + mem + } else { + return Ok(()); + }; // If shrinking to 0, deallocate the buffer. We don't reach this point // for the T::IS_ZST case since current_memory() will have returned @@ -748,7 +811,8 @@ impl RawVecInner { /// Ideally this function would take `self` by move, but it cannot because it exists to be /// called from a `Drop` impl. unsafe fn deallocate(&mut self, elem_layout: Layout) { - if let Some((ptr, layout)) = self.current_memory(elem_layout) { + // SAFETY: Precondition passed to caller + if let Some((ptr, layout)) = unsafe { self.current_memory(elem_layout) } { unsafe { self.alloc.deallocate(ptr, layout); } @@ -756,10 +820,17 @@ impl RawVecInner { } } +/// # Safety +/// If `current_memory` matches `Some((ptr, old_layout))`: +/// - `ptr` must denote a block of memory *currently allocated* via `alloc` +/// - `old_layout` must *fit* that block of memory +/// - `new_layout` must have the same alignment as `old_layout` +/// - `new_layout.size()` must be greater than or equal to `old_layout.size()` +/// If `current_memory` is `None`, this function is safe. // not marked inline(never) since we want optimizers to be able to observe the specifics of this // function, see tests/codegen-llvm/vec-reserve-extend.rs. #[cold] -fn finish_grow( +unsafe fn finish_grow( new_layout: Layout, current_memory: Option<(NonNull, Layout)>, alloc: &mut A, @@ -785,7 +856,6 @@ where #[cfg(not(no_global_oom_handling))] #[cold] #[optimize(size)] -#[track_caller] fn handle_error(e: TryReserveError) -> ! { match e.kind() { CapacityOverflow => capacity_overflow(),