diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index 3c4802f80e..b550a60a0b 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -391,11 +391,44 @@ impl ImmixSpace { debug_assert!(!super::BLOCK_ONLY); let forwarding_status = ForwardingWord::attempt_to_forward::(object); if ForwardingWord::state_is_forwarded_or_being_forwarded(forwarding_status) { - ForwardingWord::spin_and_get_forwarded_object::(object, forwarding_status) + // We lost the forwarding race as some other thread has set the forwarding word; wait + // until the object has been forwarded by the winner. Note that the object may not + // necessarily get forwarded since Immix opportunistically moves objects. + #[allow(clippy::let_and_return)] + let new_object = + ForwardingWord::spin_and_get_forwarded_object::(object, forwarding_status); + #[cfg(debug_assertions)] + { + if new_object == object { + debug_assert!( + self.is_marked(object, self.mark_state) || self.defrag.space_exhausted() || Self::is_pinned(object), + "Forwarded object is the same as original object {} even though it should have been copied", + object, + ); + } else { + // new_object != object + debug_assert!( + !Block::containing::(new_object).is_defrag_source(), + "Block {:?} containing forwarded object {} should not be a defragmentation source", + Block::containing::(new_object), + new_object, + ); + } + } + new_object } else if self.is_marked(object, self.mark_state) { + // We won the forwarding race but the object is already marked so we clear the + // forwarding status and return the unmoved object + debug_assert!( + self.defrag.space_exhausted() || Self::is_pinned(object), + "Forwarded object is the same as original object {} even though it should have been copied", + object, + ); ForwardingWord::clear_forwarding_bits::(object); object } else { + // We won the forwarding race; actually forward and copy the object if it is not pinned + // and we have sufficient space in our copy allocator let new_object = if Self::is_pinned(object) || self.defrag.space_exhausted() { self.attempt_mark(object, self.mark_state); ForwardingWord::clear_forwarding_bits::(object); diff --git a/src/util/object_forwarding.rs b/src/util/object_forwarding.rs index 4b167cb463..1253de8582 100644 --- a/src/util/object_forwarding.rs +++ b/src/util/object_forwarding.rs @@ -63,12 +63,16 @@ pub fn spin_and_get_forwarded_object( if forwarding_bits == FORWARDED { read_forwarding_pointer::(object) } else { - panic!( - "Invalid forwarding state 0x{:x} 0x{:x} for object {}", + // For some policies (such as Immix), we can have interleaving such that one thread clears + // the forwarding word while another thread was stuck spinning in the above loop. + // See: https://github.com/mmtk/mmtk-core/issues/579 + debug_assert!( + forwarding_bits == FORWARDING_NOT_TRIGGERED_YET, + "Invalid/Corrupted forwarding word {:x} for object {}", forwarding_bits, - read_forwarding_pointer::(object), - object - ) + object, + ); + object } }