diff --git a/Cargo.toml b/Cargo.toml index 51cb2cc69a..b3c3f97949 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,7 +58,8 @@ perf_counter = ["pfm"] # spaces vm_space = [] ro_space = [] -code_space = [] +code_space = [] +malloc_space = [] # metadata global_alloc_bit = [] diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 907042828e..c40807465a 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -139,6 +139,16 @@ pub fn post_alloc( mutator.post_alloc(refer, bytes, semantics); } +#[cfg(feature = "malloc_space")] +pub fn free(mmtk: &MMTK, address: Address) { + mmtk.plan.base().malloc_space.free_manually(address) +} + +#[cfg(feature = "malloc_space")] +pub fn malloc_usable_size(mmtk: &MMTK, address: Address) -> usize { + mmtk.plan.base().malloc_space.malloc_usable_size(address) +} + /// Return an AllocatorSelector for the given allocation semantic. This method is provided /// so that VM compilers may call it to help generate allocation fast-path. /// diff --git a/src/plan/global.rs b/src/plan/global.rs index 2c990cd115..07bdc1fae9 100644 --- a/src/plan/global.rs +++ b/src/plan/global.rs @@ -8,6 +8,8 @@ use crate::plan::transitive_closure::TransitiveClosure; use crate::plan::Mutator; use crate::policy::immortalspace::ImmortalSpace; use crate::policy::largeobjectspace::LargeObjectSpace; +#[cfg(feature = "malloc_space")] +use crate::policy::mallocspace::MallocSpace; use crate::policy::space::Space; use crate::scheduler::*; use crate::util::alloc::allocators::AllocatorSelector; @@ -376,13 +378,23 @@ pub struct BasePlan { pub analysis_manager: AnalysisManager, // Spaces in base plan + /// Code space allows execution. + // TODO: a proper CodeSpace. #[cfg(feature = "code_space")] pub code_space: ImmortalSpace, #[cfg(feature = "code_space")] pub code_lo_space: ImmortalSpace, + + /// Read-only space does not allow write once read-only is enabled. + // TODO: a proper ReadOnlySpace #[cfg(feature = "ro_space")] pub ro_space: ImmortalSpace, + /// Malloc space allows manual malloc/free-alike API. + /// A user allocate into a malloc space with [`AllocationSemantics::Malloc`](crate::plan::AllocationSemantics) + #[cfg(feature = "malloc_space")] + pub malloc_space: MallocSpace, + /// A VM space is a space allocated and populated by the VM. Currently it is used by JikesRVM /// for boot image. /// @@ -478,6 +490,8 @@ impl BasePlan { &mut heap, constraints, ), + #[cfg(feature = "malloc_space")] + malloc_space: MallocSpace::new(false, global_side_metadata_specs.clone()), #[cfg(feature = "vm_space")] vm_space: create_vm_space( vm_map, @@ -532,6 +546,8 @@ impl BasePlan { self.code_lo_space.init(vm_map); #[cfg(feature = "ro_space")] self.ro_space.init(vm_map); + #[cfg(feature = "malloc_space")] + self.malloc_space.init(vm_map); #[cfg(feature = "vm_space")] { self.vm_space.init(vm_map); @@ -588,6 +604,10 @@ impl BasePlan { { pages += self.ro_space.reserved_pages(); } + #[cfg(feature = "malloc_space")] + { + pages += self.malloc_space.reserved_pages(); + } // The VM space may be used as an immutable boot image, in which case, we should not count // it as part of the heap size. @@ -617,6 +637,11 @@ impl BasePlan { return self.ro_space.trace_object(_trace, _object); } + #[cfg(feature = "malloc_space")] + if self.malloc_space.in_space(_object) { + panic!("We cannot trace object in malloc_space"); + } + #[cfg(feature = "vm_space")] if self.vm_space.in_space(_object) { trace!("trace_object: object in boot space"); @@ -632,6 +657,10 @@ impl BasePlan { self.code_lo_space.prepare(); #[cfg(feature = "ro_space")] self.ro_space.prepare(); + #[cfg(feature = "malloc_space")] + { + // We do not need to prepare for malloc space + } #[cfg(feature = "vm_space")] self.vm_space.prepare(); } @@ -643,6 +672,10 @@ impl BasePlan { self.code_lo_space.release(); #[cfg(feature = "ro_space")] self.ro_space.release(); + #[cfg(feature = "malloc_space")] + { + // We do not need to release for malloc space + } #[cfg(feature = "vm_space")] self.vm_space.release(); } @@ -825,6 +858,9 @@ impl BasePlan { #[cfg(feature = "ro_space")] self.ro_space .verify_side_metadata_sanity(side_metadata_sanity_checker); + #[cfg(feature = "malloc_space")] + self.malloc_space + .verify_side_metadata_sanity(side_metadata_sanity_checker); #[cfg(feature = "vm_space")] self.vm_space .verify_side_metadata_sanity(side_metadata_sanity_checker); @@ -951,10 +987,26 @@ use enum_map::Enum; #[repr(i32)] #[derive(Clone, Copy, Debug, Enum, PartialEq, Eq)] pub enum AllocationSemantics { + /// The default allocation semantic. Most allocation should use this semantic. Default = 0, + /// This semantic guarantees the memory won't be reclaimed by GC. Immortal = 1, + /// Large object. Generally a plan defines a constant `max_non_los_default_alloc_bytes`, + /// which specifies the max object size that can be allocated with the Default allocation semantic. + /// An object that is larger than the defined max size needs to be allocated with `Los`. Los = 2, + /// This semantic guarantees the memory will not be moved and has the execution permission. Code = 3, + /// This semantic allows the memory to be made read-only after allocation. + /// This semantic is not implemented yet. ReadOnly = 4, + /// Large object + Code. LargeCode = 5, + /// Malloc and free through MMTk. Note that when this is used, the user should treat the result + /// as if it is returned from malloc(): + /// * The memory needs to be manually freed by the user. + /// * The user should not use any other API other than `free()` and `malloc_usable_size()` for the + /// memory address we return. MMTk may panic if the user casts the memory address into an object + /// reference, and pass the object reference in MMTk's API. + Malloc = 6, } diff --git a/src/plan/marksweep/global.rs b/src/plan/marksweep/global.rs index 1bf8ef0611..21cf52f405 100644 --- a/src/plan/marksweep/global.rs +++ b/src/plan/marksweep/global.rs @@ -108,8 +108,13 @@ impl MarkSweep { ACTIVE_CHUNK_METADATA_SPEC, ]); + if cfg!(feature = "malloc_space") { + // We can remove this check once we no longer use malloc marksweep. + panic!("We only allow one malloc space in use (marksweep currently uses malloc space and performs marksweep on it"); + } + let res = MarkSweep { - ms: MallocSpace::new(global_metadata_specs.clone()), + ms: MallocSpace::new(true, global_metadata_specs.clone()), common: CommonPlan::new( vm_map, mmapper, diff --git a/src/plan/mutator_context.rs b/src/plan/mutator_context.rs index 0f7349dc3a..32b4a8519d 100644 --- a/src/plan/mutator_context.rs +++ b/src/plan/mutator_context.rs @@ -225,6 +225,12 @@ pub(crate) fn create_allocator_mapping( reserved.n_bump_pointer += 1; } + #[cfg(feature = "malloc_space")] + { + map[AllocationSemantics::Malloc] = AllocatorSelector::Malloc(reserved.n_malloc); + reserved.n_malloc += 1; + } + // spaces in common plan if include_common_plan { @@ -282,6 +288,15 @@ pub(crate) fn create_space_mapping( reserved.n_bump_pointer += 1; } + #[cfg(feature = "malloc_space")] + { + vec.push(( + AllocatorSelector::Malloc(reserved.n_malloc), + &plan.base().malloc_space, + )); + reserved.n_malloc += 1; + } + // spaces in CommonPlan if include_common_plan { diff --git a/src/policy/mallocspace/global.rs b/src/policy/mallocspace/global.rs index 136a212ed8..2e7d756e75 100644 --- a/src/policy/mallocspace/global.rs +++ b/src/policy/mallocspace/global.rs @@ -20,7 +20,7 @@ use crate::{policy::space::Space, util::heap::layout::vm_layout_constants::BYTES use std::marker::PhantomData; #[cfg(debug_assertions)] use std::sync::atomic::AtomicU32; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; // only used for debugging use crate::policy::space::*; #[cfg(debug_assertions)] @@ -33,12 +33,37 @@ use std::sync::Mutex; #[cfg(debug_assertions)] const ASSERT_ALLOCATION: bool = false; +/// A malloc space. This space is special in a number of ways: +/// 1. This space does not use a page resource. Instead, this space use malloc to back up its +/// allocation. So MMTk does not manage the mmapping for its data, but MMTk still manages +/// the mmapping for the necessary metadata it needs. +/// 2. This space can be a GC space, for which we use mark sweep to do GC on the space. And +/// it can be a non-GC space, the user will manage its allocation and free through a +/// malloc/free API. In this case, we do not perform any GC work on the space, and it is +/// purely the user's responsibility to manage objects in the space. +/// 3. We do not allow more than one MallocSpace in a plan. MMTk has an assumption that +/// each chunk belongs to one space. However, we cannot control the behavior of the underlying +/// malloc, and two malloc spaces may break the assumption. This should not be a big issue +/// as we would expect only one malloc space for `AllocationSemantics::Malloc`, and in the long +/// term, we will not use `MallocSpace` for GC. pub struct MallocSpace { phantom: PhantomData, active_bytes: AtomicUsize, pub chunk_addr_min: AtomicUsize, // XXX: have to use AtomicUsize to represent an Address pub chunk_addr_max: AtomicUsize, metadata: SideMetadataContext, + + /// Do we perform GC on this space? If true, we perform mark sweep on the space. Otherwise, + /// objects are freed manually. + /// To be specific, the differences are: + /// GC | non-GC/manual + /// mark/trace_object yes | no + /// sweep/release yes | no + /// object reference normal | no object reference, any method related with object reference cannot be used + /// set alloc bit in post_alloc | in alloc if global_alloc_bit is set + /// clear alloc bit in sweep | when freed manually if global_alloc_bit is set + is_gc_space: AtomicBool, + // Mapping between allocated address and its size - this is used to check correctness. // Size will be set to zero when the memory is freed. #[cfg(debug_assertions)] @@ -54,12 +79,21 @@ pub struct MallocSpace { pub work_live_bytes: AtomicUsize, } +const NON_GC_MALLOC_SPACE_CANNOT_USE_OBJECTREFERENCE: &str = + "Manual malloc space cannot use methods with an ObjectReference argument"; + impl SFT for MallocSpace { fn name(&self) -> &str { self.get_name() } fn is_live(&self, object: ObjectReference) -> bool { + debug_assert!( + self.is_gc_space(), + "{}", + NON_GC_MALLOC_SPACE_CANNOT_USE_OBJECTREFERENCE + ); + // If this is a GC space, we use mark bit to tell whether it is alive is_marked::(object, Some(Ordering::SeqCst)) } @@ -74,6 +108,11 @@ impl SFT for MallocSpace { // For malloc space, we need to further check the alloc bit. fn is_in_space(&self, object: ObjectReference) -> bool { + debug_assert!( + self.is_gc_space(), + "{}", + NON_GC_MALLOC_SPACE_CANNOT_USE_OBJECTREFERENCE + ); is_alloced_by_malloc(object) } @@ -81,6 +120,11 @@ impl SFT for MallocSpace { #[cfg(feature = "is_mmtk_object")] #[inline(always)] fn is_mmtk_object(&self, addr: Address) -> bool { + debug_assert!( + self.is_gc_space(), + "{}", + NON_GC_MALLOC_SPACE_CANNOT_USE_OBJECTREFERENCE + ); debug_assert!(!addr.is_zero()); // `addr` cannot be mapped by us. It should be mapped by the malloc library. debug_assert!(!addr.is_mapped()); @@ -88,6 +132,11 @@ impl SFT for MallocSpace { } fn initialize_object_metadata(&self, object: ObjectReference, _alloc: bool) { + debug_assert!( + self.is_gc_space(), + "{}", + NON_GC_MALLOC_SPACE_CANNOT_USE_OBJECTREFERENCE + ); trace!("initialize_object_metadata for object {}", object); let page_addr = conversions::page_align_down(object.to_address()); set_page_mark(page_addr); @@ -101,6 +150,11 @@ impl SFT for MallocSpace { object: ObjectReference, _worker: GCWorkerMutRef, ) -> ObjectReference { + debug_assert!( + self.is_gc_space(), + "{}", + NON_GC_MALLOC_SPACE_CANNOT_USE_OBJECTREFERENCE + ); let trace = trace.into_mut::(); self.trace_object(trace, object) } @@ -134,6 +188,12 @@ impl Space for MallocSpace { // We have assertions in a debug build. We allow this pattern for the release build. #[allow(clippy::let_and_return)] fn in_space(&self, object: ObjectReference) -> bool { + debug_assert!( + self.is_gc_space(), + "{}", + NON_GC_MALLOC_SPACE_CANNOT_USE_OBJECTREFERENCE + ); + let ret = is_alloced_by_malloc(object); #[cfg(debug_assertions)] @@ -190,7 +250,7 @@ impl Space for MallocSpace { } impl MallocSpace { - pub fn new(global_side_metadata_specs: Vec) -> Self { + pub fn new(gc: bool, global_side_metadata_specs: Vec) -> Self { MallocSpace { phantom: PhantomData, active_bytes: AtomicUsize::new(0), @@ -204,6 +264,7 @@ impl MallocSpace { *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, ]), }, + is_gc_space: AtomicBool::new(gc), #[cfg(debug_assertions)] active_mem: Mutex::new(HashMap::new()), #[cfg(debug_assertions)] @@ -215,6 +276,10 @@ impl MallocSpace { } } + fn is_gc_space(&self) -> bool { + self.is_gc_space.load(Ordering::Relaxed) + } + pub fn alloc(&self, tls: VMThread, size: usize, align: usize, offset: isize) -> Address { // TODO: Should refactor this and Space.acquire() if VM::VMActivePlan::global().poll(false, self) { @@ -240,6 +305,11 @@ impl MallocSpace { set_offset_malloc_bit(address); } + if !self.is_gc_space() && cfg!(feature = "global_alloc_bit") { + // if this is a non-GC space, we directly set alloc bit for the address + set_alloc_bit(unsafe { address.to_object_reference() }); + } + #[cfg(debug_assertions)] if ASSERT_ALLOCATION { debug_assert!(actual_size != 0); @@ -250,7 +320,32 @@ impl MallocSpace { address } - pub fn free(&self, addr: Address) { + /// This implements the free() API for the malloc space. + pub fn free_manually(&self, address: Address) { + debug_assert!( + !self.is_gc_space(), + "we can only manually free objects if it is non-GC malloc space" + ); + // Free the result + self.free_malloc_result(address); + // Unset alloc bit + if cfg!(feature = "global_alloc_bit") { + // This is non-GC malloc space. We assume address === object reference. + unset_alloc_bit(unsafe { address.to_object_reference() }); + } + } + + /// This implements the malloc_usable_size() API for the malloc. + pub fn malloc_usable_size(&self, address: Address) -> usize { + let offset_malloc_bit = is_offset_malloc(address); + get_malloc_usable_size(address, offset_malloc_bit) + } + + /// Frees an address result that we get from maloc, but have ot yet returned to the user. + /// This is used for retrying malloc in the case that malloc may give us some undesired addresses. + /// This will clear offset bit if appriopriate, but this does not clear alloc bit + /// (as the alloc bit should not be set for it yet) + pub fn free_malloc_result(&self, addr: Address) { let offset_malloc_bit = is_offset_malloc(addr); let bytes = get_malloc_usable_size(addr, offset_malloc_bit); self.free_internal(addr, bytes, offset_malloc_bit); @@ -285,6 +380,12 @@ impl MallocSpace { trace: &mut T, object: ObjectReference, ) -> ObjectReference { + debug_assert!( + self.is_gc_space(), + "{}", + NON_GC_MALLOC_SPACE_CANNOT_USE_OBJECTREFERENCE + ); + if object.is_null() { return object; } @@ -351,6 +452,10 @@ impl MallocSpace { } pub fn sweep_chunk(&self, chunk_start: Address) { + debug_assert!( + self.is_gc_space(), + "sweep_chunk() should only be called in a GC malloc space" + ); // Call the relevant sweep function depending on the location of the mark bits match *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC { MetadataSpec::OnSide(local_mark_bit_side_spec) => { @@ -365,6 +470,7 @@ impl MallocSpace { /// Given an object in MallocSpace, return its malloc address, whether it is an offset malloc, and malloc size #[inline(always)] fn get_malloc_addr_size(object: ObjectReference) -> (Address, bool, usize) { + // This method is only used for GC. So we assume we can use object model normally. let obj_start = VM::VMObjectModel::object_start_ref(object); let offset_malloc_bit = is_offset_malloc(obj_start); let bytes = get_malloc_usable_size(obj_start, offset_malloc_bit); diff --git a/src/policy/mallocspace/mod.rs b/src/policy/mallocspace/mod.rs index 07dec884ac..b24fdd5f2e 100644 --- a/src/policy/mallocspace/mod.rs +++ b/src/policy/mallocspace/mod.rs @@ -1,4 +1,4 @@ -///! A marksweep space that allocates from malloc. +///! Malloc space that allocates from malloc. mod global; pub mod metadata; diff --git a/src/util/alloc/malloc_allocator.rs b/src/util/alloc/malloc_allocator.rs index 41200889b7..3eb409f880 100644 --- a/src/util/alloc/malloc_allocator.rs +++ b/src/util/alloc/malloc_allocator.rs @@ -53,7 +53,7 @@ impl Allocator for MallocAllocator { } else { // The result passes the check. We free all the cached results, and return the new result. for addr in to_free.iter() { - self.space.free(*addr); + self.space.free_malloc_result(*addr); } return ret; } diff --git a/vmbindings/dummyvm/Cargo.toml b/vmbindings/dummyvm/Cargo.toml index d0bea8a8b5..8e4c075fee 100644 --- a/vmbindings/dummyvm/Cargo.toml +++ b/vmbindings/dummyvm/Cargo.toml @@ -21,3 +21,5 @@ atomic_refcell = "0.1.7" [features] default = [] is_mmtk_object = ["mmtk/is_mmtk_object"] +malloc_space = ["mmtk/malloc_space"] +nogc_multi_space = ["mmtk/nogc_multi_space"] diff --git a/vmbindings/dummyvm/src/api.rs b/vmbindings/dummyvm/src/api.rs index 3c5aa55a8e..e42f15628f 100644 --- a/vmbindings/dummyvm/src/api.rs +++ b/vmbindings/dummyvm/src/api.rs @@ -50,6 +50,18 @@ pub extern "C" fn mmtk_post_alloc(mutator: *mut Mutator, refer: ObjectR memory_manager::post_alloc::(unsafe { &mut *mutator }, refer, bytes, semantics) } +#[cfg(feature = "malloc_space")] +#[no_mangle] +pub extern "C" fn mmtk_free(address: Address) { + memory_manager::free::(&SINGLETON, address) +} + +#[cfg(feature = "malloc_space")] +#[no_mangle] +pub extern "C" fn mmtk_malloc_usable_size(address: Address) -> usize { + memory_manager::malloc_usable_size::(&SINGLETON, address) +} + #[no_mangle] pub extern "C" fn mmtk_will_never_move(object: ObjectReference) -> bool { !object.is_movable() diff --git a/vmbindings/dummyvm/src/tests/fixtures/mod.rs b/vmbindings/dummyvm/src/tests/fixtures/mod.rs index 2eb249b1d0..d55194a2cc 100644 --- a/vmbindings/dummyvm/src/tests/fixtures/mod.rs +++ b/vmbindings/dummyvm/src/tests/fixtures/mod.rs @@ -3,9 +3,11 @@ use std::sync::Once; use mmtk::AllocationSemantics; use mmtk::util::{ObjectReference, VMThread, VMMutatorThread}; +use mmtk::Mutator; use crate::api::*; use crate::object_model::OBJECT_REF_OFFSET; +use crate::DummyVM; pub trait FixtureContent { fn create() -> Self; @@ -66,3 +68,19 @@ impl FixtureContent for SingleObject { SingleObject { objref } } } + +pub struct MutatorInstance { + pub mutator: *mut Mutator, +} + +impl FixtureContent for MutatorInstance { + fn create() -> Self { + const MB: usize = 1024 * 1024; + // 1MB heap + mmtk_gc_init(MB); + mmtk_initialize_collection(VMThread::UNINITIALIZED); + let mutator = mmtk_bind_mutator(VMMutatorThread(VMThread::UNINITIALIZED)); + + MutatorInstance { mutator } + } +} diff --git a/vmbindings/dummyvm/src/tests/malloc.rs b/vmbindings/dummyvm/src/tests/malloc_impl.rs similarity index 100% rename from vmbindings/dummyvm/src/tests/malloc.rs rename to vmbindings/dummyvm/src/tests/malloc_impl.rs diff --git a/vmbindings/dummyvm/src/tests/malloc_space.rs b/vmbindings/dummyvm/src/tests/malloc_space.rs new file mode 100644 index 0000000000..6325be4eb4 --- /dev/null +++ b/vmbindings/dummyvm/src/tests/malloc_space.rs @@ -0,0 +1,52 @@ +// This runs with plan that is not malloc MS +// GITHUB-CI: MMTK_PLAN=NoGC +// GITHUB-CI: MMTK_PLAN=Immix +// GITHUB-CI: MMTK_PLAN=GenImmix +// GITHUB-CI: MMTK_PLAN=GenCopy +// GITHUB-CI: MMTK_PLAN=MarkCompact +// GITHUB-CI: FEATURES=malloc_space,nogc_multi_space + +use crate::api::*; +use crate::tests::fixtures::{Fixture, MutatorInstance}; + +use mmtk::AllocationSemantics; + +lazy_static! { + static ref MUTATOR: Fixture = Fixture::new(); +} + +const SIZE: usize = 40; +const NO_OFFSET: isize = 0; +const ALIGN: usize = 8; +const OFFSET: isize = 4; + +#[test] +pub fn malloc_free() { + MUTATOR.with_fixture(|fixture| { + let res = mmtk_alloc(fixture.mutator, SIZE, ALIGN, NO_OFFSET, AllocationSemantics::Malloc); + assert!(!res.is_zero()); + assert!(res.is_aligned_to(ALIGN)); + mmtk_free(res); + }) +} + +#[test] +pub fn malloc_offset() { + MUTATOR.with_fixture(|fixture| { + let res = mmtk_alloc(fixture.mutator, SIZE, ALIGN, OFFSET, AllocationSemantics::Malloc); + assert!(!res.is_zero()); + assert!((res + OFFSET).is_aligned_to(ALIGN)); + mmtk_free(res); + }) +} + +#[test] +pub fn malloc_usable_size() { + MUTATOR.with_fixture(|fixture| { + let res = mmtk_alloc(fixture.mutator, SIZE, ALIGN, NO_OFFSET, AllocationSemantics::Malloc); + assert!(!res.is_zero()); + let size = mmtk_malloc_usable_size(res); + assert!(size >= SIZE); + mmtk_free(res); + }) +} diff --git a/vmbindings/dummyvm/src/tests/mod.rs b/vmbindings/dummyvm/src/tests/mod.rs index d8f2a202b9..adffeab2d4 100644 --- a/vmbindings/dummyvm/src/tests/mod.rs +++ b/vmbindings/dummyvm/src/tests/mod.rs @@ -11,7 +11,9 @@ mod allocate_without_initialize_collection; mod allocate_with_initialize_collection; mod allocate_with_disable_collection; mod allocate_with_re_enable_collection; -mod malloc; +mod malloc_impl; +#[cfg(feature = "malloc_space")] +mod malloc_space; #[cfg(feature = "is_mmtk_object")] mod conservatism; mod is_in_mmtk_spaces;