Malloc API

.github/scripts/ci-test.sh

@@ -18,7 +18,7 @@ cd vmbindings/dummyvm
 for fn in $(ls src/tests/*.rs); do
     t=$(basename -s .rs $fn)
 
-    if [[ $t == "mod.rs" ]]; then
+    if [[ $t == "mod" ]]; then
         continue
     fi
 

Cargo.toml

@@ -24,7 +24,7 @@ mmtk-macros = { version = "0.12.0", path = "macros/" }
 libc = "0.2"
 jemalloc-sys = { version = "0.3.2", features = ["disable_initial_exec_tls"], optional = true }
 mimalloc-sys = { version = "0.1.6", optional = true }
-hoard-sys = { version = "0.1.1", optional = true }
+hoard-sys = { version = "0.1.2", optional = true }
 lazy_static = "1.1"
 log = { version = "0.4", features = ["max_level_trace", "release_max_level_off"] }
 crossbeam = "0.8.1"
@@ -89,6 +89,9 @@ nogc_multi_space = []
 # To collect statistics for each GC work packet. Enabling this may introduce a small overhead (several percentage slowdown on benchmark time).
 work_packet_stats = []
 
+# Count the malloc'd memory into the heap size
+malloc_counted_size = []
+
 # Do not modify the following line - ci-common.sh matches it
 # -- Mutally exclusive features --
 # Only one feature from each group can be provided. Otherwise build will fail.

docs/tutorial/code/mygc_semispace/global.rs

@@ -102,8 +102,8 @@ impl<VM: VMBinding> Plan for MyGC<VM> {
     // ANCHOR_END: schedule_collection
 
     // ANCHOR: collection_required()
-    fn collection_required(&self, space_full: bool, space: &dyn Space<Self::VM>) -> bool {
-        self.base().collection_required(self, space_full, space)
+    fn collection_required(&self, space_full: bool, _space: Option<&dyn Space<Self::VM>>) -> bool {
+        self.base().collection_required(self, space_full)
     }
     // ANCHOR_END: collection_required()
 

src/memory_manager.rs

@@ -153,6 +153,84 @@ pub fn get_allocator_mapping<VM: VMBinding>(
     mmtk.plan.get_allocator_mapping()[semantics]
 }
 
+/// The standard malloc. MMTk either uses its own allocator, or forward the call to a
+/// library malloc.
+pub fn malloc(size: usize) -> Address {
+    crate::util::malloc::malloc(size)
+}
+
+/// The standard malloc except that with the feature `malloc_counted_size`, MMTk will count the allocated memory into its heap size.
+/// Thus the method requires a reference to an MMTk instance. MMTk either uses its own allocator, or forward the call to a
+/// library malloc.
+#[cfg(feature = "malloc_counted_size")]
+pub fn counted_malloc<VM: VMBinding>(mmtk: &MMTK<VM>, size: usize) -> Address {
+    crate::util::malloc::counted_malloc(mmtk, size)
+}
+
+/// The standard calloc.
+pub fn calloc(num: usize, size: usize) -> Address {
+    crate::util::malloc::calloc(num, size)
+}
+
+/// The standard calloc except that with the feature `malloc_counted_size`, MMTk will count the allocated memory into its heap size.
+/// Thus the method requires a reference to an MMTk instance.
+#[cfg(feature = "malloc_counted_size")]
+pub fn counted_calloc<VM: VMBinding>(mmtk: &MMTK<VM>, num: usize, size: usize) -> Address {
+    crate::util::malloc::counted_calloc(mmtk, num, size)
+}
+
+/// The standard realloc.
+pub fn realloc(addr: Address, size: usize) -> Address {
+    crate::util::malloc::realloc(addr, size)
+}
+
+/// The standard realloc except that with the feature `malloc_counted_size`, MMTk will count the allocated memory into its heap size.
+/// Thus the method requires a reference to an MMTk instance, and the size of the existing memory that will be reallocated.
+/// The `addr` in the arguments must be an address that is earlier returned from MMTk's `malloc()`, `calloc()` or `realloc()`.
+#[cfg(feature = "malloc_counted_size")]
+pub fn realloc_with_old_size<VM: VMBinding>(
+    mmtk: &MMTK<VM>,
+    addr: Address,
+    size: usize,
+    old_size: usize,
+) -> Address {
+    crate::util::malloc::realloc_with_old_size(mmtk, addr, size, old_size)
+}
+
+/// The standard free.
+/// The `addr` in the arguments must be an address that is earlier returned from MMTk's `malloc()`, `calloc()` or `realloc()`.
+pub fn free(addr: Address) {
+    crate::util::malloc::free(addr)
+}
+
+/// The standard free except that with the feature `malloc_counted_size`, MMTk will count the allocated memory into its heap size.
+/// Thus the method requires a reference to an MMTk instance, and the size of the memory to free.
+/// The `addr` in the arguments must be an address that is earlier returned from MMTk's `malloc()`, `calloc()` or `realloc()`.
+#[cfg(feature = "malloc_counted_size")]
+pub fn free_with_size<VM: VMBinding>(mmtk: &MMTK<VM>, addr: Address, old_size: usize) {
+    crate::util::malloc::free_with_size(mmtk, addr, old_size)
+}
+
+/// Poll for GC. MMTk will decide if a GC is needed. If so, this call will block
+/// the current thread, and trigger a GC. Otherwise, it will simply return.
+/// Usually a binding does not need to call this function. MMTk will poll for GC during its allocation.
+/// However, if a binding uses counted malloc (which won't poll for GC), they may want to poll for GC manually.
+/// This function should only be used by mutator threads.
+pub fn gc_poll<VM: VMBinding>(mmtk: &MMTK<VM>, tls: VMMutatorThread) {
+    use crate::vm::{ActivePlan, Collection};
+    debug_assert!(
+        VM::VMActivePlan::is_mutator(tls.0),
+        "gc_poll() can only be called by a mutator thread."
+    );
+
+    let plan = mmtk.get_plan();
+    if plan.should_trigger_gc_when_heap_is_full() && plan.poll(false, None) {
+        debug!("Collection required");
+        assert!(plan.is_initialized(), "GC is not allowed here: collection is not initialized (did you call initialize_collection()?).");
+        VM::VMCollection::block_for_gc(tls);
+    }
+}
+
 /// Run the main loop for the GC controller thread. This method does not return.
 ///
 /// Arguments:

src/plan/generational/copying/global.rs

@@ -64,7 +64,7 @@ impl<VM: VMBinding> Plan for GenCopy<VM> {
         }
     }
 
-    fn collection_required(&self, space_full: bool, space: &dyn Space<Self::VM>) -> bool
+    fn collection_required(&self, space_full: bool, space: Option<&dyn Space<Self::VM>>) -> bool
     where
         Self: Sized,
     {

src/plan/generational/global.rs

@@ -106,21 +106,26 @@ impl<VM: VMBinding> Gen<VM> {
         &self,
         plan: &P,
         space_full: bool,
-        space: &dyn Space<VM>,
+        space: Option<&dyn Space<VM>>,
     ) -> bool {
         let nursery_full = self.nursery.reserved_pages()
             >= (conversions::bytes_to_pages_up(*self.common.base.options.max_nursery));
         if nursery_full {
             return true;
         }
 
-        if space_full && space.common().descriptor != self.nursery.common().descriptor {
+        // Is the GC triggered by nursery?
+        // - if space is none, it is not. Return false immediately.
+        // - if space is some, we further check its descriptor.
+        let is_triggered_by_nursery = space.map_or(false, |s| {
+            s.common().descriptor == self.nursery.common().descriptor
+        });
+        // If space is full and the GC is not triggered by nursery, next GC will be full heap GC.
+        if space_full && !is_triggered_by_nursery {
             self.next_gc_full_heap.store(true, Ordering::SeqCst);
         }
 
-        self.common
-            .base
-            .collection_required(plan, space_full, space)
+        self.common.base.collection_required(plan, space_full)
     }
 
     pub fn force_full_heap_collection(&self) {

src/plan/generational/immix/global.rs

@@ -96,7 +96,7 @@ impl<VM: VMBinding> Plan for GenImmix<VM> {
         self.gen.last_collection_full_heap()
     }
 
-    fn collection_required(&self, space_full: bool, space: &dyn Space<Self::VM>) -> bool
+    fn collection_required(&self, space_full: bool, space: Option<&dyn Space<Self::VM>>) -> bool
     where
         Self: Sized,
     {

src/plan/global.rs

@@ -203,7 +203,14 @@ pub trait Plan: 'static + Sync + Downcast {
     /// This is invoked once per GC by one worker thread. 'tls' is the worker thread that executes this method.
     fn release(&mut self, tls: VMWorkerThread);
 
-    fn poll(&self, space_full: bool, space: &dyn Space<Self::VM>) -> bool {
+    /// This method is called periodically by the allocation subsystem
+    /// (by default, each time a page is consumed), and provides the
+    /// collector with an opportunity to collect.
+    ///
+    /// Arguments:
+    /// * `space_full`: Space request failed, must recover pages within 'space'.
+    /// * `space`: The space that triggered the poll. This could `None` if the poll is not triggered by a space.
+    fn poll(&self, space_full: bool, space: Option<&dyn Space<Self::VM>>) -> bool {
         if self.collection_required(space_full, space) {
             // FIXME
             /*if space == META_DATA_SPACE {
@@ -236,8 +243,12 @@ pub trait Plan: 'static + Sync + Downcast {
         false
     }
 
-    fn log_poll(&self, space: &dyn Space<Self::VM>, message: &'static str) {
-        info!("  [POLL] {}: {}", space.get_name(), message);
+    fn log_poll(&self, space: Option<&dyn Space<Self::VM>>, message: &'static str) {
+        if let Some(space) = space {
+            info!("  [POLL] {}: {}", space.get_name(), message);
+        } else {
+            info!("  [POLL] {}", message);
+        }
     }
 
     /**
@@ -248,7 +259,7 @@ pub trait Plan: 'static + Sync + Downcast {
      * @param space TODO
      * @return <code>true</code> if a collection is requested by the plan.
      */
-    fn collection_required(&self, space_full: bool, _space: &dyn Space<Self::VM>) -> bool;
+    fn collection_required(&self, space_full: bool, _space: Option<&dyn Space<Self::VM>>) -> bool;
 
     // Note: The following methods are about page accounting. The default implementation should
     // work fine for non-copying plans. For copying plans, the plan should override any of these methods
@@ -372,9 +383,12 @@ pub struct BasePlan<VM: VMBinding> {
     /// Have we scanned all the stacks?
     stacks_prepared: AtomicBool,
     pub mutator_iterator_lock: Mutex<()>,
-    // A counter that keeps tracks of the number of bytes allocated since last stress test
-    pub allocation_bytes: AtomicUsize,
-    // Wrapper around analysis counters
+    /// A counter that keeps tracks of the number of bytes allocated since last stress test
+    allocation_bytes: AtomicUsize,
+    /// A counteer that keeps tracks of the number of bytes allocated by malloc
+    #[cfg(feature = "malloc_counted_size")]
+    malloc_bytes: AtomicUsize,
+    /// Wrapper around analysis counters
     #[cfg(feature = "analysis")]
     pub analysis_manager: AnalysisManager<VM>,
 
@@ -518,6 +532,8 @@ impl<VM: VMBinding> BasePlan<VM> {
             scanned_stacks: AtomicUsize::new(0),
             mutator_iterator_lock: Mutex::new(()),
             allocation_bytes: AtomicUsize::new(0),
+            #[cfg(feature = "malloc_counted_size")]
+            malloc_bytes: AtomicUsize::new(0),
             #[cfg(feature = "analysis")]
             analysis_manager,
         }
@@ -596,6 +612,14 @@ impl<VM: VMBinding> BasePlan<VM> {
             pages += self.ro_space.reserved_pages();
         }
 
+        // If we need to count malloc'd size as part of our heap, we add it here.
+        #[cfg(feature = "malloc_counted_size")]
+        {
+            pages += crate::util::conversions::bytes_to_pages_up(
+                self.malloc_bytes.load(Ordering::SeqCst),
+            );
+        }
+
         // The VM space may be used as an immutable boot image, in which case, we should not count
         // it as part of the heap size.
         pages
@@ -794,12 +818,7 @@ impl<VM: VMBinding> BasePlan<VM> {
             && (self.allocation_bytes.load(Ordering::SeqCst) > *self.options.stress_factor)
     }
 
-    pub(super) fn collection_required<P: Plan>(
-        &self,
-        plan: &P,
-        space_full: bool,
-        _space: &dyn Space<VM>,
-    ) -> bool {
+    pub(super) fn collection_required<P: Plan>(&self, plan: &P, space_full: bool) -> bool {
         let stress_force_gc = self.should_do_stress_gc();
         if stress_force_gc {
             debug!(
@@ -838,6 +857,19 @@ impl<VM: VMBinding> BasePlan<VM> {
         self.vm_space
             .verify_side_metadata_sanity(side_metadata_sanity_checker);
     }
+
+    #[cfg(feature = "malloc_counted_size")]
+    pub(crate) fn increase_malloc_bytes_by(&self, size: usize) {
+        self.malloc_bytes.fetch_add(size, Ordering::SeqCst);
+    }
+    #[cfg(feature = "malloc_counted_size")]
+    pub(crate) fn decrease_malloc_bytes_by(&self, size: usize) {
+        self.malloc_bytes.fetch_sub(size, Ordering::SeqCst);
+    }
+    #[cfg(feature = "malloc_counted_size")]
+    pub fn get_malloc_bytes(&self) -> usize {
+        self.malloc_bytes.load(Ordering::SeqCst)
+    }
 }
 
 /**

src/plan/immix/global.rs

@@ -51,8 +51,8 @@ pub const IMMIX_CONSTRAINTS: PlanConstraints = PlanConstraints {
 impl<VM: VMBinding> Plan for Immix<VM> {
     type VM = VM;
 
-    fn collection_required(&self, space_full: bool, space: &dyn Space<Self::VM>) -> bool {
-        self.base().collection_required(self, space_full, space)
+    fn collection_required(&self, space_full: bool, _space: Option<&dyn Space<Self::VM>>) -> bool {
+        self.base().collection_required(self, space_full)
     }
 
     fn last_collection_was_exhaustive(&self) -> bool {

src/plan/markcompact/global.rs

@@ -163,8 +163,8 @@ impl<VM: VMBinding> Plan for MarkCompact<VM> {
             .add(crate::util::sanity::sanity_checker::ScheduleSanityGC::<Self>::new(self));
     }
 
-    fn collection_required(&self, space_full: bool, space: &dyn Space<Self::VM>) -> bool {
-        self.base().collection_required(self, space_full, space)
+    fn collection_required(&self, space_full: bool, _space: Option<&dyn Space<Self::VM>>) -> bool {
+        self.base().collection_required(self, space_full)
     }
 
     fn get_used_pages(&self) -> usize {

src/plan/marksweep/global.rs

@@ -72,8 +72,8 @@ impl<VM: VMBinding> Plan for MarkSweep<VM> {
         self.common.release(tls, true);
     }
 
-    fn collection_required(&self, space_full: bool, space: &dyn Space<Self::VM>) -> bool {
-        self.base().collection_required(self, space_full, space)
+    fn collection_required(&self, space_full: bool, _space: Option<&dyn Space<Self::VM>>) -> bool {
+        self.base().collection_required(self, space_full)
     }
 
     fn get_used_pages(&self) -> usize {

src/plan/nogc/global.rs

@@ -48,8 +48,8 @@ impl<VM: VMBinding> Plan for NoGC<VM> {
         self.nogc_space.init(vm_map);
     }
 
-    fn collection_required(&self, space_full: bool, space: &dyn Space<Self::VM>) -> bool {
-        self.base().collection_required(self, space_full, space)
+    fn collection_required(&self, space_full: bool, _space: Option<&dyn Space<Self::VM>>) -> bool {
+        self.base().collection_required(self, space_full)
     }
 
     fn base(&self) -> &BasePlan<VM> {

src/plan/pageprotect/global.rs

@@ -79,8 +79,8 @@ impl<VM: VMBinding> Plan for PageProtect<VM> {
         self.space.release(true);
     }
 
-    fn collection_required(&self, space_full: bool, space: &dyn Space<Self::VM>) -> bool {
-        self.base().collection_required(self, space_full, space)
+    fn collection_required(&self, space_full: bool, _space: Option<&dyn Space<Self::VM>>) -> bool {
+        self.base().collection_required(self, space_full)
     }
 
     fn get_used_pages(&self) -> usize {

src/plan/semispace/global.rs

@@ -110,8 +110,8 @@ impl<VM: VMBinding> Plan for SemiSpace<VM> {
         self.fromspace().release();
     }
 
-    fn collection_required(&self, space_full: bool, space: &dyn Space<Self::VM>) -> bool {
-        self.base().collection_required(self, space_full, space)
+    fn collection_required(&self, space_full: bool, _space: Option<&dyn Space<Self::VM>>) -> bool {
+        self.base().collection_required(self, space_full)
     }
 
     fn get_collection_reserved_pages(&self) -> usize {

src/policy/mallocspace/global.rs

@@ -6,7 +6,7 @@ use crate::policy::space::SFT;
 use crate::util::constants::BYTES_IN_PAGE;
 use crate::util::heap::layout::heap_layout::VMMap;
 use crate::util::heap::PageResource;
-use crate::util::malloc::*;
+use crate::util::malloc::malloc_ms_util::*;
 use crate::util::metadata::side_metadata::{
     bzero_metadata, SideMetadataContext, SideMetadataSanity, SideMetadataSpec,
 };
@@ -238,7 +238,7 @@ impl<VM: VMBinding> MallocSpace<VM> {
 
     pub fn alloc(&self, tls: VMThread, size: usize, align: usize, offset: isize) -> Address {
         // TODO: Should refactor this and Space.acquire()
-        if VM::VMActivePlan::global().poll(false, self) {
+        if VM::VMActivePlan::global().poll(false, Some(self)) {
             assert!(VM::VMActivePlan::is_mutator(tls), "Polling in GC worker");
             VM::VMCollection::block_for_gc(VMMutatorThread(tls));
             return unsafe { Address::zero() };

src/policy/space.rs

@@ -407,7 +407,7 @@ pub trait Space<VM: VMBinding>: 'static + SFT + Sync + Downcast {
         trace!("Pages reserved");
         trace!("Polling ..");
 
-        if should_poll && VM::VMActivePlan::global().poll(false, self.as_space()) {
+        if should_poll && VM::VMActivePlan::global().poll(false, Some(self.as_space())) {
             debug!("Collection required");
             assert!(allow_gc, "GC is not allowed here: collection is not initialized (did you call initialize_collection()?).");
             pr.clear_request(pages_reserved);
@@ -485,7 +485,7 @@ pub trait Space<VM: VMBinding>: 'static + SFT + Sync + Downcast {
                         "Physical allocation failed when GC is not allowed!"
                     );
 
-                    let gc_performed = VM::VMActivePlan::global().poll(true, self.as_space());
+                    let gc_performed = VM::VMActivePlan::global().poll(true, Some(self.as_space()));
                     debug_assert!(gc_performed, "GC not performed when forced.");
                     pr.clear_request(pages_reserved);
                     drop(lock); // drop the lock before block