Support moving

.github/scripts/ci-test-stdlib.sh

@@ -22,6 +22,8 @@ declare -a tests_to_skip=(
     "SparseArrays"
     # Running LinearAlgebra in a separate job
     "LinearAlgebra"
+    # Skipping Distributed tests
+    "Distributed"
 )
 # These tests need multiple workers.
 declare -a tests_with_multi_workers=(

.github/workflows/ci.yml

@@ -40,7 +40,7 @@ jobs:
       fail-fast: false
       matrix:
         gc_plan: [Immix]
-        moving: [Non_Moving]
+        moving: [Non_Moving, Moving]
     uses: ./.github/workflows/binding-tests.yml
     with:
       gc_plan: ${{ matrix.gc_plan }}

.gitignore

@@ -1,4 +1,5 @@
 .idea/
 julia/*.o
 julia/*.dbj.obj
-.vscode
+.vscode
+mmtk/src/julia_types.rs

mmtk/Cargo.toml

@@ -40,15 +40,17 @@ memoffset = "*"
 # ykstackmaps = { git = "https://github.com/udesou/ykstackmaps.git", branch = "udesou-master", version = "*" }
 
 [features]
-default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning"]
+# We must build with default features
+default = ["mmtk/vm_space", "julia_copy_stack", "mmtk/object_pinning", "mmtk/is_mmtk_object", "mmtk/vo_bit_access"]
 
-# Plans
+# Default features
+julia_copy_stack = []
+
+# Plans: choose one
 nogc = []
 immix = []
 stickyimmix = ["mmtk/sticky_immix_non_moving_nursery", "mmtk/immix_smaller_block"]
 marksweep = []
-object_pinning = ["mmtk/object_pinning"]
 
 # This feature disables moving
 non_moving = ["mmtk/immix_non_moving", "mmtk/immix_smaller_block"]
-julia_copy_stack = []

mmtk/api/mmtk.h

@@ -49,7 +49,9 @@ extern int mmtk_object_is_managed_by_mmtk(void* addr);
 extern void mmtk_runtime_panic(void);
 extern void mmtk_unreachable(void);
 extern unsigned char mmtk_pin_object(void* obj);
-extern bool mmtk_is_pinned(void* obj);
+extern bool mmtk_is_object_pinned(void* obj);
+extern unsigned char mmtk_pin_pointer(void* ptr);
+extern bool mmtk_is_pointer_pinned(void* ptr);
 extern const char* get_mmtk_version(void);
 
 extern void mmtk_set_vm_space(void* addr, size_t size);
@@ -85,7 +87,7 @@ extern void mmtk_block_thread_for_gc(void);
 extern void* mmtk_new_mutator_iterator(void);
 extern void* mmtk_get_next_mutator_tls(void*);
 extern void* mmtk_close_mutator_iterator(void*);
-
+extern void mmtk_store_obj_size_c(void* obj, size_t size);
 
 /**
  * VM Accounting
@@ -106,6 +108,11 @@ extern void mmtk_add_phantom_candidate(void* ref);
 extern void mmtk_harness_begin(void *tls);
 extern void mmtk_harness_end(void);
 
+/**
+ * Copying
+ */
+extern const void* MMTK_SIDE_VO_BIT_BASE_ADDRESS;
+
 #ifdef __cplusplus
 }
 #endif

mmtk/build.rs

@@ -54,6 +54,7 @@ fn main() {
         let bindings = bindgen::Builder::default()
             .header(format!("{}/src/julia.h", julia_dir))
             .header(format!("{}/src/julia_internal.h", julia_dir))
+            .header(format!("{}/src/gc-common.h", julia_dir))
             // Including the paths to depending .h files
             .clang_arg("-I")
             .clang_arg(format!("{}/mmtk/api", mmtk_dir))
@@ -77,6 +78,7 @@ fn main() {
             .allowlist_item("jl_bt_element_t")
             .allowlist_item("jl_taggedvalue_t")
             .allowlist_item("MMTkMutatorContext")
+            .allowlist_item("_bigval_t")
             // --opaque-type MMTkMutatorContext
             .opaque_type("MMTkMutatorContext")
             // compile using c++

mmtk/src/api.rs

@@ -350,6 +350,7 @@ pub extern "C" fn mmtk_set_vm_space(start: Address, size: usize) {
 
     #[cfg(feature = "stickyimmix")]
     set_side_log_bit_for_region(start, size);
+    set_side_vo_bit_for_region(start, size);
 }
 
 #[no_mangle]
@@ -381,6 +382,8 @@ pub extern "C" fn mmtk_memory_region_copy(
 pub extern "C" fn mmtk_immortal_region_post_alloc(start: Address, size: usize) {
     #[cfg(feature = "stickyimmix")]
     set_side_log_bit_for_region(start, size);
+
+    set_side_vo_bit_for_region(start, size);
 }
 
 #[cfg(feature = "stickyimmix")]
@@ -393,6 +396,19 @@ fn set_side_log_bit_for_region(start: Address, size: usize) {
     }
 }
 
+// We have to set VO bit even if this is a non_moving build. Otherwise, assertions in mmtk-core
+// will complain about seeing objects without VO bit.
+fn set_side_vo_bit_for_region(start: Address, size: usize) {
+    debug!(
+        "Bulk set VO bit {} to {} ({} bytes)",
+        start,
+        start + size,
+        size
+    );
+
+    mmtk::util::metadata::vo_bit::VO_BIT_SIDE_METADATA_SPEC.bset_metadata(start, size);
+}
+
 #[no_mangle]
 pub extern "C" fn mmtk_object_reference_write_post(
     mutator: *mut Mutator<JuliaVM>,
@@ -427,6 +443,11 @@ pub extern "C" fn mmtk_object_reference_write_slow(
 pub static MMTK_SIDE_LOG_BIT_BASE_ADDRESS: Address =
     mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS;
 
+/// VO bit base address
+#[no_mangle]
+pub static MMTK_SIDE_VO_BIT_BASE_ADDRESS: Address =
+    mmtk::util::metadata::side_metadata::VO_BIT_SIDE_METADATA_ADDR;
+
 #[no_mangle]
 pub extern "C" fn mmtk_object_is_managed_by_mmtk(addr: usize) -> bool {
     crate::api::mmtk_is_mapped_address(unsafe { Address::from_usize(addr) })
@@ -464,57 +485,133 @@ pub extern "C" fn mmtk_get_obj_size(obj: ObjectReference) -> usize {
     }
 }
 
-#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))]
+#[allow(unused_variables)]
+fn assert_is_object(object: ObjectReference) {
+    // The checks are quite expensive. Dont run it in normal builds.
+    const ASSERT_OBJECT: bool = false;
+    if ASSERT_OBJECT {
+        #[cfg(debug_assertions)]
+        {
+            use crate::object_model::{is_object_in_immixspace, is_object_in_los};
+            if !mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) {
+                panic!("{} is not managed by MMTk", object);
+            }
+            if !is_object_in_immixspace(&object) && !is_object_in_los(&object) {
+                // We will use VO bit in the following check. But if the object is not in immix space or LOS, we cannot do the check.
+                return;
+            }
+            if !object
+                .to_raw_address()
+                .is_aligned_to(ObjectReference::ALIGNMENT)
+            {
+                panic!(
+                    "{} is not aligned, it cannot be an object reference",
+                    object
+                )
+            }
+            if memory_manager::is_mmtk_object(object.to_raw_address()).is_none() {
+                error!("{} is not an object", object);
+                if let Some(base_ref) = memory_manager::find_object_from_internal_pointer(
+                    object.to_raw_address(),
+                    usize::MAX,
+                ) {
+                    panic!("{} is an internal pointer of {}", object, base_ref);
+                } else {
+                    panic!(
+                        "{} is not recognised as an object reference, or an internal reference",
+                        object
+                    );
+                }
+            }
+        }
+    }
+}
 #[no_mangle]
 pub extern "C" fn mmtk_pin_object(object: ObjectReference) -> bool {
-    // We may in the future replace this with a check for the immix space (bound check), which should be much cheaper.
-    if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) {
-        memory_manager::pin_object(object)
-    } else {
-        debug!("Object is not managed by mmtk - (un)pinning it via this function isn't supported.");
-        false
-    }
+    assert_is_object(object);
+    crate::early_return_for_non_moving_build!(false);
+    memory_manager::pin_object(object)
 }
 
-#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))]
 #[no_mangle]
 pub extern "C" fn mmtk_unpin_object(object: ObjectReference) -> bool {
-    if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) {
-        memory_manager::unpin_object(object)
-    } else {
-        debug!("Object is not managed by mmtk - (un)pinning it via this function isn't supported.");
-        false
-    }
+    assert_is_object(object);
+    crate::early_return_for_non_moving_build!(false);
+    memory_manager::unpin_object(object)
+}
+
+#[no_mangle]
+pub extern "C" fn mmtk_is_object_pinned(object: ObjectReference) -> bool {
+    assert_is_object(object);
+    crate::early_return_for_non_moving_build!(false);
+
+    memory_manager::is_pinned(object)
+}
+
+macro_rules! handle_potential_internal_pointer {
+    ($func: path, $addr: expr) => {{
+        if $addr.is_aligned_to(ObjectReference::ALIGNMENT) {
+            if let Some(obj) = memory_manager::is_mmtk_object($addr) {
+                return $func(obj);
+            }
+        }
+        let maybe_objref = memory_manager::find_object_from_internal_pointer($addr, usize::MAX);
+        if let Some(obj) = maybe_objref {
+            trace!(
+                "Attempt to pin {:?}, but it is an internal reference of {:?}",
+                $addr,
+                obj
+            );
+            $func(obj)
+        } else {
+            warn!(
+                "Attempt to pin {:?}, but it is not recognised as a object",
+                $addr
+            );
+            false
+        }
+    }};
 }
 
-#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))]
 #[no_mangle]
-pub extern "C" fn mmtk_is_pinned(object: ObjectReference) -> bool {
-    if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) {
-        memory_manager::is_pinned(object)
+pub extern "C" fn mmtk_pin_pointer(addr: Address) -> bool {
+    crate::early_return_for_non_moving_build!(false);
+
+    if crate::object_model::is_addr_in_immixspace(addr) {
+        handle_potential_internal_pointer!(memory_manager::pin_object, addr)
     } else {
-        debug!("Object is not managed by mmtk - checking via this function isn't supported.");
+        debug!("Object is not in Immix space. MMTk will not move the object. No need to pin the object.");
         false
     }
 }
 
-// If the `non-moving` feature is selected, pinning/unpinning is a noop and simply returns false
-#[cfg(all(feature = "object_pinning", feature = "non_moving"))]
 #[no_mangle]
-pub extern "C" fn mmtk_pin_object(_object: ObjectReference) -> bool {
-    false
-}
+pub extern "C" fn mmtk_unpin_pointer(addr: Address) -> bool {
+    crate::early_return_for_non_moving_build!(false);
 
-#[cfg(all(feature = "object_pinning", feature = "non_moving"))]
-#[no_mangle]
-pub extern "C" fn mmtk_unpin_object(_object: ObjectReference) -> bool {
-    false
+    if crate::object_model::is_addr_in_immixspace(addr) {
+        handle_potential_internal_pointer!(memory_manager::unpin_object, addr)
+    } else {
+        debug!("Object is not in Immix space. MMTk will not move the object. No need to unpin the object.");
+        false
+    }
 }
 
-#[cfg(all(feature = "object_pinning", feature = "non_moving"))]
 #[no_mangle]
-pub extern "C" fn mmtk_is_pinned(_object: ObjectReference) -> bool {
-    false
+pub extern "C" fn mmtk_is_pointer_pinned(addr: Address) -> bool {
+    crate::early_return_for_non_moving_build!(false);
+
+    if crate::object_model::is_addr_in_immixspace(addr) {
+        handle_potential_internal_pointer!(memory_manager::is_pinned, addr)
+    } else if !mmtk_object_is_managed_by_mmtk(addr.as_usize()) {
+        debug!(
+            "Object is not in Immix space. MMTk will not move the object. We assume it is pinned."
+        );
+        true
+    } else {
+        debug!("Object is not managed by mmtk - checking pinning state via this function isn't supported.");
+        false
+    }
 }
 
 #[no_mangle]