diff --git a/.github/scripts/Make.user b/.github/scripts/Make.user index 59ab603..28495e8 100644 --- a/.github/scripts/Make.user +++ b/.github/scripts/Make.user @@ -1,3 +1,4 @@ FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1 USE_BINARYBUILDER_MMTK_JULIA=0 +WITH_THIRD_PARTY_GC=mmtk diff --git a/.github/scripts/ci-test-LinearAlgebra.sh b/.github/scripts/ci-test-LinearAlgebra.sh index 5d099dd..ef6c143 100755 --- a/.github/scripts/ci-test-LinearAlgebra.sh +++ b/.github/scripts/ci-test-LinearAlgebra.sh @@ -5,6 +5,11 @@ set -e . $(dirname "$0")/common.sh +# plan to use +plan=$1 + +export MMTK_PLAN=$plan + export MMTK_MAX_HSIZE_G=8 total_mem=$(free -m | awk '/^Mem:/ {print $2}') mem_threshold=512 # use 0.5Gb as a threshold for the max rss based on the total free memory diff --git a/.github/scripts/ci-test-other.sh b/.github/scripts/ci-test-other.sh index 09e04d8..662ad29 100755 --- a/.github/scripts/ci-test-other.sh +++ b/.github/scripts/ci-test-other.sh @@ -2,6 +2,11 @@ set -xe . $(dirname "$0")/common.sh +# plan to use +plan=$1 + +export MMTK_PLAN=$plan + # Get all the tests CHOOSE_TESTS_JL_PATH=$JULIA_PATH/test/choosetests.jl CHOOSE_TESTS_JL_CONTENT=`cat $CHOOSE_TESTS_JL_PATH` @@ -28,6 +33,12 @@ if [[ $CHOOSE_TESTS_JL_CONTENT =~ $REGEX_PATTERN ]]; then continue fi + if [[ $test =~ "compiler_extras" ]]; then + # Skipping compiler_extras for now + echo "-> Skip compiler_extras" + continue + fi + echo "-> Run" ci_run_jl_test $test fi diff --git a/.github/scripts/ci-test-patching.sh b/.github/scripts/ci-test-patching.sh index 8597684..c03b067 100755 --- a/.github/scripts/ci-test-patching.sh +++ b/.github/scripts/ci-test-patching.sh @@ -52,8 +52,10 @@ declare -a tests_to_skip=( '@test contains(sshot, "redact_this")' "$JULIA_PATH/stdlib/Profile/test/runtests.jl" # This test checks GC logging - '@test occursin("GC: pause", read(tmppath, String))' "$JULIA_PATH/test/misc.jl" + '@test !isempty(test_complete("?("))' "$JULIA_PATH/stdlib/REPL/test/replcompletions.jl" + # This test seems to be failing in the new version of Julia + '@test occursin("int.jl", code)' "$JULIA_PATH/test/cmdlineargs.jl" # These tests check for the number of stock GC threads (which we set to 0 with mmtk) '@test (cpu_threads == 1 ? "1" : string(div(cpu_threads, 2))) ==' "$JULIA_PATH/test/cmdlineargs.jl" '@test read(`$exename --gcthreads=2 -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl" diff --git a/.github/scripts/ci-test-stdlib.sh b/.github/scripts/ci-test-stdlib.sh index f285876..5669ada 100755 --- a/.github/scripts/ci-test-stdlib.sh +++ b/.github/scripts/ci-test-stdlib.sh @@ -5,6 +5,11 @@ set -e . $(dirname "$0")/common.sh +# plan to use +plan=$1 + +export MMTK_PLAN=$plan + # These tests seem to fail. We skip them. declare -a tests_to_skip=( # Test Failed at /home/runner/work/mmtk-julia/mmtk-julia/vm/julia/usr/share/julia/stdlib/v1.8/Dates/test/io.jl:45 @@ -22,6 +27,8 @@ declare -a tests_to_skip=( "SparseArrays" # Running LinearAlgebra in a separate job "LinearAlgebra" + # Skipping distributed tests because of https://github.com/JuliaLang/Distributed.jl/issues/132 + "Distributed" ) # These tests need multiple workers. declare -a tests_with_multi_workers=( diff --git a/.github/scripts/ci-test.sh b/.github/scripts/ci-test.sh index 24ccc26..0d007cf 100755 --- a/.github/scripts/ci-test.sh +++ b/.github/scripts/ci-test.sh @@ -13,6 +13,8 @@ cur=$(realpath $(dirname "$0")) cd $cur ./ci-build.sh release Immix +export MMTK_PLAN=Immix + # Use release build to run tests cd $cur ./ci-test-gc-core.sh diff --git a/.github/workflows/binding-tests.yml b/.github/workflows/binding-tests.yml index 1f595fc..403fcdf 100644 --- a/.github/workflows/binding-tests.yml +++ b/.github/workflows/binding-tests.yml @@ -109,7 +109,7 @@ jobs: $JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()" - name: Test Julia run: | - ./.github/scripts/ci-test-other.sh + ./.github/scripts/ci-test-other.sh ${{ inputs.gc_plan }} build-test-stdlib: runs-on: ubuntu-22.04 @@ -135,7 +135,7 @@ jobs: $JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()" - name: Test Julia run: | - ./.github/scripts/ci-test-stdlib.sh + ./.github/scripts/ci-test-stdlib.sh ${{ inputs.gc_plan }} build-test-LinearAlgebra: runs-on: ubuntu-22.04 @@ -158,4 +158,4 @@ jobs: $JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()" - name: Test Julia run: | - ./.github/scripts/ci-test-LinearAlgebra.sh + ./.github/scripts/ci-test-LinearAlgebra.sh ${{ inputs.gc_plan }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 467f73b..4baa88d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ on: - v1.8.2\+RAI - v1.9.2\+RAI - dev + - upstream-ready/sticky concurrency: # Cancels pending runs when a PR gets updated. diff --git a/mmtk/build.rs b/mmtk/build.rs index 3871e37..68891ce 100644 --- a/mmtk/build.rs +++ b/mmtk/build.rs @@ -84,7 +84,9 @@ fn main() { .clang_arg("c++") .clang_arg("-std=c++14") // using MMTK types - .clang_arg("-DMMTK_GC") + .clang_arg("-DWITH_THIRD_PARTY_HEAP=1") + // using sticky, but it should not matter for the FFI bindings + .clang_arg("-DMMTK_PLAN_STICKYIMMIX") // Finish the builder and generate the bindings. .generate() // Unwrap the Result and panic on failure. diff --git a/mmtk/src/julia_scanning.rs b/mmtk/src/julia_scanning.rs index ca9a6de..c3a6de2 100644 --- a/mmtk/src/julia_scanning.rs +++ b/mmtk/src/julia_scanning.rs @@ -132,6 +132,18 @@ pub unsafe fn scan_julia_object>(obj: Address, clos } process_slot(closure, Address::from_ptr(parent_slot)); + let usings_backeges_slot = ::std::ptr::addr_of!((*m).usings_backedges); + if PRINT_OBJ_TYPE { + println!(" - scan parent: {:?}\n", usings_backeges_slot); + } + process_slot(closure, Address::from_ptr(usings_backeges_slot)); + + let scanned_methods_slot = ::std::ptr::addr_of!((*m).scanned_methods); + if PRINT_OBJ_TYPE { + println!(" - scan parent: {:?}\n", scanned_methods_slot); + } + process_slot(closure, Address::from_ptr(scanned_methods_slot)); + // m.usings.items may be inlined in the module when the array list size <= AL_N_INLINE (cf. arraylist_new) // In that case it may be an mmtk object and not a malloced address. // If it is an mmtk object, (*m).usings.items will then be an internal pointer to the module @@ -311,15 +323,6 @@ pub unsafe fn scan_julia_object>(obj: Address, clos let layout = (*vt).layout; let npointers = (*layout).npointers; if npointers != 0 { - if vt == jl_binding_partition_type { - let bpart_ptr = obj.to_mut_ptr::(); - let restriction = (*bpart_ptr).restriction._M_i; - let offset = mmtk_decode_restriction_value(restriction); - let slot = Address::from_ptr(::std::ptr::addr_of!((*bpart_ptr).restriction)); - if restriction - offset != 0 { - process_offset_slot(closure, slot, offset); - } - } debug_assert!( (*layout).nfields > 0 && (*layout).fielddesc_type_custom() != 3, "opaque types should have been handled specially" @@ -520,27 +523,6 @@ pub fn process_slot>(closure: &mut EV, slot: Addres closure.visit_slot(JuliaVMSlot::Simple(simple_slot)); } -// This is based on the function decode_restriction_value from julia_internal.h. -// However, since MMTk uses the slot to load the object, we get the offset from pku using -// that offset to pass to process_offset_edge and get the right address. -#[inline(always)] -pub fn mmtk_decode_restriction_value(pku: usize) -> usize { - #[cfg(target_pointer_width = "64")] - { - // need to apply (pku & ~0x7) to get the object to be traced - // so we use (pku & 0x7) to get the offset from the object - // and pass it to process_offset_slot - pku & 0x7 - } - - #[cfg(not(target_pointer_width = "64"))] - { - // when not #ifdef _P64 we simply return pku.val - // i.e., the offset is 0, since val is the first field - 0 - } -} - #[inline(always)] pub fn process_offset_slot>( closure: &mut EV, diff --git a/mmtk/src/julia_types.rs b/mmtk/src/julia_types.rs index 530c2d9..cb192cd 100644 --- a/mmtk/src/julia_types.rs +++ b/mmtk/src/julia_types.rs @@ -248,12 +248,12 @@ const _: () = { pub type sigjmp_buf = [__jmp_buf_tag; 1usize]; pub type jl_taggedvalue_t = _jl_taggedvalue_t; pub type jl_ptls_t = *mut _jl_tls_states_t; +pub type sig_atomic_t = __sig_atomic_t; #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct _jl_value_t { _unused: [u8; 0], } -pub type sig_atomic_t = __sig_atomic_t; pub type jl_value_t = _jl_value_t; #[repr(C)] #[repr(align(8))] @@ -930,19 +930,20 @@ const _: () = { }; #[repr(C)] #[derive(Debug, Copy, Clone)] -pub struct jl_genericmemory_t { +pub struct _jl_genericmemory_t { pub length: usize, pub ptr: *mut ::std::os::raw::c_void, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of jl_genericmemory_t"][::std::mem::size_of::() - 16usize]; - ["Alignment of jl_genericmemory_t"][::std::mem::align_of::() - 8usize]; - ["Offset of field: jl_genericmemory_t::length"] - [::std::mem::offset_of!(jl_genericmemory_t, length) - 0usize]; - ["Offset of field: jl_genericmemory_t::ptr"] - [::std::mem::offset_of!(jl_genericmemory_t, ptr) - 8usize]; + ["Size of _jl_genericmemory_t"][::std::mem::size_of::<_jl_genericmemory_t>() - 16usize]; + ["Alignment of _jl_genericmemory_t"][::std::mem::align_of::<_jl_genericmemory_t>() - 8usize]; + ["Offset of field: _jl_genericmemory_t::length"] + [::std::mem::offset_of!(_jl_genericmemory_t, length) - 0usize]; + ["Offset of field: _jl_genericmemory_t::ptr"] + [::std::mem::offset_of!(_jl_genericmemory_t, ptr) - 8usize]; }; +pub type jl_genericmemory_t = _jl_genericmemory_t; #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct jl_genericmemoryref_t { @@ -1936,15 +1937,14 @@ const _: () = { [::std::mem::offset_of!(_jl_weakref_t, value) - 0usize]; }; pub type jl_weakref_t = _jl_weakref_t; -pub type jl_ptr_kind_union_t = usize; #[repr(C)] #[derive(Debug)] pub struct _jl_binding_partition_t { - pub restriction: std_atomic, + pub restriction: *mut jl_value_t, pub min_world: usize, pub max_world: std_atomic, pub next: u64, - pub reserved: usize, + pub kind: usize, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { @@ -1959,8 +1959,8 @@ const _: () = { [::std::mem::offset_of!(_jl_binding_partition_t, max_world) - 16usize]; ["Offset of field: _jl_binding_partition_t::next"] [::std::mem::offset_of!(_jl_binding_partition_t, next) - 24usize]; - ["Offset of field: _jl_binding_partition_t::reserved"] - [::std::mem::offset_of!(_jl_binding_partition_t, reserved) - 32usize]; + ["Offset of field: _jl_binding_partition_t::kind"] + [::std::mem::offset_of!(_jl_binding_partition_t, kind) - 32usize]; }; pub type jl_binding_partition_t = _jl_binding_partition_t; #[repr(C)] @@ -1985,6 +1985,8 @@ pub struct _jl_module_t { pub bindingkeyset: u64, pub file: *mut jl_sym_t, pub line: i32, + pub usings_backedges: *mut jl_value_t, + pub scanned_methods: *mut jl_value_t, pub usings: arraylist_t, pub build_id: jl_uuid_t, pub uuid: jl_uuid_t, @@ -1995,12 +1997,13 @@ pub struct _jl_module_t { pub infer: i8, pub istopmod: u8, pub max_methods: i8, + pub export_set_changed_since_require_world: std_atomic, pub lock: jl_mutex_t, pub hash: isize, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of _jl_module_t"][::std::mem::size_of::<_jl_module_t>() - 376usize]; + ["Size of _jl_module_t"][::std::mem::size_of::<_jl_module_t>() - 392usize]; ["Alignment of _jl_module_t"][::std::mem::align_of::<_jl_module_t>() - 8usize]; ["Offset of field: _jl_module_t::name"][::std::mem::offset_of!(_jl_module_t, name) - 0usize]; ["Offset of field: _jl_module_t::parent"] @@ -2011,27 +2014,33 @@ const _: () = { [::std::mem::offset_of!(_jl_module_t, bindingkeyset) - 24usize]; ["Offset of field: _jl_module_t::file"][::std::mem::offset_of!(_jl_module_t, file) - 32usize]; ["Offset of field: _jl_module_t::line"][::std::mem::offset_of!(_jl_module_t, line) - 40usize]; + ["Offset of field: _jl_module_t::usings_backedges"] + [::std::mem::offset_of!(_jl_module_t, usings_backedges) - 48usize]; + ["Offset of field: _jl_module_t::scanned_methods"] + [::std::mem::offset_of!(_jl_module_t, scanned_methods) - 56usize]; ["Offset of field: _jl_module_t::usings"] - [::std::mem::offset_of!(_jl_module_t, usings) - 48usize]; + [::std::mem::offset_of!(_jl_module_t, usings) - 64usize]; ["Offset of field: _jl_module_t::build_id"] - [::std::mem::offset_of!(_jl_module_t, build_id) - 304usize]; - ["Offset of field: _jl_module_t::uuid"][::std::mem::offset_of!(_jl_module_t, uuid) - 320usize]; + [::std::mem::offset_of!(_jl_module_t, build_id) - 320usize]; + ["Offset of field: _jl_module_t::uuid"][::std::mem::offset_of!(_jl_module_t, uuid) - 336usize]; ["Offset of field: _jl_module_t::counter"] - [::std::mem::offset_of!(_jl_module_t, counter) - 336usize]; + [::std::mem::offset_of!(_jl_module_t, counter) - 352usize]; ["Offset of field: _jl_module_t::nospecialize"] - [::std::mem::offset_of!(_jl_module_t, nospecialize) - 340usize]; + [::std::mem::offset_of!(_jl_module_t, nospecialize) - 356usize]; ["Offset of field: _jl_module_t::optlevel"] - [::std::mem::offset_of!(_jl_module_t, optlevel) - 344usize]; + [::std::mem::offset_of!(_jl_module_t, optlevel) - 360usize]; ["Offset of field: _jl_module_t::compile"] - [::std::mem::offset_of!(_jl_module_t, compile) - 345usize]; + [::std::mem::offset_of!(_jl_module_t, compile) - 361usize]; ["Offset of field: _jl_module_t::infer"] - [::std::mem::offset_of!(_jl_module_t, infer) - 346usize]; + [::std::mem::offset_of!(_jl_module_t, infer) - 362usize]; ["Offset of field: _jl_module_t::istopmod"] - [::std::mem::offset_of!(_jl_module_t, istopmod) - 347usize]; + [::std::mem::offset_of!(_jl_module_t, istopmod) - 363usize]; ["Offset of field: _jl_module_t::max_methods"] - [::std::mem::offset_of!(_jl_module_t, max_methods) - 348usize]; - ["Offset of field: _jl_module_t::lock"][::std::mem::offset_of!(_jl_module_t, lock) - 352usize]; - ["Offset of field: _jl_module_t::hash"][::std::mem::offset_of!(_jl_module_t, hash) - 368usize]; + [::std::mem::offset_of!(_jl_module_t, max_methods) - 364usize]; + ["Offset of field: _jl_module_t::export_set_changed_since_require_world"] + [::std::mem::offset_of!(_jl_module_t, export_set_changed_since_require_world) - 365usize]; + ["Offset of field: _jl_module_t::lock"][::std::mem::offset_of!(_jl_module_t, lock) - 368usize]; + ["Offset of field: _jl_module_t::hash"][::std::mem::offset_of!(_jl_module_t, hash) - 384usize]; }; pub type jl_module_t = _jl_module_t; #[repr(C)] @@ -2337,13 +2346,6 @@ const _: () = { [::std::mem::align_of::>() - 2usize]; }; #[allow(clippy::unnecessary_operation, clippy::identity_op)] -const _: () = { - ["Size of template specialization: std_atomic_open0_jl_ptr_kind_union_t_close0"] - [::std::mem::size_of::>() - 8usize]; - ["Align of template specialization: std_atomic_open0_jl_ptr_kind_union_t_close0"] - [::std::mem::align_of::>() - 8usize]; -}; -#[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { ["Size of template specialization: std_atomic_open0_size_t_close0"] [::std::mem::size_of::>() - 8usize]; @@ -2358,6 +2360,13 @@ const _: () = { [::std::mem::align_of::>() - 4usize]; }; #[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of template specialization: std_atomic_open0_int8_t_close0"] + [::std::mem::size_of::>() - 1usize]; + ["Align of template specialization: std_atomic_open0_int8_t_close0"] + [::std::mem::align_of::>() - 1usize]; +}; +#[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { ["Size of template specialization: std_atomic_open0_intptr_t_close0"] [::std::mem::size_of::>() - 8usize];