From 3e4e8f2040f297d81cc56cdc571eb248a94a94ef Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Thu, 13 Feb 2025 18:42:19 +0000 Subject: [PATCH] Migrate all in-repo crates to edition 2024 Most of this just means wrapping the bodies of `mem` functions in `unsafe` and updating `no_mangle` / `extern`. --- Cargo.toml | 2 +- crates/panic-handler/Cargo.toml | 2 +- examples/intrinsics.rs | 18 +-- src/arm.rs | 110 +++++++------- src/macros.rs | 18 +-- src/mem/impls.rs | 113 +++++++------- src/mem/mod.rs | 32 ++-- src/mem/x86_64.rs | 257 +++++++++++++++++--------------- src/probestack.rs | 2 +- src/x86_64.rs | 44 +++--- testcrate/Cargo.toml | 2 +- testcrate/src/bench.rs | 2 +- testcrate/tests/aeabi_memclr.rs | 2 +- testcrate/tests/aeabi_memcpy.rs | 2 +- testcrate/tests/aeabi_memset.rs | 2 +- 15 files changed, 327 insertions(+), 281 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c93ca563..baa1d0a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ readme = "README.md" repository = "https://github.com/rust-lang/compiler-builtins" homepage = "https://github.com/rust-lang/compiler-builtins" documentation = "https://docs.rs/compiler_builtins" -edition = "2021" +edition = "2024" description = """ Compiler intrinsics used by the Rust compiler. Also available for other targets if necessary! diff --git a/crates/panic-handler/Cargo.toml b/crates/panic-handler/Cargo.toml index 2ad85840..96b83eaa 100644 --- a/crates/panic-handler/Cargo.toml +++ b/crates/panic-handler/Cargo.toml @@ -2,7 +2,7 @@ name = "panic-handler" version = "0.1.0" authors = ["Alex Crichton "] -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/examples/intrinsics.rs b/examples/intrinsics.rs index 59a70e20..3818dc62 100644 --- a/examples/intrinsics.rs +++ b/examples/intrinsics.rs @@ -17,7 +17,7 @@ extern crate panic_handler; #[cfg(all(not(thumb), not(windows), not(target_arch = "wasm32")))] #[link(name = "c")] -extern "C" {} +unsafe extern "C" {} // Every function in this module maps will be lowered to an intrinsic by LLVM, if the platform // doesn't have native support for the operation used in the function. ARM has a naming convention @@ -626,7 +626,7 @@ fn run() { something_with_a_dtor(&|| assert_eq!(bb(1), 1)); - extern "C" { + unsafe extern "C" { fn rust_begin_unwind(x: usize); } @@ -647,14 +647,14 @@ fn something_with_a_dtor(f: &dyn Fn()) { f(); } -#[no_mangle] +#[unsafe(no_mangle)] #[cfg(not(thumb))] fn main(_argc: core::ffi::c_int, _argv: *const *const u8) -> core::ffi::c_int { run(); 0 } -#[no_mangle] +#[unsafe(no_mangle)] #[cfg(thumb)] pub fn _start() -> ! { run(); @@ -664,23 +664,23 @@ pub fn _start() -> ! { #[cfg(windows)] #[link(name = "kernel32")] #[link(name = "msvcrt")] -extern "C" {} +unsafe extern "C" {} // ARM targets need these symbols -#[no_mangle] +#[unsafe(no_mangle)] pub fn __aeabi_unwind_cpp_pr0() {} -#[no_mangle] +#[unsafe(no_mangle)] pub fn __aeabi_unwind_cpp_pr1() {} #[cfg(not(windows))] #[allow(non_snake_case)] -#[no_mangle] +#[unsafe(no_mangle)] pub fn _Unwind_Resume() {} #[cfg(not(windows))] #[lang = "eh_personality"] -#[no_mangle] +#[unsafe(no_mangle)] pub extern "C" fn eh_personality() {} #[cfg(all(windows, target_env = "gnu"))] diff --git a/src/arm.rs b/src/arm.rs index 9e660839..b836ee54 100644 --- a/src/arm.rs +++ b/src/arm.rs @@ -23,64 +23,72 @@ intrinsics! { #[naked] #[cfg(not(target_env = "msvc"))] pub unsafe extern "C" fn __aeabi_uidivmod() { - core::arch::naked_asm!( - "push {{lr}}", - "sub sp, sp, #4", - "mov r2, sp", - bl!("__udivmodsi4"), - "ldr r1, [sp]", - "add sp, sp, #4", - "pop {{pc}}", - ); + unsafe { + core::arch::naked_asm!( + "push {{lr}}", + "sub sp, sp, #4", + "mov r2, sp", + bl!("__udivmodsi4"), + "ldr r1, [sp]", + "add sp, sp, #4", + "pop {{pc}}", + ); + } } #[naked] pub unsafe extern "C" fn __aeabi_uldivmod() { - core::arch::naked_asm!( - "push {{r4, lr}}", - "sub sp, sp, #16", - "add r4, sp, #8", - "str r4, [sp]", - bl!("__udivmoddi4"), - "ldr r2, [sp, #8]", - "ldr r3, [sp, #12]", - "add sp, sp, #16", - "pop {{r4, pc}}", - ); + unsafe { + core::arch::naked_asm!( + "push {{r4, lr}}", + "sub sp, sp, #16", + "add r4, sp, #8", + "str r4, [sp]", + bl!("__udivmoddi4"), + "ldr r2, [sp, #8]", + "ldr r3, [sp, #12]", + "add sp, sp, #16", + "pop {{r4, pc}}", + ); + } } #[naked] pub unsafe extern "C" fn __aeabi_idivmod() { - core::arch::naked_asm!( - "push {{r0, r1, r4, lr}}", - bl!("__aeabi_idiv"), - "pop {{r1, r2}}", - "muls r2, r2, r0", - "subs r1, r1, r2", - "pop {{r4, pc}}", - ); + unsafe { + core::arch::naked_asm!( + "push {{r0, r1, r4, lr}}", + bl!("__aeabi_idiv"), + "pop {{r1, r2}}", + "muls r2, r2, r0", + "subs r1, r1, r2", + "pop {{r4, pc}}", + ); + } } #[naked] pub unsafe extern "C" fn __aeabi_ldivmod() { - core::arch::naked_asm!( - "push {{r4, lr}}", - "sub sp, sp, #16", - "add r4, sp, #8", - "str r4, [sp]", - bl!("__divmoddi4"), - "ldr r2, [sp, #8]", - "ldr r3, [sp, #12]", - "add sp, sp, #16", - "pop {{r4, pc}}", - ); + unsafe { + core::arch::naked_asm!( + "push {{r4, lr}}", + "sub sp, sp, #16", + "add r4, sp, #8", + "str r4, [sp]", + bl!("__divmoddi4"), + "ldr r2, [sp, #8]", + "ldr r3, [sp, #12]", + "add sp, sp, #16", + "pop {{r4, pc}}", + ); + } } // FIXME: The `*4` and `*8` variants should be defined as aliases. #[cfg(not(target_vendor = "apple"))] pub unsafe extern "aapcs" fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize) { - crate::mem::memcpy(dest, src, n); + unsafe { crate::mem::memcpy(dest, src, n) }; } #[cfg(not(target_vendor = "apple"))] @@ -97,33 +105,33 @@ intrinsics! { n -= 4; } - __aeabi_memcpy(dest as *mut u8, src as *const u8, n); + unsafe { __aeabi_memcpy(dest as *mut u8, src as *const u8, n) }; } #[cfg(not(target_vendor = "apple"))] pub unsafe extern "aapcs" fn __aeabi_memcpy8(dest: *mut u8, src: *const u8, n: usize) { - __aeabi_memcpy4(dest, src, n); + unsafe { __aeabi_memcpy4(dest, src, n) }; } #[cfg(not(target_vendor = "apple"))] pub unsafe extern "aapcs" fn __aeabi_memmove(dest: *mut u8, src: *const u8, n: usize) { - crate::mem::memmove(dest, src, n); + unsafe { crate::mem::memmove(dest, src, n) }; } #[cfg(not(any(target_vendor = "apple", target_env = "msvc")))] pub unsafe extern "aapcs" fn __aeabi_memmove4(dest: *mut u8, src: *const u8, n: usize) { - __aeabi_memmove(dest, src, n); + unsafe { __aeabi_memmove(dest, src, n) }; } #[cfg(not(any(target_vendor = "apple", target_env = "msvc")))] pub unsafe extern "aapcs" fn __aeabi_memmove8(dest: *mut u8, src: *const u8, n: usize) { - __aeabi_memmove(dest, src, n); + unsafe { __aeabi_memmove(dest, src, n) }; } #[cfg(not(target_vendor = "apple"))] pub unsafe extern "aapcs" fn __aeabi_memset(dest: *mut u8, n: usize, c: i32) { // Note the different argument order - crate::mem::memset(dest, c, n); + unsafe { crate::mem::memset(dest, c, n) }; } #[cfg(not(target_vendor = "apple"))] @@ -140,26 +148,26 @@ intrinsics! { n -= 4; } - __aeabi_memset(dest as *mut u8, n, byte as i32); + unsafe { __aeabi_memset(dest as *mut u8, n, byte as i32) }; } #[cfg(not(target_vendor = "apple"))] pub unsafe extern "aapcs" fn __aeabi_memset8(dest: *mut u8, n: usize, c: i32) { - __aeabi_memset4(dest, n, c); + unsafe { __aeabi_memset4(dest, n, c) }; } #[cfg(not(target_vendor = "apple"))] pub unsafe extern "aapcs" fn __aeabi_memclr(dest: *mut u8, n: usize) { - __aeabi_memset(dest, n, 0); + unsafe { __aeabi_memset(dest, n, 0) }; } #[cfg(not(any(target_vendor = "apple", target_env = "msvc")))] pub unsafe extern "aapcs" fn __aeabi_memclr4(dest: *mut u8, n: usize) { - __aeabi_memset4(dest, n, 0); + unsafe { __aeabi_memset4(dest, n, 0) }; } #[cfg(not(any(target_vendor = "apple", target_env = "msvc")))] pub unsafe extern "aapcs" fn __aeabi_memclr8(dest: *mut u8, n: usize) { - __aeabi_memset4(dest, n, 0); + unsafe { __aeabi_memset4(dest, n, 0) }; } } diff --git a/src/macros.rs b/src/macros.rs index f51e49e9..4c7282be 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -255,7 +255,7 @@ macro_rules! intrinsics { #[cfg(all(any(windows, target_os = "uefi"), target_arch = "x86_64", not(feature = "mangled-names")))] mod $name { - #[no_mangle] + #[unsafe(no_mangle)] #[cfg_attr(not(all(windows, target_env = "gnu")), linkage = "weak")] extern $abi fn $name( $($argname: $ty),* ) -> $crate::macros::win64_128bit_abi_hack::U64x2 @@ -320,7 +320,7 @@ macro_rules! intrinsics { #[cfg(all(target_vendor = "apple", any(target_arch = "x86", target_arch = "x86_64"), not(feature = "mangled-names")))] mod $name { - #[no_mangle] + #[unsafe(no_mangle)] #[cfg_attr(not(all(windows, target_env = "gnu")), linkage = "weak")] $(#[$($attr)*])* extern $abi fn $name( $($argname: u16),* ) $(-> $ret)? { @@ -356,7 +356,7 @@ macro_rules! intrinsics { #[cfg(all(target_vendor = "apple", any(target_arch = "x86", target_arch = "x86_64"), not(feature = "mangled-names")))] mod $name { - #[no_mangle] + #[unsafe(no_mangle)] #[cfg_attr(not(all(windows, target_env = "gnu")), linkage = "weak")] $(#[$($attr)*])* extern $abi fn $name( $($argname: $ty),* ) -> u16 { @@ -397,7 +397,7 @@ macro_rules! intrinsics { #[cfg(all(target_arch = "arm", not(feature = "mangled-names")))] mod $name { - #[no_mangle] + #[unsafe(no_mangle)] #[cfg_attr(not(all(windows, target_env = "gnu")), linkage = "weak")] $(#[$($attr)*])* extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { @@ -407,7 +407,7 @@ macro_rules! intrinsics { #[cfg(all(target_arch = "arm", not(feature = "mangled-names")))] mod $alias { - #[no_mangle] + #[unsafe(no_mangle)] #[cfg_attr(not(all(windows, target_env = "gnu")), linkage = "weak")] $(#[$($attr)*])* extern "aapcs" fn $alias( $($argname: $ty),* ) $(-> $ret)? { @@ -474,7 +474,7 @@ macro_rules! intrinsics { #[cfg(all(feature = "mem", not(feature = "mangled-names")))] mod $name { $(#[$($attr)*])* - #[no_mangle] + #[unsafe(no_mangle)] #[cfg_attr(not(all(windows, target_env = "gnu")), linkage = "weak")] unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { super::$name($($argname),*) @@ -499,7 +499,7 @@ macro_rules! intrinsics { pub mod $name { #[naked] $(#[$($attr)*])* - #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + #[cfg_attr(not(feature = "mangled-names"), unsafe(no_mangle))] #[cfg_attr(not(all(windows, target_env = "gnu")), linkage = "weak")] pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* @@ -566,10 +566,10 @@ macro_rules! intrinsics { #[cfg(not(feature = "mangled-names"))] mod $name { $(#[$($attr)*])* - #[no_mangle] + #[unsafe(no_mangle)] #[cfg_attr(not(all(windows, target_env = "gnu")), linkage = "weak")] $(unsafe $($empty)?)? extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { - super::$name($($argname),*) + $(unsafe $($empty)?)? { super::$name($($argname),*) } } } diff --git a/src/mem/impls.rs b/src/mem/impls.rs index c602a67d..5ee294c1 100644 --- a/src/mem/impls.rs +++ b/src/mem/impls.rs @@ -33,12 +33,16 @@ const WORD_COPY_THRESHOLD: usize = if 2 * WORD_SIZE > 16 { 16 }; +/// `x` must be valid for a read of `usize`. #[cfg(feature = "mem-unaligned")] unsafe fn read_usize_unaligned(x: *const usize) -> usize { + type Arr = [u8; core::mem::size_of::()]; // Do not use `core::ptr::read_unaligned` here, since it calls `copy_nonoverlapping` which // is translated to memcpy in LLVM. - let x_read = (x as *const [u8; core::mem::size_of::()]).read(); - core::mem::transmute(x_read) + // SAFETY: x is valid for reads by preconditions. + let x_read = unsafe { (x as *const Arr).read() }; + // SAFETY: Same-sized POD cast + unsafe { core::mem::transmute::(x_read) } } #[inline(always)] @@ -47,7 +51,7 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) unsafe fn copy_forward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) { let dest_end = dest.wrapping_add(n); while dest < dest_end { - *dest = *src; + unsafe { *dest = *src }; dest = dest.wrapping_add(1); src = src.wrapping_add(1); } @@ -60,7 +64,7 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) let dest_end = dest.wrapping_add(n) as *mut usize; while dest_usize < dest_end { - *dest_usize = *src_usize; + unsafe { *dest_usize = *src_usize }; dest_usize = dest_usize.wrapping_add(1); src_usize = src_usize.wrapping_add(1); } @@ -108,7 +112,7 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) let dest_end = dest.wrapping_add(n) as *mut usize; while dest_usize < dest_end { - *dest_usize = read_usize_unaligned(src_usize); + unsafe { *dest_usize = read_usize_unaligned(src_usize) }; dest_usize = dest_usize.wrapping_add(1); src_usize = src_usize.wrapping_add(1); } @@ -117,24 +121,26 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) if n >= WORD_COPY_THRESHOLD { // Align dest // Because of n >= 2 * WORD_SIZE, dst_misalignment < n - let dest_misalignment = (dest as usize).wrapping_neg() & WORD_MASK; - copy_forward_bytes(dest, src, dest_misalignment); - dest = dest.wrapping_add(dest_misalignment); - src = src.wrapping_add(dest_misalignment); - n -= dest_misalignment; - - let n_words = n & !WORD_MASK; - let src_misalignment = src as usize & WORD_MASK; - if likely(src_misalignment == 0) { - copy_forward_aligned_words(dest, src, n_words); - } else { - copy_forward_misaligned_words(dest, src, n_words); + unsafe { + let dest_misalignment = (dest as usize).wrapping_neg() & WORD_MASK; + copy_forward_bytes(dest, src, dest_misalignment); + dest = dest.wrapping_add(dest_misalignment); + src = src.wrapping_add(dest_misalignment); + n -= dest_misalignment; + let n_words = n & !WORD_MASK; + let src_misalignment = src as usize & WORD_MASK; + if likely(src_misalignment == 0) { + copy_forward_aligned_words(dest, src, n_words); + } else { + copy_forward_misaligned_words(dest, src, n_words); + } + dest = dest.wrapping_add(n_words); + src = src.wrapping_add(n_words); + n -= n_words; } - dest = dest.wrapping_add(n_words); - src = src.wrapping_add(n_words); - n -= n_words; } - copy_forward_bytes(dest, src, n); + + unsafe { copy_forward_bytes(dest, src, n) }; } #[inline(always)] @@ -147,7 +153,7 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) { while dest_start < dest { dest = dest.wrapping_sub(1); src = src.wrapping_sub(1); - *dest = *src; + unsafe { *dest = *src }; } } @@ -160,7 +166,7 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) { while dest_start < dest_usize { dest_usize = dest_usize.wrapping_sub(1); src_usize = src_usize.wrapping_sub(1); - *dest_usize = *src_usize; + unsafe { *dest_usize = *src_usize }; } } @@ -180,13 +186,13 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) { // cfg needed because not all targets will have atomic loads that can be lowered // (e.g. BPF, MSP430), or provided by an external library (e.g. RV32I) #[cfg(target_has_atomic_load_store = "ptr")] - let mut prev_word = core::intrinsics::atomic_load_unordered(src_aligned); + let mut prev_word = unsafe { core::intrinsics::atomic_load_unordered(src_aligned) }; #[cfg(not(target_has_atomic_load_store = "ptr"))] - let mut prev_word = core::ptr::read_volatile(src_aligned); + let mut prev_word = unsafe { core::ptr::read_volatile(src_aligned) }; while dest_start < dest_usize { src_aligned = src_aligned.wrapping_sub(1); - let cur_word = *src_aligned; + let cur_word = unsafe { *src_aligned }; #[cfg(target_endian = "little")] let resembled = prev_word << (WORD_SIZE * 8 - shift) | cur_word >> shift; #[cfg(target_endian = "big")] @@ -208,7 +214,7 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) { while dest_start < dest_usize { dest_usize = dest_usize.wrapping_sub(1); src_usize = src_usize.wrapping_sub(1); - *dest_usize = read_usize_unaligned(src_usize); + unsafe { *dest_usize = read_usize_unaligned(src_usize) }; } } @@ -218,24 +224,26 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) { if n >= WORD_COPY_THRESHOLD { // Align dest // Because of n >= 2 * WORD_SIZE, dst_misalignment < n - let dest_misalignment = dest as usize & WORD_MASK; - copy_backward_bytes(dest, src, dest_misalignment); - dest = dest.wrapping_sub(dest_misalignment); - src = src.wrapping_sub(dest_misalignment); - n -= dest_misalignment; - - let n_words = n & !WORD_MASK; - let src_misalignment = src as usize & WORD_MASK; - if likely(src_misalignment == 0) { - copy_backward_aligned_words(dest, src, n_words); - } else { - copy_backward_misaligned_words(dest, src, n_words); + unsafe { + let dest_misalignment = dest as usize & WORD_MASK; + copy_backward_bytes(dest, src, dest_misalignment); + dest = dest.wrapping_sub(dest_misalignment); + src = src.wrapping_sub(dest_misalignment); + n -= dest_misalignment; + + let n_words = n & !WORD_MASK; + let src_misalignment = src as usize & WORD_MASK; + if likely(src_misalignment == 0) { + copy_backward_aligned_words(dest, src, n_words); + } else { + copy_backward_misaligned_words(dest, src, n_words); + } + dest = dest.wrapping_sub(n_words); + src = src.wrapping_sub(n_words); + n -= n_words; } - dest = dest.wrapping_sub(n_words); - src = src.wrapping_sub(n_words); - n -= n_words; } - copy_backward_bytes(dest, src, n); + unsafe { copy_backward_bytes(dest, src, n) }; } #[inline(always)] @@ -244,7 +252,7 @@ pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) { pub unsafe fn set_bytes_bytes(mut s: *mut u8, c: u8, n: usize) { let end = s.wrapping_add(n); while s < end { - *s = c; + unsafe { *s = c }; s = s.wrapping_add(1); } } @@ -262,7 +270,7 @@ pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) { let end = s.wrapping_add(n) as *mut usize; while s_usize < end { - *s_usize = broadcast; + unsafe { *s_usize = broadcast }; s_usize = s_usize.wrapping_add(1); } } @@ -271,24 +279,25 @@ pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) { // Align s // Because of n >= 2 * WORD_SIZE, dst_misalignment < n let misalignment = (s as usize).wrapping_neg() & WORD_MASK; - set_bytes_bytes(s, c, misalignment); + unsafe { set_bytes_bytes(s, c, misalignment) }; s = s.wrapping_add(misalignment); n -= misalignment; let n_words = n & !WORD_MASK; - set_bytes_words(s, c, n_words); + unsafe { set_bytes_words(s, c, n_words) }; s = s.wrapping_add(n_words); n -= n_words; } - set_bytes_bytes(s, c, n); + unsafe { set_bytes_bytes(s, c, n) }; } +/// `memcmp` implementation. `s1` and `s2` must be valid for `n`. #[inline(always)] pub unsafe fn compare_bytes(s1: *const u8, s2: *const u8, n: usize) -> i32 { let mut i = 0; while i < n { - let a = *s1.wrapping_add(i); - let b = *s2.wrapping_add(i); + // SAFETY: `i < n`, thus the reads are valid by preconditions. + let (a, b) = unsafe { (*s1.wrapping_add(i), *s2.wrapping_add(i)) }; if a != b { return a as i32 - b as i32; } @@ -297,10 +306,12 @@ pub unsafe fn compare_bytes(s1: *const u8, s2: *const u8, n: usize) -> i32 { 0 } +/// `strlen` implementation. `s` must be valid for reads up to and including a null character. #[inline(always)] pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { let mut n = 0; - while *s != 0 { + // SAFETY: safe to read until after the first `*s == 0` + while unsafe { *s } != 0 { n += 1; s = s.wrapping_add(1); } diff --git a/src/mem/mod.rs b/src/mem/mod.rs index f10439e2..2e610a17 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -22,42 +22,50 @@ mod impls; intrinsics! { #[mem_builtin] pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { - impls::copy_forward(dest, src, n); + // SAFETY: same preconditions + unsafe { impls::copy_forward(dest, src, n) }; dest } #[mem_builtin] pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { - let delta = (dest as usize).wrapping_sub(src as usize); - if delta >= n { - // We can copy forwards because either dest is far enough ahead of src, - // or src is ahead of dest (and delta overflowed). - impls::copy_forward(dest, src, n); - } else { - impls::copy_backward(dest, src, n); + // SAFETY: same preconditions + unsafe { + let delta = (dest as usize).wrapping_sub(src as usize); + if delta >= n { + // We can copy forwards because either dest is far enough ahead of src, + // or src is ahead of dest (and delta overflowed). + impls::copy_forward(dest, src, n); + } else { + impls::copy_backward(dest, src, n); + } } dest } #[mem_builtin] pub unsafe extern "C" fn memset(s: *mut u8, c: crate::mem::c_int, n: usize) -> *mut u8 { - impls::set_bytes(s, c as u8, n); + // SAFETY: same preconditions + unsafe { impls::set_bytes(s, c as u8, n) }; s } #[mem_builtin] pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { - impls::compare_bytes(s1, s2, n) + // SAFETY: same preconditions + unsafe { impls::compare_bytes(s1, s2, n) } } #[mem_builtin] pub unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { - memcmp(s1, s2, n) + // SAFETY: same preconditions + unsafe { memcmp(s1, s2, n) } } #[mem_builtin] pub unsafe extern "C" fn strlen(s: *const core::ffi::c_char) -> usize { - impls::c_string_length(s) + // SAFETY: same preconditions + unsafe { impls::c_string_length(s) } } } diff --git a/src/mem/x86_64.rs b/src/mem/x86_64.rs index 40b67093..d0294cf9 100644 --- a/src/mem/x86_64.rs +++ b/src/mem/x86_64.rs @@ -24,13 +24,15 @@ use core::mem; #[cfg(target_feature = "ermsb")] pub unsafe fn copy_forward(dest: *mut u8, src: *const u8, count: usize) { // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. - core::arch::asm!( - "repe movsb (%rsi), (%rdi)", - inout("rcx") count => _, - inout("rdi") dest => _, - inout("rsi") src => _, - options(att_syntax, nostack, preserves_flags) - ); + unsafe { + core::arch::asm!( + "repe movsb (%rsi), (%rdi)", + inout("rcx") count => _, + inout("rdi") dest => _, + inout("rsi") src => _, + options(att_syntax, nostack, preserves_flags) + ) + }; } #[inline(always)] @@ -38,67 +40,73 @@ pub unsafe fn copy_forward(dest: *mut u8, src: *const u8, count: usize) { pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, count: usize) { let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); // Separating the blocks gives the compiler more freedom to reorder instructions. - asm!( - "rep movsb", - inout("ecx") pre_byte_count => _, - inout("rdi") dest => dest, - inout("rsi") src => src, - options(att_syntax, nostack, preserves_flags) - ); - asm!( - "rep movsq", - inout("rcx") qword_count => _, - inout("rdi") dest => dest, - inout("rsi") src => src, - options(att_syntax, nostack, preserves_flags) - ); - asm!( - "rep movsb", - inout("ecx") byte_count => _, - inout("rdi") dest => _, - inout("rsi") src => _, - options(att_syntax, nostack, preserves_flags) - ); + unsafe { + asm!( + "rep movsb", + inout("ecx") pre_byte_count => _, + inout("rdi") dest => dest, + inout("rsi") src => src, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep movsq", + inout("rcx") qword_count => _, + inout("rdi") dest => dest, + inout("rsi") src => src, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep movsb", + inout("ecx") byte_count => _, + inout("rdi") dest => _, + inout("rsi") src => _, + options(att_syntax, nostack, preserves_flags) + ); + } } #[inline(always)] pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, count: usize) { let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); // We can't separate this block due to std/cld - asm!( - "std", - "rep movsb", - "sub $7, %rsi", - "sub $7, %rdi", - "mov {qword_count}, %rcx", - "rep movsq", - "test {pre_byte_count:e}, {pre_byte_count:e}", - "add $7, %rsi", - "add $7, %rdi", - "mov {pre_byte_count:e}, %ecx", - "rep movsb", - "cld", - pre_byte_count = in(reg) pre_byte_count, - qword_count = in(reg) qword_count, - inout("ecx") byte_count => _, - inout("rdi") dest.add(count - 1) => _, - inout("rsi") src.add(count - 1) => _, - // We modify flags, but we restore it afterwards - options(att_syntax, nostack, preserves_flags) - ); + unsafe { + asm!( + "std", + "rep movsb", + "sub $7, %rsi", + "sub $7, %rdi", + "mov {qword_count}, %rcx", + "rep movsq", + "test {pre_byte_count:e}, {pre_byte_count:e}", + "add $7, %rsi", + "add $7, %rdi", + "mov {pre_byte_count:e}, %ecx", + "rep movsb", + "cld", + pre_byte_count = in(reg) pre_byte_count, + qword_count = in(reg) qword_count, + inout("ecx") byte_count => _, + inout("rdi") dest.add(count - 1) => _, + inout("rsi") src.add(count - 1) => _, + // we modify flags, but we restore it afterwards + options(att_syntax, nostack, preserves_flags) + ); + } } #[inline(always)] #[cfg(target_feature = "ermsb")] pub unsafe fn set_bytes(dest: *mut u8, c: u8, count: usize) { // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. - core::arch::asm!( - "repe stosb %al, (%rdi)", - inout("rcx") count => _, - inout("rdi") dest => _, - inout("al") c => _, - options(att_syntax, nostack, preserves_flags) - ) + unsafe { + core::arch::asm!( + "repe stosb %al, (%rdi)", + inout("rcx") count => _, + inout("rdi") dest => _, + inout("al") c => _, + options(att_syntax, nostack, preserves_flags) + ) + } } #[inline(always)] @@ -107,27 +115,29 @@ pub unsafe fn set_bytes(mut dest: *mut u8, c: u8, count: usize) { let c = c as u64 * 0x0101_0101_0101_0101; let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); // Separating the blocks gives the compiler more freedom to reorder instructions. - asm!( - "rep stosb", - inout("ecx") pre_byte_count => _, - inout("rdi") dest => dest, - in("rax") c, - options(att_syntax, nostack, preserves_flags) - ); - asm!( - "rep stosq", - inout("rcx") qword_count => _, - inout("rdi") dest => dest, - in("rax") c, - options(att_syntax, nostack, preserves_flags) - ); - asm!( - "rep stosb", - inout("ecx") byte_count => _, - inout("rdi") dest => _, - in("rax") c, - options(att_syntax, nostack, preserves_flags) - ); + unsafe { + asm!( + "rep stosb", + inout("ecx") pre_byte_count => _, + inout("rdi") dest => dest, + in("rax") c, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep stosq", + inout("rcx") qword_count => _, + inout("rdi") dest => dest, + in("rax") c, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep stosb", + inout("ecx") byte_count => _, + inout("rdi") dest => _, + in("rax") c, + options(att_syntax, nostack, preserves_flags) + ); + } } #[inline(always)] @@ -142,35 +152,41 @@ pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 { // Ensure T is not a ZST. const { assert!(mem::size_of::() != 0) }; - let end = a.add(intrinsics::unchecked_div(n, mem::size_of::())); - while a != end { - if a.read_unaligned() != b.read_unaligned() { - return f(a.cast(), b.cast(), mem::size_of::()); + unsafe { + let end = a.add(intrinsics::unchecked_div(n, mem::size_of::())); + while a != end { + if a.read_unaligned() != b.read_unaligned() { + return f(a.cast(), b.cast(), mem::size_of::()); + } + a = a.add(1); + b = b.add(1); } - a = a.add(1); - b = b.add(1); + f( + a.cast(), + b.cast(), + intrinsics::unchecked_rem(n, mem::size_of::()), + ) } - f( - a.cast(), - b.cast(), - intrinsics::unchecked_rem(n, mem::size_of::()), - ) } let c1 = |mut a: *const u8, mut b: *const u8, n| { for _ in 0..n { - if a.read() != b.read() { - return i32::from(a.read()) - i32::from(b.read()); + unsafe { + if a.read() != b.read() { + return i32::from(a.read()) - i32::from(b.read()); + } + a = a.add(1); + b = b.add(1); } - a = a.add(1); - b = b.add(1); } 0 }; - let c2 = |a: *const u16, b, n| cmp(a, b, n, c1); - let c4 = |a: *const u32, b, n| cmp(a, b, n, c2); - let c8 = |a: *const u64, b, n| cmp(a, b, n, c4); - let c16 = |a: *const u128, b, n| cmp(a, b, n, c8); - c16(a.cast(), b.cast(), n) + unsafe { + let c2 = |a: *const u16, b, n| cmp(a, b, n, c1); + let c4 = |a: *const u32, b, n| cmp(a, b, n, c2); + let c8 = |a: *const u64, b, n| cmp(a, b, n, c4); + let c16 = |a: *const u128, b, n| cmp(a, b, n, c8); + c16(a.cast(), b.cast(), n) + } } // In order to process more than on byte simultaneously when executing strlen, @@ -195,12 +211,13 @@ pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { // are handled in simple loops. for _ in 0..4 { - if *s == 0 { + if unsafe { *s == 0 } { return n; } n += 1; - s = s.add(1); + // SAFETY: + s = unsafe { s.add(1) }; } // Shave of the least significand bits to align the address to a 16 @@ -208,42 +225,40 @@ pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { let align = s as usize & 15; let mut s = ((s as usize) - align) as *const __m128i; - let zero = _mm_set1_epi8(0); + let zero = unsafe { _mm_set1_epi8(0) }; - let x = { - let r; + let x; + unsafe { asm!( "movdqa ({addr}), {dest}", addr = in(reg) s, - dest = out(xmm_reg) r, + dest = out(xmm_reg) x, options(att_syntax, nostack), ); - r - }; - let cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(x, zero)) >> align; + } + let cmp = unsafe { _mm_movemask_epi8(_mm_cmpeq_epi8(x, zero)) >> align }; if cmp != 0 { return n + cmp.trailing_zeros() as usize; } n += 16 - align; - s = s.add(1); + s = unsafe { s.add(1) }; loop { - let x = { - let r; + let x; + unsafe { asm!( "movdqa ({addr}), {dest}", addr = in(reg) s, - dest = out(xmm_reg) r, + dest = out(xmm_reg) x, options(att_syntax, nostack), ); - r - }; - let cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(x, zero)) as u32; + } + let cmp = unsafe { _mm_movemask_epi8(_mm_cmpeq_epi8(x, zero)) as u32 }; if cmp == 0 { n += 16; - s = s.add(1); + s = unsafe { s.add(1) }; } else { return n + cmp.trailing_zeros() as usize; } @@ -277,12 +292,14 @@ pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { loop { let mut cs = { let r: u64; - asm!( - "mov ({addr}), {dest}", - addr = in(reg) s, - dest = out(reg) r, - options(att_syntax, nostack), - ); + unsafe { + asm!( + "mov ({addr}), {dest}", + addr = in(reg) s, + dest = out(reg) r, + options(att_syntax, nostack), + ); + } r }; // Detect if a word has a zero byte, taken from diff --git a/src/probestack.rs b/src/probestack.rs index 0c30384d..f0d16115 100644 --- a/src/probestack.rs +++ b/src/probestack.rs @@ -49,7 +49,7 @@ // We only define stack probing for these architectures today. #![cfg(any(target_arch = "x86_64", target_arch = "x86"))] -extern "C" { +unsafe extern "C" { pub fn __rust_probestack(); } diff --git a/src/x86_64.rs b/src/x86_64.rs index 9c91a455..2ed3ff04 100644 --- a/src/x86_64.rs +++ b/src/x86_64.rs @@ -14,33 +14,35 @@ intrinsics! { not(feature = "no-asm") ))] pub unsafe extern "C" fn ___chkstk_ms() { - core::arch::naked_asm!( - "push %rcx", - "push %rax", - "cmp $0x1000,%rax", - "lea 24(%rsp),%rcx", - "jb 1f", - "2:", - "sub $0x1000,%rcx", - "test %rcx,(%rcx)", - "sub $0x1000,%rax", - "cmp $0x1000,%rax", - "ja 2b", - "1:", - "sub %rax,%rcx", - "test %rcx,(%rcx)", - "pop %rax", - "pop %rcx", - "ret", - options(att_syntax) - ); + unsafe { + core::arch::naked_asm!( + "push %rcx", + "push %rax", + "cmp $0x1000,%rax", + "lea 24(%rsp),%rcx", + "jb 1f", + "2:", + "sub $0x1000,%rcx", + "test %rcx,(%rcx)", + "sub $0x1000,%rax", + "cmp $0x1000,%rax", + "ja 2b", + "1:", + "sub %rax,%rcx", + "test %rcx,(%rcx)", + "pop %rax", + "pop %rcx", + "ret", + options(att_syntax) + ); + } } } // HACK(https://github.com/rust-lang/rust/issues/62785): x86_64-unknown-uefi needs special LLVM // support unless we emit the _fltused mod _fltused { - #[no_mangle] + #[unsafe(no_mangle)] #[used] #[cfg(target_os = "uefi")] static _fltused: i32 = 0; diff --git a/testcrate/Cargo.toml b/testcrate/Cargo.toml index 21cec170..4265f6ba 100644 --- a/testcrate/Cargo.toml +++ b/testcrate/Cargo.toml @@ -2,7 +2,7 @@ name = "testcrate" version = "0.1.0" authors = ["Alex Crichton "] -edition = "2021" +edition = "2024" publish = false [lib] diff --git a/testcrate/src/bench.rs b/testcrate/src/bench.rs index f5da1f3a..97f69881 100644 --- a/testcrate/src/bench.rs +++ b/testcrate/src/bench.rs @@ -121,7 +121,7 @@ macro_rules! float_bench { $(,)? ) => {paste::paste! { #[cfg($sys_available)] - extern "C" { + unsafe extern "C" { /// Binding for the system function #[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] fn $sys_fn($($arg: $arg_ty),*) -> $ret_ty; diff --git a/testcrate/tests/aeabi_memclr.rs b/testcrate/tests/aeabi_memclr.rs index bfd15a39..5ffa1b06 100644 --- a/testcrate/tests/aeabi_memclr.rs +++ b/testcrate/tests/aeabi_memclr.rs @@ -24,7 +24,7 @@ macro_rules! panic { }; } -extern "C" { +unsafe extern "C" { fn __aeabi_memclr4(dest: *mut u8, n: usize); fn __aeabi_memset4(dest: *mut u8, n: usize, c: u32); } diff --git a/testcrate/tests/aeabi_memcpy.rs b/testcrate/tests/aeabi_memcpy.rs index c892c5ab..3499bfb0 100644 --- a/testcrate/tests/aeabi_memcpy.rs +++ b/testcrate/tests/aeabi_memcpy.rs @@ -22,7 +22,7 @@ macro_rules! panic { }; } -extern "C" { +unsafe extern "C" { fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize); fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, n: usize); } diff --git a/testcrate/tests/aeabi_memset.rs b/testcrate/tests/aeabi_memset.rs index 34ab3acc..9ed2f8d6 100644 --- a/testcrate/tests/aeabi_memset.rs +++ b/testcrate/tests/aeabi_memset.rs @@ -24,7 +24,7 @@ macro_rules! panic { }; } -extern "C" { +unsafe extern "C" { fn __aeabi_memset4(dest: *mut u8, n: usize, c: u32); }