From a1731adb8c954c6b41f549e2db3d92039c4774cb Mon Sep 17 00:00:00 2001 From: Misaki Kasumi Date: Sat, 12 Oct 2024 13:53:38 +0800 Subject: [PATCH 1/3] Adds priority-inheritance futexes for mutexex This uses FUTEX_LOCK_PI and FUTEX_UNLOCK_PI on Linux. --- library/std/src/sys/pal/unix/mod.rs | 1 + library/std/src/sys/pal/unix/pi_futex.rs | 52 ++++++++++++++ library/std/src/sys/sync/mutex/mod.rs | 6 +- library/std/src/sys/sync/mutex/pi_futex.rs | 83 ++++++++++++++++++++++ 4 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 library/std/src/sys/pal/unix/pi_futex.rs create mode 100644 library/std/src/sys/sync/mutex/pi_futex.rs diff --git a/library/std/src/sys/pal/unix/mod.rs b/library/std/src/sys/pal/unix/mod.rs index 4fe18daa2040f..12785481e6958 100644 --- a/library/std/src/sys/pal/unix/mod.rs +++ b/library/std/src/sys/pal/unix/mod.rs @@ -23,6 +23,7 @@ pub mod net; #[cfg(target_os = "l4re")] pub use self::l4re::net; pub mod os; +pub mod pi_futex; pub mod pipe; pub mod process; pub mod stack_overflow; diff --git a/library/std/src/sys/pal/unix/pi_futex.rs b/library/std/src/sys/pal/unix/pi_futex.rs new file mode 100644 index 0000000000000..c70200a80e68d --- /dev/null +++ b/library/std/src/sys/pal/unix/pi_futex.rs @@ -0,0 +1,52 @@ +#![cfg(any(target_os = "linux", target_os = "android"))] + +use crate::sync::atomic::AtomicU32; +use crate::sys::cvt; +use crate::{io, ptr}; + +pub const fn unlocked() -> u32 { + 0 +} + +pub fn locked() -> u32 { + (unsafe { libc::gettid() }) as _ +} + +pub fn is_contended(futex_val: u32) -> bool { + (futex_val & libc::FUTEX_WAITERS) != 0 +} + +pub fn is_owned_died(futex_val: u32) -> bool { + (futex_val & libc::FUTEX_OWNER_DIED) != 0 +} + +pub fn futex_lock(futex: &AtomicU32) -> io::Result<()> { + loop { + match cvt(unsafe { + libc::syscall( + libc::SYS_futex, + ptr::from_ref(futex), + libc::FUTEX_LOCK_PI | libc::FUTEX_PRIVATE_FLAG, + 0, + ptr::null::(), + // remaining args are unused + ) + }) { + Ok(_) => return Ok(()), + Err(e) if e.raw_os_error() == Some(libc::EINTR) => continue, + Err(e) => return Err(e), + } + } +} + +pub fn futex_unlock(futex: &AtomicU32) -> io::Result<()> { + cvt(unsafe { + libc::syscall( + libc::SYS_futex, + ptr::from_ref(futex), + libc::FUTEX_UNLOCK_PI | libc::FUTEX_PRIVATE_FLAG, + // remaining args are unused + ) + }) + .map(|_| ()) +} diff --git a/library/std/src/sys/sync/mutex/mod.rs b/library/std/src/sys/sync/mutex/mod.rs index 360df3fc4b55d..0b97013b30fa8 100644 --- a/library/std/src/sys/sync/mutex/mod.rs +++ b/library/std/src/sys/sync/mutex/mod.rs @@ -1,8 +1,12 @@ cfg_if::cfg_if! { if #[cfg(any( - all(target_os = "windows", not(target_vendor = "win7")), target_os = "linux", target_os = "android", + ))] { + mod pi_futex; + pub use pi_futex::Mutex; + } else if #[cfg(any( + all(target_os = "windows", not(target_vendor = "win7")), target_os = "freebsd", target_os = "openbsd", target_os = "dragonfly", diff --git a/library/std/src/sys/sync/mutex/pi_futex.rs b/library/std/src/sys/sync/mutex/pi_futex.rs new file mode 100644 index 0000000000000..c2220b7323fe2 --- /dev/null +++ b/library/std/src/sys/sync/mutex/pi_futex.rs @@ -0,0 +1,83 @@ +use crate::sync::atomic::AtomicU32; +use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release}; +use crate::sys::pi_futex as pi; + +pub struct Mutex { + futex: AtomicU32, +} + +impl Mutex { + #[inline] + pub const fn new() -> Self { + Self { futex: AtomicU32::new(pi::unlocked()) } + } + + #[inline] + pub fn try_lock(&self) -> bool { + self.futex.compare_exchange(pi::unlocked(), pi::locked(), Acquire, Relaxed).is_ok() + } + + #[inline] + pub fn lock(&self) { + if self.futex.compare_exchange(pi::unlocked(), pi::locked(), Acquire, Relaxed).is_err() { + self.lock_contended(); + } + } + + #[cold] + fn lock_contended(&self) { + // Spin first to speed things up if the lock is released quickly. + let state = self.spin(); + + // If it's unlocked now, attempt to take the lock. + if state == pi::unlocked() { + if self.try_lock() { + return; + } + }; + + pi::futex_lock(&self.futex).expect("failed to lock mutex"); + + let state = self.futex.load(Relaxed); + if pi::is_owned_died(state) { + panic!( + "failed to lock mutex because the thread owning it finished without unlocking it" + ); + } + } + + fn spin(&self) -> u32 { + let mut spin = 100; + loop { + // We only use `load` (and not `swap` or `compare_exchange`) + // while spinning, to be easier on the caches. + let state = self.futex.load(Relaxed); + + // We stop spinning when the mutex is unlocked, + // but also when it's contended. + if state == pi::unlocked() || pi::is_contended(state) || spin == 0 { + return state; + } + + crate::hint::spin_loop(); + spin -= 1; + } + } + + #[inline] + pub unsafe fn unlock(&self) { + if self.futex.compare_exchange(pi::locked(), pi::unlocked(), Release, Relaxed).is_err() { + // We only wake up one thread. When that thread locks the mutex, + // the kernel will mark the mutex as contended automatically + // (futex != pi::locked() in this case), + // which makes sure that any other waiting threads will also be + // woken up eventually. + self.wake(); + } + } + + #[cold] + fn wake(&self) { + pi::futex_unlock(&self.futex).unwrap(); + } +} From a360d5468990007f378ea10372d8c16fc1bac3ec Mon Sep 17 00:00:00 2001 From: Misaki Kasumi Date: Mon, 14 Oct 2024 09:44:19 +0800 Subject: [PATCH 2/3] Adds PI futex for FreeBSD --- library/std/src/sys/pal/unix/pi_futex.rs | 195 +++++++++++++++++---- library/std/src/sys/sync/mutex/mod.rs | 2 +- library/std/src/sys/sync/mutex/pi_futex.rs | 7 +- 3 files changed, 161 insertions(+), 43 deletions(-) diff --git a/library/std/src/sys/pal/unix/pi_futex.rs b/library/std/src/sys/pal/unix/pi_futex.rs index c70200a80e68d..721076c2633da 100644 --- a/library/std/src/sys/pal/unix/pi_futex.rs +++ b/library/std/src/sys/pal/unix/pi_futex.rs @@ -1,52 +1,171 @@ -#![cfg(any(target_os = "linux", target_os = "android"))] +#[cfg(any(target_os = "linux", target_os = "android"))] +mod linux { + use crate::ops::Deref; + use crate::sync::atomic::AtomicU32; + use crate::sys::cvt; + use crate::{io, ptr}; -use crate::sync::atomic::AtomicU32; -use crate::sys::cvt; -use crate::{io, ptr}; + pub type State = u32; -pub const fn unlocked() -> u32 { - 0 -} + pub struct Futex(AtomicU32); -pub fn locked() -> u32 { - (unsafe { libc::gettid() }) as _ -} + impl Futex { + pub const fn new() -> Futex { + Futex(AtomicU32::new(0)) + } + } -pub fn is_contended(futex_val: u32) -> bool { - (futex_val & libc::FUTEX_WAITERS) != 0 -} + impl Deref for Futex { + type Target = AtomicU32; + fn deref(&self) -> &AtomicU32 { + &self.0 + } + } -pub fn is_owned_died(futex_val: u32) -> bool { - (futex_val & libc::FUTEX_OWNER_DIED) != 0 -} + pub const fn unlocked() -> State { + 0 + } + + pub fn locked() -> State { + (unsafe { libc::gettid() }) as _ + } + + pub fn is_contended(futex_val: State) -> bool { + (futex_val & libc::FUTEX_WAITERS) != 0 + } -pub fn futex_lock(futex: &AtomicU32) -> io::Result<()> { - loop { - match cvt(unsafe { + pub fn is_owned_died(futex_val: State) -> bool { + (futex_val & libc::FUTEX_OWNER_DIED) != 0 + } + + pub fn futex_lock(futex: &Futex) -> io::Result<()> { + loop { + match cvt(unsafe { + libc::syscall( + libc::SYS_futex, + ptr::from_ref(futex.deref()), + libc::FUTEX_LOCK_PI | libc::FUTEX_PRIVATE_FLAG, + 0, + ptr::null::(), + // remaining args are unused + ) + }) { + Ok(_) => return Ok(()), + Err(e) if e.raw_os_error() == Some(libc::EINTR) => continue, + Err(e) => return Err(e), + } + } + } + + pub fn futex_unlock(futex: &Futex) -> io::Result<()> { + cvt(unsafe { libc::syscall( libc::SYS_futex, - ptr::from_ref(futex), - libc::FUTEX_LOCK_PI | libc::FUTEX_PRIVATE_FLAG, - 0, - ptr::null::(), + ptr::from_ref(futex.deref()), + libc::FUTEX_UNLOCK_PI | libc::FUTEX_PRIVATE_FLAG, // remaining args are unused ) - }) { - Ok(_) => return Ok(()), - Err(e) if e.raw_os_error() == Some(libc::EINTR) => continue, - Err(e) => return Err(e), - } + }) + .map(|_| ()) } } -pub fn futex_unlock(futex: &AtomicU32) -> io::Result<()> { - cvt(unsafe { - libc::syscall( - libc::SYS_futex, - ptr::from_ref(futex), - libc::FUTEX_UNLOCK_PI | libc::FUTEX_PRIVATE_FLAG, - // remaining args are unused - ) - }) - .map(|_| ()) +#[cfg(any(target_os = "linux", target_os = "android"))] +pub use linux::*; + +#[cfg(target_os = "freebsd")] +mod freebsd { + use crate::mem::transmute; + use crate::ops::Deref; + use crate::sync::atomic::AtomicU32; + use crate::sys::cvt; + use crate::{io, ptr}; + + pub type State = u32; + + #[repr(C)] + pub struct umutex { + m_owner: libc::lwpid_t, + m_flags: u32, + m_ceilings: [u32; 2], + m_rb_link: libc::uintptr_t, + #[cfg(target_pointer_width = "32")] + m_pad: u32, + m_spare: [u32; 2], + } + + pub struct Futex(umutex); + + impl Futex { + pub const fn new() -> Futex { + Futex(umutex { + m_owner: 0, + m_flags: UMUTEX_PRIO_INHERIT, + m_ceilings: [0, 0], + m_rb_link: 0, + #[cfg(target_pointer_width = "32")] + m_pad: 0, + m_spare: [0, 0], + }) + } + } + + impl Deref for Futex { + type Target = AtomicU32; + fn deref(&self) -> &AtomicU32 { + unsafe { transmute(&self.0.m_owner) } + } + } + + const UMUTEX_PRIO_INHERIT: u32 = 0x0004; + const UMUTEX_CONTESTED: u32 = 0x80000000; + + pub const fn unlocked() -> State { + 0 + } + + pub fn locked() -> State { + let mut tid: libc::c_long = 0; + let _ = unsafe { libc::thr_self(ptr::from_mut(&mut tid)) }; + tid as _ + } + + pub fn is_contended(futex_val: State) -> bool { + (futex_val & UMUTEX_CONTESTED) != 0 + } + + pub fn is_owned_died(futex_val: State) -> bool { + // never happens for non-robust mutex + let _ = futex_val; + false + } + + pub fn futex_lock(futex: &Futex) -> io::Result<()> { + cvt(unsafe { + libc::_umtx_op( + ptr::from_ref(futex.deref()) as _, + libc::UMTX_OP_MUTEX_LOCK, + 0, + ptr::null_mut::(), + ptr::null_mut::(), + ) + }) + .map(|_| ()) + } + + pub fn futex_unlock(futex: &Futex) -> io::Result<()> { + cvt(unsafe { + libc::_umtx_op( + ptr::from_ref(futex.deref()) as _, + libc::UMTX_OP_MUTEX_UNLOCK, + 0, + ptr::null_mut::(), + ptr::null_mut::(), + ) + }) + .map(|_| ()) + } } + +#[cfg(target_os = "freebsd")] +pub use freebsd::*; diff --git a/library/std/src/sys/sync/mutex/mod.rs b/library/std/src/sys/sync/mutex/mod.rs index 0b97013b30fa8..937cc88aa04b6 100644 --- a/library/std/src/sys/sync/mutex/mod.rs +++ b/library/std/src/sys/sync/mutex/mod.rs @@ -2,12 +2,12 @@ cfg_if::cfg_if! { if #[cfg(any( target_os = "linux", target_os = "android", + target_os = "freebsd", ))] { mod pi_futex; pub use pi_futex::Mutex; } else if #[cfg(any( all(target_os = "windows", not(target_vendor = "win7")), - target_os = "freebsd", target_os = "openbsd", target_os = "dragonfly", all(target_family = "wasm", target_feature = "atomics"), diff --git a/library/std/src/sys/sync/mutex/pi_futex.rs b/library/std/src/sys/sync/mutex/pi_futex.rs index c2220b7323fe2..d7d1e61c81464 100644 --- a/library/std/src/sys/sync/mutex/pi_futex.rs +++ b/library/std/src/sys/sync/mutex/pi_futex.rs @@ -1,15 +1,14 @@ -use crate::sync::atomic::AtomicU32; use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release}; use crate::sys::pi_futex as pi; pub struct Mutex { - futex: AtomicU32, + futex: pi::Futex, } impl Mutex { #[inline] pub const fn new() -> Self { - Self { futex: AtomicU32::new(pi::unlocked()) } + Self { futex: pi::Futex::new() } } #[inline] @@ -46,7 +45,7 @@ impl Mutex { } } - fn spin(&self) -> u32 { + fn spin(&self) -> pi::State { let mut spin = 100; loop { // We only use `load` (and not `swap` or `compare_exchange`) From 8f33536393e62c71e3585e796a47e2085384aba7 Mon Sep 17 00:00:00 2001 From: Misaki Kasumi Date: Fri, 22 Nov 2024 05:47:08 +0800 Subject: [PATCH 3/3] Cache tid in futex tls --- library/std/src/sys/pal/unix/pi_futex.rs | 36 ++++++++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/library/std/src/sys/pal/unix/pi_futex.rs b/library/std/src/sys/pal/unix/pi_futex.rs index 721076c2633da..0ebc056ad3931 100644 --- a/library/std/src/sys/pal/unix/pi_futex.rs +++ b/library/std/src/sys/pal/unix/pi_futex.rs @@ -1,9 +1,14 @@ #[cfg(any(target_os = "linux", target_os = "android"))] mod linux { + use crate::cell::Cell; use crate::ops::Deref; use crate::sync::atomic::AtomicU32; use crate::sys::cvt; - use crate::{io, ptr}; + use crate::{io, ptr, thread_local}; + + thread_local! { + static TID: Cell = Cell::new(0); + } pub type State = u32; @@ -27,7 +32,14 @@ mod linux { } pub fn locked() -> State { - (unsafe { libc::gettid() }) as _ + let tid = TID.get(); + if tid == 0 { + let tid = (unsafe { libc::gettid() }) as u32; + TID.set(tid); + tid + } else { + tid + } } pub fn is_contended(futex_val: State) -> bool { @@ -75,11 +87,16 @@ pub use linux::*; #[cfg(target_os = "freebsd")] mod freebsd { + use crate::cell::Cell; use crate::mem::transmute; use crate::ops::Deref; use crate::sync::atomic::AtomicU32; use crate::sys::cvt; - use crate::{io, ptr}; + use crate::{io, ptr, thread_local}; + + thread_local! { + static TID: Cell = Cell::new(0); + } pub type State = u32; @@ -125,9 +142,16 @@ mod freebsd { } pub fn locked() -> State { - let mut tid: libc::c_long = 0; - let _ = unsafe { libc::thr_self(ptr::from_mut(&mut tid)) }; - tid as _ + let tid = TID.get(); + if tid == 0 { + let mut tid: libc::c_long = 0; + let _ = unsafe { libc::thr_self(ptr::from_mut(&mut tid)) }; + let tid = tid as u32; + TID.set(tid); + tid + } else { + tid + } } pub fn is_contended(futex_val: State) -> bool {