Adds priority-inheritance futexes

library/std/src/sys/pal/unix/mod.rs

@@ -23,6 +23,7 @@ pub mod net;
 #[cfg(target_os = "l4re")]
 pub use self::l4re::net;
 pub mod os;
+pub mod pi_futex;
 pub mod pipe;
 pub mod process;
 pub mod stack_overflow;

library/std/src/sys/pal/unix/pi_futex.rs

@@ -0,0 +1,195 @@
+#[cfg(any(target_os = "linux", target_os = "android"))]
+mod linux {
+    use crate::cell::Cell;
+    use crate::ops::Deref;
+    use crate::sync::atomic::AtomicU32;
+    use crate::sys::cvt;
+    use crate::{io, ptr, thread_local};
+
+    thread_local! {
+        static TID: Cell<u32> = Cell::new(0);
+    }
+
+    pub type State = u32;
+
+    pub struct Futex(AtomicU32);
+
+    impl Futex {
+        pub const fn new() -> Futex {
+            Futex(AtomicU32::new(0))
+        }
+    }
+
+    impl Deref for Futex {
+        type Target = AtomicU32;
+        fn deref(&self) -> &AtomicU32 {
+            &self.0
+        }
+    }
+
+    pub const fn unlocked() -> State {
+        0
+    }
+
+    pub fn locked() -> State {
+        let tid = TID.get();
+        if tid == 0 {
+            let tid = (unsafe { libc::gettid() }) as u32;
+            TID.set(tid);
+            tid
+        } else {
+            tid
+        }
+    }
+
+    pub fn is_contended(futex_val: State) -> bool {
+        (futex_val & libc::FUTEX_WAITERS) != 0
+    }
+
+    pub fn is_owned_died(futex_val: State) -> bool {
+        (futex_val & libc::FUTEX_OWNER_DIED) != 0
+    }
+
+    pub fn futex_lock(futex: &Futex) -> io::Result<()> {
+        loop {
+            match cvt(unsafe {
+                libc::syscall(
+                    libc::SYS_futex,
+                    ptr::from_ref(futex.deref()),
+                    libc::FUTEX_LOCK_PI | libc::FUTEX_PRIVATE_FLAG,
+                    0,
+                    ptr::null::<u32>(),
+                    // remaining args are unused
+                )
+            }) {
+                Ok(_) => return Ok(()),
+                Err(e) if e.raw_os_error() == Some(libc::EINTR) => continue,
+                Err(e) => return Err(e),
+            }
+        }
+    }
+
+    pub fn futex_unlock(futex: &Futex) -> io::Result<()> {
+        cvt(unsafe {
+            libc::syscall(
+                libc::SYS_futex,
+                ptr::from_ref(futex.deref()),
+                libc::FUTEX_UNLOCK_PI | libc::FUTEX_PRIVATE_FLAG,
+                // remaining args are unused
+            )
+        })
+        .map(|_| ())
+    }
+}
+
+#[cfg(any(target_os = "linux", target_os = "android"))]
+pub use linux::*;
+
+#[cfg(target_os = "freebsd")]
+mod freebsd {
+    use crate::cell::Cell;
+    use crate::mem::transmute;
+    use crate::ops::Deref;
+    use crate::sync::atomic::AtomicU32;
+    use crate::sys::cvt;
+    use crate::{io, ptr, thread_local};
+
+    thread_local! {
+        static TID: Cell<u32> = Cell::new(0);
+    }
+
+    pub type State = u32;
+
+    #[repr(C)]
+    pub struct umutex {
+        m_owner: libc::lwpid_t,
+        m_flags: u32,
+        m_ceilings: [u32; 2],
+        m_rb_link: libc::uintptr_t,
+        #[cfg(target_pointer_width = "32")]
+        m_pad: u32,
+        m_spare: [u32; 2],
+    }
+
+    pub struct Futex(umutex);
+
+    impl Futex {
+        pub const fn new() -> Futex {
+            Futex(umutex {
+                m_owner: 0,
+                m_flags: UMUTEX_PRIO_INHERIT,
+                m_ceilings: [0, 0],
+                m_rb_link: 0,
+                #[cfg(target_pointer_width = "32")]
+                m_pad: 0,
+                m_spare: [0, 0],
+            })
+        }
+    }
+
+    impl Deref for Futex {
+        type Target = AtomicU32;
+        fn deref(&self) -> &AtomicU32 {
+            unsafe { transmute(&self.0.m_owner) }
+        }
+    }
+
+    const UMUTEX_PRIO_INHERIT: u32 = 0x0004;
+    const UMUTEX_CONTESTED: u32 = 0x80000000;
+
+    pub const fn unlocked() -> State {
+        0
+    }
+
+    pub fn locked() -> State {
+        let tid = TID.get();
+        if tid == 0 {
+            let mut tid: libc::c_long = 0;
+            let _ = unsafe { libc::thr_self(ptr::from_mut(&mut tid)) };
+            let tid = tid as u32;
+            TID.set(tid);
+            tid
+        } else {
+            tid
+        }
+    }
+
+    pub fn is_contended(futex_val: State) -> bool {
+        (futex_val & UMUTEX_CONTESTED) != 0
+    }
+
+    pub fn is_owned_died(futex_val: State) -> bool {
+        // never happens for non-robust mutex
+        let _ = futex_val;
+        false
+    }
+
+    pub fn futex_lock(futex: &Futex) -> io::Result<()> {
+        cvt(unsafe {
+            libc::_umtx_op(
+                ptr::from_ref(futex.deref()) as _,
+                libc::UMTX_OP_MUTEX_LOCK,
+                0,
+                ptr::null_mut::<libc::c_void>(),
+                ptr::null_mut::<libc::c_void>(),
+            )
+        })
+        .map(|_| ())
+    }
+
+    pub fn futex_unlock(futex: &Futex) -> io::Result<()> {
+        cvt(unsafe {
+            libc::_umtx_op(
+                ptr::from_ref(futex.deref()) as _,
+                libc::UMTX_OP_MUTEX_UNLOCK,
+                0,
+                ptr::null_mut::<libc::c_void>(),
+                ptr::null_mut::<libc::c_void>(),
+            )
+        })
+        .map(|_| ())
+    }
+}
+
+#[cfg(target_os = "freebsd")]
+pub use freebsd::*;

library/std/src/sys/sync/mutex/mod.rs

@@ -1,9 +1,13 @@
 cfg_if::cfg_if! {
     if #[cfg(any(
-        all(target_os = "windows", not(target_vendor = "win7")),
         target_os = "linux",
         target_os = "android",
         target_os = "freebsd",
+    ))] {
+        mod pi_futex;
+        pub use pi_futex::Mutex;
+    } else if #[cfg(any(
+        all(target_os = "windows", not(target_vendor = "win7")),
         target_os = "openbsd",
         target_os = "dragonfly",
         all(target_family = "wasm", target_feature = "atomics"),

library/std/src/sys/sync/mutex/pi_futex.rs

@@ -0,0 +1,82 @@
+use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release};
+use crate::sys::pi_futex as pi;
+
+pub struct Mutex {
+    futex: pi::Futex,
+}
+
+impl Mutex {
+    #[inline]
+    pub const fn new() -> Self {
+        Self { futex: pi::Futex::new() }
+    }
+
+    #[inline]
+    pub fn try_lock(&self) -> bool {
+        self.futex.compare_exchange(pi::unlocked(), pi::locked(), Acquire, Relaxed).is_ok()
+    }
+
+    #[inline]
+    pub fn lock(&self) {
+        if self.futex.compare_exchange(pi::unlocked(), pi::locked(), Acquire, Relaxed).is_err() {
+            self.lock_contended();
+        }
+    }
+
+    #[cold]
+    fn lock_contended(&self) {
+        // Spin first to speed things up if the lock is released quickly.
+        let state = self.spin();
+
+        // If it's unlocked now, attempt to take the lock.
+        if state == pi::unlocked() {
+            if self.try_lock() {
+                return;
+            }
+        };
+
+        pi::futex_lock(&self.futex).expect("failed to lock mutex");
+
+        let state = self.futex.load(Relaxed);
+        if pi::is_owned_died(state) {
+            panic!(
+                "failed to lock mutex because the thread owning it finished without unlocking it"
+            );
+        }
+    }
+
+    fn spin(&self) -> pi::State {
+        let mut spin = 100;
+        loop {
+            // We only use `load` (and not `swap` or `compare_exchange`)
+            // while spinning, to be easier on the caches.
+            let state = self.futex.load(Relaxed);
+
+            // We stop spinning when the mutex is unlocked,
+            // but also when it's contended.
+            if state == pi::unlocked() || pi::is_contended(state) || spin == 0 {
+                return state;
+            }
+
+            crate::hint::spin_loop();
+            spin -= 1;
+        }
+    }
+
+    #[inline]
+    pub unsafe fn unlock(&self) {
+        if self.futex.compare_exchange(pi::locked(), pi::unlocked(), Release, Relaxed).is_err() {
+            // We only wake up one thread. When that thread locks the mutex,
+            // the kernel will mark the mutex as contended automatically
+            // (futex != pi::locked() in this case),
+            // which makes sure that any other waiting threads will also be
+            // woken up eventually.
+            self.wake();
+        }
+    }
+
+    #[cold]
+    fn wake(&self) {
+        pi::futex_unlock(&self.futex).unwrap();
+    }
+}