diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b770c5ba6..9f1d275ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -987,6 +987,8 @@ jobs: # if: matrix.rust == 'stable' - run: tools/no-std.sh +esp xtensa-esp32s2-none-elf if: matrix.rust == 'stable' + - run: tools/no-std.sh +esp xtensa-esp32s3-none-elf + if: matrix.rust == 'stable' miri: needs: tidy diff --git a/build.rs b/build.rs index 03bd291fe..96989b3fe 100644 --- a/build.rs +++ b/build.rs @@ -56,13 +56,17 @@ fn main() { // Custom cfgs set by build script. Not public API. // grep -F 'cargo:rustc-cfg=' build.rs | grep -Ev '^ *//' | sed -E 's/^.*cargo:rustc-cfg=//; s/(=\\)?".*$//' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/' println!( - "cargo:rustc-check-cfg=cfg(portable_atomic_atomic_intrinsics,portable_atomic_disable_fiq,portable_atomic_force_amo,portable_atomic_ll_sc_rmw,portable_atomic_no_asm,portable_atomic_no_asm_maybe_uninit,portable_atomic_no_atomic_64,portable_atomic_no_atomic_cas,portable_atomic_no_atomic_load_store,portable_atomic_no_atomic_min_max,portable_atomic_no_cfg_target_has_atomic,portable_atomic_no_cmpxchg16b_intrinsic,portable_atomic_no_cmpxchg16b_target_feature,portable_atomic_no_const_mut_refs,portable_atomic_no_const_raw_ptr_deref,portable_atomic_no_const_transmute,portable_atomic_no_core_unwind_safe,portable_atomic_no_diagnostic_namespace,portable_atomic_no_strict_provenance,portable_atomic_no_strict_provenance_atomic_ptr,portable_atomic_no_stronger_failure_ordering,portable_atomic_no_track_caller,portable_atomic_no_unsafe_op_in_unsafe_fn,portable_atomic_pre_llvm_15,portable_atomic_pre_llvm_16,portable_atomic_pre_llvm_18,portable_atomic_pre_llvm_20,portable_atomic_s_mode,portable_atomic_sanitize_thread,portable_atomic_target_feature,portable_atomic_unsafe_assume_privileged,portable_atomic_unsafe_assume_single_core,portable_atomic_unstable_asm,portable_atomic_unstable_asm_experimental_arch,portable_atomic_unstable_cfg_target_has_atomic,portable_atomic_unstable_isa_attribute)" + "cargo:rustc-check-cfg=cfg(portable_atomic_atomic_intrinsics,portable_atomic_disable_fiq,portable_atomic_force_amo,portable_atomic_ll_sc_rmw,portable_atomic_no_asm,portable_atomic_no_asm_maybe_uninit,portable_atomic_no_atomic_64,portable_atomic_no_atomic_cas,portable_atomic_no_atomic_load_store,portable_atomic_no_atomic_min_max,portable_atomic_no_cfg_target_has_atomic,portable_atomic_no_cmpxchg16b_intrinsic,portable_atomic_no_cmpxchg16b_target_feature,portable_atomic_no_const_mut_refs,portable_atomic_no_const_raw_ptr_deref,portable_atomic_no_const_transmute,portable_atomic_no_core_unwind_safe,portable_atomic_no_diagnostic_namespace,portable_atomic_no_outline_atomics,portable_atomic_no_strict_provenance,portable_atomic_no_strict_provenance_atomic_ptr,portable_atomic_no_stronger_failure_ordering,portable_atomic_no_track_caller,portable_atomic_no_unsafe_op_in_unsafe_fn,portable_atomic_pre_llvm_15,portable_atomic_pre_llvm_16,portable_atomic_pre_llvm_18,portable_atomic_pre_llvm_20,portable_atomic_s_mode,portable_atomic_sanitize_thread,portable_atomic_target_cpu,portable_atomic_target_feature,portable_atomic_unsafe_assume_privileged,portable_atomic_unsafe_assume_single_core,portable_atomic_unstable_asm,portable_atomic_unstable_asm_experimental_arch,portable_atomic_unstable_cfg_target_has_atomic,portable_atomic_unstable_isa_attribute)" ); // TODO: handle multi-line target_feature_fallback // grep -F 'target_feature_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/' println!( r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","lsfe","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","rmw","v6","v7","zaamo","zabha","zacas"))"# ); + // grep -F 'target_cpu_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_cpu_fallback\(//; s/"\).*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/' + println!( + r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_cpu,values("esp32","esp32s3"))"# + ); } // https://github.com/rust-lang/rust/pull/123745 (includes https://github.com/rust-lang/cargo/pull/13560) merged in Rust 1.79 (nightly-2024-04-11). @@ -193,6 +197,12 @@ fn main() { println!("cargo:rustc-cfg=portable_atomic_unstable_asm_experimental_arch"); } } + "xtensa" => { + // https://github.com/rust-lang/rust/pull/93868 merged in Rust 1.60 (nightly-2022-02-13). + if is_allowed_feature("asm_experimental_arch") { + println!("cargo:rustc-cfg=portable_atomic_unstable_asm_experimental_arch"); + } + } _ => {} } } @@ -502,6 +512,27 @@ fn main() { } target_feature_fallback("rmw", xmegau); } + "xtensa" => { + // Some Xtensa CPUs have CAS for internal memory, but also have an external address space, + // which does not correctly provide atomic access. The affected CPUs have their own build targets, + // but may also be specified with `-C target-cpu`. + // Xtensa targets have been introduced in Rust 1.81.0. + if let Some(cpu) = target_cpu() { + if cpu == "esp32" || cpu == "esp32s3" { + target_cpu_fallback(&cpu); + } + } else { + match target { + "xtensa-esp32-none-elf" | "xtensa-esp32-espidf" => target_cpu_fallback("esp32"), + "xtensa-esp32s3-none-elf" | "xtensa-esp32s3-espidf" => { + target_cpu_fallback("esp32s3") + } + // ESP32-S2 does not have atomic CAS, so it is not affected by the issue the same way. + // For other Xtensa CPUs, assume they are not affected. + _ => {} + } + } + } _ => {} } } @@ -560,6 +591,11 @@ fn target_cpu() -> Option { cpu.map(str::to_owned) } +// `target_cpu` is not a valid cfg option. Where there is absolutely no other option, inject a cfg fallback. +fn target_cpu_fallback(cpu: &str) { + println!("cargo:rustc-cfg=portable_atomic_target_cpu=\"{}\"", cpu); +} + fn is_allowed_feature(name: &str) -> bool { // https://github.com/dtolnay/thiserror/pull/248 if env::var_os("RUSTC_STAGE").is_some() { diff --git a/src/imp/interrupt/mod.rs b/src/imp/interrupt/mod.rs index 16631c250..78f21d48a 100644 --- a/src/imp/interrupt/mod.rs +++ b/src/imp/interrupt/mod.rs @@ -9,7 +9,11 @@ Fallback implementation based on disabling interrupts or critical-section See README.md of this directory for details. */ -#[cfg(not(feature = "critical-section"))] +#[cfg(any( + not(feature = "critical-section"), + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", +))] #[cfg_attr( all( target_arch = "arm", @@ -30,13 +34,26 @@ See README.md of this directory for details. #[cfg_attr(target_arch = "xtensa", path = "xtensa.rs")] pub(super) mod arch; +// ESP32 and ESP32-S3 require a critical-section based implementation for some of their address spaces. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, - cfg(any(test, portable_atomic_no_atomic_cas, portable_atomic_unsafe_assume_single_core)) + cfg(any( + test, + portable_atomic_no_atomic_cas, + portable_atomic_unsafe_assume_single_core, + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", + )) )] #[cfg_attr( not(portable_atomic_no_cfg_target_has_atomic), - cfg(any(test, not(target_has_atomic = "ptr"), portable_atomic_unsafe_assume_single_core)) + cfg(any( + test, + not(target_has_atomic = "ptr"), + portable_atomic_unsafe_assume_single_core, + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", + )) )] items!({ use core::{cell::UnsafeCell, sync::atomic::Ordering}; @@ -243,6 +260,8 @@ items!({ test, target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", not(target_has_atomic = "ptr") )) )] @@ -616,6 +635,46 @@ items!({ } }); + // Current target data unsoundly enables CAS for these devices. Generate the necessary + // critical-section based implementations that we'll wrap in `imp/xtensa.rs`. + #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(all()))] + #[cfg_attr( + not(portable_atomic_no_cfg_target_has_atomic), + cfg(all( + target_has_atomic, + any(portable_atomic_target_cpu = "esp32", portable_atomic_target_cpu = "esp32s3") + )) + )] + items!({ + use self::arch::atomic; + + atomic_int!(load_store_atomic, AtomicIsize, isize, 4); + atomic_int!(load_store_atomic, AtomicUsize, usize, 4); + + atomic_int!(load_store_atomic[sub_word], AtomicI8, i8, 1); + atomic_int!(load_store_atomic[sub_word], AtomicU8, u8, 1); + + atomic_int!(load_store_atomic[sub_word], AtomicI16, i16, 2); + atomic_int!(load_store_atomic[sub_word], AtomicU16, u16, 2); + + atomic_int!(load_store_atomic, AtomicI32, i32, 4); + atomic_int!(load_store_atomic, AtomicU32, u32, 4); + + atomic_base!(native_load_store, [T] AtomicPtr, *mut T); + impl AtomicPtr { + #[cfg(test)] + #[inline] + fn fetch_ptr_add(&self, val: usize, order: Ordering) -> *mut T { + self.fetch_byte_add(val.wrapping_mul(core::mem::size_of::()), order) + } + #[cfg(test)] + #[inline] + fn fetch_ptr_sub(&self, val: usize, order: Ordering) -> *mut T { + self.fetch_byte_sub(val.wrapping_mul(core::mem::size_of::()), order) + } + } + }); + // Double or more width atomics (require fallback feature for consistency with other situations). #[cfg(target_pointer_width = "16")] #[cfg(any(test, feature = "fallback"))] diff --git a/src/imp/interrupt/xtensa.rs b/src/imp/interrupt/xtensa.rs index da400cdd1..507109238 100644 --- a/src/imp/interrupt/xtensa.rs +++ b/src/imp/interrupt/xtensa.rs @@ -15,10 +15,20 @@ use core::arch::asm; )] #[cfg_attr( not(portable_atomic_no_cfg_target_has_atomic), - cfg(any(test, not(target_has_atomic = "ptr"))) + cfg(any( + test, + not(any( + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", + target_has_atomic = "ptr" + )) + )) )] pub(super) use core::sync::atomic; +#[cfg(any(portable_atomic_target_cpu = "esp32", portable_atomic_target_cpu = "esp32s3"))] +pub(super) use crate::imp::xtensa as atomic; + pub(crate) type State = u32; /// Disables interrupts and returns the previous interrupt state. diff --git a/src/imp/mod.rs b/src/imp/mod.rs index 54fbd1555..3cd252714 100644 --- a/src/imp/mod.rs +++ b/src/imp/mod.rs @@ -35,6 +35,10 @@ not(target_has_atomic = "ptr"), ))) )] +#[cfg_attr( + target_arch = "xtensa", + cfg(not(any(portable_atomic_target_cpu = "esp32", portable_atomic_target_cpu = "esp32s3"))) +)] mod core_atomic; // AVR @@ -47,6 +51,10 @@ mod avr; #[cfg(target_arch = "msp430")] pub(crate) mod msp430; +// Xtensas with address ranges that do not support atomic operations +#[cfg(any(portable_atomic_target_cpu = "esp32", portable_atomic_target_cpu = "esp32s3"))] +pub(crate) mod xtensa; + // RISC-V without A-extension #[cfg(any(test, not(feature = "critical-section")))] #[cfg_attr( @@ -97,6 +105,8 @@ mod atomic128; #[cfg(not(any( target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", portable_atomic_unsafe_assume_single_core, )))] #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))] @@ -161,10 +171,13 @@ mod fallback; // On AVR, we always use critical section based fallback implementation. // AVR can be safely assumed to be single-core, so this is sound. // MSP430 as well. +// On ESP32, we always use critical section based fallback implementation. // See the module-level comments of interrupt module for more. #[cfg(any( target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", feature = "critical-section", portable_atomic_unsafe_assume_single_core, portable_atomic_unsafe_assume_privileged, @@ -176,6 +189,8 @@ mod fallback; portable_atomic_no_atomic_cas, portable_atomic_unsafe_assume_single_core, portable_atomic_unsafe_assume_privileged, + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", )) )] #[cfg_attr( @@ -185,6 +200,8 @@ mod fallback; not(target_has_atomic = "ptr"), portable_atomic_unsafe_assume_single_core, portable_atomic_unsafe_assume_privileged, + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", )) )] #[cfg(any( @@ -208,7 +225,7 @@ pub(crate) mod float; // {8,16,32}-bit & ptr-sized atomics cfg_sel!({ - // has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic + // has CAS | (has core atomic & !(avr | msp430 | critical section | esp32)) => core atomic #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(not(any( @@ -224,6 +241,8 @@ cfg_sel!({ ), portable_atomic_no_atomic_cas, ), + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", ))) )] #[cfg_attr( @@ -241,6 +260,8 @@ cfg_sel!({ ), not(target_has_atomic = "ptr"), ), + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", ))) )] { @@ -253,7 +274,13 @@ cfg_sel!({ #[cfg(any( target_arch = "avr", target_arch = "msp430", - feature = "critical-section", + all( + not(any( + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3" + )), + feature = "critical-section", + ), portable_atomic_unsafe_assume_single_core, ))] { @@ -271,6 +298,14 @@ cfg_sel!({ AtomicUsize, }; } + // Xtensa with quirky external data bus address range + #[cfg(any(portable_atomic_target_cpu = "esp32", portable_atomic_target_cpu = "esp32s3"))] + { + pub(crate) use self::xtensa::{ + AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicU32, + AtomicUsize, + }; + } // bpf & !(critical section) => core atomic #[cfg(target_arch = "bpf")] { @@ -284,7 +319,7 @@ cfg_sel!({ // 64-bit atomics cfg_sel!({ - // has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic + // (has CAS & !esp32) | (has core atomic & !(avr | msp430 | critical section)) => core atomic #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(all( @@ -300,7 +335,9 @@ cfg_sel!({ portable_atomic_unsafe_assume_single_core, ), portable_atomic_no_atomic_cas, - ) + ), + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", )), any( not(portable_atomic_no_atomic_64), @@ -323,7 +360,9 @@ cfg_sel!({ portable_atomic_unsafe_assume_single_core, ), not(target_has_atomic = "ptr"), - ) + ), + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", )), any( target_has_atomic = "64", @@ -365,12 +404,14 @@ cfg_sel!({ { pub(crate) use self::atomic64::riscv32::{AtomicI64, AtomicU64}; } - // no native atomic CAS & (assume single core | critical section) => critical section based fallback + // (no native atomic CAS | esp32) & (assume single core | critical section) => critical section based fallback #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any( target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", all(feature = "critical-section", portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core, )) @@ -380,6 +421,8 @@ cfg_sel!({ cfg(any( target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", all(feature = "critical-section", not(target_has_atomic = "ptr")), portable_atomic_unsafe_assume_single_core, )) @@ -548,12 +591,14 @@ cfg_sel!({ { pub(crate) use self::atomic128::s390x::{AtomicI128, AtomicU128}; } - // no native atomic CAS & (assume single core | critical section) => critical section based fallback + // (no native atomic CAS | esp32) & (assume single core | critical section) => critical section based fallback #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any( target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", all(feature = "critical-section", portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core, )) @@ -563,6 +608,8 @@ cfg_sel!({ cfg(any( target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", all(feature = "critical-section", not(target_has_atomic = "ptr")), portable_atomic_unsafe_assume_single_core, )) diff --git a/src/imp/xtensa.rs b/src/imp/xtensa.rs new file mode 100644 index 000000000..3ca1a46f6 --- /dev/null +++ b/src/imp/xtensa.rs @@ -0,0 +1,759 @@ +//! Implementations for Xtensa CPUs that have atomic CAS, but also have an address range where it is not working properly. +//! +//! For these CPUs rustc may, but should not provide atomic access unconditionally. portable-atomic generates code which selects +//! the appropriate implementation based on the address of the atomic variable. +#![allow(unreachable_code)] +use core::{arch::asm, ops::Range, sync::atomic::Ordering}; + +// https://documentation.espressif.com/esp32_technical_reference_manual_en.pdf, Table 3.3-4. External Memory Address Mapping +#[cfg(portable_atomic_target_cpu = "esp32")] +const EXTERNAL_DATA_BUS_ADDRESS_RANGE: Range = 0x3F80_0000..0x3FC0_0000; + +// https://documentation.espressif.com/esp32-s3_technical_reference_manual_en.pdf, Table 4.3-2. External Memory Address Mapping +#[cfg(portable_atomic_target_cpu = "esp32s3")] +const EXTERNAL_DATA_BUS_ADDRESS_RANGE: Range = 0x3C00_0000..0x3E00_0000; + +macro_rules! dispatch_impl { + ($self:expr, $atomic:expr, $fallback:expr) => { + { + let addr = $self.as_ptr() as usize; + if addr >= EXTERNAL_DATA_BUS_ADDRESS_RANGE.start && addr < EXTERNAL_DATA_BUS_ADDRESS_RANGE.end { + return $fallback; + } else { + return $atomic; + } + } + }; +} + +items!({ + macro_rules! atomic { + (base, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt) => { + #[repr(transparent)] + pub(crate) struct $atomic_type $(<$($generics)*>)? { + v: crate::imp::interrupt::$atomic_type $(<$($generics)*>)?, + } + + impl_default_bit_opts!($atomic_type, $int_type); + + // Send is implicitly implemented for atomic integers, but not for atomic pointers. + // SAFETY: any data races are prevented by atomic operations. + unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {} + // SAFETY: any data races are prevented by atomic operations. + unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {} + + impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? { + #[inline] + pub(crate) const fn new(v: $value_type) -> Self { + Self { v: crate::imp::interrupt::$atomic_type::new(v) } + } + + #[inline] + pub(crate) const fn as_ptr(&self) -> *mut $value_type { + self.v.as_ptr() + } + + #[inline] + pub(crate) fn is_lock_free() -> bool { + )?>::is_lock_free() + } + pub(crate) const IS_ALWAYS_LOCK_FREE: bool = )?>::IS_ALWAYS_LOCK_FREE; + + #[inline] + pub(crate) fn swap(&self, val: $value_type, order: Ordering) -> $value_type { + dispatch_impl!(self, + unsafe { <$value_type>::atomic_swap(self.v.as_ptr(), val, order) }, + self.v.swap(val, order) + ) + } + + #[inline] + pub(crate) fn compare_exchange( + &self, + current: $value_type, + new: $value_type, + success: Ordering, + failure: Ordering, + ) -> Result<$value_type, $value_type> { + dispatch_impl!(self, + to_result(unsafe { <$value_type>::atomic_compare_exchange(self.v.as_ptr(), current, new, success, failure) }), + self.v.compare_exchange(current, new, success, failure) + ) + } + + #[inline] + pub(crate) fn compare_exchange_weak( + &self, + current: $value_type, + new: $value_type, + success: Ordering, + failure: Ordering, + ) -> Result<$value_type, $value_type> { + dispatch_impl!(self, + to_result(unsafe { <$value_type>::atomic_compare_exchange_weak(self.v.as_ptr(), current, new, success, failure) }), + self.v.compare_exchange_weak(current, new, success, failure) + ) + } + } + }; + (load_store, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt) => { + atomic!(base, $([$($generics)*])? $atomic_type, $value_type, $int_type, $size); + impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? { + #[inline] + #[cfg_attr( + all(debug_assertions, not(portable_atomic_no_track_caller)), + track_caller + )] + pub(crate) fn load(&self, order: Ordering) -> $value_type { + dispatch_impl!(self, + unsafe { <$value_type>::atomic_load(self.v.as_ptr(), order) }, + self.v.load(order) + ) + } + + #[inline] + #[cfg_attr( + all(debug_assertions, not(portable_atomic_no_track_caller)), + track_caller + )] + pub(crate) fn store(&self, val: $value_type, order: Ordering) { + dispatch_impl!(self, + unsafe { <$value_type>::atomic_store(self.v.as_ptr(), val, order) }, + self.v.store(val, order) + ) + } + } + }; + (boolean_fetch, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt) => { + atomic!(load_store, $([$($generics)*])? $atomic_type, $value_type, $int_type, $size); + impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? { + #[inline] + pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $value_type { + dispatch_impl!(self, + loop {}, + self.v.fetch_and(val, order) + ) + } + + #[inline] + pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $value_type { + dispatch_impl!(self, + loop {}, + self.v.fetch_or(val, order) + ) + } + + #[inline] + pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $value_type { + dispatch_impl!(self, + loop {}, + self.v.fetch_xor(val, order) + ) + } + + #[inline] + pub(crate) fn fetch_not(&self, order: Ordering) -> $value_type { + dispatch_impl!(self, + loop {}, + self.v.fetch_not(order) + ) + } + + #[inline] + pub(crate) fn fetch_neg(&self, order: Ordering) -> $value_type { + dispatch_impl!(self, + loop {}, + self.v.fetch_neg(order) + ) + } + + #[inline] + pub(crate) fn fetch_nand(&self, val: $int_type, order: Ordering) -> $value_type { + dispatch_impl!(self, + loop {}, + self.v.fetch_nand(val, order) + ) + } + + #[inline] + pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $value_type { + dispatch_impl!(self, + loop {}, + self.v.fetch_min(val, order) + ) + } + + #[inline] + pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $value_type { + dispatch_impl!(self, + loop {}, + self.v.fetch_max(val, order) + ) + } + } + }; + (boolean, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt) => { + atomic!(boolean_fetch, $([$($generics)*])? $atomic_type, $value_type, $int_type, $size); + impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? { + #[inline] + pub(crate) fn and(&self, val: $int_type, order: Ordering) { + self.fetch_and(val, order); + } + + #[inline] + pub(crate) fn or(&self, val: $int_type, order: Ordering) { + self.fetch_or(val, order); + } + + #[inline] + pub(crate) fn xor(&self, val: $int_type, order: Ordering) { + self.fetch_xor(val, order); + } + + #[inline] + pub(crate) fn not(&self, order: Ordering) { + self.fetch_not(order); + } + + #[inline] + pub(crate) fn neg(&self, order: Ordering) { + self.fetch_neg(order); + } + } + }; + (ptr, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt) => { + atomic!(boolean_fetch, $([$($generics)*])? $atomic_type, $value_type, $int_type, $size); + impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? { + #[inline] + pub(crate) fn fetch_byte_sub(&self, val: usize, order: Ordering) -> $value_type { + dispatch_impl!(self, + loop {}, + self.v.fetch_byte_sub(val, order) + ) + } + + #[inline] + pub(crate) fn fetch_byte_add(&self, val: usize, order: Ordering) -> $value_type { + dispatch_impl!(self, + loop {}, + self.v.fetch_byte_add(val, order) + ) + } + } + }; + ($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt) => { + atomic!(boolean, $([$($generics)*])? $atomic_type, $value_type, $int_type, $size); + impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? { + #[inline] + pub(crate) fn add(&self, val: $value_type, order: Ordering) { + self.fetch_add(val, order); + } + + #[inline] + pub(crate) fn sub(&self, val: $value_type, order: Ordering) { + self.fetch_sub(val, order); + } + + #[inline] + pub(crate) fn fetch_sub(&self, val: $value_type, order: Ordering) -> $value_type { + dispatch_impl!(self, + loop {}, + self.v.fetch_sub(val, order) + ) + } + + #[inline] + pub(crate) fn fetch_add(&self, val: $value_type, order: Ordering) -> $value_type { + dispatch_impl!(self, + loop {}, + self.v.fetch_add(val, order) + ) + } + } + }; + } + + atomic!(AtomicI8, i8, i8, "b"); + atomic!(AtomicU8, u8, u8, "b"); + atomic!(AtomicI16, i16, i16, "h"); + atomic!(AtomicU16, u16, u16, "h"); + atomic!(AtomicI32, i32, i32, "w"); + atomic!(AtomicU32, u32, u32, "w"); + atomic!(AtomicIsize, isize, isize, "w"); + atomic!(AtomicUsize, usize, usize, "w"); + atomic!(ptr, [T] AtomicPtr, *mut T, usize, "w"); +}); + +#[cfg(feature = "fallback")] +items!({ + pub(crate) use crate::imp::interrupt::{AtomicI64, AtomicU64}; +}); + +macro_rules! atomic_asm_rmw { + ($op:ident, $order:ident) => { + match $order { + Ordering::Relaxed => $op!("", ""), + Ordering::Acquire => $op!("memw", ""), + Ordering::Release => $op!("", "memw"), + Ordering::AcqRel | Ordering::SeqCst => $op!("memw", "memw"), + _ => unreachable!(), + } + }; +} + +#[rustfmt::skip] +macro_rules! atomic_asm_load_store { + (($($ty:tt)*), $bits:tt, $narrow:tt, $unsigned:tt $(, [$generic:ident])?) => { + delegate_signed!(delegate_all, $($ty)*); + impl$(<$generic>)? AtomicLoad for $($ty)* { + #[inline] + unsafe fn atomic_load( + src: *const Self, + order: Ordering, + ) -> Self { + let out: Self; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + macro_rules! atomic_load { + ($acquire:tt) => { + asm!( + concat!("l", $bits, $unsigned, "i", $narrow, " {out}, {src}, 0"), // atomic { out = *src } + $acquire, // fence + src = in(reg) ptr_reg!(src), + out = lateout(reg) out, + options(nostack, preserves_flags), + ) + }; + } + match order { + Ordering::Relaxed => atomic_load!(""), + Ordering::Acquire | Ordering::SeqCst => atomic_load!("memw"), + _ => unreachable!(), + } + } + out + } + } + impl$(<$generic>)? AtomicStore for $($ty)* { + #[inline] + unsafe fn atomic_store( + dst: *mut Self, + val: Self, + order: Ordering, + ) { + // SAFETY: the caller must uphold the safety contract. + unsafe { + macro_rules! store { + ($acquire:tt, $release:tt) => { + asm!( + $release, // fence + concat!("s", $bits, "i", $narrow, " {val}, {dst}, 0"), // atomic { *dst = val } + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + val = in(reg) val, + options(nostack, preserves_flags), + ) + }; + } + atomic_asm_rmw!(store, order); + } + } + } + }; +} + +trait AtomicLoad: Sized { + unsafe fn atomic_load( + src: *const Self, + order: Ordering, + ) -> Self; +} + +trait AtomicStore: Sized { + unsafe fn atomic_store( + dst: *mut Self, + val: Self, + order: Ordering, + ); +} + +trait AtomicSwap: Sized { + unsafe fn atomic_swap( + dst: *mut Self, + val: Self, + order: Ordering, + ) -> Self; +} + +trait AtomicCompareExchange: Sized { + unsafe fn atomic_compare_exchange( + dst: *mut Self, + expected: Self, + desired: Self, + success: Ordering, + failure: Ordering, + ) -> (Self, bool); + + #[inline] + unsafe fn atomic_compare_exchange_weak( + dst: *mut Self, + current: Self, + new: Self, + success: Ordering, + failure: Ordering, + ) -> (Self, bool) { + // SAFETY: the caller must uphold the safety contract. + unsafe { Self::atomic_compare_exchange(dst, current, new, success, failure) } + } +} + +#[rustfmt::skip] +macro_rules! atomic_asm { + (($($ty:tt)*) $(, [$generic:ident])?) => { + atomic_asm_load_store!(($($ty)*), "32", ".n", "" $(, [$generic])? ); + + impl$(<$generic>)? AtomicSwap for $($ty)* { + #[inline] + unsafe fn atomic_swap( + dst: *mut Self, + val: Self, + order: Ordering, + ) -> Self { + let mut out: Self; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + macro_rules! swap { + ($acquire:tt, $release:tt) => { + asm!( + $release, // fence + "l32i.n {out}, {dst}, 0", // atomic { out = *dst } + "2:", // 'retry: + "mov.n {tmp}, {out}", // tmp = out + "wsr {tmp}, scompare1", // scompare1 = tmp + "mov.n {out}, {val}", // out = val + "s32c1i {out}, {dst}, 0", // atomic { _x = *dst; if _x == scompare1 { *dst = out }; out = _x } + "bne {tmp}, {out}, 2b", // if tmp != out { jump 'retry } + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + val = in(reg) val, + out = out(reg) out, + tmp = out(reg) _, + out("scompare1") _, + options(nostack, preserves_flags), + ) + }; + } + atomic_asm_rmw!(swap, order); + } + out + } + } + + impl$(<$generic>)? AtomicCompareExchange for $($ty)* { + #[inline] + unsafe fn atomic_compare_exchange( + dst: *mut Self, + old: Self, + new: Self, + success: Ordering, + failure: Ordering, + ) -> (Self, bool) { + let order = crate::utils::upgrade_success_ordering(success, failure); + let out: Self; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + let mut r: u32 = 1; + macro_rules! cmpxchg { + ($acquire:tt, $release:tt) => { + asm!( + $release, // fence + "wsr {old}, scompare1", // scompare1 = old + "s32c1i {out}, {dst}, 0", // atomic { _x = *dst; if _x == scompare1 { *dst = out }; out = _x } + $acquire, // fence + "beq {old}, {out}, 2f", // if old == out { jump 'success } + "movi {r}, 0", // r = 0 + "2:", // 'success: + dst = in(reg) ptr_reg!(dst), + old = in(reg) old, + out = inout(reg) new => out, + r = inout(reg) r, + out("scompare1") _, + options(nostack, preserves_flags), + ) + }; + } + atomic_asm_rmw!(cmpxchg, order); + crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test + (out, r != 0) + } + } + } + }; +} + +#[rustfmt::skip] +macro_rules! atomic_asm_sub_word { + (($($ty:tt)*), $bits:tt) => { + atomic_asm_load_store!(($($ty)*), $bits, "", "u"); + + impl AtomicSwap for $($ty)* { + #[inline] + unsafe fn atomic_swap( + dst: *mut Self, + val: Self, + order: Ordering, + ) -> Self { + let (dst, shift, mask) = crate::utils::create_sub_word_mask_values(dst); + let mut out: Self; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + macro_rules! swap { + ($acquire:tt, $release:tt) => { + // Implement sub-word atomic operations using word-sized CAS loop. + // See also create_sub_word_mask_values. + asm!( + "ssl {shift}", // sar = for_sll(shift & 31) + "sll {mask}, {mask}", // mask <<= sar + "sll {val}, {val}", // val <<= sar + $release, // fence + "l32i.n {out}, {dst}, 0", // atomic { out = *dst } + "2:", // 'retry: + "mov.n {tmp}, {out}", // tmp = out + "wsr {tmp}, scompare1", // scompare1 = tmp + "xor {out}, {tmp}, {val}", // out = tmp ^ val + "and {out}, {out}, {mask}", // out &= mask + "xor {out}, {out}, {tmp}", // out ^= out + "s32c1i {out}, {dst}, 0", // atomic { _x = *dst; if _x == scompare1 { *dst = out }; out = _x } + "bne {tmp}, {out}, 2b", // if tmp != out { jump 'retry } + "ssr {shift}", // sar = for_srl(shift & 31) + "srl {out}, {out}", // out >>= sar + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + val = inout(reg) val as u32 => _, + out = out(reg) out, + shift = in(reg) shift, + mask = inout(reg) mask => _, + tmp = out(reg) _, + out("scompare1") _, + out("sar") _, + options(nostack, preserves_flags), + ) + }; + } + atomic_asm_rmw!(swap, order); + } + out + } + } + + impl AtomicCompareExchange for $($ty)* { + #[inline] + unsafe fn atomic_compare_exchange( + dst: *mut Self, + old: Self, + new: Self, + success: Ordering, + failure: Ordering, + ) -> (Self, bool) { + let order = crate::utils::upgrade_success_ordering(success, failure); + let (dst, shift, mask) = crate::utils::create_sub_word_mask_values(dst); + let mut out: Self; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + let mut r: u32 = 0; + macro_rules! cmpxchg { + ($acquire:tt, $release:tt) => { + // Implement sub-word atomic operations using word-sized CAS loop. + // See also create_sub_word_mask_values. + asm!( + "ssl {shift}", // sar = for_sll(shift & 31) + "sll {mask}, {mask}", // mask <<= sar + "sll {old}, {old}", // old <<= sar + "sll {new}, {new}", // new <<= sar + $release, // fence + "l32i.n {out}, {dst}, 0", // atomic { out = *dst } + "2:", // 'retry: + "and {tmp}, {out}, {mask}", // tmp = out & mask + "bne {tmp}, {old}, 3f", // if tmp != old { jump 'cmp-fail } + "mov.n {tmp}, {out}", // tmp = out + "wsr {tmp}, scompare1", // scompare1 = tmp + "xor {out}, {tmp}, {new}", // out = tmp ^ new + "and {out}, {out}, {mask}", // out &= mask + "xor {out}, {out}, {tmp}", // out ^= tmp + "s32c1i {out}, {dst}, 0", // atomic { _x = *dst; if _x == scompare1 { *dst = out }; out = _x } + "bne {tmp}, {out}, 2b", // if tmp != out { jump 'retry } + "movi {r}, 1", // r = 1 + "3:", // 'cmp-fail: + "ssr {shift}", // sar = for_srl(shift & 31) + "srl {out}, {out}", // out >>= sar + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + old = inout(reg) old as u32 => _, + new = inout(reg) new as u32 => _, + out = out(reg) out, + shift = in(reg) shift, + mask = inout(reg) mask => _, + tmp = out(reg) _, + r = inout(reg) r, + out("scompare1") _, + out("sar") _, + options(nostack, preserves_flags), + ) + }; + } + atomic_asm_rmw!(cmpxchg, order); + crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test + (out, r != 0) + } + } + } + }; +} + + +macro_rules! delegate_load_store { + ($ty:ty, $base:ident $(, [$generic:ident])?) => { + const _: () = { + use core::mem; + assert!(mem::size_of::<$ty>() == mem::size_of::<$base>()); + assert!(mem::align_of::<$ty>() == mem::align_of::<$base>()); + }; + impl$(<$generic>)? AtomicLoad for $ty { + #[inline] + unsafe fn atomic_load( + src: *const Self, + order: Ordering, + ) -> Self { + // SAFETY: the caller must uphold the safety contract. + // cast and transmute are okay because $ty and $base implement the same layout. + unsafe { + <$base as AtomicLoad>::atomic_load(src.cast::<$base>(), order) as Self + } + } + } + impl$(<$generic>)? AtomicStore for $ty { + #[inline] + unsafe fn atomic_store( + dst: *mut Self, + val: Self, + order: Ordering, + ) { + // SAFETY: the caller must uphold the safety contract. + // cast and transmute are okay because $ty and $base implement the same layout. + unsafe { + <$base as AtomicStore>::atomic_store( + dst.cast::<$base>(), + val as $base, + order, + ); + } + } + } + }; +} +#[allow(unused_macros)] +macro_rules! delegate_swap { + ($ty:ty, $base:ident $(, [$generic:ident])?) => { + const _: () = { + use core::mem; + assert!(mem::size_of::<$ty>() == mem::size_of::<$base>()); + assert!(mem::align_of::<$ty>() == mem::align_of::<$base>()); + }; + impl$(<$generic:ty>)? AtomicSwap for $ty { + #[inline] + unsafe fn atomic_swap( + dst: *mut Self, + val: Self, + order: Ordering, + ) -> Self { + // SAFETY: the caller must uphold the safety contract. + // cast and transmute are okay because $ty and $base implement the same layout. + unsafe { + <$base as AtomicSwap>::atomic_swap( + dst.cast::<$base>(), + val as $base, + order, + ) as Self + } + } + } + }; +} + +macro_rules! delegate_cas { + ($ty:ty, $base:ident $(, [$generic:ident])?) => { + const _: () = { + use core::mem; + assert!(mem::size_of::<$ty>() == mem::size_of::<$base>()); + assert!(mem::align_of::<$ty>() == mem::align_of::<$base>()); + }; + impl$(<$generic:ty>)? AtomicCompareExchange for $ty { + #[inline] + unsafe fn atomic_compare_exchange( + dst: *mut Self, + current: Self, + new: Self, + success: Ordering, + failure: Ordering, + ) -> (Self, bool) { + // SAFETY: the caller must uphold the safety contract. + // cast and transmute are okay because $ty and $base implement the same layout. + unsafe { + let (out, ok) = <$base as AtomicCompareExchange>::atomic_compare_exchange( + dst.cast::<$base>(), + current as $base, + new as $base, + success, + failure, + ); + (out as Self, ok) + } + } + } + }; +} + +macro_rules! delegate_all { + ($ty:ident, $base:ident) => { + delegate_load_store!($ty, $base); + delegate_swap!($ty, $base); + delegate_cas!($ty, $base); + }; +} + +macro_rules! delegate_signed { + ($delegate:ident, u8) => { + $delegate!(i8, u8); + }; + ($delegate:ident, u16) => { + $delegate!(i16, u16); + }; + ($delegate:ident, u32) => { + $delegate!(i32, u32); + }; + ($delegate:ident, usize) => { + $delegate!(isize, usize); + }; + ($delegate:ident, u64) => { + $delegate!(i64, u64); + }; + ($delegate:ident, u128) => { + $delegate!(i128, u128); + }; + ($delegate:ident, *mut T) => {}; +} + +atomic_asm_sub_word!((u8), "8"); +atomic_asm_sub_word!((u16), "16"); +atomic_asm!((u32)); +atomic_asm!((usize)); +atomic_asm!((*mut T), [T]); + +fn to_result((value, ok): (T, bool)) -> Result { + if ok { Ok(value) } else { Err(value) } +} diff --git a/src/lib.rs b/src/lib.rs index 3dfdaa8e3..6d77db89b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -310,6 +310,8 @@ See also the [`interrupt` module's readme](https://github.com/taiki-e/portable-a any( portable_atomic_unsafe_assume_single_core, portable_atomic_unsafe_assume_privileged, + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", ), ), all(target_arch = "powerpc64", portable_atomic_unstable_asm_experimental_arch), diff --git a/src/utils.rs b/src/utils.rs index f53afe584..d7c513a60 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -356,7 +356,7 @@ pub(crate) fn unlikely(b: bool) -> bool { } // Equivalent to core::hint::assert_unchecked, but compatible with pre-1.81 rustc. -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "xtensa"))] #[allow(dead_code)] #[inline(always)] #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] @@ -507,16 +507,16 @@ pub(crate) struct Pair { pub(crate) lo: T, } -#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "xtensa"))] type MinWord = u32; -#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "xtensa"))] type RetInt = u32; // Adapted from https://github.com/taiki-e/atomic-maybe-uninit/blob/v0.3.6/src/utils.rs#L255. // Helper for implementing sub-word atomic operations using word-sized LL/SC loop or CAS loop. // // Refs: https://github.com/llvm/llvm-project/blob/llvmorg-21.1.0/llvm/lib/CodeGen/AtomicExpandPass.cpp#L812 // (aligned_ptr, shift, mask) -#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "xtensa"))] #[allow(dead_code)] #[inline] pub(crate) fn create_sub_word_mask_values(ptr: *mut T) -> (*mut MinWord, RetInt, RetInt) { diff --git a/tests/xtensa/Cargo.toml b/tests/xtensa/Cargo.toml index 4e18d7745..1ab0e18ff 100644 --- a/tests/xtensa/Cargo.toml +++ b/tests/xtensa/Cargo.toml @@ -14,6 +14,11 @@ paste = "1" esp-println = { default-features = false, features = ["uart", "esp32s2"], git = "https://github.com/taiki-e/esp-hal.git", rev = "293a19f" } esp-hal = { features = ["esp32s2"], git = "https://github.com/taiki-e/esp-hal.git", rev = "293a19f" } +[target.xtensa-esp32s3-none-elf.dependencies] +esp-println = { default-features = false, features = ["uart", "esp32s3"], git = "https://github.com/taiki-e/esp-hal.git", rev = "293a19f" } +esp-hal = { features = ["esp32s3"], git = "https://github.com/taiki-e/esp-hal.git", rev = "293a19f" } +portable-atomic = { path = "../..", features = ["critical-section"] } + [lints] workspace = true diff --git a/tools/no-std.sh b/tools/no-std.sh index 1f9ec4650..ee9a8f261 100755 --- a/tools/no-std.sh +++ b/tools/no-std.sh @@ -209,6 +209,9 @@ run() { armv[4-5]t* | thumbv[4-5]t* | thumbv6m* | xtensa-esp32s2-*) target_rustflags+=" --cfg portable_atomic_unsafe_assume_single_core" ;; + xtensa-esp32*) + # At this time, these chips require critical-section to be enabled, which is incompatible with the single-core and privileged assumptions. + ;; arm* | thumb* | xtensa*) assume_single_core_target_rustflags+=" --cfg portable_atomic_unsafe_assume_single_core" assume_privileged_target_rustflags+=" --cfg portable_atomic_unsafe_assume_privileged"