From 18b6cb07b8d35bb9e4345511d422025c6a7ae086 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Wed, 1 Mar 2023 01:59:49 +0900 Subject: [PATCH 1/3] x86_64: Update to stabilized cmpxchg16b_target_feature --- README.md | 2 +- bench/benches/bench.rs | 2 +- build.rs | 32 ++++++++++++++++----------- src/imp/atomic128/README.md | 2 +- src/imp/atomic128/x86_64.rs | 12 +--------- src/imp/fallback/imp.rs | 34 ++++++++-------------------- src/imp/fallback/mod.rs | 13 +++++++++-- src/imp/fallback/seq_lock.rs | 2 -- src/imp/mod.rs | 18 ++++++++++++--- src/lib.rs | 43 ++++++++++++++++++------------------ src/tests/mod.rs | 7 ++++-- 11 files changed, 84 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index b39e6b3c0..a8bc94102 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ See also [the `atomic128` module's readme](https://github.com/taiki-e/portable-a If dynamic dispatching by run-time CPU feature detection is enabled, it allows maintaining support for older CPUs while using features that are not supported on older CPUs, such as CMPXCHG16B (x86_64) and FEAT_LSE (aarch64). Note: - - Dynamic detection is currently only enabled in Rust 1.61+ for aarch64, in 1.59+ (AVX) or nightly (CMPXCHG16B) for x86_64, and in nightly for other platforms, otherwise it works the same as when this cfg is set. + - Dynamic detection is currently only enabled in Rust 1.61+ for aarch64, in Rust 1.59+ (AVX) or 1.69+ (CMPXCHG16B) for x86_64, otherwise it works the same as when this cfg is set. - If the required target features are enabled at compile-time, the atomic operations are inlined. - This is compatible with no-std (as with all features except `std`). - Some aarch64 targets enable LLVM's `outline-atomics` target feature by default, so if you set this cfg, you may want to disable that as well. diff --git a/bench/benches/bench.rs b/bench/benches/bench.rs index 4233f1d15..6a4b25788 100644 --- a/bench/benches/bench.rs +++ b/bench/benches/bench.rs @@ -1,7 +1,7 @@ #![warn(rust_2018_idioms, single_use_lifetimes, unsafe_op_in_unsafe_fn)] #![allow(dead_code, unused_extern_crates)] #![allow(clippy::only_used_in_recursion)] -#![feature(asm_experimental_arch, cmpxchg16b_target_feature, core_intrinsics, stdsimd)] +#![feature(asm_experimental_arch, core_intrinsics, stdsimd)] use std::{ hint::black_box, diff --git a/build.rs b/build.rs index 0e40c7d51..05eff645f 100644 --- a/build.rs +++ b/build.rs @@ -30,7 +30,6 @@ fn main() { let mut target_upper = target.replace(|c: char| c == '-' || c == '.', "_"); target_upper.make_ascii_uppercase(); println!("cargo:rerun-if-env-changed=CARGO_TARGET_{}_RUSTFLAGS", target_upper); - println!("cargo:rerun-if-env-changed=CARGO_CFG_PORTABLE_ATOMIC_NO_OUTLINE_ATOMICS"); println!("cargo:rerun-if-env-changed=RUSTC"); let version = match rustc_version() { @@ -70,7 +69,6 @@ fn main() { } // asm stabilized in Rust 1.59 (nightly-2021-12-16): https://github.com/rust-lang/rust/pull/91728 let no_asm = !version.probe(59, 2021, 12, 15); - let mut unstable_asm = false; if no_asm { if version.nightly && version.probe(46, 2020, 6, 20) @@ -85,7 +83,6 @@ fn main() { // until it was stabilized in nightly-2021-12-16, so it can be safely enabled in // nightly, which is older than nightly-2021-12-16. println!("cargo:rustc-cfg=portable_atomic_unstable_asm"); - unstable_asm = true; } println!("cargo:rustc-cfg=portable_atomic_no_asm"); } @@ -157,19 +154,28 @@ fn main() { match target_arch { "x86_64" => { + // cmpxchg16b_target_feature stabilized in Rust 1.69 (nightly-2023-03-01): https://github.com/rust-lang/rust/pull/106774 + if !version.probe(69, 2023, 2, 28) { + if version.nightly && is_allowed_feature("cmpxchg16b_target_feature") { + // This feature has not been changed since 1.33 + // (https://github.com/rust-lang/rust/commit/fbb56bcf44d28e65a9495decf091b6d0386e540c) + // until it was stabilized in nightly-2023-03-01, so it can be safely enabled in + // nightly, which is older than nightly-2023-03-01. + println!("cargo:rustc-cfg=portable_atomic_unstable_cmpxchg16b_target_feature"); + println!("cargo:rustc-cfg=portable_atomic_cmpxchg16b_target_feature"); + } else { + // println!("cargo:rustc-cfg=portable_atomic_no_cmpxchg16b_target_feature"); + } + } else { + // TODO: invert flag once 1.69 became stable + println!("cargo:rustc-cfg=portable_atomic_cmpxchg16b_target_feature"); + } + // x86_64 macos always support CMPXCHG16B: https://github.com/rust-lang/rust/blob/1.67.0/compiler/rustc_target/src/spec/x86_64_apple_darwin.rs#L8 let has_cmpxchg16b = target_os == "macos"; // LLVM recognizes this also as cx16 target feature: https://godbolt.org/z/6dszGeYsf - // It is unlikely that rustc will support that name, so we will ignore it for now. - target_feature_if("cmpxchg16b", has_cmpxchg16b, &version, None, true); - if version.nightly - && (!no_asm || unstable_asm) - && cfg!(feature = "fallback") - && env::var_os("CARGO_CFG_PORTABLE_ATOMIC_NO_OUTLINE_ATOMICS").is_none() - && is_allowed_feature("cmpxchg16b_target_feature") - { - println!("cargo:rustc-cfg=portable_atomic_unstable_cmpxchg16b_target_feature"); - } + // It is unlikely that rustc will support that name, so we ignore it. + target_feature_if("cmpxchg16b", has_cmpxchg16b, &version, Some(69), true); } "aarch64" => { // aarch64_target_feature stabilized in Rust 1.61 (nightly-2022-03-16): https://github.com/rust-lang/rust/pull/90621 diff --git a/src/imp/atomic128/README.md b/src/imp/atomic128/README.md index 2c9a9ab46..48ad7a0fa 100644 --- a/src/imp/atomic128/README.md +++ b/src/imp/atomic128/README.md @@ -4,7 +4,7 @@ The table of targets that support 128-bit atomics and the instructions used: | target_arch | load | store | CAS | RMW | note | | ----------- | ----- | ----- | ---- | ---- | ---- | -| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel or AMD CPU with AVX.
Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only.
Requires rustc 1.59+ when cmpxchg16b target feature is enabled at compile-time, otherwise requires nightly | +| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel or AMD CPU with AVX.
Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only.
Requires rustc 1.59+ when cmpxchg16b target feature is enabled at compile-time, otherwise requires rustc 1.69+ | | aarch64 | ldxp/stxp or casp or ldp | ldxp/stxp or casp or stp | ldxp/stxp or casp | ldxp/stxp or casp | casp requires lse target feature, ldp/stp requires lse2 target feature.
Both compile-time and run-time detection are supported for lse. lse2 is currently compile-time detection only.
Requires rustc 1.59+ | | powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Little endian or target CPU pwr8+.
Requires nightly | | s390x | lpq | stpq | cdsg | cdsg | Requires nightly | diff --git a/src/imp/atomic128/x86_64.rs b/src/imp/atomic128/x86_64.rs index 313cc4a9f..71bf7389b 100644 --- a/src/imp/atomic128/x86_64.rs +++ b/src/imp/atomic128/x86_64.rs @@ -102,16 +102,7 @@ unsafe fn cmpxchg16b( failure: Ordering, ) -> (u128, bool) { #[cfg_attr( - all( - any( - all(test, portable_atomic_nightly), - portable_atomic_unstable_cmpxchg16b_target_feature, - ), - not(any( - target_feature = "cmpxchg16b", - portable_atomic_target_feature = "cmpxchg16b", - )), - ), + not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), target_feature(enable = "cmpxchg16b") )] #[cfg_attr( @@ -151,7 +142,6 @@ unsafe fn cmpxchg16b( unsafe { _cmpxchg16b(dst, old, new, success, failure) } - #[cfg(portable_atomic_unstable_cmpxchg16b_target_feature)] #[cfg(not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")))] { #[cold] diff --git a/src/imp/fallback/imp.rs b/src/imp/fallback/imp.rs index 39e398f1c..14d8b4496 100644 --- a/src/imp/fallback/imp.rs +++ b/src/imp/fallback/imp.rs @@ -1,3 +1,12 @@ +#![cfg_attr( + all( + target_arch = "x86_64", + portable_atomic_cmpxchg16b_target_feature, + not(portable_atomic_no_outline_atomics), + ), + allow(dead_code) +)] + use core::{cell::UnsafeCell, mem, sync::atomic::Ordering}; use super::{super::utils::CachePadded, SeqLock, SeqLockWriteGuard}; @@ -49,7 +58,6 @@ macro_rules! atomic { unsafe { &*(self.v.get() as *const $int_type as *const [AtomicChunk; Self::LEN]) } } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] fn optimistic_read(&self) -> $int_type { // Using `MaybeUninit<[usize; Self::LEN]>` here doesn't change codegen: https://godbolt.org/z/86f8s733M @@ -120,27 +128,22 @@ macro_rules! atomic { // SAFETY: any data races are prevented by the lock and atomic operation. unsafe impl Sync for $atomic_type {} - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] no_fetch_ops_impl!($atomic_type, $int_type); impl $atomic_type { - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) const fn new(v: $int_type) -> Self { Self { v: UnsafeCell::new(v) } } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn is_lock_free() -> bool { Self::is_always_lock_free() } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) const fn is_always_lock_free() -> bool { false } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn get_mut(&mut self) -> &mut $int_type { // SAFETY: the mutable reference guarantees unique ownership. @@ -148,13 +151,11 @@ macro_rules! atomic { unsafe { &mut *self.v.get() } } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn into_inner(self) -> $int_type { self.v.into_inner() } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] pub(crate) fn load(&self, order: Ordering) -> $int_type { @@ -178,7 +179,6 @@ macro_rules! atomic { val } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] pub(crate) fn store(&self, val: $int_type, order: Ordering) { @@ -187,7 +187,6 @@ macro_rules! atomic { self.write(val, &guard) } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn swap(&self, val: $int_type, _order: Ordering) -> $int_type { let guard = lock(self.v.get() as usize).write(); @@ -218,7 +217,6 @@ macro_rules! atomic { } } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] pub(crate) fn compare_exchange_weak( @@ -231,7 +229,6 @@ macro_rules! atomic { self.compare_exchange(current, new, success, failure) } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn fetch_add(&self, val: $int_type, _order: Ordering) -> $int_type { let guard = lock(self.v.get() as usize).write(); @@ -240,7 +237,6 @@ macro_rules! atomic { result } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn fetch_sub(&self, val: $int_type, _order: Ordering) -> $int_type { let guard = lock(self.v.get() as usize).write(); @@ -249,7 +245,6 @@ macro_rules! atomic { result } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn fetch_and(&self, val: $int_type, _order: Ordering) -> $int_type { let guard = lock(self.v.get() as usize).write(); @@ -258,7 +253,6 @@ macro_rules! atomic { result } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn fetch_nand(&self, val: $int_type, _order: Ordering) -> $int_type { let guard = lock(self.v.get() as usize).write(); @@ -267,7 +261,6 @@ macro_rules! atomic { result } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn fetch_or(&self, val: $int_type, _order: Ordering) -> $int_type { let guard = lock(self.v.get() as usize).write(); @@ -276,7 +269,6 @@ macro_rules! atomic { result } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn fetch_xor(&self, val: $int_type, _order: Ordering) -> $int_type { let guard = lock(self.v.get() as usize).write(); @@ -285,7 +277,6 @@ macro_rules! atomic { result } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn fetch_max(&self, val: $int_type, _order: Ordering) -> $int_type { let guard = lock(self.v.get() as usize).write(); @@ -294,7 +285,6 @@ macro_rules! atomic { result } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn fetch_min(&self, val: $int_type, _order: Ordering) -> $int_type { let guard = lock(self.v.get() as usize).write(); @@ -303,7 +293,6 @@ macro_rules! atomic { result } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn fetch_not(&self, _order: Ordering) -> $int_type { let guard = lock(self.v.get() as usize).write(); @@ -311,13 +300,11 @@ macro_rules! atomic { self.write(!result, &guard); result } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn not(&self, order: Ordering) { self.fetch_not(order); } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) const fn as_ptr(&self) -> *mut $int_type { self.v.get() @@ -327,7 +314,6 @@ macro_rules! atomic { (int, $atomic_type:ident, $int_type:ident, $align:literal) => { atomic!(uint, $atomic_type, $int_type, $align); impl $atomic_type { - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn fetch_neg(&self, _order: Ordering) -> $int_type { let guard = lock(self.v.get() as usize).write(); @@ -335,7 +321,6 @@ macro_rules! atomic { self.write(result.wrapping_neg(), &guard); result } - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] pub(crate) fn neg(&self, order: Ordering) { self.fetch_neg(order); @@ -383,7 +368,6 @@ atomic!(int, AtomicI64, i64, 8); )] atomic!(uint, AtomicU64, u64, 8); -#[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] atomic!(int, AtomicI128, i128, 16); atomic!(uint, AtomicU128, u128, 16); diff --git a/src/imp/fallback/mod.rs b/src/imp/fallback/mod.rs index dfd433828..71af7235c 100644 --- a/src/imp/fallback/mod.rs +++ b/src/imp/fallback/mod.rs @@ -79,8 +79,17 @@ mod seq_lock; #[cfg_attr(test, allow(unused_imports))] pub(crate) use seq_lock::imp::{AtomicI64, AtomicU64}; -#[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] -#[cfg_attr(test, allow(unused_imports))] +#[cfg_attr( + any( + test, + all( + target_arch = "x86_64", + portable_atomic_cmpxchg16b_target_feature, + not(portable_atomic_no_outline_atomics), + ), + ), + allow(unused_imports) +)] pub(crate) use seq_lock::imp::AtomicI128; #[cfg_attr(test, allow(unused_imports))] pub(crate) use seq_lock::imp::AtomicU128; diff --git a/src/imp/fallback/seq_lock.rs b/src/imp/fallback/seq_lock.rs index b91180625..e4ad83033 100644 --- a/src/imp/fallback/seq_lock.rs +++ b/src/imp/fallback/seq_lock.rs @@ -42,7 +42,6 @@ impl SeqLock { /// If not locked, returns the current stamp. /// /// This method should be called before optimistic reads. - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] fn optimistic_read(&self) -> Option { let state = self.state.load(Ordering::Acquire); @@ -57,7 +56,6 @@ impl SeqLock { /// /// This method should be called after optimistic reads to check whether they are valid. The /// argument `stamp` should correspond to the one returned by method `optimistic_read`. - #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))] #[inline] fn validate_read(&self, stamp: Stamp) -> bool { atomic::fence(Ordering::Acquire); diff --git a/src/imp/mod.rs b/src/imp/mod.rs index 480ef31f8..6d66121f1 100644 --- a/src/imp/mod.rs +++ b/src/imp/mod.rs @@ -39,7 +39,11 @@ mod aarch64; #[cfg(any( target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b", - portable_atomic_unstable_cmpxchg16b_target_feature, + all( + feature = "fallback", + portable_atomic_cmpxchg16b_target_feature, + not(portable_atomic_no_outline_atomics), + ), ))] #[cfg(target_arch = "x86_64")] #[path = "atomic128/x86_64.rs"] @@ -315,7 +319,11 @@ pub(crate) use self::aarch64::{AtomicI128, AtomicU128}; any( target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b", - portable_atomic_unstable_cmpxchg16b_target_feature, + all( + feature = "fallback", + portable_atomic_cmpxchg16b_target_feature, + not(portable_atomic_no_outline_atomics), + ), ), target_arch = "x86_64", ))] @@ -341,7 +349,11 @@ pub(crate) use self::s390x::{AtomicI128, AtomicU128}; any( target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b", - portable_atomic_unstable_cmpxchg16b_target_feature, + all( + feature = "fallback", + portable_atomic_cmpxchg16b_target_feature, + not(portable_atomic_no_outline_atomics), + ), ), target_arch = "x86_64", ), diff --git a/src/lib.rs b/src/lib.rs index bda288c76..35f82c6cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -117,7 +117,7 @@ See also [the `atomic128` module's readme](https://github.com/taiki-e/portable-a If dynamic dispatching by run-time CPU feature detection is enabled, it allows maintaining support for older CPUs while using features that are not supported on older CPUs, such as CMPXCHG16B (x86_64) and FEAT_LSE (aarch64). Note: - - Dynamic detection is currently only enabled in Rust 1.61+ for aarch64, in 1.59+ (AVX) or nightly (CMPXCHG16B) for x86_64, and in nightly for other platforms, otherwise it works the same as when this cfg is set. + - Dynamic detection is currently only enabled in Rust 1.61+ for aarch64, in Rust 1.59+ (AVX) or 1.69+ (CMPXCHG16B) for x86_64, otherwise it works the same as when this cfg is set. - If the required target features are enabled at compile-time, the atomic operations are inlined. - This is compatible with no-std (as with all features except `std`). - Some aarch64 targets enable LLVM's `outline-atomics` target feature by default, so if you set this cfg, you may want to disable that as well. @@ -187,25 +187,6 @@ See also [the `atomic128` module's readme](https://github.com/taiki-e/portable-a clippy::single_match, clippy::type_complexity )] -// x86_64 128-bit atomic (fallback + dynamic detection only) -// We use cfg set by build script to determine whether this feature is available or not. -// This feature will be stabilized in https://github.com/rust-lang/rust/pull/106774. -#![cfg_attr( - all( - target_arch = "x86_64", - any( - all(test, portable_atomic_nightly), - all( - portable_atomic_unstable_cmpxchg16b_target_feature, - not(any( - target_feature = "cmpxchg16b", - portable_atomic_target_feature = "cmpxchg16b", - )), - ), - ), - ), - feature(cmpxchg16b_target_feature) -)] // asm_experimental_arch // AVR and MSP430 are tier 3 platforms and require nightly anyway. // On tier 2 platforms (powerpc64 and s390x), we use cfg set by build script to @@ -234,6 +215,7 @@ See also [the `atomic128` module's readme](https://github.com/taiki-e/portable-a // and can safely be enabled for old nightly as long as version detection works. // - cfg(target_has_atomic) // - #[target_feature(enable = "lse")] on AArch64 +// - #[target_feature(enable = "cmpxchg16b")] on x86_64 // - asm! on ARM, AArch64, RISC-V, x86_64 // - llvm_asm! on AVR (tier 3) and MSP430 (tier 3) // - #[instruction_set] on non-Linux pre-v6 ARM (tier 3) @@ -246,6 +228,15 @@ See also [the `atomic128` module's readme](https://github.com/taiki-e/portable-a ), feature(aarch64_target_feature) )] +#![cfg_attr( + all( + target_arch = "x86_64", + portable_atomic_unstable_cmpxchg16b_target_feature, + not(portable_atomic_no_outline_atomics), + feature = "fallback", + ), + feature(cmpxchg16b_target_feature) +)] #![cfg_attr( all( portable_atomic_unstable_asm, @@ -4602,7 +4593,11 @@ atomic_int!(AtomicU64, u64, 8); any( target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b", - portable_atomic_unstable_cmpxchg16b_target_feature, + all( + feature = "fallback", + portable_atomic_cmpxchg16b_target_feature, + not(portable_atomic_no_outline_atomics), + ), ), target_arch = "x86_64", ), @@ -4650,7 +4645,11 @@ atomic_int!(AtomicI128, i128, 16); any( target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b", - portable_atomic_unstable_cmpxchg16b_target_feature, + all( + feature = "fallback", + portable_atomic_cmpxchg16b_target_feature, + not(portable_atomic_no_outline_atomics), + ), ), target_arch = "x86_64", ), diff --git a/src/tests/mod.rs b/src/tests/mod.rs index d34970722..24c35c79e 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -163,8 +163,11 @@ fn test_is_lock_free() { // Miri doesn't support inline assembly used in is_x86_feature_detected #[cfg(not(miri))] { - let has_cmpxchg16b = cfg!(portable_atomic_unstable_cmpxchg16b_target_feature) - && std::is_x86_feature_detected!("cmpxchg16b"); + let has_cmpxchg16b = cfg!(all( + feature = "fallback", + portable_atomic_cmpxchg16b_target_feature, + not(portable_atomic_no_outline_atomics), + )) && std::is_x86_feature_detected!("cmpxchg16b"); assert_eq!(AtomicI128::is_lock_free(), has_cmpxchg16b); assert_eq!(AtomicU128::is_lock_free(), has_cmpxchg16b); } From bdcf02cc3f178a57244fb64672920b92822c66b9 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Wed, 1 Mar 2023 03:26:29 +0900 Subject: [PATCH 2/3] x86_64: Merge _cmpxchg16b and __cmpxchg16b --- src/imp/atomic128/x86_64.rs | 82 ++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/src/imp/atomic128/x86_64.rs b/src/imp/atomic128/x86_64.rs index 71bf7389b..12c61579b 100644 --- a/src/imp/atomic128/x86_64.rs +++ b/src/imp/atomic128/x86_64.rs @@ -45,10 +45,35 @@ struct Pair { hi: u64, } -#[inline(always)] -unsafe fn __cmpxchg16b(dst: *mut u128, old: u128, new: u128) -> (u128, bool) { +#[cfg_attr( + not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), + target_feature(enable = "cmpxchg16b") +)] +#[cfg_attr( + any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"), + inline +)] +#[cfg_attr( + not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), + inline(never) +)] +unsafe fn _cmpxchg16b( + dst: *mut u128, + old: u128, + new: u128, + success: Ordering, + failure: Ordering, +) -> (u128, bool) { debug_assert!(dst as usize % 16 == 0); + // Miri and Sanitizer do not support inline assembly. + #[cfg(any(miri, portable_atomic_sanitize_thread))] + // SAFETY: the caller must uphold the safety contract for `_cmpxchg16b`. + unsafe { + let res = core::arch::x86_64::cmpxchg16b(dst, old, new, success, failure); + (res, res == old) + } + #[cfg(not(any(miri, portable_atomic_sanitize_thread)))] // SAFETY: the caller must guarantee that `dst` is valid for both writes and // reads, 16-byte aligned (required by CMPXCHG16B), that there are no // concurrent non-atomic operations, and that the CPU supports CMPXCHG16B. @@ -62,6 +87,8 @@ unsafe fn __cmpxchg16b(dst: *mut u128, old: u128, new: u128) -> (u128, bool) { // // Refs: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b unsafe { + // cmpxchg16b is always SeqCst. + let _ = (success, failure); let r: u8; let old = U128 { whole: old }; let new = U128 { whole: new }; @@ -101,40 +128,6 @@ unsafe fn cmpxchg16b( success: Ordering, failure: Ordering, ) -> (u128, bool) { - #[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), - target_feature(enable = "cmpxchg16b") - )] - #[cfg_attr( - any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"), - inline - )] - #[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), - inline(never) - )] - unsafe fn _cmpxchg16b( - dst: *mut u128, - old: u128, - new: u128, - success: Ordering, - failure: Ordering, - ) -> (u128, bool) { - // Miri and Sanitizer do not support inline assembly. - #[cfg(any(miri, portable_atomic_sanitize_thread))] - // SAFETY: the caller must uphold the safety contract for `_cmpxchg16b`. - unsafe { - let res = core::arch::x86_64::cmpxchg16b(dst, old, new, success, failure); - (res, res == old) - } - #[cfg(not(any(miri, portable_atomic_sanitize_thread)))] - // SAFETY: the caller must uphold the safety contract for `_cmpxchg16b`. - unsafe { - let _ = (success, failure); - __cmpxchg16b(dst, old, new) - } - } - #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] // SAFETY: the caller must guarantee that `dst` is valid for both writes and // reads, 16-byte aligned, that there are no concurrent non-atomic operations, @@ -434,9 +427,12 @@ mod tests { test_atomic_int!(u128); #[test] - #[cfg_attr(miri, ignore)] // Miri doesn't support inline assembly fn test() { - assert!(std::is_x86_feature_detected!("cmpxchg16b")); + // Miri doesn't support inline assembly used in is_x86_feature_detected + #[cfg(not(miri))] + { + assert!(std::is_x86_feature_detected!("cmpxchg16b")); + } assert!(AtomicI128::is_lock_free()); assert!(AtomicU128::is_lock_free()); } @@ -450,13 +446,15 @@ mod tests { use super::super::*; ::quickcheck::quickcheck! { - #[cfg_attr(miri, ignore)] // Miri doesn't support inline assembly - #[cfg_attr(portable_atomic_sanitize_thread, ignore)] // TSan doesn't know the semantics of the asm synchronization instructions. fn test(x: u128, y: u128, z: u128) -> bool { - assert!(std::is_x86_feature_detected!("cmpxchg16b")); + // Miri doesn't support inline assembly used in is_x86_feature_detected + #[cfg(not(miri))] + { + assert!(std::is_x86_feature_detected!("cmpxchg16b")); + } unsafe { let a = Align16(UnsafeCell::new(x)); - let (res, ok) = __cmpxchg16b(a.get(), y, z); + let (res, ok) = _cmpxchg16b(a.get(), y, z, Ordering::SeqCst, Ordering::SeqCst); if x == y { assert!(ok); assert_eq!(res, x); From 5e38444e013551235dbdb177bcee63124015f9e4 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Wed, 1 Mar 2023 12:10:08 +0900 Subject: [PATCH 3/3] x86_64: Do not enable outline-atomics on target where CPUID is not available SGX doesn't support CPUID. Currently, on this target, if the cmpxchg16b feature is not enabled at compile time, detect::has_cmpxchg16b will always return false. So, enabling outline-atomics on this target does not make sense since the fallback implementation is always used. --- src/imp/atomic128/detect/x86_64.rs | 8 ++++---- src/imp/atomic128/x86_64.rs | 14 ++++++++++---- src/imp/fallback/imp.rs | 1 + src/imp/fallback/mod.rs | 1 + src/imp/mod.rs | 3 +++ src/lib.rs | 3 +++ src/tests/mod.rs | 1 + src/utils.rs | 1 + 8 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/imp/atomic128/detect/x86_64.rs b/src/imp/atomic128/detect/x86_64.rs index defd01796..939bbea96 100644 --- a/src/imp/atomic128/detect/x86_64.rs +++ b/src/imp/atomic128/detect/x86_64.rs @@ -2,8 +2,8 @@ #![cfg_attr( any( - portable_atomic_no_outline_atomics, not(target_feature = "sse"), + portable_atomic_no_outline_atomics, target_env = "sgx", miri, portable_atomic_sanitize_thread, @@ -69,7 +69,7 @@ fn _detect(info: &mut CpuInfo) { { info.set(CpuInfo::HAS_CMPXCHG16B); } - // sgx doesn't support `cpuid`: https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L105 + // SGX doesn't support CPUID: https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L105 #[cfg(not(any(target_env = "sgx", miri)))] { use core::arch::x86_64::_xgetbv; @@ -137,8 +137,8 @@ mod tests { } #[test] - // Miri doesn't support inline assembly - // sgx doesn't support `cpuid` + // SGX doesn't support CPUID. + // Miri doesn't support inline assembly. #[cfg_attr(any(target_env = "sgx", miri), ignore)] fn test_cpuid() { assert_eq!(std::is_x86_feature_detected!("cmpxchg16b"), has_cmpxchg16b()); diff --git a/src/imp/atomic128/x86_64.rs b/src/imp/atomic128/x86_64.rs index 12c61579b..e332937ac 100644 --- a/src/imp/atomic128/x86_64.rs +++ b/src/imp/atomic128/x86_64.rs @@ -273,10 +273,12 @@ unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 { // Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled. // https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html + // SGX doesn't support CPUID. // Miri and Sanitizer do not support inline assembly. #[cfg(any( - portable_atomic_no_outline_atomics, not(target_feature = "sse"), + portable_atomic_no_outline_atomics, + target_env = "sgx", miri, portable_atomic_sanitize_thread, ))] @@ -285,8 +287,9 @@ unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 { _atomic_load_cmpxchg16b(src, order) } #[cfg(not(any( - portable_atomic_no_outline_atomics, not(target_feature = "sse"), + portable_atomic_no_outline_atomics, + target_env = "sgx", miri, portable_atomic_sanitize_thread, )))] @@ -316,10 +319,12 @@ unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) { // Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled. // https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html + // SGX doesn't support CPUID. // Miri and Sanitizer do not support inline assembly. #[cfg(any( - portable_atomic_no_outline_atomics, not(target_feature = "sse"), + portable_atomic_no_outline_atomics, + target_env = "sgx", miri, portable_atomic_sanitize_thread, ))] @@ -328,8 +333,9 @@ unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) { _atomic_store_cmpxchg16b(dst, val, order); } #[cfg(not(any( - portable_atomic_no_outline_atomics, not(target_feature = "sse"), + portable_atomic_no_outline_atomics, + target_env = "sgx", miri, portable_atomic_sanitize_thread, )))] diff --git a/src/imp/fallback/imp.rs b/src/imp/fallback/imp.rs index 14d8b4496..d216b0b47 100644 --- a/src/imp/fallback/imp.rs +++ b/src/imp/fallback/imp.rs @@ -3,6 +3,7 @@ target_arch = "x86_64", portable_atomic_cmpxchg16b_target_feature, not(portable_atomic_no_outline_atomics), + not(target_env = "sgx"), ), allow(dead_code) )] diff --git a/src/imp/fallback/mod.rs b/src/imp/fallback/mod.rs index 71af7235c..28cc14da4 100644 --- a/src/imp/fallback/mod.rs +++ b/src/imp/fallback/mod.rs @@ -86,6 +86,7 @@ pub(crate) use seq_lock::imp::{AtomicI64, AtomicU64}; target_arch = "x86_64", portable_atomic_cmpxchg16b_target_feature, not(portable_atomic_no_outline_atomics), + not(target_env = "sgx"), ), ), allow(unused_imports) diff --git a/src/imp/mod.rs b/src/imp/mod.rs index 6d66121f1..8eb690a02 100644 --- a/src/imp/mod.rs +++ b/src/imp/mod.rs @@ -43,6 +43,7 @@ mod aarch64; feature = "fallback", portable_atomic_cmpxchg16b_target_feature, not(portable_atomic_no_outline_atomics), + not(target_env = "sgx"), ), ))] #[cfg(target_arch = "x86_64")] @@ -323,6 +324,7 @@ pub(crate) use self::aarch64::{AtomicI128, AtomicU128}; feature = "fallback", portable_atomic_cmpxchg16b_target_feature, not(portable_atomic_no_outline_atomics), + not(target_env = "sgx"), ), ), target_arch = "x86_64", @@ -353,6 +355,7 @@ pub(crate) use self::s390x::{AtomicI128, AtomicU128}; feature = "fallback", portable_atomic_cmpxchg16b_target_feature, not(portable_atomic_no_outline_atomics), + not(target_env = "sgx"), ), ), target_arch = "x86_64", diff --git a/src/lib.rs b/src/lib.rs index 35f82c6cb..1fb9a41d5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -233,6 +233,7 @@ See also [the `atomic128` module's readme](https://github.com/taiki-e/portable-a target_arch = "x86_64", portable_atomic_unstable_cmpxchg16b_target_feature, not(portable_atomic_no_outline_atomics), + not(target_env = "sgx"), feature = "fallback", ), feature(cmpxchg16b_target_feature) @@ -4597,6 +4598,7 @@ atomic_int!(AtomicU64, u64, 8); feature = "fallback", portable_atomic_cmpxchg16b_target_feature, not(portable_atomic_no_outline_atomics), + not(target_env = "sgx"), ), ), target_arch = "x86_64", @@ -4649,6 +4651,7 @@ atomic_int!(AtomicI128, i128, 16); feature = "fallback", portable_atomic_cmpxchg16b_target_feature, not(portable_atomic_no_outline_atomics), + not(target_env = "sgx"), ), ), target_arch = "x86_64", diff --git a/src/tests/mod.rs b/src/tests/mod.rs index 24c35c79e..ad26cfa54 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -167,6 +167,7 @@ fn test_is_lock_free() { feature = "fallback", portable_atomic_cmpxchg16b_target_feature, not(portable_atomic_no_outline_atomics), + not(target_env = "sgx"), )) && std::is_x86_feature_detected!("cmpxchg16b"); assert_eq!(AtomicI128::is_lock_free(), has_cmpxchg16b); assert_eq!(AtomicU128::is_lock_free(), has_cmpxchg16b); diff --git a/src/utils.rs b/src/utils.rs index 4c2e15f06..d5c172d8c 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -141,6 +141,7 @@ macro_rules! serde_impls { // Adapted from https://github.com/BurntSushi/memchr/blob/2.4.1/src/memchr/x86/mod.rs#L9-L71. #[allow(unused_macros)] #[cfg(not(portable_atomic_no_outline_atomics))] +#[cfg(any(target_arch = "aarch64", all(target_arch = "x86_64", not(target_env = "sgx"))))] macro_rules! ifunc { // if the functions are unsafe, this macro is also unsafe. (unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)? { $($if_block:tt)* }) => {{