diff --git a/README.md b/README.md
index b39e6b3c0..a8bc94102 100644
--- a/README.md
+++ b/README.md
@@ -125,7 +125,7 @@ See also [the `atomic128` module's readme](https://github.com/taiki-e/portable-a
If dynamic dispatching by run-time CPU feature detection is enabled, it allows maintaining support for older CPUs while using features that are not supported on older CPUs, such as CMPXCHG16B (x86_64) and FEAT_LSE (aarch64).
Note:
- - Dynamic detection is currently only enabled in Rust 1.61+ for aarch64, in 1.59+ (AVX) or nightly (CMPXCHG16B) for x86_64, and in nightly for other platforms, otherwise it works the same as when this cfg is set.
+ - Dynamic detection is currently only enabled in Rust 1.61+ for aarch64, in Rust 1.59+ (AVX) or 1.69+ (CMPXCHG16B) for x86_64, otherwise it works the same as when this cfg is set.
- If the required target features are enabled at compile-time, the atomic operations are inlined.
- This is compatible with no-std (as with all features except `std`).
- Some aarch64 targets enable LLVM's `outline-atomics` target feature by default, so if you set this cfg, you may want to disable that as well.
diff --git a/bench/benches/bench.rs b/bench/benches/bench.rs
index 4233f1d15..6a4b25788 100644
--- a/bench/benches/bench.rs
+++ b/bench/benches/bench.rs
@@ -1,7 +1,7 @@
#![warn(rust_2018_idioms, single_use_lifetimes, unsafe_op_in_unsafe_fn)]
#![allow(dead_code, unused_extern_crates)]
#![allow(clippy::only_used_in_recursion)]
-#![feature(asm_experimental_arch, cmpxchg16b_target_feature, core_intrinsics, stdsimd)]
+#![feature(asm_experimental_arch, core_intrinsics, stdsimd)]
use std::{
hint::black_box,
diff --git a/build.rs b/build.rs
index 0e40c7d51..05eff645f 100644
--- a/build.rs
+++ b/build.rs
@@ -30,7 +30,6 @@ fn main() {
let mut target_upper = target.replace(|c: char| c == '-' || c == '.', "_");
target_upper.make_ascii_uppercase();
println!("cargo:rerun-if-env-changed=CARGO_TARGET_{}_RUSTFLAGS", target_upper);
- println!("cargo:rerun-if-env-changed=CARGO_CFG_PORTABLE_ATOMIC_NO_OUTLINE_ATOMICS");
println!("cargo:rerun-if-env-changed=RUSTC");
let version = match rustc_version() {
@@ -70,7 +69,6 @@ fn main() {
}
// asm stabilized in Rust 1.59 (nightly-2021-12-16): https://github.com/rust-lang/rust/pull/91728
let no_asm = !version.probe(59, 2021, 12, 15);
- let mut unstable_asm = false;
if no_asm {
if version.nightly
&& version.probe(46, 2020, 6, 20)
@@ -85,7 +83,6 @@ fn main() {
// until it was stabilized in nightly-2021-12-16, so it can be safely enabled in
// nightly, which is older than nightly-2021-12-16.
println!("cargo:rustc-cfg=portable_atomic_unstable_asm");
- unstable_asm = true;
}
println!("cargo:rustc-cfg=portable_atomic_no_asm");
}
@@ -157,19 +154,28 @@ fn main() {
match target_arch {
"x86_64" => {
+ // cmpxchg16b_target_feature stabilized in Rust 1.69 (nightly-2023-03-01): https://github.com/rust-lang/rust/pull/106774
+ if !version.probe(69, 2023, 2, 28) {
+ if version.nightly && is_allowed_feature("cmpxchg16b_target_feature") {
+ // This feature has not been changed since 1.33
+ // (https://github.com/rust-lang/rust/commit/fbb56bcf44d28e65a9495decf091b6d0386e540c)
+ // until it was stabilized in nightly-2023-03-01, so it can be safely enabled in
+ // nightly, which is older than nightly-2023-03-01.
+ println!("cargo:rustc-cfg=portable_atomic_unstable_cmpxchg16b_target_feature");
+ println!("cargo:rustc-cfg=portable_atomic_cmpxchg16b_target_feature");
+ } else {
+ // println!("cargo:rustc-cfg=portable_atomic_no_cmpxchg16b_target_feature");
+ }
+ } else {
+ // TODO: invert flag once 1.69 became stable
+ println!("cargo:rustc-cfg=portable_atomic_cmpxchg16b_target_feature");
+ }
+
// x86_64 macos always support CMPXCHG16B: https://github.com/rust-lang/rust/blob/1.67.0/compiler/rustc_target/src/spec/x86_64_apple_darwin.rs#L8
let has_cmpxchg16b = target_os == "macos";
// LLVM recognizes this also as cx16 target feature: https://godbolt.org/z/6dszGeYsf
- // It is unlikely that rustc will support that name, so we will ignore it for now.
- target_feature_if("cmpxchg16b", has_cmpxchg16b, &version, None, true);
- if version.nightly
- && (!no_asm || unstable_asm)
- && cfg!(feature = "fallback")
- && env::var_os("CARGO_CFG_PORTABLE_ATOMIC_NO_OUTLINE_ATOMICS").is_none()
- && is_allowed_feature("cmpxchg16b_target_feature")
- {
- println!("cargo:rustc-cfg=portable_atomic_unstable_cmpxchg16b_target_feature");
- }
+ // It is unlikely that rustc will support that name, so we ignore it.
+ target_feature_if("cmpxchg16b", has_cmpxchg16b, &version, Some(69), true);
}
"aarch64" => {
// aarch64_target_feature stabilized in Rust 1.61 (nightly-2022-03-16): https://github.com/rust-lang/rust/pull/90621
diff --git a/src/imp/atomic128/README.md b/src/imp/atomic128/README.md
index 2c9a9ab46..48ad7a0fa 100644
--- a/src/imp/atomic128/README.md
+++ b/src/imp/atomic128/README.md
@@ -4,7 +4,7 @@ The table of targets that support 128-bit atomics and the instructions used:
| target_arch | load | store | CAS | RMW | note |
| ----------- | ----- | ----- | ---- | ---- | ---- |
-| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel or AMD CPU with AVX.
Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only.
Requires rustc 1.59+ when cmpxchg16b target feature is enabled at compile-time, otherwise requires nightly |
+| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel or AMD CPU with AVX.
Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only.
Requires rustc 1.59+ when cmpxchg16b target feature is enabled at compile-time, otherwise requires rustc 1.69+ |
| aarch64 | ldxp/stxp or casp or ldp | ldxp/stxp or casp or stp | ldxp/stxp or casp | ldxp/stxp or casp | casp requires lse target feature, ldp/stp requires lse2 target feature.
Both compile-time and run-time detection are supported for lse. lse2 is currently compile-time detection only.
Requires rustc 1.59+ |
| powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Little endian or target CPU pwr8+.
Requires nightly |
| s390x | lpq | stpq | cdsg | cdsg | Requires nightly |
diff --git a/src/imp/atomic128/detect/x86_64.rs b/src/imp/atomic128/detect/x86_64.rs
index defd01796..939bbea96 100644
--- a/src/imp/atomic128/detect/x86_64.rs
+++ b/src/imp/atomic128/detect/x86_64.rs
@@ -2,8 +2,8 @@
#![cfg_attr(
any(
- portable_atomic_no_outline_atomics,
not(target_feature = "sse"),
+ portable_atomic_no_outline_atomics,
target_env = "sgx",
miri,
portable_atomic_sanitize_thread,
@@ -69,7 +69,7 @@ fn _detect(info: &mut CpuInfo) {
{
info.set(CpuInfo::HAS_CMPXCHG16B);
}
- // sgx doesn't support `cpuid`: https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L105
+ // SGX doesn't support CPUID: https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L105
#[cfg(not(any(target_env = "sgx", miri)))]
{
use core::arch::x86_64::_xgetbv;
@@ -137,8 +137,8 @@ mod tests {
}
#[test]
- // Miri doesn't support inline assembly
- // sgx doesn't support `cpuid`
+ // SGX doesn't support CPUID.
+ // Miri doesn't support inline assembly.
#[cfg_attr(any(target_env = "sgx", miri), ignore)]
fn test_cpuid() {
assert_eq!(std::is_x86_feature_detected!("cmpxchg16b"), has_cmpxchg16b());
diff --git a/src/imp/atomic128/x86_64.rs b/src/imp/atomic128/x86_64.rs
index 313cc4a9f..e332937ac 100644
--- a/src/imp/atomic128/x86_64.rs
+++ b/src/imp/atomic128/x86_64.rs
@@ -45,10 +45,35 @@ struct Pair {
hi: u64,
}
-#[inline(always)]
-unsafe fn __cmpxchg16b(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
+#[cfg_attr(
+ not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
+ target_feature(enable = "cmpxchg16b")
+)]
+#[cfg_attr(
+ any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
+ inline
+)]
+#[cfg_attr(
+ not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
+ inline(never)
+)]
+unsafe fn _cmpxchg16b(
+ dst: *mut u128,
+ old: u128,
+ new: u128,
+ success: Ordering,
+ failure: Ordering,
+) -> (u128, bool) {
debug_assert!(dst as usize % 16 == 0);
+ // Miri and Sanitizer do not support inline assembly.
+ #[cfg(any(miri, portable_atomic_sanitize_thread))]
+ // SAFETY: the caller must uphold the safety contract for `_cmpxchg16b`.
+ unsafe {
+ let res = core::arch::x86_64::cmpxchg16b(dst, old, new, success, failure);
+ (res, res == old)
+ }
+ #[cfg(not(any(miri, portable_atomic_sanitize_thread)))]
// SAFETY: the caller must guarantee that `dst` is valid for both writes and
// reads, 16-byte aligned (required by CMPXCHG16B), that there are no
// concurrent non-atomic operations, and that the CPU supports CMPXCHG16B.
@@ -62,6 +87,8 @@ unsafe fn __cmpxchg16b(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
//
// Refs: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b
unsafe {
+ // cmpxchg16b is always SeqCst.
+ let _ = (success, failure);
let r: u8;
let old = U128 { whole: old };
let new = U128 { whole: new };
@@ -101,49 +128,6 @@ unsafe fn cmpxchg16b(
success: Ordering,
failure: Ordering,
) -> (u128, bool) {
- #[cfg_attr(
- all(
- any(
- all(test, portable_atomic_nightly),
- portable_atomic_unstable_cmpxchg16b_target_feature,
- ),
- not(any(
- target_feature = "cmpxchg16b",
- portable_atomic_target_feature = "cmpxchg16b",
- )),
- ),
- target_feature(enable = "cmpxchg16b")
- )]
- #[cfg_attr(
- any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
- inline
- )]
- #[cfg_attr(
- not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
- inline(never)
- )]
- unsafe fn _cmpxchg16b(
- dst: *mut u128,
- old: u128,
- new: u128,
- success: Ordering,
- failure: Ordering,
- ) -> (u128, bool) {
- // Miri and Sanitizer do not support inline assembly.
- #[cfg(any(miri, portable_atomic_sanitize_thread))]
- // SAFETY: the caller must uphold the safety contract for `_cmpxchg16b`.
- unsafe {
- let res = core::arch::x86_64::cmpxchg16b(dst, old, new, success, failure);
- (res, res == old)
- }
- #[cfg(not(any(miri, portable_atomic_sanitize_thread)))]
- // SAFETY: the caller must uphold the safety contract for `_cmpxchg16b`.
- unsafe {
- let _ = (success, failure);
- __cmpxchg16b(dst, old, new)
- }
- }
-
#[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
// SAFETY: the caller must guarantee that `dst` is valid for both writes and
// reads, 16-byte aligned, that there are no concurrent non-atomic operations,
@@ -151,7 +135,6 @@ unsafe fn cmpxchg16b(
unsafe {
_cmpxchg16b(dst, old, new, success, failure)
}
- #[cfg(portable_atomic_unstable_cmpxchg16b_target_feature)]
#[cfg(not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")))]
{
#[cold]
@@ -290,10 +273,12 @@ unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
// Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled.
// https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
+ // SGX doesn't support CPUID.
// Miri and Sanitizer do not support inline assembly.
#[cfg(any(
- portable_atomic_no_outline_atomics,
not(target_feature = "sse"),
+ portable_atomic_no_outline_atomics,
+ target_env = "sgx",
miri,
portable_atomic_sanitize_thread,
))]
@@ -302,8 +287,9 @@ unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
_atomic_load_cmpxchg16b(src, order)
}
#[cfg(not(any(
- portable_atomic_no_outline_atomics,
not(target_feature = "sse"),
+ portable_atomic_no_outline_atomics,
+ target_env = "sgx",
miri,
portable_atomic_sanitize_thread,
)))]
@@ -333,10 +319,12 @@ unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
// Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled.
// https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
+ // SGX doesn't support CPUID.
// Miri and Sanitizer do not support inline assembly.
#[cfg(any(
- portable_atomic_no_outline_atomics,
not(target_feature = "sse"),
+ portable_atomic_no_outline_atomics,
+ target_env = "sgx",
miri,
portable_atomic_sanitize_thread,
))]
@@ -345,8 +333,9 @@ unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
_atomic_store_cmpxchg16b(dst, val, order);
}
#[cfg(not(any(
- portable_atomic_no_outline_atomics,
not(target_feature = "sse"),
+ portable_atomic_no_outline_atomics,
+ target_env = "sgx",
miri,
portable_atomic_sanitize_thread,
)))]
@@ -444,9 +433,12 @@ mod tests {
test_atomic_int!(u128);
#[test]
- #[cfg_attr(miri, ignore)] // Miri doesn't support inline assembly
fn test() {
- assert!(std::is_x86_feature_detected!("cmpxchg16b"));
+ // Miri doesn't support inline assembly used in is_x86_feature_detected
+ #[cfg(not(miri))]
+ {
+ assert!(std::is_x86_feature_detected!("cmpxchg16b"));
+ }
assert!(AtomicI128::is_lock_free());
assert!(AtomicU128::is_lock_free());
}
@@ -460,13 +452,15 @@ mod tests {
use super::super::*;
::quickcheck::quickcheck! {
- #[cfg_attr(miri, ignore)] // Miri doesn't support inline assembly
- #[cfg_attr(portable_atomic_sanitize_thread, ignore)] // TSan doesn't know the semantics of the asm synchronization instructions.
fn test(x: u128, y: u128, z: u128) -> bool {
- assert!(std::is_x86_feature_detected!("cmpxchg16b"));
+ // Miri doesn't support inline assembly used in is_x86_feature_detected
+ #[cfg(not(miri))]
+ {
+ assert!(std::is_x86_feature_detected!("cmpxchg16b"));
+ }
unsafe {
let a = Align16(UnsafeCell::new(x));
- let (res, ok) = __cmpxchg16b(a.get(), y, z);
+ let (res, ok) = _cmpxchg16b(a.get(), y, z, Ordering::SeqCst, Ordering::SeqCst);
if x == y {
assert!(ok);
assert_eq!(res, x);
diff --git a/src/imp/fallback/imp.rs b/src/imp/fallback/imp.rs
index 39e398f1c..d216b0b47 100644
--- a/src/imp/fallback/imp.rs
+++ b/src/imp/fallback/imp.rs
@@ -1,3 +1,13 @@
+#![cfg_attr(
+ all(
+ target_arch = "x86_64",
+ portable_atomic_cmpxchg16b_target_feature,
+ not(portable_atomic_no_outline_atomics),
+ not(target_env = "sgx"),
+ ),
+ allow(dead_code)
+)]
+
use core::{cell::UnsafeCell, mem, sync::atomic::Ordering};
use super::{super::utils::CachePadded, SeqLock, SeqLockWriteGuard};
@@ -49,7 +59,6 @@ macro_rules! atomic {
unsafe { &*(self.v.get() as *const $int_type as *const [AtomicChunk; Self::LEN]) }
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
fn optimistic_read(&self) -> $int_type {
// Using `MaybeUninit<[usize; Self::LEN]>` here doesn't change codegen: https://godbolt.org/z/86f8s733M
@@ -120,27 +129,22 @@ macro_rules! atomic {
// SAFETY: any data races are prevented by the lock and atomic operation.
unsafe impl Sync for $atomic_type {}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
no_fetch_ops_impl!($atomic_type, $int_type);
impl $atomic_type {
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) const fn new(v: $int_type) -> Self {
Self { v: UnsafeCell::new(v) }
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn is_lock_free() -> bool {
Self::is_always_lock_free()
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) const fn is_always_lock_free() -> bool {
false
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn get_mut(&mut self) -> &mut $int_type {
// SAFETY: the mutable reference guarantees unique ownership.
@@ -148,13 +152,11 @@ macro_rules! atomic {
unsafe { &mut *self.v.get() }
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn into_inner(self) -> $int_type {
self.v.into_inner()
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn load(&self, order: Ordering) -> $int_type {
@@ -178,7 +180,6 @@ macro_rules! atomic {
val
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn store(&self, val: $int_type, order: Ordering) {
@@ -187,7 +188,6 @@ macro_rules! atomic {
self.write(val, &guard)
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn swap(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = lock(self.v.get() as usize).write();
@@ -218,7 +218,6 @@ macro_rules! atomic {
}
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn compare_exchange_weak(
@@ -231,7 +230,6 @@ macro_rules! atomic {
self.compare_exchange(current, new, success, failure)
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn fetch_add(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = lock(self.v.get() as usize).write();
@@ -240,7 +238,6 @@ macro_rules! atomic {
result
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn fetch_sub(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = lock(self.v.get() as usize).write();
@@ -249,7 +246,6 @@ macro_rules! atomic {
result
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn fetch_and(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = lock(self.v.get() as usize).write();
@@ -258,7 +254,6 @@ macro_rules! atomic {
result
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn fetch_nand(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = lock(self.v.get() as usize).write();
@@ -267,7 +262,6 @@ macro_rules! atomic {
result
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn fetch_or(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = lock(self.v.get() as usize).write();
@@ -276,7 +270,6 @@ macro_rules! atomic {
result
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn fetch_xor(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = lock(self.v.get() as usize).write();
@@ -285,7 +278,6 @@ macro_rules! atomic {
result
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn fetch_max(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = lock(self.v.get() as usize).write();
@@ -294,7 +286,6 @@ macro_rules! atomic {
result
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn fetch_min(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = lock(self.v.get() as usize).write();
@@ -303,7 +294,6 @@ macro_rules! atomic {
result
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn fetch_not(&self, _order: Ordering) -> $int_type {
let guard = lock(self.v.get() as usize).write();
@@ -311,13 +301,11 @@ macro_rules! atomic {
self.write(!result, &guard);
result
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn not(&self, order: Ordering) {
self.fetch_not(order);
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) const fn as_ptr(&self) -> *mut $int_type {
self.v.get()
@@ -327,7 +315,6 @@ macro_rules! atomic {
(int, $atomic_type:ident, $int_type:ident, $align:literal) => {
atomic!(uint, $atomic_type, $int_type, $align);
impl $atomic_type {
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn fetch_neg(&self, _order: Ordering) -> $int_type {
let guard = lock(self.v.get() as usize).write();
@@ -335,7 +322,6 @@ macro_rules! atomic {
self.write(result.wrapping_neg(), &guard);
result
}
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
pub(crate) fn neg(&self, order: Ordering) {
self.fetch_neg(order);
@@ -383,7 +369,6 @@ atomic!(int, AtomicI64, i64, 8);
)]
atomic!(uint, AtomicU64, u64, 8);
-#[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
atomic!(int, AtomicI128, i128, 16);
atomic!(uint, AtomicU128, u128, 16);
diff --git a/src/imp/fallback/mod.rs b/src/imp/fallback/mod.rs
index dfd433828..28cc14da4 100644
--- a/src/imp/fallback/mod.rs
+++ b/src/imp/fallback/mod.rs
@@ -79,8 +79,18 @@ mod seq_lock;
#[cfg_attr(test, allow(unused_imports))]
pub(crate) use seq_lock::imp::{AtomicI64, AtomicU64};
-#[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
-#[cfg_attr(test, allow(unused_imports))]
+#[cfg_attr(
+ any(
+ test,
+ all(
+ target_arch = "x86_64",
+ portable_atomic_cmpxchg16b_target_feature,
+ not(portable_atomic_no_outline_atomics),
+ not(target_env = "sgx"),
+ ),
+ ),
+ allow(unused_imports)
+)]
pub(crate) use seq_lock::imp::AtomicI128;
#[cfg_attr(test, allow(unused_imports))]
pub(crate) use seq_lock::imp::AtomicU128;
diff --git a/src/imp/fallback/seq_lock.rs b/src/imp/fallback/seq_lock.rs
index b91180625..e4ad83033 100644
--- a/src/imp/fallback/seq_lock.rs
+++ b/src/imp/fallback/seq_lock.rs
@@ -42,7 +42,6 @@ impl SeqLock {
/// If not locked, returns the current stamp.
///
/// This method should be called before optimistic reads.
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
fn optimistic_read(&self) -> Option {
let state = self.state.load(Ordering::Acquire);
@@ -57,7 +56,6 @@ impl SeqLock {
///
/// This method should be called after optimistic reads to check whether they are valid. The
/// argument `stamp` should correspond to the one returned by method `optimistic_read`.
- #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
#[inline]
fn validate_read(&self, stamp: Stamp) -> bool {
atomic::fence(Ordering::Acquire);
diff --git a/src/imp/mod.rs b/src/imp/mod.rs
index 480ef31f8..8eb690a02 100644
--- a/src/imp/mod.rs
+++ b/src/imp/mod.rs
@@ -39,7 +39,12 @@ mod aarch64;
#[cfg(any(
target_feature = "cmpxchg16b",
portable_atomic_target_feature = "cmpxchg16b",
- portable_atomic_unstable_cmpxchg16b_target_feature,
+ all(
+ feature = "fallback",
+ portable_atomic_cmpxchg16b_target_feature,
+ not(portable_atomic_no_outline_atomics),
+ not(target_env = "sgx"),
+ ),
))]
#[cfg(target_arch = "x86_64")]
#[path = "atomic128/x86_64.rs"]
@@ -315,7 +320,12 @@ pub(crate) use self::aarch64::{AtomicI128, AtomicU128};
any(
target_feature = "cmpxchg16b",
portable_atomic_target_feature = "cmpxchg16b",
- portable_atomic_unstable_cmpxchg16b_target_feature,
+ all(
+ feature = "fallback",
+ portable_atomic_cmpxchg16b_target_feature,
+ not(portable_atomic_no_outline_atomics),
+ not(target_env = "sgx"),
+ ),
),
target_arch = "x86_64",
))]
@@ -341,7 +351,12 @@ pub(crate) use self::s390x::{AtomicI128, AtomicU128};
any(
target_feature = "cmpxchg16b",
portable_atomic_target_feature = "cmpxchg16b",
- portable_atomic_unstable_cmpxchg16b_target_feature,
+ all(
+ feature = "fallback",
+ portable_atomic_cmpxchg16b_target_feature,
+ not(portable_atomic_no_outline_atomics),
+ not(target_env = "sgx"),
+ ),
),
target_arch = "x86_64",
),
diff --git a/src/lib.rs b/src/lib.rs
index bda288c76..1fb9a41d5 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -117,7 +117,7 @@ See also [the `atomic128` module's readme](https://github.com/taiki-e/portable-a
If dynamic dispatching by run-time CPU feature detection is enabled, it allows maintaining support for older CPUs while using features that are not supported on older CPUs, such as CMPXCHG16B (x86_64) and FEAT_LSE (aarch64).
Note:
- - Dynamic detection is currently only enabled in Rust 1.61+ for aarch64, in 1.59+ (AVX) or nightly (CMPXCHG16B) for x86_64, and in nightly for other platforms, otherwise it works the same as when this cfg is set.
+ - Dynamic detection is currently only enabled in Rust 1.61+ for aarch64, in Rust 1.59+ (AVX) or 1.69+ (CMPXCHG16B) for x86_64, otherwise it works the same as when this cfg is set.
- If the required target features are enabled at compile-time, the atomic operations are inlined.
- This is compatible with no-std (as with all features except `std`).
- Some aarch64 targets enable LLVM's `outline-atomics` target feature by default, so if you set this cfg, you may want to disable that as well.
@@ -187,25 +187,6 @@ See also [the `atomic128` module's readme](https://github.com/taiki-e/portable-a
clippy::single_match,
clippy::type_complexity
)]
-// x86_64 128-bit atomic (fallback + dynamic detection only)
-// We use cfg set by build script to determine whether this feature is available or not.
-// This feature will be stabilized in https://github.com/rust-lang/rust/pull/106774.
-#![cfg_attr(
- all(
- target_arch = "x86_64",
- any(
- all(test, portable_atomic_nightly),
- all(
- portable_atomic_unstable_cmpxchg16b_target_feature,
- not(any(
- target_feature = "cmpxchg16b",
- portable_atomic_target_feature = "cmpxchg16b",
- )),
- ),
- ),
- ),
- feature(cmpxchg16b_target_feature)
-)]
// asm_experimental_arch
// AVR and MSP430 are tier 3 platforms and require nightly anyway.
// On tier 2 platforms (powerpc64 and s390x), we use cfg set by build script to
@@ -234,6 +215,7 @@ See also [the `atomic128` module's readme](https://github.com/taiki-e/portable-a
// and can safely be enabled for old nightly as long as version detection works.
// - cfg(target_has_atomic)
// - #[target_feature(enable = "lse")] on AArch64
+// - #[target_feature(enable = "cmpxchg16b")] on x86_64
// - asm! on ARM, AArch64, RISC-V, x86_64
// - llvm_asm! on AVR (tier 3) and MSP430 (tier 3)
// - #[instruction_set] on non-Linux pre-v6 ARM (tier 3)
@@ -246,6 +228,16 @@ See also [the `atomic128` module's readme](https://github.com/taiki-e/portable-a
),
feature(aarch64_target_feature)
)]
+#![cfg_attr(
+ all(
+ target_arch = "x86_64",
+ portable_atomic_unstable_cmpxchg16b_target_feature,
+ not(portable_atomic_no_outline_atomics),
+ not(target_env = "sgx"),
+ feature = "fallback",
+ ),
+ feature(cmpxchg16b_target_feature)
+)]
#![cfg_attr(
all(
portable_atomic_unstable_asm,
@@ -4602,7 +4594,12 @@ atomic_int!(AtomicU64, u64, 8);
any(
target_feature = "cmpxchg16b",
portable_atomic_target_feature = "cmpxchg16b",
- portable_atomic_unstable_cmpxchg16b_target_feature,
+ all(
+ feature = "fallback",
+ portable_atomic_cmpxchg16b_target_feature,
+ not(portable_atomic_no_outline_atomics),
+ not(target_env = "sgx"),
+ ),
),
target_arch = "x86_64",
),
@@ -4650,7 +4647,12 @@ atomic_int!(AtomicI128, i128, 16);
any(
target_feature = "cmpxchg16b",
portable_atomic_target_feature = "cmpxchg16b",
- portable_atomic_unstable_cmpxchg16b_target_feature,
+ all(
+ feature = "fallback",
+ portable_atomic_cmpxchg16b_target_feature,
+ not(portable_atomic_no_outline_atomics),
+ not(target_env = "sgx"),
+ ),
),
target_arch = "x86_64",
),
diff --git a/src/tests/mod.rs b/src/tests/mod.rs
index d34970722..ad26cfa54 100644
--- a/src/tests/mod.rs
+++ b/src/tests/mod.rs
@@ -163,8 +163,12 @@ fn test_is_lock_free() {
// Miri doesn't support inline assembly used in is_x86_feature_detected
#[cfg(not(miri))]
{
- let has_cmpxchg16b = cfg!(portable_atomic_unstable_cmpxchg16b_target_feature)
- && std::is_x86_feature_detected!("cmpxchg16b");
+ let has_cmpxchg16b = cfg!(all(
+ feature = "fallback",
+ portable_atomic_cmpxchg16b_target_feature,
+ not(portable_atomic_no_outline_atomics),
+ not(target_env = "sgx"),
+ )) && std::is_x86_feature_detected!("cmpxchg16b");
assert_eq!(AtomicI128::is_lock_free(), has_cmpxchg16b);
assert_eq!(AtomicU128::is_lock_free(), has_cmpxchg16b);
}
diff --git a/src/utils.rs b/src/utils.rs
index 4c2e15f06..d5c172d8c 100644
--- a/src/utils.rs
+++ b/src/utils.rs
@@ -141,6 +141,7 @@ macro_rules! serde_impls {
// Adapted from https://github.com/BurntSushi/memchr/blob/2.4.1/src/memchr/x86/mod.rs#L9-L71.
#[allow(unused_macros)]
#[cfg(not(portable_atomic_no_outline_atomics))]
+#[cfg(any(target_arch = "aarch64", all(target_arch = "x86_64", not(target_env = "sgx"))))]
macro_rules! ifunc {
// if the functions are unsafe, this macro is also unsafe.
(unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)? { $($if_block:tt)* }) => {{