diff --git a/libc/src/__support/CPP/atomic.h b/libc/src/__support/CPP/atomic.h index a9fc7e61610cc..287dcac98fbb6 100644 --- a/libc/src/__support/CPP/atomic.h +++ b/libc/src/__support/CPP/atomic.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_CPP_ATOMIC_H #define LLVM_LIBC_SRC___SUPPORT_CPP_ATOMIC_H +#include "src/__support/CPP/type_traits/has_unique_object_representations.h" #include "src/__support/macros/attributes.h" #include "src/__support/macros/config.h" #include "src/__support/macros/properties/architectures.h" @@ -47,12 +48,11 @@ template struct Atomic { "constructible, move constructible, copy assignable, " "and move assignable."); + static_assert(cpp::has_unique_object_representations_v, + "atomic in libc only support types whose values has unique " + "object representations."); + private: - // The value stored should be appropriately aligned so that - // hardware instructions used to perform atomic operations work - // correctly. - static constexpr int ALIGNMENT = sizeof(T) > alignof(T) ? sizeof(T) - : alignof(T); // type conversion helper to avoid long c++ style casts LIBC_INLINE static int order(MemoryOrder mem_ord) { return static_cast(mem_ord); @@ -62,6 +62,17 @@ template struct Atomic { return static_cast(mem_scope); } + LIBC_INLINE static T *addressof(T &ref) { return __builtin_addressof(ref); } + + // Require types that are 1, 2, 4, 8, or 16 bytes in length to be aligned to + // at least their size to be potentially used lock-free. + LIBC_INLINE_VAR static constexpr size_t MIN_ALIGNMENT = + (sizeof(T) & (sizeof(T) - 1)) || (sizeof(T) > 16) ? 0 : sizeof(T); + + LIBC_INLINE_VAR static constexpr size_t ALIGNMENT = alignof(T) > MIN_ALIGNMENT + ? alignof(T) + : MIN_ALIGNMENT; + public: using value_type = T; @@ -87,9 +98,10 @@ template struct Atomic { [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { T res; #if __has_builtin(__scoped_atomic_load) - __scoped_atomic_load(&val, &res, order(mem_ord), scope(mem_scope)); + __scoped_atomic_load(addressof(val), addressof(res), order(mem_ord), + scope(mem_scope)); #else - __atomic_load(&val, &res, order(mem_ord)); + __atomic_load(addressof(val), addressof(res), order(mem_ord)); #endif return res; } @@ -104,9 +116,10 @@ template struct Atomic { store(T rhs, MemoryOrder mem_ord = MemoryOrder::SEQ_CST, [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { #if __has_builtin(__scoped_atomic_store) - __scoped_atomic_store(&val, &rhs, order(mem_ord), scope(mem_scope)); + __scoped_atomic_store(addressof(val), addressof(rhs), order(mem_ord), + scope(mem_scope)); #else - __atomic_store(&val, &rhs, order(mem_ord)); + __atomic_store(addressof(val), addressof(rhs), order(mem_ord)); #endif } @@ -114,8 +127,9 @@ template struct Atomic { LIBC_INLINE bool compare_exchange_strong( T &expected, T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST, [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { - return __atomic_compare_exchange(&val, &expected, &desired, false, - order(mem_ord), order(mem_ord)); + return __atomic_compare_exchange(addressof(val), addressof(expected), + addressof(desired), false, order(mem_ord), + order(mem_ord)); } // Atomic compare exchange (separate success and failure memory orders) @@ -123,17 +137,18 @@ template struct Atomic { T &expected, T desired, MemoryOrder success_order, MemoryOrder failure_order, [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { - return __atomic_compare_exchange(&val, &expected, &desired, false, - order(success_order), - order(failure_order)); + return __atomic_compare_exchange( + addressof(val), addressof(expected), addressof(desired), false, + order(success_order), order(failure_order)); } // Atomic compare exchange (weak version) LIBC_INLINE bool compare_exchange_weak( T &expected, T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST, [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { - return __atomic_compare_exchange(&val, &expected, &desired, true, - order(mem_ord), order(mem_ord)); + return __atomic_compare_exchange(addressof(val), addressof(expected), + addressof(desired), true, order(mem_ord), + order(mem_ord)); } // Atomic compare exchange (weak version with separate success and failure @@ -142,9 +157,9 @@ template struct Atomic { T &expected, T desired, MemoryOrder success_order, MemoryOrder failure_order, [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { - return __atomic_compare_exchange(&val, &expected, &desired, true, - order(success_order), - order(failure_order)); + return __atomic_compare_exchange( + addressof(val), addressof(expected), addressof(desired), true, + order(success_order), order(failure_order)); } LIBC_INLINE T @@ -152,10 +167,11 @@ template struct Atomic { [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { T ret; #if __has_builtin(__scoped_atomic_exchange) - __scoped_atomic_exchange(&val, &desired, &ret, order(mem_ord), - scope(mem_scope)); + __scoped_atomic_exchange(addressof(val), addressof(desired), addressof(ret), + order(mem_ord), scope(mem_scope)); #else - __atomic_exchange(&val, &desired, &ret, order(mem_ord)); + __atomic_exchange(addressof(val), addressof(desired), addressof(ret), + order(mem_ord)); #endif return ret; } @@ -165,10 +181,10 @@ template struct Atomic { [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { static_assert(cpp::is_integral_v, "T must be an integral type."); #if __has_builtin(__scoped_atomic_fetch_add) - return __scoped_atomic_fetch_add(&val, increment, order(mem_ord), + return __scoped_atomic_fetch_add(addressof(val), increment, order(mem_ord), scope(mem_scope)); #else - return __atomic_fetch_add(&val, increment, order(mem_ord)); + return __atomic_fetch_add(addressof(val), increment, order(mem_ord)); #endif } @@ -177,10 +193,10 @@ template struct Atomic { [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { static_assert(cpp::is_integral_v, "T must be an integral type."); #if __has_builtin(__scoped_atomic_fetch_or) - return __scoped_atomic_fetch_or(&val, mask, order(mem_ord), + return __scoped_atomic_fetch_or(addressof(val), mask, order(mem_ord), scope(mem_scope)); #else - return __atomic_fetch_or(&val, mask, order(mem_ord)); + return __atomic_fetch_or(addressof(val), mask, order(mem_ord)); #endif } @@ -189,10 +205,10 @@ template struct Atomic { [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { static_assert(cpp::is_integral_v, "T must be an integral type."); #if __has_builtin(__scoped_atomic_fetch_and) - return __scoped_atomic_fetch_and(&val, mask, order(mem_ord), + return __scoped_atomic_fetch_and(addressof(val), mask, order(mem_ord), scope(mem_scope)); #else - return __atomic_fetch_and(&val, mask, order(mem_ord)); + return __atomic_fetch_and(addressof(val), mask, order(mem_ord)); #endif } @@ -201,10 +217,10 @@ template struct Atomic { [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { static_assert(cpp::is_integral_v, "T must be an integral type."); #if __has_builtin(__scoped_atomic_fetch_sub) - return __scoped_atomic_fetch_sub(&val, decrement, order(mem_ord), + return __scoped_atomic_fetch_sub(addressof(val), decrement, order(mem_ord), scope(mem_scope)); #else - return __atomic_fetch_sub(&val, decrement, order(mem_ord)); + return __atomic_fetch_sub(addressof(val), decrement, order(mem_ord)); #endif } diff --git a/libc/src/__support/CPP/type_traits.h b/libc/src/__support/CPP/type_traits.h index b9bc5b8568441..910cebbb8d059 100644 --- a/libc/src/__support/CPP/type_traits.h +++ b/libc/src/__support/CPP/type_traits.h @@ -18,6 +18,7 @@ #include "src/__support/CPP/type_traits/decay.h" #include "src/__support/CPP/type_traits/enable_if.h" #include "src/__support/CPP/type_traits/false_type.h" +#include "src/__support/CPP/type_traits/has_unique_object_representations.h" #include "src/__support/CPP/type_traits/integral_constant.h" #include "src/__support/CPP/type_traits/invoke.h" #include "src/__support/CPP/type_traits/invoke_result.h" diff --git a/libc/src/__support/CPP/type_traits/has_unique_object_representations.h b/libc/src/__support/CPP/type_traits/has_unique_object_representations.h new file mode 100644 index 0000000000000..639fb69d27203 --- /dev/null +++ b/libc/src/__support/CPP/type_traits/has_unique_object_representations.h @@ -0,0 +1,30 @@ +//===-- has_unique_object_representations type_traits ------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATIONS_H +#define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATIONS_H + +#include "src/__support/CPP/type_traits/integral_constant.h" +#include "src/__support/CPP/type_traits/remove_all_extents.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { +namespace cpp { + +template +struct has_unique_object_representations + : public integral_constant)> {}; + +template +LIBC_INLINE_VAR constexpr bool has_unique_object_representations_v = + has_unique_object_representations::value; + +} // namespace cpp +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATIONS_H diff --git a/libc/test/src/__support/CPP/type_traits_test.cpp b/libc/test/src/__support/CPP/type_traits_test.cpp index fa5298a12d3fc..4b3e48c6a6c0f 100644 --- a/libc/test/src/__support/CPP/type_traits_test.cpp +++ b/libc/test/src/__support/CPP/type_traits_test.cpp @@ -439,6 +439,28 @@ TEST(LlvmLibcTypeTraitsTest, is_object) { TEST(LlvmLibcTypeTraitsTest, true_type) { EXPECT_TRUE((true_type::value)); } +struct CompilerLeadingPadded { + char b; + int a; +}; + +struct CompilerTrailingPadded { + int a; + char b; +}; + +struct alignas(long long) ManuallyPadded { + int b; + char padding[sizeof(long long) - sizeof(int)]; +}; + +TEST(LlvmLibcTypeTraitsTest, has_unique_object_representations) { + EXPECT_TRUE(has_unique_object_representations::value); + EXPECT_FALSE(has_unique_object_representations_v); + EXPECT_FALSE(has_unique_object_representations_v); + EXPECT_TRUE(has_unique_object_representations_v); +} + // TODO type_identity // TODO void_t