diff options
Diffstat (limited to 'contrib/llvm-project/libcxx/include/__atomic')
16 files changed, 2576 insertions, 0 deletions
diff --git a/contrib/llvm-project/libcxx/include/__atomic/aliases.h b/contrib/llvm-project/libcxx/include/__atomic/aliases.h new file mode 100644 index 000000000000..e27e09af6b77 --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/aliases.h @@ -0,0 +1,110 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ALIASES_H +#define _LIBCPP___ATOMIC_ALIASES_H + +#include <__atomic/atomic.h> +#include <__atomic/atomic_lock_free.h> +#include <__atomic/contention_t.h> +#include <__atomic/is_always_lock_free.h> +#include <__config> +#include <__type_traits/conditional.h> +#include <__type_traits/make_unsigned.h> +#include <cstddef> +#include <cstdint> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +using atomic_bool = atomic<bool>; +using atomic_char = atomic<char>; +using atomic_schar = atomic<signed char>; +using atomic_uchar = atomic<unsigned char>; +using atomic_short = atomic<short>; +using atomic_ushort = atomic<unsigned short>; +using atomic_int = atomic<int>; +using atomic_uint = atomic<unsigned int>; +using atomic_long = atomic<long>; +using atomic_ulong = atomic<unsigned long>; +using atomic_llong = atomic<long long>; +using atomic_ullong = atomic<unsigned long long>; +#ifndef _LIBCPP_HAS_NO_CHAR8_T +using atomic_char8_t = atomic<char8_t>; +#endif +using atomic_char16_t = atomic<char16_t>; +using atomic_char32_t = atomic<char32_t>; +#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS +using atomic_wchar_t = atomic<wchar_t>; +#endif + +using atomic_int_least8_t = atomic<int_least8_t>; +using atomic_uint_least8_t = atomic<uint_least8_t>; +using atomic_int_least16_t = atomic<int_least16_t>; +using atomic_uint_least16_t = atomic<uint_least16_t>; +using atomic_int_least32_t = atomic<int_least32_t>; +using atomic_uint_least32_t = atomic<uint_least32_t>; +using atomic_int_least64_t = atomic<int_least64_t>; +using atomic_uint_least64_t = atomic<uint_least64_t>; + +using atomic_int_fast8_t = atomic<int_fast8_t>; +using atomic_uint_fast8_t = atomic<uint_fast8_t>; +using atomic_int_fast16_t = atomic<int_fast16_t>; +using atomic_uint_fast16_t = atomic<uint_fast16_t>; +using atomic_int_fast32_t = atomic<int_fast32_t>; +using atomic_uint_fast32_t = atomic<uint_fast32_t>; +using atomic_int_fast64_t = atomic<int_fast64_t>; +using atomic_uint_fast64_t = atomic<uint_fast64_t>; + +using atomic_int8_t = atomic< int8_t>; +using atomic_uint8_t = atomic<uint8_t>; +using atomic_int16_t = atomic< int16_t>; +using atomic_uint16_t = atomic<uint16_t>; +using atomic_int32_t = atomic< int32_t>; +using atomic_uint32_t = atomic<uint32_t>; +using atomic_int64_t = atomic< int64_t>; +using atomic_uint64_t = atomic<uint64_t>; + +using atomic_intptr_t = atomic<intptr_t>; +using atomic_uintptr_t = atomic<uintptr_t>; +using atomic_size_t = atomic<size_t>; +using atomic_ptrdiff_t = atomic<ptrdiff_t>; +using atomic_intmax_t = atomic<intmax_t>; +using atomic_uintmax_t = atomic<uintmax_t>; + +// C++20 atomic_{signed,unsigned}_lock_free: prefer the contention type most highly, then the largest lock-free type +#if _LIBCPP_STD_VER >= 20 +# if ATOMIC_LLONG_LOCK_FREE == 2 +using __largest_lock_free_type = long long; +# elif ATOMIC_INT_LOCK_FREE == 2 +using __largest_lock_free_type = int; +# elif ATOMIC_SHORT_LOCK_FREE == 2 +using __largest_lock_free_type = short; +# elif ATOMIC_CHAR_LOCK_FREE == 2 +using __largest_lock_free_type = char; +# else +# define _LIBCPP_NO_LOCK_FREE_TYPES // There are no lockfree types (this can happen on unusual platforms) +# endif + +# ifndef _LIBCPP_NO_LOCK_FREE_TYPES +using __contention_t_or_largest = + __conditional_t<__libcpp_is_always_lock_free<__cxx_contention_t>::__value, + __cxx_contention_t, + __largest_lock_free_type>; + +using atomic_signed_lock_free = atomic<__contention_t_or_largest>; +using atomic_unsigned_lock_free = atomic<make_unsigned_t<__contention_t_or_largest>>; +# endif // !_LIBCPP_NO_LOCK_FREE_TYPES +#endif // C++20 + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_ALIASES_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/atomic.h b/contrib/llvm-project/libcxx/include/__atomic/atomic.h new file mode 100644 index 000000000000..bd3f659c22df --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/atomic.h @@ -0,0 +1,622 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_H +#define _LIBCPP___ATOMIC_ATOMIC_H + +#include <__atomic/atomic_base.h> +#include <__atomic/check_memory_order.h> +#include <__atomic/cxx_atomic_impl.h> +#include <__atomic/memory_order.h> +#include <__config> +#include <__functional/operations.h> +#include <__memory/addressof.h> +#include <__type_traits/is_floating_point.h> +#include <__type_traits/is_function.h> +#include <__type_traits/is_same.h> +#include <__type_traits/remove_const.h> +#include <__type_traits/remove_pointer.h> +#include <__type_traits/remove_volatile.h> +#include <__utility/forward.h> +#include <cstddef> +#include <cstring> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _Tp> +struct atomic : public __atomic_base<_Tp> { + using __base = __atomic_base<_Tp>; + using value_type = _Tp; + using difference_type = value_type; + +#if _LIBCPP_STD_VER >= 20 + _LIBCPP_HIDE_FROM_ABI atomic() = default; +#else + _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default; +#endif + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} + + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile _NOEXCEPT { + __base::store(__d); + return __d; + } + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) _NOEXCEPT { + __base::store(__d); + return __d; + } + + atomic& operator=(const atomic&) = delete; + atomic& operator=(const atomic&) volatile = delete; +}; + +// atomic<T*> + +template <class _Tp> +struct atomic<_Tp*> : public __atomic_base<_Tp*> { + using __base = __atomic_base<_Tp*>; + using value_type = _Tp*; + using difference_type = ptrdiff_t; + + _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default; + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} + + _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) volatile _NOEXCEPT { + __base::store(__d); + return __d; + } + _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) _NOEXCEPT { + __base::store(__d); + return __d; + } + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + // __atomic_fetch_add accepts function pointers, guard against them. + static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); + return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + // __atomic_fetch_add accepts function pointers, guard against them. + static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); + return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + // __atomic_fetch_add accepts function pointers, guard against them. + static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); + return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + // __atomic_fetch_add accepts function pointers, guard against them. + static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); + return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) volatile _NOEXCEPT { return fetch_add(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) _NOEXCEPT { return fetch_add(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) volatile _NOEXCEPT { return fetch_sub(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) _NOEXCEPT { return fetch_sub(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator++() volatile _NOEXCEPT { return fetch_add(1) + 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator++() _NOEXCEPT { return fetch_add(1) + 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--() volatile _NOEXCEPT { return fetch_sub(1) - 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--() _NOEXCEPT { return fetch_sub(1) - 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT { return fetch_add(__op) + __op; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT { return fetch_sub(__op) - __op; } + + atomic& operator=(const atomic&) = delete; + atomic& operator=(const atomic&) volatile = delete; +}; + +#if _LIBCPP_STD_VER >= 20 +template <class _Tp> + requires is_floating_point_v<_Tp> +struct atomic<_Tp> : __atomic_base<_Tp> { +private: + _LIBCPP_HIDE_FROM_ABI static constexpr bool __is_fp80_long_double() { + // Only x87-fp80 long double has 64-bit mantissa + return __LDBL_MANT_DIG__ == 64 && std::is_same_v<_Tp, long double>; + } + + _LIBCPP_HIDE_FROM_ABI static constexpr bool __has_rmw_builtin() { +# ifndef _LIBCPP_COMPILER_CLANG_BASED + return false; +# else + // The builtin __cxx_atomic_fetch_add errors during compilation for + // long double on platforms with fp80 format. + // For more details, see + // lib/Sema/SemaChecking.cpp function IsAllowedValueType + // LLVM Parser does not allow atomicrmw with x86_fp80 type. + // if (ValType->isSpecificBuiltinType(BuiltinType::LongDouble) && + // &Context.getTargetInfo().getLongDoubleFormat() == + // &llvm::APFloat::x87DoubleExtended()) + // For more info + // https://github.com/llvm/llvm-project/issues/68602 + // https://reviews.llvm.org/D53965 + return !__is_fp80_long_double(); +# endif + } + + template <class _This, class _Operation, class _BuiltinOp> + _LIBCPP_HIDE_FROM_ABI static _Tp + __rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) { + if constexpr (__has_rmw_builtin()) { + return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m); + } else { + _Tp __old = __self.load(memory_order_relaxed); + _Tp __new = __operation(__old, __operand); + while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) { +# ifdef _LIBCPP_COMPILER_CLANG_BASED + if constexpr (__is_fp80_long_double()) { + // https://github.com/llvm/llvm-project/issues/47978 + // clang bug: __old is not updated on failure for atomic<long double>::compare_exchange_weak + // Note __old = __self.load(memory_order_relaxed) will not work + std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), &__old, memory_order_relaxed); + } +# endif + __new = __operation(__old, __operand); + } + return __old; + } + } + + template <class _This> + _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) { + auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) { + return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order); + }; + return __rmw_op(std::forward<_This>(__self), __operand, __m, std::plus<>{}, __builtin_op); + } + + template <class _This> + _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) { + auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) { + return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order); + }; + return __rmw_op(std::forward<_This>(__self), __operand, __m, std::minus<>{}, __builtin_op); + } + +public: + using __base = __atomic_base<_Tp>; + using value_type = _Tp; + using difference_type = value_type; + + _LIBCPP_HIDE_FROM_ABI constexpr atomic() noexcept = default; + _LIBCPP_HIDE_FROM_ABI constexpr atomic(_Tp __d) noexcept : __base(__d) {} + + atomic(const atomic&) = delete; + atomic& operator=(const atomic&) = delete; + atomic& operator=(const atomic&) volatile = delete; + + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile noexcept + requires __base::is_always_lock_free + { + __base::store(__d); + return __d; + } + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) noexcept { + __base::store(__d); + return __d; + } + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept + requires __base::is_always_lock_free + { + return __fetch_add(*this, __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept { + return __fetch_add(*this, __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept + requires __base::is_always_lock_free + { + return __fetch_sub(*this, __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept { + return __fetch_sub(*this, __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile noexcept + requires __base::is_always_lock_free + { + return fetch_add(__op) + __op; + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; } + + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile noexcept + requires __base::is_always_lock_free + { + return fetch_sub(__op) - __op; + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; } +}; + +#endif // _LIBCPP_STD_VER >= 20 + +// atomic_is_lock_free + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT { + return __o->is_lock_free(); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT { + return __o->is_lock_free(); +} + +// atomic_init + +template <class _Tp> +_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void +atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + std::__cxx_atomic_init(std::addressof(__o->__a_), __d); +} + +template <class _Tp> +_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void +atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + std::__cxx_atomic_init(std::addressof(__o->__a_), __d); +} + +// atomic_store + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + __o->store(__d); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + __o->store(__d); +} + +// atomic_store_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { + __o->store(__d, __m); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { + __o->store(__d, __m); +} + +// atomic_load + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT { + return __o->load(); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const atomic<_Tp>* __o) _NOEXCEPT { + return __o->load(); +} + +// atomic_load_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return __o->load(__m); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return __o->load(__m); +} + +// atomic_exchange + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->exchange(__d); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->exchange(__d); +} + +// atomic_exchange_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT { + return __o->exchange(__d, __m); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT { + return __o->exchange(__d, __m); +} + +// atomic_compare_exchange_weak + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->compare_exchange_weak(*__e, __d); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak( + atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->compare_exchange_weak(*__e, __d); +} + +// atomic_compare_exchange_strong + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->compare_exchange_strong(*__e, __d); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong( + atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->compare_exchange_strong(*__e, __d); +} + +// atomic_compare_exchange_weak_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit( + volatile atomic<_Tp>* __o, + typename atomic<_Tp>::value_type* __e, + typename atomic<_Tp>::value_type __d, + memory_order __s, + memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return __o->compare_exchange_weak(*__e, __d, __s, __f); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit( + atomic<_Tp>* __o, + typename atomic<_Tp>::value_type* __e, + typename atomic<_Tp>::value_type __d, + memory_order __s, + memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return __o->compare_exchange_weak(*__e, __d, __s, __f); +} + +// atomic_compare_exchange_strong_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit( + volatile atomic<_Tp>* __o, + typename atomic<_Tp>::value_type* __e, + typename atomic<_Tp>::value_type __d, + memory_order __s, + memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return __o->compare_exchange_strong(*__e, __d, __s, __f); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit( + atomic<_Tp>* __o, + typename atomic<_Tp>::value_type* __e, + typename atomic<_Tp>::value_type __d, + memory_order __s, + memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return __o->compare_exchange_strong(*__e, __d, __s, __f); +} + +// atomic_wait + +template <class _Tp> +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +atomic_wait(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT { + return __o->wait(__v); +} + +template <class _Tp> +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +atomic_wait(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT { + return __o->wait(__v); +} + +// atomic_wait_explicit + +template <class _Tp> +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +atomic_wait_explicit(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return __o->wait(__v, __m); +} + +template <class _Tp> +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +atomic_wait_explicit(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return __o->wait(__v, __m); +} + +// atomic_notify_one + +template <class _Tp> +_LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT { + __o->notify_one(); +} +template <class _Tp> +_LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT { + __o->notify_one(); +} + +// atomic_notify_all + +template <class _Tp> +_LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT { + __o->notify_all(); +} +template <class _Tp> +_LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT { + __o->notify_all(); +} + +// atomic_fetch_add + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { + return __o->fetch_add(__op); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { + return __o->fetch_add(__op); +} + +// atomic_fetch_add_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add_explicit( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_add(__op, __m); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_add(__op, __m); +} + +// atomic_fetch_sub + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { + return __o->fetch_sub(__op); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { + return __o->fetch_sub(__op); +} + +// atomic_fetch_sub_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub_explicit( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_sub(__op, __m); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_sub(__op, __m); +} + +// atomic_fetch_and + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_and(__op); +} + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_and(__op); +} + +// atomic_fetch_and_explicit + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and_explicit( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_and(__op, __m); +} + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_and(__op, __m); +} + +// atomic_fetch_or + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_or(__op); +} + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_or(__op); +} + +// atomic_fetch_or_explicit + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_or(__op, __m); +} + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_or(__op, __m); +} + +// atomic_fetch_xor + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_xor(__op); +} + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_xor(__op); +} + +// atomic_fetch_xor_explicit + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor_explicit( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_xor(__op, __m); +} + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_xor(__op, __m); +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_ATOMIC_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/atomic_base.h b/contrib/llvm-project/libcxx/include/__atomic/atomic_base.h new file mode 100644 index 000000000000..7e26434c9c3a --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/atomic_base.h @@ -0,0 +1,221 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_BASE_H +#define _LIBCPP___ATOMIC_ATOMIC_BASE_H + +#include <__atomic/atomic_sync.h> +#include <__atomic/check_memory_order.h> +#include <__atomic/cxx_atomic_impl.h> +#include <__atomic/is_always_lock_free.h> +#include <__atomic/memory_order.h> +#include <__config> +#include <__memory/addressof.h> +#include <__type_traits/is_integral.h> +#include <__type_traits/is_nothrow_constructible.h> +#include <__type_traits/is_same.h> +#include <version> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value> +struct __atomic_base // false +{ + mutable __cxx_atomic_impl<_Tp> __a_; + +#if _LIBCPP_STD_VER >= 17 + static constexpr bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value; +#endif + + _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const volatile _NOEXCEPT { + return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>)); + } + _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT { + return static_cast<__atomic_base const volatile*>(this)->is_lock_free(); + } + _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { + std::__cxx_atomic_store(std::addressof(__a_), __d, __m); + } + _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { + std::__cxx_atomic_store(std::addressof(__a_), __d, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return std::__cxx_atomic_load(std::addressof(__a_), __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return std::__cxx_atomic_load(std::addressof(__a_), __m); + } + _LIBCPP_HIDE_FROM_ABI operator _Tp() const volatile _NOEXCEPT { return load(); } + _LIBCPP_HIDE_FROM_ABI operator _Tp() const _NOEXCEPT { return load(); } + _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f); + } + _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f); + } + _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m); + } + + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const + volatile _NOEXCEPT { + std::__atomic_wait(*this, __v, __m); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void + wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT { + std::__atomic_wait(*this, __v, __m); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { + std::__atomic_notify_one(*this); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { + std::__atomic_notify_all(*this); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); } + +#if _LIBCPP_STD_VER >= 20 + _LIBCPP_HIDE_FROM_ABI constexpr __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {} +#else + _LIBCPP_HIDE_FROM_ABI __atomic_base() _NOEXCEPT = default; +#endif + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} + + __atomic_base(const __atomic_base&) = delete; +}; + +// atomic<Integral> + +template <class _Tp> +struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> { + using __base = __atomic_base<_Tp, false>; + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __atomic_base() _NOEXCEPT = default; + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) volatile _NOEXCEPT { return fetch_add(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) _NOEXCEPT { return fetch_add(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) volatile _NOEXCEPT { return fetch_sub(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) _NOEXCEPT { return fetch_sub(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator++() volatile _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator++() _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--() volatile _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--() _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) _NOEXCEPT { return fetch_add(__op) + __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) _NOEXCEPT { return fetch_sub(__op) - __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) volatile _NOEXCEPT { return fetch_and(__op) & __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) _NOEXCEPT { return fetch_and(__op) & __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) volatile _NOEXCEPT { return fetch_or(__op) | __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) _NOEXCEPT { return fetch_or(__op) | __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) volatile _NOEXCEPT { return fetch_xor(__op) ^ __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; } +}; + +// Here we need _IsIntegral because the default template argument is not enough +// e.g __atomic_base<int> is __atomic_base<int, true>, which inherits from +// __atomic_base<int, false> and the caller of the wait function is +// __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work +template <class _Tp, bool _IsIntegral> +struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > { + static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) { + return __a.load(__order); + } + + static _LIBCPP_HIDE_FROM_ABI _Tp + __atomic_load(const volatile __atomic_base<_Tp, _IsIntegral>& __this, memory_order __order) { + return __this.load(__order); + } + + static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_Tp>* + __atomic_contention_address(const __atomic_base<_Tp, _IsIntegral>& __a) { + return std::addressof(__a.__a_); + } + + static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_Tp>* + __atomic_contention_address(const volatile __atomic_base<_Tp, _IsIntegral>& __this) { + return std::addressof(__this.__a_); + } +}; + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_ATOMIC_BASE_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/atomic_flag.h b/contrib/llvm-project/libcxx/include/__atomic/atomic_flag.h new file mode 100644 index 000000000000..00b157cdff78 --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/atomic_flag.h @@ -0,0 +1,189 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_FLAG_H +#define _LIBCPP___ATOMIC_ATOMIC_FLAG_H + +#include <__atomic/atomic_sync.h> +#include <__atomic/contention_t.h> +#include <__atomic/cxx_atomic_impl.h> +#include <__atomic/memory_order.h> +#include <__chrono/duration.h> +#include <__config> +#include <__memory/addressof.h> +#include <__thread/support.h> +#include <cstdint> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +struct atomic_flag { + __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; + + _LIBCPP_HIDE_FROM_ABI bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT { + return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m); + } + _LIBCPP_HIDE_FROM_ABI bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT { + return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m); + } + + _LIBCPP_HIDE_FROM_ABI bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m); + } + _LIBCPP_HIDE_FROM_ABI bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m); + } + _LIBCPP_HIDE_FROM_ABI void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + __cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m); + } + _LIBCPP_HIDE_FROM_ABI void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT { + __cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m); + } + + _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void + wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT { + std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m); + } + _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void + wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT { + std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m); + } + _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { + std::__atomic_notify_one(*this); + } + _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { + std::__atomic_notify_one(*this); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { + std::__atomic_notify_all(*this); + } + _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { + std::__atomic_notify_all(*this); + } + +#if _LIBCPP_STD_VER >= 20 + _LIBCPP_HIDE_FROM_ABI constexpr atomic_flag() _NOEXCEPT : __a_(false) {} +#else + atomic_flag() _NOEXCEPT = default; +#endif + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION + + atomic_flag(const atomic_flag&) = delete; + atomic_flag& operator=(const atomic_flag&) = delete; + atomic_flag& operator=(const atomic_flag&) volatile = delete; +}; + +template <> +struct __atomic_waitable_traits<atomic_flag> { + static _LIBCPP_HIDE_FROM_ABI _LIBCPP_ATOMIC_FLAG_TYPE __atomic_load(const atomic_flag& __a, memory_order __order) { + return std::__cxx_atomic_load(&__a.__a_, __order); + } + + static _LIBCPP_HIDE_FROM_ABI _LIBCPP_ATOMIC_FLAG_TYPE + __atomic_load(const volatile atomic_flag& __a, memory_order __order) { + return std::__cxx_atomic_load(&__a.__a_, __order); + } + + static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE>* + __atomic_contention_address(const atomic_flag& __a) { + return std::addressof(__a.__a_); + } + + static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE>* + __atomic_contention_address(const volatile atomic_flag& __a) { + return std::addressof(__a.__a_); + } +}; + +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT { return __o->test(); } + +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const atomic_flag* __o) _NOEXCEPT { return __o->test(); } + +inline _LIBCPP_HIDE_FROM_ABI bool +atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT { + return __o->test(__m); +} + +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT { + return __o->test(__m); +} + +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT { + return __o->test_and_set(); +} + +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT { return __o->test_and_set(); } + +inline _LIBCPP_HIDE_FROM_ABI bool +atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT { + return __o->test_and_set(__m); +} + +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT { + return __o->test_and_set(__m); +} + +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT { __o->clear(); } + +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear(atomic_flag* __o) _NOEXCEPT { __o->clear(); } + +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT { + __o->clear(__m); +} + +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT { + __o->clear(__m); +} + +inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT { + __o->wait(__v); +} + +inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT { + __o->wait(__v); +} + +inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_wait_explicit(const volatile atomic_flag* __o, bool __v, memory_order __m) _NOEXCEPT { + __o->wait(__v, __m); +} + +inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_wait_explicit(const atomic_flag* __o, bool __v, memory_order __m) _NOEXCEPT { + __o->wait(__v, __m); +} + +inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT { + __o->notify_one(); +} + +inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT { + __o->notify_one(); +} + +inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT { + __o->notify_all(); +} + +inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT { + __o->notify_all(); +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_ATOMIC_FLAG_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/atomic_init.h b/contrib/llvm-project/libcxx/include/__atomic/atomic_init.h new file mode 100644 index 000000000000..8e86ba31b4ac --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/atomic_init.h @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_INIT_H +#define _LIBCPP___ATOMIC_ATOMIC_INIT_H + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#define ATOMIC_FLAG_INIT {false} +#define ATOMIC_VAR_INIT(__v) {__v} + +#if _LIBCPP_STD_VER >= 20 && defined(_LIBCPP_COMPILER_CLANG_BASED) && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS) +# pragma clang deprecated(ATOMIC_VAR_INIT) +#endif + +#endif // _LIBCPP___ATOMIC_ATOMIC_INIT_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/atomic_lock_free.h b/contrib/llvm-project/libcxx/include/__atomic/atomic_lock_free.h new file mode 100644 index 000000000000..0715439db450 --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/atomic_lock_free.h @@ -0,0 +1,48 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_LOCK_FREE_H +#define _LIBCPP___ATOMIC_ATOMIC_LOCK_FREE_H + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) +# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE +# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE +# ifndef _LIBCPP_HAS_NO_CHAR8_T +# define ATOMIC_CHAR8_T_LOCK_FREE __CLANG_ATOMIC_CHAR8_T_LOCK_FREE +# endif +# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE +# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE +# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE +# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE +# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE +# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE +# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE +# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE +#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE) +# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE +# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE +# ifndef _LIBCPP_HAS_NO_CHAR8_T +# define ATOMIC_CHAR8_T_LOCK_FREE __GCC_ATOMIC_CHAR8_T_LOCK_FREE +# endif +# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE +# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE +# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE +# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE +# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE +# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE +# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE +# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE +#endif + +#endif // _LIBCPP___ATOMIC_ATOMIC_LOCK_FREE_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/atomic_ref.h b/contrib/llvm-project/libcxx/include/__atomic/atomic_ref.h new file mode 100644 index 000000000000..2db88d9f0654 --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/atomic_ref.h @@ -0,0 +1,378 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +//===---------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_REF_H +#define _LIBCPP___ATOMIC_ATOMIC_REF_H + +#include <__assert> +#include <__atomic/atomic_sync.h> +#include <__atomic/check_memory_order.h> +#include <__atomic/to_gcc_order.h> +#include <__concepts/arithmetic.h> +#include <__concepts/same_as.h> +#include <__config> +#include <__memory/addressof.h> +#include <__type_traits/has_unique_object_representation.h> +#include <__type_traits/is_trivially_copyable.h> +#include <cstddef> +#include <cstdint> +#include <cstring> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if _LIBCPP_STD_VER >= 20 + +// These types are required to make __atomic_is_always_lock_free work across GCC and Clang. +// The purpose of this trick is to make sure that we provide an object with the correct alignment +// to __atomic_is_always_lock_free, since that answer depends on the alignment. +template <size_t _Alignment> +struct __alignment_checker_type { + alignas(_Alignment) char __data; +}; + +template <size_t _Alignment> +struct __get_aligner_instance { + static constexpr __alignment_checker_type<_Alignment> __instance{}; +}; + +template <class _Tp> +struct __atomic_ref_base { +private: + _LIBCPP_HIDE_FROM_ABI static _Tp* __clear_padding(_Tp& __val) noexcept { + _Tp* __ptr = std::addressof(__val); +# if __has_builtin(__builtin_clear_padding) + __builtin_clear_padding(__ptr); +# endif + return __ptr; + } + + _LIBCPP_HIDE_FROM_ABI static bool __compare_exchange( + _Tp* __ptr, _Tp* __expected, _Tp* __desired, bool __is_weak, int __success, int __failure) noexcept { + if constexpr ( +# if __has_builtin(__builtin_clear_padding) + has_unique_object_representations_v<_Tp> || floating_point<_Tp> +# else + true // NOLINT(readability-simplify-boolean-expr) +# endif + ) { + return __atomic_compare_exchange(__ptr, __expected, __desired, __is_weak, __success, __failure); + } else { // _Tp has padding bits and __builtin_clear_padding is available + __clear_padding(*__desired); + _Tp __copy = *__expected; + __clear_padding(__copy); + // The algorithm we use here is basically to perform `__atomic_compare_exchange` on the + // values until it has either succeeded, or failed because the value representation of the + // objects involved was different. This is why we loop around __atomic_compare_exchange: + // we basically loop until its failure is caused by the value representation of the objects + // being different, not only their object representation. + while (true) { + _Tp __prev = __copy; + if (__atomic_compare_exchange(__ptr, std::addressof(__copy), __desired, __is_weak, __success, __failure)) { + return true; + } + _Tp __curr = __copy; + if (std::memcmp(__clear_padding(__prev), __clear_padding(__curr), sizeof(_Tp)) != 0) { + // Value representation without padding bits do not compare equal -> + // write the current content of *ptr into *expected + std::memcpy(__expected, std::addressof(__copy), sizeof(_Tp)); + return false; + } + } + } + } + + friend struct __atomic_waitable_traits<__atomic_ref_base<_Tp>>; + + // require types that are 1, 2, 4, 8, or 16 bytes in length to be aligned to at least their size to be potentially + // used lock-free + static constexpr size_t __min_alignment = (sizeof(_Tp) & (sizeof(_Tp) - 1)) || (sizeof(_Tp) > 16) ? 0 : sizeof(_Tp); + +public: + using value_type = _Tp; + + static constexpr size_t required_alignment = alignof(_Tp) > __min_alignment ? alignof(_Tp) : __min_alignment; + + // The __atomic_always_lock_free builtin takes into account the alignment of the pointer if provided, + // so we create a fake pointer with a suitable alignment when querying it. Note that we are guaranteed + // that the pointer is going to be aligned properly at runtime because that is a (checked) precondition + // of atomic_ref's constructor. + static constexpr bool is_always_lock_free = + __atomic_always_lock_free(sizeof(_Tp), &__get_aligner_instance<required_alignment>::__instance); + + _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return __atomic_is_lock_free(sizeof(_Tp), __ptr_); } + + _LIBCPP_HIDE_FROM_ABI void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst, + "atomic_ref: memory order argument to atomic store operation is invalid"); + __atomic_store(__ptr_, __clear_padding(__desired), std::__to_gcc_order(__order)); + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { + store(__desired); + return __desired; + } + + _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __order = memory_order::seq_cst) const noexcept + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__order) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire || + __order == memory_order::seq_cst, + "atomic_ref: memory order argument to atomic load operation is invalid"); + alignas(_Tp) byte __mem[sizeof(_Tp)]; + auto* __ret = reinterpret_cast<_Tp*>(__mem); + __atomic_load(__ptr_, __ret, std::__to_gcc_order(__order)); + return *__ret; + } + + _LIBCPP_HIDE_FROM_ABI operator _Tp() const noexcept { return load(); } + + _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept { + alignas(_Tp) byte __mem[sizeof(_Tp)]; + auto* __ret = reinterpret_cast<_Tp*>(__mem); + __atomic_exchange(__ptr_, __clear_padding(__desired), __ret, std::__to_gcc_order(__order)); + return *__ret; + } + + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + __failure == memory_order::relaxed || __failure == memory_order::consume || + __failure == memory_order::acquire || __failure == memory_order::seq_cst, + "atomic_ref: failure memory order argument to weak atomic compare-and-exchange operation is invalid"); + return __compare_exchange( + __ptr_, + std::addressof(__expected), + std::addressof(__desired), + true, + std::__to_gcc_order(__success), + std::__to_gcc_order(__failure)); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + __failure == memory_order::relaxed || __failure == memory_order::consume || + __failure == memory_order::acquire || __failure == memory_order::seq_cst, + "atomic_ref: failure memory order argument to strong atomic compare-and-exchange operation is invalid"); + return __compare_exchange( + __ptr_, + std::addressof(__expected), + std::addressof(__desired), + false, + std::__to_gcc_order(__success), + std::__to_gcc_order(__failure)); + } + + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept { + return __compare_exchange( + __ptr_, + std::addressof(__expected), + std::addressof(__desired), + true, + std::__to_gcc_order(__order), + std::__to_gcc_failure_order(__order)); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept { + return __compare_exchange( + __ptr_, + std::addressof(__expected), + std::addressof(__desired), + false, + std::__to_gcc_order(__order), + std::__to_gcc_failure_order(__order)); + } + + _LIBCPP_HIDE_FROM_ABI void wait(_Tp __old, memory_order __order = memory_order::seq_cst) const noexcept + _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__order) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire || + __order == memory_order::seq_cst, + "atomic_ref: memory order argument to atomic wait operation is invalid"); + std::__atomic_wait(*this, __old, __order); + } + _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { std::__atomic_notify_one(*this); } + _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { std::__atomic_notify_all(*this); } + +protected: + typedef _Tp _Aligned_Tp __attribute__((aligned(required_alignment))); + _Aligned_Tp* __ptr_; + + _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __ptr_(std::addressof(__obj)) {} +}; + +template <class _Tp> +struct __atomic_waitable_traits<__atomic_ref_base<_Tp>> { + static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_ref_base<_Tp>& __a, memory_order __order) { + return __a.load(__order); + } + static _LIBCPP_HIDE_FROM_ABI const _Tp* __atomic_contention_address(const __atomic_ref_base<_Tp>& __a) { + return __a.__ptr_; + } +}; + +template <class _Tp> +struct atomic_ref : public __atomic_ref_base<_Tp> { + static_assert(is_trivially_copyable_v<_Tp>, "std::atomic_ref<T> requires that 'T' be a trivially copyable type"); + + using __base = __atomic_ref_base<_Tp>; + + _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + reinterpret_cast<uintptr_t>(std::addressof(__obj)) % __base::required_alignment == 0, + "atomic_ref ctor: referenced object must be aligned to required_alignment"); + } + + _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default; + + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); } + + atomic_ref& operator=(const atomic_ref&) = delete; +}; + +template <class _Tp> + requires(std::integral<_Tp> && !std::same_as<bool, _Tp>) +struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> { + using __base = __atomic_ref_base<_Tp>; + + using difference_type = typename __base::value_type; + + _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + reinterpret_cast<uintptr_t>(std::addressof(__obj)) % __base::required_alignment == 0, + "atomic_ref ctor: referenced object must be aligned to required_alignment"); + } + + _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default; + + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); } + + atomic_ref& operator=(const atomic_ref&) = delete; + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order)); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_sub(this->__ptr_, __arg, std::__to_gcc_order(__order)); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_and(this->__ptr_, __arg, std::__to_gcc_order(__order)); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_or(this->__ptr_, __arg, std::__to_gcc_order(__order)); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_xor(this->__ptr_, __arg, std::__to_gcc_order(__order)); + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) const noexcept { return fetch_add(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) const noexcept { return fetch_sub(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator++() const noexcept { return fetch_add(_Tp(1)) + _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--() const noexcept { return fetch_sub(_Tp(1)) - _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; } + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; } + _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __arg) const noexcept { return fetch_and(__arg) & __arg; } + _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __arg) const noexcept { return fetch_or(__arg) | __arg; } + _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __arg) const noexcept { return fetch_xor(__arg) ^ __arg; } +}; + +template <class _Tp> + requires std::floating_point<_Tp> +struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> { + using __base = __atomic_ref_base<_Tp>; + + using difference_type = typename __base::value_type; + + _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + reinterpret_cast<uintptr_t>(std::addressof(__obj)) % __base::required_alignment == 0, + "atomic_ref ctor: referenced object must be aligned to required_alignment"); + } + + _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default; + + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); } + + atomic_ref& operator=(const atomic_ref&) = delete; + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + _Tp __old = this->load(memory_order_relaxed); + _Tp __new = __old + __arg; + while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) { + __new = __old + __arg; + } + return __old; + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + _Tp __old = this->load(memory_order_relaxed); + _Tp __new = __old - __arg; + while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) { + __new = __old - __arg; + } + return __old; + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; } + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; } +}; + +template <class _Tp> +struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> { + using __base = __atomic_ref_base<_Tp*>; + + using difference_type = ptrdiff_t; + + _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp*& __ptr) : __base(__ptr) {} + + _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __desired) const noexcept { return __base::operator=(__desired); } + + atomic_ref& operator=(const atomic_ref&) = delete; + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_add(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order)); + } + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_sub(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order)); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) const noexcept { return fetch_add(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) const noexcept { return fetch_sub(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator++() const noexcept { return fetch_add(1) + 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--() const noexcept { return fetch_sub(1) - 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __arg) const noexcept { return fetch_add(__arg) + __arg; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __arg) const noexcept { return fetch_sub(__arg) - __arg; } +}; + +_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(atomic_ref); + +#endif // _LIBCPP_STD_VER >= 20 + +_LIBCPP_END_NAMESPACE_STD + +_LIBCPP_POP_MACROS + +#endif // _LIBCPP__ATOMIC_ATOMIC_REF_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/atomic_sync.h b/contrib/llvm-project/libcxx/include/__atomic/atomic_sync.h new file mode 100644 index 000000000000..aaf81f58731a --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/atomic_sync.h @@ -0,0 +1,205 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_H +#define _LIBCPP___ATOMIC_ATOMIC_SYNC_H + +#include <__atomic/contention_t.h> +#include <__atomic/cxx_atomic_impl.h> +#include <__atomic/memory_order.h> +#include <__atomic/to_gcc_order.h> +#include <__chrono/duration.h> +#include <__config> +#include <__memory/addressof.h> +#include <__thread/poll_with_backoff.h> +#include <__thread/support.h> +#include <__type_traits/conjunction.h> +#include <__type_traits/decay.h> +#include <__type_traits/invoke.h> +#include <__type_traits/void_t.h> +#include <__utility/declval.h> +#include <cstring> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +// The customisation points to enable the following functions: +// - __atomic_wait +// - __atomic_wait_unless +// - __atomic_notify_one +// - __atomic_notify_all +// Note that std::atomic<T>::wait was back-ported to C++03 +// The below implementations look ugly to support C++03 +template <class _Tp, class = void> +struct __atomic_waitable_traits { + template <class _AtomicWaitable> + static void __atomic_load(_AtomicWaitable&&, memory_order) = delete; + + template <class _AtomicWaitable> + static void __atomic_contention_address(_AtomicWaitable&&) = delete; +}; + +template <class _Tp, class = void> +struct __atomic_waitable : false_type {}; + +template <class _Tp> +struct __atomic_waitable< _Tp, + __void_t<decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_load( + std::declval<const _Tp&>(), std::declval<memory_order>())), + decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_contention_address( + std::declval<const _Tp&>()))> > : true_type {}; + +template <class _AtomicWaitable, class _Poll> +struct __atomic_wait_poll_impl { + const _AtomicWaitable& __a_; + _Poll __poll_; + memory_order __order_; + + _LIBCPP_HIDE_FROM_ABI bool operator()() const { + auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a_, __order_); + return __poll_(__current_val); + } +}; + +#ifndef _LIBCPP_HAS_NO_THREADS + +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*) _NOEXCEPT; +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*) _NOEXCEPT; +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t +__libcpp_atomic_monitor(void const volatile*) _NOEXCEPT; +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void +__libcpp_atomic_wait(void const volatile*, __cxx_contention_t) _NOEXCEPT; + +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void +__cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*) _NOEXCEPT; +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void +__cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*) _NOEXCEPT; +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t +__libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*) _NOEXCEPT; +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void +__libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t) _NOEXCEPT; + +template <class _AtomicWaitable, class _Poll> +struct __atomic_wait_backoff_impl { + const _AtomicWaitable& __a_; + _Poll __poll_; + memory_order __order_; + + using __waitable_traits = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >; + + _LIBCPP_AVAILABILITY_SYNC + _LIBCPP_HIDE_FROM_ABI bool + __update_monitor_val_and_poll(__cxx_atomic_contention_t const volatile*, __cxx_contention_t& __monitor_val) const { + // In case the contention type happens to be __cxx_atomic_contention_t, i.e. __cxx_atomic_impl<int64_t>, + // the platform wait is directly monitoring the atomic value itself. + // `__poll_` takes the current value of the atomic as an in-out argument + // to potentially modify it. After it returns, `__monitor` has a value + // which can be safely waited on by `std::__libcpp_atomic_wait` without any + // ABA style issues. + __monitor_val = __waitable_traits::__atomic_load(__a_, __order_); + return __poll_(__monitor_val); + } + + _LIBCPP_AVAILABILITY_SYNC + _LIBCPP_HIDE_FROM_ABI bool + __update_monitor_val_and_poll(void const volatile* __contention_address, __cxx_contention_t& __monitor_val) const { + // In case the contention type is anything else, platform wait is monitoring a __cxx_atomic_contention_t + // from the global pool, the monitor comes from __libcpp_atomic_monitor + __monitor_val = std::__libcpp_atomic_monitor(__contention_address); + auto __current_val = __waitable_traits::__atomic_load(__a_, __order_); + return __poll_(__current_val); + } + + _LIBCPP_AVAILABILITY_SYNC + _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const { + if (__elapsed > chrono::microseconds(64)) { + auto __contention_address = __waitable_traits::__atomic_contention_address(__a_); + __cxx_contention_t __monitor_val; + if (__update_monitor_val_and_poll(__contention_address, __monitor_val)) + return true; + std::__libcpp_atomic_wait(__contention_address, __monitor_val); + } else if (__elapsed > chrono::microseconds(4)) + __libcpp_thread_yield(); + else { + } // poll + return false; + } +}; + +// The semantics of this function are similar to `atomic`'s +// `.wait(T old, std::memory_order order)`, but instead of having a hardcoded +// predicate (is the loaded value unequal to `old`?), the predicate function is +// specified as an argument. The loaded value is given as an in-out argument to +// the predicate. If the predicate function returns `true`, +// `__atomic_wait_unless` will return. If the predicate function returns +// `false`, it must set the argument to its current understanding of the atomic +// value. The predicate function must not return `false` spuriously. +template <class _AtomicWaitable, class _Poll> +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +__atomic_wait_unless(const _AtomicWaitable& __a, _Poll&& __poll, memory_order __order) { + static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + __atomic_wait_poll_impl<_AtomicWaitable, __decay_t<_Poll> > __poll_impl = {__a, __poll, __order}; + __atomic_wait_backoff_impl<_AtomicWaitable, __decay_t<_Poll> > __backoff_fn = {__a, __poll, __order}; + std::__libcpp_thread_poll_with_backoff(__poll_impl, __backoff_fn); +} + +template <class _AtomicWaitable> +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) { + static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + std::__cxx_atomic_notify_one(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a)); +} + +template <class _AtomicWaitable> +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) { + static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + std::__cxx_atomic_notify_all(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a)); +} + +#else // _LIBCPP_HAS_NO_THREADS + +template <class _AtomicWaitable, class _Poll> +_LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, _Poll&& __poll, memory_order __order) { + __atomic_wait_poll_impl<_AtomicWaitable, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order}; + std::__libcpp_thread_poll_with_backoff(__poll_fn, __spinning_backoff_policy()); +} + +template <class _AtomicWaitable> +_LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable&) {} + +template <class _AtomicWaitable> +_LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable&) {} + +#endif // _LIBCPP_HAS_NO_THREADS + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { + return std::memcmp(std::addressof(__lhs), std::addressof(__rhs), sizeof(_Tp)) == 0; +} + +template <class _Tp> +struct __atomic_compare_unequal_to { + _Tp __val_; + _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __arg) const { + return !std::__cxx_nonatomic_compare_equal(__arg, __val_); + } +}; + +template <class _AtomicWaitable, class _Up> +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +__atomic_wait(_AtomicWaitable& __a, _Up __val, memory_order __order) { + static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + __atomic_compare_unequal_to<_Up> __nonatomic_equal = {__val}; + std::__atomic_wait_unless(__a, __nonatomic_equal, __order); +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/check_memory_order.h b/contrib/llvm-project/libcxx/include/__atomic/check_memory_order.h new file mode 100644 index 000000000000..536f764a6190 --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/check_memory_order.h @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H +#define _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ + _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || __m == memory_order_acquire || __m == memory_order_acq_rel, \ + "memory order argument to atomic operation is invalid") + +#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \ + _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || __m == memory_order_acq_rel, \ + "memory order argument to atomic operation is invalid") + +#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \ + _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || __f == memory_order_acq_rel, \ + "memory order argument to atomic operation is invalid") + +#define _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m) \ + _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || __m == memory_order_acq_rel, \ + "memory order argument to atomic operation is invalid") + +#endif // _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/contention_t.h b/contrib/llvm-project/libcxx/include/__atomic/contention_t.h new file mode 100644 index 000000000000..65890f338ce9 --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/contention_t.h @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_CONTENTION_T_H +#define _LIBCPP___ATOMIC_CONTENTION_T_H + +#include <__atomic/cxx_atomic_impl.h> +#include <__config> +#include <cstdint> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if defined(__linux__) || (defined(_AIX) && !defined(__64BIT__)) +using __cxx_contention_t = int32_t; +#else +using __cxx_contention_t = int64_t; +#endif // __linux__ || (_AIX && !__64BIT__) + +using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>; + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_CONTENTION_T_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/cxx_atomic_impl.h b/contrib/llvm-project/libcxx/include/__atomic/cxx_atomic_impl.h new file mode 100644 index 000000000000..18e88aa97bec --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/cxx_atomic_impl.h @@ -0,0 +1,510 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H +#define _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H + +#include <__atomic/memory_order.h> +#include <__atomic/to_gcc_order.h> +#include <__config> +#include <__memory/addressof.h> +#include <__type_traits/is_assignable.h> +#include <__type_traits/is_trivially_copyable.h> +#include <__type_traits/remove_const.h> +#include <cstddef> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) + +// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because +// the default operator= in an object is not volatile, a byte-by-byte copy +// is required. +template <typename _Tp, typename _Tv, __enable_if_t<is_assignable<_Tp&, _Tv>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { + __a_value = __val; +} +template <typename _Tp, typename _Tv, __enable_if_t<is_assignable<_Tp&, _Tv>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { + volatile char* __to = reinterpret_cast<volatile char*>(std::addressof(__a_value)); + volatile char* __end = __to + sizeof(_Tp); + volatile const char* __from = reinterpret_cast<volatile const char*>(std::addressof(__val)); + while (__to != __end) + *__to++ = *__from++; +} + +template <typename _Tp> +struct __cxx_atomic_base_impl { + _LIBCPP_HIDE_FROM_ABI +# ifndef _LIBCPP_CXX03_LANG + __cxx_atomic_base_impl() _NOEXCEPT = default; +# else + __cxx_atomic_base_impl() _NOEXCEPT : __a_value() { + } +# endif // _LIBCPP_CXX03_LANG + _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {} + _Tp __a_value; +}; + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { + __cxx_atomic_assign_volatile(__a->__a_value, __val); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { + __a->__a_value = __val; +} + +_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) { + __atomic_thread_fence(__to_gcc_order(__order)); +} + +_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) { + __atomic_signal_fence(__to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) { + __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) { + __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { + _Tp __ret; + __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order)); + return __ret; +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_load_inplace(const volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) { + __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_load_inplace(const __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) { + __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { + _Tp __ret; + __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order)); + return __ret; +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) { + _Tp __ret; + __atomic_exchange( + std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order)); + return __ret; +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) { + _Tp __ret; + __atomic_exchange( + std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order)); + return __ret; +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + volatile __cxx_atomic_base_impl<_Tp>* __a, + _Tp* __expected, + _Tp __value, + memory_order __success, + memory_order __failure) { + return __atomic_compare_exchange( + std::addressof(__a->__a_value), + __expected, + std::addressof(__value), + false, + __to_gcc_order(__success), + __to_gcc_failure_order(__failure)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { + return __atomic_compare_exchange( + std::addressof(__a->__a_value), + __expected, + std::addressof(__value), + false, + __to_gcc_order(__success), + __to_gcc_failure_order(__failure)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + volatile __cxx_atomic_base_impl<_Tp>* __a, + _Tp* __expected, + _Tp __value, + memory_order __success, + memory_order __failure) { + return __atomic_compare_exchange( + std::addressof(__a->__a_value), + __expected, + std::addressof(__value), + true, + __to_gcc_order(__success), + __to_gcc_failure_order(__failure)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { + return __atomic_compare_exchange( + std::addressof(__a->__a_value), + __expected, + std::addressof(__value), + true, + __to_gcc_order(__success), + __to_gcc_failure_order(__failure)); +} + +template <typename _Tp> +struct __skip_amt { + enum { value = 1 }; +}; + +template <typename _Tp> +struct __skip_amt<_Tp*> { + enum { value = sizeof(_Tp) }; +}; + +// FIXME: Haven't figured out what the spec says about using arrays with +// atomic_fetch_add. Force a failure rather than creating bad behavior. +template <typename _Tp> +struct __skip_amt<_Tp[]> {}; +template <typename _Tp, int n> +struct __skip_amt<_Tp[n]> {}; + +template <typename _Tp, typename _Td> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) { + return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); +} + +template <typename _Tp, typename _Td> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) { + return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); +} + +template <typename _Tp, typename _Td> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) { + return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); +} + +template <typename _Tp, typename _Td> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) { + return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { + return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { + return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { + return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { + return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { + return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { + return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); +} + +# define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) + +#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP) + +template <typename _Tp> +struct __cxx_atomic_base_impl { + _LIBCPP_HIDE_FROM_ABI +# ifndef _LIBCPP_CXX03_LANG + __cxx_atomic_base_impl() _NOEXCEPT = default; +# else + __cxx_atomic_base_impl() _NOEXCEPT : __a_value() { + } +# endif // _LIBCPP_CXX03_LANG + _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {} + _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; +}; + +# define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) + +_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { + __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); +} + +_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { + __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { + __c11_atomic_init(std::addressof(__a->__a_value), __val); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT { + __c11_atomic_init(std::addressof(__a->__a_value), __val); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { + __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT { + __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { + using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*; + return __c11_atomic_load( + const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { + using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*; + return __c11_atomic_load( + const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT { + using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*; + *__dst = __c11_atomic_load( + const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT { + using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*; + *__dst = __c11_atomic_load( + const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { + return __c11_atomic_exchange( + std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT { + return __c11_atomic_exchange( + std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order)); +} + +_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) { + // Avoid switch statement to make this a constexpr. + return __order == memory_order_release + ? memory_order_relaxed + : (__order == memory_order_acq_rel ? memory_order_acquire : __order); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + __cxx_atomic_base_impl<_Tp> volatile* __a, + _Tp* __expected, + _Tp __value, + memory_order __success, + memory_order __failure) _NOEXCEPT { + return __c11_atomic_compare_exchange_strong( + std::addressof(__a->__a_value), + __expected, + __value, + static_cast<__memory_order_underlying_t>(__success), + static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) + _NOEXCEPT { + return __c11_atomic_compare_exchange_strong( + std::addressof(__a->__a_value), + __expected, + __value, + static_cast<__memory_order_underlying_t>(__success), + static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + __cxx_atomic_base_impl<_Tp> volatile* __a, + _Tp* __expected, + _Tp __value, + memory_order __success, + memory_order __failure) _NOEXCEPT { + return __c11_atomic_compare_exchange_weak( + std::addressof(__a->__a_value), + __expected, + __value, + static_cast<__memory_order_underlying_t>(__success), + static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) + _NOEXCEPT { + return __c11_atomic_compare_exchange_weak( + std::addressof(__a->__a_value), + __expected, + __value, + static_cast<__memory_order_underlying_t>(__success), + static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_add( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_add( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp* +__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_add( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp* +__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_add( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_sub( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_sub( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp* +__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_sub( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp* +__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_sub( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_and( + std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_and( + std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_or( + std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_or( + std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_xor( + std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_xor( + std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); +} + +#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP + +template <typename _Tp, typename _Base = __cxx_atomic_base_impl<_Tp> > +struct __cxx_atomic_impl : public _Base { + static_assert(is_trivially_copyable<_Tp>::value, "std::atomic<T> requires that 'T' be a trivially copyable type"); + + _LIBCPP_HIDE_FROM_ABI __cxx_atomic_impl() _NOEXCEPT = default; + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT : _Base(__value) {} +}; + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/fence.h b/contrib/llvm-project/libcxx/include/__atomic/fence.h new file mode 100644 index 000000000000..8c27ea54d62d --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/fence.h @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_FENCE_H +#define _LIBCPP___ATOMIC_FENCE_H + +#include <__atomic/cxx_atomic_impl.h> +#include <__atomic/memory_order.h> +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +inline _LIBCPP_HIDE_FROM_ABI void atomic_thread_fence(memory_order __m) _NOEXCEPT { __cxx_atomic_thread_fence(__m); } + +inline _LIBCPP_HIDE_FROM_ABI void atomic_signal_fence(memory_order __m) _NOEXCEPT { __cxx_atomic_signal_fence(__m); } + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_FENCE_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/is_always_lock_free.h b/contrib/llvm-project/libcxx/include/__atomic/is_always_lock_free.h new file mode 100644 index 000000000000..f928e79f70ce --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/is_always_lock_free.h @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_IS_ALWAYS_LOCK_FREE_H +#define _LIBCPP___ATOMIC_IS_ALWAYS_LOCK_FREE_H + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _Tp> +struct __libcpp_is_always_lock_free { + // __atomic_always_lock_free is available in all Standard modes + static const bool __value = __atomic_always_lock_free(sizeof(_Tp), nullptr); +}; + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_IS_ALWAYS_LOCK_FREE_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/kill_dependency.h b/contrib/llvm-project/libcxx/include/__atomic/kill_dependency.h new file mode 100644 index 000000000000..103d52d35787 --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/kill_dependency.h @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_KILL_DEPENDENCY_H +#define _LIBCPP___ATOMIC_KILL_DEPENDENCY_H + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp kill_dependency(_Tp __y) _NOEXCEPT { + return __y; +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_KILL_DEPENDENCY_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/memory_order.h b/contrib/llvm-project/libcxx/include/__atomic/memory_order.h new file mode 100644 index 000000000000..294121d1c4e7 --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/memory_order.h @@ -0,0 +1,65 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_MEMORY_ORDER_H +#define _LIBCPP___ATOMIC_MEMORY_ORDER_H + +#include <__config> +#include <__type_traits/is_same.h> +#include <__type_traits/underlying_type.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +// Figure out what the underlying type for `memory_order` would be if it were +// declared as an unscoped enum (accounting for -fshort-enums). Use this result +// to pin the underlying type in C++20. +enum __legacy_memory_order { __mo_relaxed, __mo_consume, __mo_acquire, __mo_release, __mo_acq_rel, __mo_seq_cst }; + +using __memory_order_underlying_t = underlying_type<__legacy_memory_order>::type; + +#if _LIBCPP_STD_VER >= 20 + +enum class memory_order : __memory_order_underlying_t { + relaxed = __mo_relaxed, + consume = __mo_consume, + acquire = __mo_acquire, + release = __mo_release, + acq_rel = __mo_acq_rel, + seq_cst = __mo_seq_cst +}; + +static_assert(is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value, + "unexpected underlying type for std::memory_order"); + +inline constexpr auto memory_order_relaxed = memory_order::relaxed; +inline constexpr auto memory_order_consume = memory_order::consume; +inline constexpr auto memory_order_acquire = memory_order::acquire; +inline constexpr auto memory_order_release = memory_order::release; +inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; +inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; + +#else + +enum memory_order { + memory_order_relaxed = __mo_relaxed, + memory_order_consume = __mo_consume, + memory_order_acquire = __mo_acquire, + memory_order_release = __mo_release, + memory_order_acq_rel = __mo_acq_rel, + memory_order_seq_cst = __mo_seq_cst, +}; + +#endif // _LIBCPP_STD_VER >= 20 + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_MEMORY_ORDER_H diff --git a/contrib/llvm-project/libcxx/include/__atomic/to_gcc_order.h b/contrib/llvm-project/libcxx/include/__atomic/to_gcc_order.h new file mode 100644 index 000000000000..d04c111addd3 --- /dev/null +++ b/contrib/llvm-project/libcxx/include/__atomic/to_gcc_order.h @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_TO_GCC_ORDER_H +#define _LIBCPP___ATOMIC_TO_GCC_ORDER_H + +#include <__atomic/memory_order.h> +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if defined(__ATOMIC_RELAXED) && defined(__ATOMIC_CONSUME) && defined(__ATOMIC_ACQUIRE) && \ + defined(__ATOMIC_RELEASE) && defined(__ATOMIC_ACQ_REL) && defined(__ATOMIC_SEQ_CST) + +_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { + // Avoid switch statement to make this a constexpr. + return __order == memory_order_relaxed + ? __ATOMIC_RELAXED + : (__order == memory_order_acquire + ? __ATOMIC_ACQUIRE + : (__order == memory_order_release + ? __ATOMIC_RELEASE + : (__order == memory_order_seq_cst + ? __ATOMIC_SEQ_CST + : (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL : __ATOMIC_CONSUME)))); +} + +_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { + // Avoid switch statement to make this a constexpr. + return __order == memory_order_relaxed + ? __ATOMIC_RELAXED + : (__order == memory_order_acquire + ? __ATOMIC_ACQUIRE + : (__order == memory_order_release + ? __ATOMIC_RELAXED + : (__order == memory_order_seq_cst + ? __ATOMIC_SEQ_CST + : (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE : __ATOMIC_CONSUME)))); +} + +#endif + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_TO_GCC_ORDER_H |