summaryrefslogtreecommitdiff
path: root/lib/Headers/intrin.h
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Headers/intrin.h')
-rw-r--r--lib/Headers/intrin.h158
1 files changed, 54 insertions, 104 deletions
diff --git a/lib/Headers/intrin.h b/lib/Headers/intrin.h
index b30aa215a452e..91914214e2994 100644
--- a/lib/Headers/intrin.h
+++ b/lib/Headers/intrin.h
@@ -38,7 +38,7 @@
#include <armintr.h>
#endif
-#if defined(_M_ARM64)
+#if defined(__aarch64__)
#include <arm64intr.h>
#endif
@@ -83,6 +83,7 @@ void __incfsdword(unsigned long);
void __incfsword(unsigned long);
unsigned long __indword(unsigned short);
void __indwordstring(unsigned short, unsigned long *, unsigned long);
+void __int2c(void);
void __invlpg(void *);
unsigned short __inword(unsigned short);
void __inwordstring(unsigned short, unsigned short *, unsigned long);
@@ -140,6 +141,7 @@ void __svm_stgi(void);
void __svm_vmload(size_t);
void __svm_vmrun(size_t);
void __svm_vmsave(size_t);
+void __ud2(void);
unsigned __int64 __ull_rshift(unsigned __int64, int);
void __vmx_off(void);
void __vmx_vmptrst(unsigned __int64 *);
@@ -161,25 +163,15 @@ static __inline__
unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask);
static __inline__
unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask);
-static __inline__
unsigned char _bittest(long const *, long);
-static __inline__
unsigned char _bittestandcomplement(long *, long);
-static __inline__
unsigned char _bittestandreset(long *, long);
-static __inline__
unsigned char _bittestandset(long *, long);
void __cdecl _disable(void);
void __cdecl _enable(void);
long _InterlockedAddLargeStatistic(__int64 volatile *_Addend, long _Value);
unsigned char _interlockedbittestandreset(long volatile *, long);
unsigned char _interlockedbittestandset(long volatile *, long);
-long _InterlockedCompareExchange_HLEAcquire(long volatile *, long, long);
-long _InterlockedCompareExchange_HLERelease(long volatile *, long, long);
-__int64 _InterlockedcompareExchange64_HLEAcquire(__int64 volatile *, __int64,
- __int64);
-__int64 _InterlockedCompareExchange64_HLERelease(__int64 volatile *, __int64,
- __int64);
void *_InterlockedCompareExchangePointer_HLEAcquire(void *volatile *, void *,
void *);
void *_InterlockedCompareExchangePointer_HLERelease(void *volatile *, void *,
@@ -256,24 +248,15 @@ void __writegsbyte(unsigned long, unsigned char);
void __writegsdword(unsigned long, unsigned long);
void __writegsqword(unsigned long, unsigned __int64);
void __writegsword(unsigned long, unsigned short);
-static __inline__
-unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
-static __inline__
-unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
-static __inline__
unsigned char _bittest64(__int64 const *, __int64);
-static __inline__
unsigned char _bittestandcomplement64(__int64 *, __int64);
-static __inline__
unsigned char _bittestandreset64(__int64 *, __int64);
-static __inline__
unsigned char _bittestandset64(__int64 *, __int64);
long _InterlockedAnd_np(long volatile *_Value, long _Mask);
short _InterlockedAnd16_np(short volatile *_Value, short _Mask);
__int64 _InterlockedAnd64_np(__int64 volatile *_Value, __int64 _Mask);
char _InterlockedAnd8_np(char volatile *_Value, char _Mask);
unsigned char _interlockedbittestandreset64(__int64 volatile *, __int64);
-static __inline__
unsigned char _interlockedbittestandset64(__int64 volatile *, __int64);
long _InterlockedCompareExchange_np(long volatile *_Destination, long _Exchange,
long _Comparand);
@@ -287,10 +270,6 @@ unsigned char _InterlockedCompareExchange128_np(__int64 volatile *_Destination,
__int64 *_ComparandResult);
short _InterlockedCompareExchange16_np(short volatile *_Destination,
short _Exchange, short _Comparand);
-__int64 _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *, __int64,
- __int64);
-__int64 _InterlockedCompareExchange64_HLERelease(__int64 volatile *, __int64,
- __int64);
__int64 _InterlockedCompareExchange64_np(__int64 volatile *_Destination,
__int64 _Exchange, __int64 _Comparand);
void *_InterlockedCompareExchangePointer_np(void *volatile *_Destination,
@@ -320,7 +299,12 @@ unsigned __int64 _umul128(unsigned __int64,
#endif /* __x86_64__ */
-#if defined(__x86_64__) || defined(__arm__)
+#if defined(__x86_64__) || defined(__arm__) || defined(__aarch64__)
+
+static __inline__
+unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
+static __inline__
+unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
static __inline__
__int64 _InterlockedDecrement64(__int64 volatile *_Addend);
@@ -342,78 +326,6 @@ __int64 _InterlockedAnd64(__int64 volatile *_Value, __int64 _Mask);
#endif
/*----------------------------------------------------------------------------*\
-|* Bit Counting and Testing
-\*----------------------------------------------------------------------------*/
-static __inline__ unsigned char __DEFAULT_FN_ATTRS
-_bittest(long const *_BitBase, long _BitPos) {
- return (*_BitBase >> _BitPos) & 1;
-}
-static __inline__ unsigned char __DEFAULT_FN_ATTRS
-_bittestandcomplement(long *_BitBase, long _BitPos) {
- unsigned char _Res = (*_BitBase >> _BitPos) & 1;
- *_BitBase = *_BitBase ^ (1 << _BitPos);
- return _Res;
-}
-static __inline__ unsigned char __DEFAULT_FN_ATTRS
-_bittestandreset(long *_BitBase, long _BitPos) {
- unsigned char _Res = (*_BitBase >> _BitPos) & 1;
- *_BitBase = *_BitBase & ~(1 << _BitPos);
- return _Res;
-}
-static __inline__ unsigned char __DEFAULT_FN_ATTRS
-_bittestandset(long *_BitBase, long _BitPos) {
- unsigned char _Res = (*_BitBase >> _BitPos) & 1;
- *_BitBase = *_BitBase | (1 << _BitPos);
- return _Res;
-}
-#if defined(__arm__) || defined(__aarch64__)
-static __inline__ unsigned char __DEFAULT_FN_ATTRS
-_interlockedbittestandset_acq(long volatile *_BitBase, long _BitPos) {
- long _PrevVal = __atomic_fetch_or(_BitBase, 1l << _BitPos, __ATOMIC_ACQUIRE);
- return (_PrevVal >> _BitPos) & 1;
-}
-static __inline__ unsigned char __DEFAULT_FN_ATTRS
-_interlockedbittestandset_nf(long volatile *_BitBase, long _BitPos) {
- long _PrevVal = __atomic_fetch_or(_BitBase, 1l << _BitPos, __ATOMIC_RELAXED);
- return (_PrevVal >> _BitPos) & 1;
-}
-static __inline__ unsigned char __DEFAULT_FN_ATTRS
-_interlockedbittestandset_rel(long volatile *_BitBase, long _BitPos) {
- long _PrevVal = __atomic_fetch_or(_BitBase, 1l << _BitPos, __ATOMIC_RELEASE);
- return (_PrevVal >> _BitPos) & 1;
-}
-#endif
-#ifdef __x86_64__
-static __inline__ unsigned char __DEFAULT_FN_ATTRS
-_bittest64(__int64 const *_BitBase, __int64 _BitPos) {
- return (*_BitBase >> _BitPos) & 1;
-}
-static __inline__ unsigned char __DEFAULT_FN_ATTRS
-_bittestandcomplement64(__int64 *_BitBase, __int64 _BitPos) {
- unsigned char _Res = (*_BitBase >> _BitPos) & 1;
- *_BitBase = *_BitBase ^ (1ll << _BitPos);
- return _Res;
-}
-static __inline__ unsigned char __DEFAULT_FN_ATTRS
-_bittestandreset64(__int64 *_BitBase, __int64 _BitPos) {
- unsigned char _Res = (*_BitBase >> _BitPos) & 1;
- *_BitBase = *_BitBase & ~(1ll << _BitPos);
- return _Res;
-}
-static __inline__ unsigned char __DEFAULT_FN_ATTRS
-_bittestandset64(__int64 *_BitBase, __int64 _BitPos) {
- unsigned char _Res = (*_BitBase >> _BitPos) & 1;
- *_BitBase = *_BitBase | (1ll << _BitPos);
- return _Res;
-}
-static __inline__ unsigned char __DEFAULT_FN_ATTRS
-_interlockedbittestandset64(__int64 volatile *_BitBase, __int64 _BitPos) {
- long long _PrevVal =
- __atomic_fetch_or(_BitBase, 1ll << _BitPos, __ATOMIC_SEQ_CST);
- return (_PrevVal >> _BitPos) & 1;
-}
-#endif
-/*----------------------------------------------------------------------------*\
|* Interlocked Exchange Add
\*----------------------------------------------------------------------------*/
#if defined(__arm__) || defined(__aarch64__)
@@ -602,6 +514,23 @@ _InterlockedAnd64_rel(__int64 volatile *_Value, __int64 _Mask) {
}
#endif
/*----------------------------------------------------------------------------*\
+|* Bit Counting and Testing
+\*----------------------------------------------------------------------------*/
+#if defined(__arm__) || defined(__aarch64__)
+unsigned char _interlockedbittestandset_acq(long volatile *_BitBase,
+ long _BitPos);
+unsigned char _interlockedbittestandset_nf(long volatile *_BitBase,
+ long _BitPos);
+unsigned char _interlockedbittestandset_rel(long volatile *_BitBase,
+ long _BitPos);
+unsigned char _interlockedbittestandreset_acq(long volatile *_BitBase,
+ long _BitPos);
+unsigned char _interlockedbittestandreset_nf(long volatile *_BitBase,
+ long _BitPos);
+unsigned char _interlockedbittestandreset_rel(long volatile *_BitBase,
+ long _BitPos);
+#endif
+/*----------------------------------------------------------------------------*\
|* Interlocked Or
\*----------------------------------------------------------------------------*/
#if defined(__arm__) || defined(__aarch64__)
@@ -868,33 +797,40 @@ _InterlockedCompareExchange64_rel(__int64 volatile *_Destination,
#if defined(__i386__) || defined(__x86_64__)
static __inline__ void __DEFAULT_FN_ATTRS
__movsb(unsigned char *__dst, unsigned char const *__src, size_t __n) {
- __asm__("rep movsb" : : "D"(__dst), "S"(__src), "c"(__n));
+ __asm__ __volatile__("rep movsb" : "+D"(__dst), "+S"(__src), "+c"(__n)
+ : : "memory");
}
static __inline__ void __DEFAULT_FN_ATTRS
__movsd(unsigned long *__dst, unsigned long const *__src, size_t __n) {
- __asm__("rep movsl" : : "D"(__dst), "S"(__src), "c"(__n));
+ __asm__ __volatile__("rep movsl" : "+D"(__dst), "+S"(__src), "+c"(__n)
+ : : "memory");
}
static __inline__ void __DEFAULT_FN_ATTRS
__movsw(unsigned short *__dst, unsigned short const *__src, size_t __n) {
- __asm__("rep movsw" : : "D"(__dst), "S"(__src), "c"(__n));
+ __asm__ __volatile__("rep movsw" : "+D"(__dst), "+S"(__src), "+c"(__n)
+ : : "memory");
}
static __inline__ void __DEFAULT_FN_ATTRS
__stosd(unsigned long *__dst, unsigned long __x, size_t __n) {
- __asm__("rep stosl" : : "D"(__dst), "a"(__x), "c"(__n));
+ __asm__ __volatile__("rep stosl" : "+D"(__dst), "+c"(__n) : "a"(__x)
+ : "memory");
}
static __inline__ void __DEFAULT_FN_ATTRS
__stosw(unsigned short *__dst, unsigned short __x, size_t __n) {
- __asm__("rep stosw" : : "D"(__dst), "a"(__x), "c"(__n));
+ __asm__ __volatile__("rep stosw" : "+D"(__dst), "+c"(__n) : "a"(__x)
+ : "memory");
}
#endif
#ifdef __x86_64__
static __inline__ void __DEFAULT_FN_ATTRS
__movsq(unsigned long long *__dst, unsigned long long const *__src, size_t __n) {
- __asm__("rep movsq" : : "D"(__dst), "S"(__src), "c"(__n));
+ __asm__ __volatile__("rep movsq" : "+D"(__dst), "+S"(__src), "+c"(__n)
+ : : "memory");
}
static __inline__ void __DEFAULT_FN_ATTRS
__stosq(unsigned __int64 *__dst, unsigned __int64 __x, size_t __n) {
- __asm__("rep stosq" : : "D"(__dst), "a"(__x), "c"(__n));
+ __asm__ __volatile__("rep stosq" : "+D"(__dst), "+c"(__n) : "a"(__x)
+ : "memory");
}
#endif
@@ -927,6 +863,20 @@ __nop(void) {
__asm__ volatile ("nop");
}
#endif
+#if defined(__x86_64__)
+static __inline__ unsigned __int64 __DEFAULT_FN_ATTRS
+__shiftleft128(unsigned __int64 __l, unsigned __int64 __h, unsigned char __d) {
+ unsigned __int128 __val = ((unsigned __int128)__h << 64) | __l;
+ unsigned __int128 __res = __val << (__d & 63);
+ return (unsigned __int64)(__res >> 64);
+}
+static __inline__ unsigned __int64 __DEFAULT_FN_ATTRS
+__shiftright128(unsigned __int64 __l, unsigned __int64 __h, unsigned char __d) {
+ unsigned __int128 __val = ((unsigned __int128)__h << 64) | __l;
+ unsigned __int128 __res = __val >> (__d & 63);
+ return (unsigned __int64)__res;
+}
+#endif
/*----------------------------------------------------------------------------*\
|* Privileged intrinsics
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-02 04:09:06 +0000