diff options
Diffstat (limited to 'compiler-rt')
| -rw-r--r-- | compiler-rt/lib/builtins/README.txt | 5 | ||||
| -rw-r--r-- | compiler-rt/lib/builtins/aarch64/sme-abi-vg.c | 28 | ||||
| -rw-r--r-- | compiler-rt/lib/builtins/aarch64/sme-abi.S | 44 | ||||
| -rw-r--r-- | compiler-rt/lib/builtins/trampoline_setup.c | 42 | ||||
| -rw-r--r-- | compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h | 46 | ||||
| -rw-r--r-- | compiler-rt/lib/ubsan/ubsan_type_hash_itanium.cpp | 2 |
6 files changed, 116 insertions, 51 deletions
diff --git a/compiler-rt/lib/builtins/README.txt b/compiler-rt/lib/builtins/README.txt index 2d213d95f333..19f26c92a0f9 100644 --- a/compiler-rt/lib/builtins/README.txt +++ b/compiler-rt/lib/builtins/README.txt @@ -272,6 +272,11 @@ switch32 switch8 switchu8 +// This function generates a custom trampoline function with the specific +// realFunc and localsPtr values. +void __trampoline_setup(uint32_t* trampOnStack, int trampSizeAllocated, + const void* realFunc, void* localsPtr); + // There is no C interface to the *_vfp_d8_d15_regs functions. There are // called in the prolog and epilog of Thumb1 functions. When the C++ ABI use // SJLJ for exceptions, each function with a catch clause or destructors needs diff --git a/compiler-rt/lib/builtins/aarch64/sme-abi-vg.c b/compiler-rt/lib/builtins/aarch64/sme-abi-vg.c index 062cf80fc684..20061012e16c 100644 --- a/compiler-rt/lib/builtins/aarch64/sme-abi-vg.c +++ b/compiler-rt/lib/builtins/aarch64/sme-abi-vg.c @@ -10,15 +10,6 @@ struct FEATURES { extern struct FEATURES __aarch64_cpu_features; -struct SME_STATE { - long PSTATE; - long TPIDR2_EL0; -}; - -extern struct SME_STATE __arm_sme_state(void) __arm_streaming_compatible; - -extern bool __aarch64_has_sme_and_tpidr2_el0; - #if __GNUC__ >= 9 #pragma GCC diagnostic ignored "-Wprio-ctor-dtor" #endif @@ -28,22 +19,3 @@ __attribute__((constructor(90))) static void get_aarch64_cpu_features(void) { __init_cpu_features(); } - -__attribute__((target("sve"))) long -__arm_get_current_vg(void) __arm_streaming_compatible { - struct SME_STATE State = __arm_sme_state(); - unsigned long long features = - __atomic_load_n(&__aarch64_cpu_features.features, __ATOMIC_RELAXED); - bool HasSVE = features & (1ULL << FEAT_SVE); - - if (!HasSVE && !__aarch64_has_sme_and_tpidr2_el0) - return 0; - - if (HasSVE || (State.PSTATE & 1)) { - long vl; - __asm__ __volatile__("cntd %0" : "=r"(vl)); - return vl; - } - - return 0; -} diff --git a/compiler-rt/lib/builtins/aarch64/sme-abi.S b/compiler-rt/lib/builtins/aarch64/sme-abi.S index 4c0ff66931db..cd8153f60670 100644 --- a/compiler-rt/lib/builtins/aarch64/sme-abi.S +++ b/compiler-rt/lib/builtins/aarch64/sme-abi.S @@ -12,11 +12,15 @@ #if !defined(__APPLE__) #define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0) #define TPIDR2_SYMBOL_OFFSET :lo12:SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0) +#define CPU_FEATS_SYMBOL SYMBOL_NAME(__aarch64_cpu_features) +#define CPU_FEATS_SYMBOL_OFFSET :lo12:SYMBOL_NAME(__aarch64_cpu_features) #else // MachO requires @page/@pageoff directives because the global is defined // in a different file. Otherwise this file may fail to build. #define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@page #define TPIDR2_SYMBOL_OFFSET SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@pageoff +#define CPU_FEATS_SYMBOL SYMBOL_NAME(__aarch64_cpu_features)@page +#define CPU_FEATS_SYMBOL_OFFSET SYMBOL_NAME(__aarch64_cpu_features)@pageoff #endif .arch armv9-a+sme @@ -180,6 +184,46 @@ DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_za_disable) ret END_COMPILERRT_OUTLINE_FUNCTION(__arm_za_disable) +DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_get_current_vg) + .variant_pcs __arm_get_current_vg + BTI_C + + stp x29, x30, [sp, #-16]! + .cfi_def_cfa_offset 16 + mov x29, sp + .cfi_def_cfa w29, 16 + .cfi_offset w30, -8 + .cfi_offset w29, -16 + adrp x17, CPU_FEATS_SYMBOL + ldr w17, [x17, CPU_FEATS_SYMBOL_OFFSET] + tbnz w17, #30, 0f + adrp x16, TPIDR2_SYMBOL + ldrb w16, [x16, TPIDR2_SYMBOL_OFFSET] + cbz w16, 1f +0: + mov x18, x1 + bl __arm_sme_state + mov x1, x18 + and x17, x17, #0x40000000 + bfxil x17, x0, #0, #1 + cbz x17, 1f + cntd x0 + .cfi_def_cfa wsp, 16 + ldp x29, x30, [sp], #16 + .cfi_def_cfa_offset 0 + .cfi_restore w30 + .cfi_restore w29 + ret +1: + mov x0, xzr + .cfi_def_cfa wsp, 16 + ldp x29, x30, [sp], #16 + .cfi_def_cfa_offset 0 + .cfi_restore w30 + .cfi_restore w29 + ret +END_COMPILERRT_OUTLINE_FUNCTION(__arm_get_current_vg) + NO_EXEC_STACK_DIRECTIVE // GNU property note for BTI and PAC diff --git a/compiler-rt/lib/builtins/trampoline_setup.c b/compiler-rt/lib/builtins/trampoline_setup.c index 844eb2794414..830e25e4c030 100644 --- a/compiler-rt/lib/builtins/trampoline_setup.c +++ b/compiler-rt/lib/builtins/trampoline_setup.c @@ -41,3 +41,45 @@ COMPILER_RT_ABI void __trampoline_setup(uint32_t *trampOnStack, __clear_cache(trampOnStack, &trampOnStack[10]); } #endif // __powerpc__ && !defined(__powerpc64__) + +// The AArch64 compiler generates calls to __trampoline_setup() when creating +// trampoline functions on the stack for use with nested functions. +// This function creates a custom 36-byte trampoline function on the stack +// which loads x18 with a pointer to the outer function's locals +// and then jumps to the target nested function. +// Note: x18 is a reserved platform register on Windows and macOS. + +#if defined(__aarch64__) && defined(__ELF__) +COMPILER_RT_ABI void __trampoline_setup(uint32_t *trampOnStack, + int trampSizeAllocated, + const void *realFunc, void *localsPtr) { + // This should never happen, but if compiler did not allocate + // enough space on stack for the trampoline, abort. + if (trampSizeAllocated < 36) + compilerrt_abort(); + + // create trampoline + // Load realFunc into x17. mov/movk 16 bits at a time. + trampOnStack[0] = + 0xd2800000u | ((((uint64_t)realFunc >> 0) & 0xffffu) << 5) | 0x11; + trampOnStack[1] = + 0xf2a00000u | ((((uint64_t)realFunc >> 16) & 0xffffu) << 5) | 0x11; + trampOnStack[2] = + 0xf2c00000u | ((((uint64_t)realFunc >> 32) & 0xffffu) << 5) | 0x11; + trampOnStack[3] = + 0xf2e00000u | ((((uint64_t)realFunc >> 48) & 0xffffu) << 5) | 0x11; + // Load localsPtr into x18 + trampOnStack[4] = + 0xd2800000u | ((((uint64_t)localsPtr >> 0) & 0xffffu) << 5) | 0x12; + trampOnStack[5] = + 0xf2a00000u | ((((uint64_t)localsPtr >> 16) & 0xffffu) << 5) | 0x12; + trampOnStack[6] = + 0xf2c00000u | ((((uint64_t)localsPtr >> 32) & 0xffffu) << 5) | 0x12; + trampOnStack[7] = + 0xf2e00000u | ((((uint64_t)localsPtr >> 48) & 0xffffu) << 5) | 0x12; + trampOnStack[8] = 0xd61f0220; // br x17 + + // Clear instruction cache. + __clear_cache(trampOnStack, &trampOnStack[9]); +} +#endif // defined(__aarch64__) && !defined(__APPLE__) && !defined(_WIN64) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h b/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h index 520035469485..265a9925a15a 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h @@ -9,31 +9,33 @@ #ifndef SANITIZER_PTRAUTH_H #define SANITIZER_PTRAUTH_H -#if __has_feature(ptrauth_calls) -#include <ptrauth.h> +#if __has_feature(ptrauth_intrinsics) +# include <ptrauth.h> #elif defined(__ARM_FEATURE_PAC_DEFAULT) && !defined(__APPLE__) -inline unsigned long ptrauth_strip(void* __value, unsigned int __key) { - // On the stack the link register is protected with Pointer - // Authentication Code when compiled with -mbranch-protection. - // Let's stripping the PAC unconditionally because xpaclri is in - // the NOP space so will do nothing when it is not enabled or not available. - unsigned long ret; - asm volatile( - "mov x30, %1\n\t" - "hint #7\n\t" // xpaclri - "mov %0, x30\n\t" - : "=r"(ret) - : "r"(__value) - : "x30"); - return ret; -} -#define ptrauth_auth_data(__value, __old_key, __old_data) __value -#define ptrauth_string_discriminator(__string) ((int)0) +// On the stack the link register is protected with Pointer +// Authentication Code when compiled with -mbranch-protection. +// Let's stripping the PAC unconditionally because xpaclri is in +// the NOP space so will do nothing when it is not enabled or not available. +# define ptrauth_strip(__value, __key) \ + ({ \ + __typeof(__value) ret; \ + asm volatile( \ + "mov x30, %1\n\t" \ + "hint #7\n\t" \ + "mov %0, x30\n\t" \ + "mov x30, xzr\n\t" \ + : "=r"(ret) \ + : "r"(__value) \ + : "x30"); \ + ret; \ + }) +# define ptrauth_auth_data(__value, __old_key, __old_data) __value +# define ptrauth_string_discriminator(__string) ((int)0) #else // Copied from <ptrauth.h> -#define ptrauth_strip(__value, __key) __value -#define ptrauth_auth_data(__value, __old_key, __old_data) __value -#define ptrauth_string_discriminator(__string) ((int)0) +# define ptrauth_strip(__value, __key) __value +# define ptrauth_auth_data(__value, __old_key, __old_data) __value +# define ptrauth_string_discriminator(__string) ((int)0) #endif #define STRIP_PAC_PC(pc) ((uptr)ptrauth_strip(pc, 0)) diff --git a/compiler-rt/lib/ubsan/ubsan_type_hash_itanium.cpp b/compiler-rt/lib/ubsan/ubsan_type_hash_itanium.cpp index 468a8fcd603f..15788574dd99 100644 --- a/compiler-rt/lib/ubsan/ubsan_type_hash_itanium.cpp +++ b/compiler-rt/lib/ubsan/ubsan_type_hash_itanium.cpp @@ -207,7 +207,7 @@ struct VtablePrefix { std::type_info *TypeInfo; }; VtablePrefix *getVtablePrefix(void *Vtable) { - Vtable = ptrauth_auth_data(Vtable, ptrauth_key_cxx_vtable_pointer, 0); + Vtable = ptrauth_strip(Vtable, ptrauth_key_cxx_vtable_pointer); VtablePrefix *Vptr = reinterpret_cast<VtablePrefix*>(Vtable); VtablePrefix *Prefix = Vptr - 1; if (!IsAccessibleMemoryRange((uptr)Prefix, sizeof(VtablePrefix))) |