diff options
Diffstat (limited to 'compiler-rt')
33 files changed, 232 insertions, 93 deletions
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 44a449800923..f5de23ff4b94 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -295,6 +295,12 @@ INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_covdata, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \ + INSTR_PROF_COVDATA_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_covname, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \ + INSTR_PROF_COVNAME_COFF, "__LLVM_COV,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -701,6 +707,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap #define INSTR_PROF_COVFUN_COMMON __llvm_covfun +#define INSTR_PROF_COVDATA_COMMON __llvm_covdata +#define INSTR_PROF_COVNAME_COMMON __llvm_covnames #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile /* Windows section names. Because these section names contain dollar characters, * they must be quoted. @@ -713,6 +721,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" +/* Since cov data and cov names sections are not allocated, we don't need to + * access them at runtime. + */ +#define INSTR_PROF_COVDATA_COFF ".lcovd" +#define INSTR_PROF_COVNAME_COFF ".lcovn" #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" #ifdef _WIN32 @@ -729,6 +742,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #else /* Runtime section names and name strings. */ @@ -744,6 +759,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) /* Order file instrumentation. */ #define INSTR_PROF_ORDERFILE_SECT_NAME \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) diff --git a/compiler-rt/include/sanitizer/hwasan_interface.h b/compiler-rt/include/sanitizer/hwasan_interface.h index abe310c06669..407f488a24a6 100644 --- a/compiler-rt/include/sanitizer/hwasan_interface.h +++ b/compiler-rt/include/sanitizer/hwasan_interface.h @@ -44,6 +44,10 @@ void SANITIZER_CDECL __hwasan_tag_memory(const volatile void *p, void *SANITIZER_CDECL __hwasan_tag_pointer(const volatile void *p, unsigned char tag); +/// Get tag from the pointer. +unsigned char SANITIZER_CDECL +__hwasan_get_tag_from_pointer(const volatile void *p); + // Set memory tag from the current SP address to the given address to zero. // This is meant to annotate longjmp and other non-local jumps. // This function needs to know the (almost) exact destination frame address; diff --git a/compiler-rt/lib/asan/asan_fuchsia.cpp b/compiler-rt/lib/asan/asan_fuchsia.cpp index 2b15504123be..12625e9d7583 100644 --- a/compiler-rt/lib/asan/asan_fuchsia.cpp +++ b/compiler-rt/lib/asan/asan_fuchsia.cpp @@ -240,6 +240,8 @@ void FlushUnneededASanShadowMemory(uptr p, uptr size) { // So this doesn't install any atexit hook like on other platforms. void InstallAtExitCheckLeaks() {} +void InstallAtForkHandler() {} + } // namespace __asan namespace __lsan { diff --git a/compiler-rt/lib/asan/asan_internal.h b/compiler-rt/lib/asan/asan_internal.h index 5b97e77882cd..2944ebe213b5 100644 --- a/compiler-rt/lib/asan/asan_internal.h +++ b/compiler-rt/lib/asan/asan_internal.h @@ -126,6 +126,7 @@ void *AsanDlSymNext(const char *sym); bool HandleDlopenInit(); void InstallAtExitCheckLeaks(); +void InstallAtForkHandler(); #define ASAN_ON_ERROR() \ if (&__asan_on_error) \ diff --git a/compiler-rt/lib/asan/asan_posix.cpp b/compiler-rt/lib/asan/asan_posix.cpp index e1f66641617c..206551b6ef91 100644 --- a/compiler-rt/lib/asan/asan_posix.cpp +++ b/compiler-rt/lib/asan/asan_posix.cpp @@ -148,6 +148,30 @@ void PlatformTSDDtor(void *tsd) { } #endif +void InstallAtForkHandler() { + auto before = []() { + if (CAN_SANITIZE_LEAKS) { + __lsan::LockGlobal(); + } + // `_lsan` functions defined regardless of `CAN_SANITIZE_LEAKS` and lock the + // stuff we need. + __lsan::LockThreads(); + __lsan::LockAllocator(); + StackDepotLockAll(); + }; + auto after = []() { + StackDepotUnlockAll(); + // `_lsan` functions defined regardless of `CAN_SANITIZE_LEAKS` and unlock + // the stuff we need. + __lsan::UnlockAllocator(); + __lsan::UnlockThreads(); + if (CAN_SANITIZE_LEAKS) { + __lsan::UnlockGlobal(); + } + }; + pthread_atfork(before, after, after); +} + void InstallAtExitCheckLeaks() { if (CAN_SANITIZE_LEAKS) { if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) { diff --git a/compiler-rt/lib/asan/asan_rtl.cpp b/compiler-rt/lib/asan/asan_rtl.cpp index 04ecd20821fa..a61deed7382b 100644 --- a/compiler-rt/lib/asan/asan_rtl.cpp +++ b/compiler-rt/lib/asan/asan_rtl.cpp @@ -71,16 +71,16 @@ static void CheckUnwind() { } // -------------------------- Globals --------------------- {{{1 -static int asan_inited = 0; -static int asan_init_is_running = 0; +static StaticSpinMutex asan_inited_mutex; +static atomic_uint8_t asan_inited = {0}; -static void SetAsanInited() { asan_inited = 1; } - -static void SetAsanInitIsRunning(u32 val) { asan_init_is_running = val; } - -bool AsanInited() { return asan_inited == 1; } +static void SetAsanInited() { + atomic_store(&asan_inited, 1, memory_order_release); +} -static bool AsanInitIsRunning() { return asan_init_is_running == 1; } +bool AsanInited() { + return atomic_load(&asan_inited, memory_order_acquire) == 1; +} bool replace_intrin_cached; @@ -390,12 +390,10 @@ void PrintAddressSpaceLayout() { kHighShadowBeg > kMidMemEnd); } -static void AsanInitInternal() { +static bool AsanInitInternal() { if (LIKELY(AsanInited())) - return; + return true; SanitizerToolName = "AddressSanitizer"; - CHECK(!AsanInitIsRunning() && "ASan init calls itself!"); - SetAsanInitIsRunning(1); CacheBinaryName(); @@ -408,9 +406,8 @@ static void AsanInitInternal() { // Stop performing init at this point if we are being loaded via // dlopen() and the platform supports it. if (SANITIZER_SUPPORTS_INIT_FOR_DLOPEN && UNLIKELY(HandleDlopenInit())) { - SetAsanInitIsRunning(0); VReport(1, "AddressSanitizer init is being performed for dlopen().\n"); - return; + return false; } AsanCheckIncompatibleRT(); @@ -471,7 +468,6 @@ static void AsanInitInternal() { // should be set to 1 prior to initializing the threads. replace_intrin_cached = flags()->replace_intrin; SetAsanInited(); - SetAsanInitIsRunning(0); if (flags()->atexit) Atexit(asan_atexit); @@ -497,6 +493,8 @@ static void AsanInitInternal() { InstallAtExitCheckLeaks(); } + InstallAtForkHandler(); + #if CAN_SANITIZE_UB __ubsan::InitAsPlugin(); #endif @@ -515,22 +513,27 @@ static void AsanInitInternal() { VReport(1, "AddressSanitizer Init done\n"); WaitForDebugger(flags()->sleep_after_init, "after init"); + + return true; } // Initialize as requested from some part of ASan runtime library (interceptors, // allocator, etc). void AsanInitFromRtl() { - CHECK(!AsanInitIsRunning()); - if (UNLIKELY(!AsanInited())) - AsanInitInternal(); + if (LIKELY(AsanInited())) + return; + SpinMutexLock lock(&asan_inited_mutex); + AsanInitInternal(); } bool TryAsanInitFromRtl() { - if (UNLIKELY(AsanInitIsRunning())) + if (LIKELY(AsanInited())) + return true; + if (!asan_inited_mutex.TryLock()) return false; - if (UNLIKELY(!AsanInited())) - AsanInitInternal(); - return true; + bool result = AsanInitInternal(); + asan_inited_mutex.Unlock(); + return result; } #if ASAN_DYNAMIC @@ -603,7 +606,7 @@ static void UnpoisonFakeStack() { using namespace __asan; void NOINLINE __asan_handle_no_return() { - if (AsanInitIsRunning()) + if (UNLIKELY(!AsanInited())) return; if (!PlatformUnpoisonStacks()) @@ -633,7 +636,7 @@ void NOINLINE __asan_set_death_callback(void (*callback)(void)) { // We use this call as a trigger to wake up ASan from deactivated state. void __asan_init() { AsanActivate(); - AsanInitInternal(); + AsanInitFromRtl(); } void __asan_version_mismatch_check() { diff --git a/compiler-rt/lib/asan/asan_win.cpp b/compiler-rt/lib/asan/asan_win.cpp index d5a30f471e2b..f16ce677618e 100644 --- a/compiler-rt/lib/asan/asan_win.cpp +++ b/compiler-rt/lib/asan/asan_win.cpp @@ -203,6 +203,8 @@ void InitializePlatformInterceptors() { void InstallAtExitCheckLeaks() {} +void InstallAtForkHandler() {} + void AsanApplyToGlobals(globals_op_fptr op, const void *needle) { UNIMPLEMENTED(); } diff --git a/compiler-rt/lib/hwasan/hwasan.cpp b/compiler-rt/lib/hwasan/hwasan.cpp index 2f6cb10caf1b..52780becbdb2 100644 --- a/compiler-rt/lib/hwasan/hwasan.cpp +++ b/compiler-rt/lib/hwasan/hwasan.cpp @@ -678,6 +678,8 @@ uptr __hwasan_tag_pointer(uptr p, u8 tag) { return AddTagToPointer(p, tag); } +u8 __hwasan_get_tag_from_pointer(uptr p) { return GetTagFromPointer(p); } + void __hwasan_handle_longjmp(const void *sp_dst) { uptr dst = (uptr)sp_dst; // HWASan does not support tagged SP. diff --git a/compiler-rt/lib/hwasan/hwasan.h b/compiler-rt/lib/hwasan/hwasan.h index 37ef48222851..df21375e8167 100644 --- a/compiler-rt/lib/hwasan/hwasan.h +++ b/compiler-rt/lib/hwasan/hwasan.h @@ -104,9 +104,9 @@ static inline void *UntagPtr(const void *tagged_ptr) { } static inline uptr AddTagToPointer(uptr p, tag_t tag) { - return InTaggableRegion(p) - ? ((p & ~kAddressTagMask) | ((uptr)tag << kAddressTagShift)) - : p; + return InTaggableRegion(p) ? ((p & ~kAddressTagMask) | + ((uptr)(tag & kTagMask) << kAddressTagShift)) + : p; } namespace __hwasan { diff --git a/compiler-rt/lib/hwasan/hwasan_interface_internal.h b/compiler-rt/lib/hwasan/hwasan_interface_internal.h index e7804cc49033..8f2f77dad917 100644 --- a/compiler-rt/lib/hwasan/hwasan_interface_internal.h +++ b/compiler-rt/lib/hwasan/hwasan_interface_internal.h @@ -161,6 +161,9 @@ SANITIZER_INTERFACE_ATTRIBUTE uptr __hwasan_tag_pointer(uptr p, u8 tag); SANITIZER_INTERFACE_ATTRIBUTE +u8 __hwasan_get_tag_from_pointer(uptr p); + +SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_tag_mismatch(uptr addr, u8 ts); SANITIZER_INTERFACE_ATTRIBUTE diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cpp b/compiler-rt/lib/hwasan/hwasan_linux.cpp index f01fa4276413..3271a955e7ed 100644 --- a/compiler-rt/lib/hwasan/hwasan_linux.cpp +++ b/compiler-rt/lib/hwasan/hwasan_linux.cpp @@ -523,12 +523,24 @@ uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { void HwasanInstallAtForkHandler() { auto before = []() { - HwasanAllocatorLock(); + if (CAN_SANITIZE_LEAKS) { + __lsan::LockGlobal(); + } + // `_lsan` functions defined regardless of `CAN_SANITIZE_LEAKS` and lock the + // stuff we need. + __lsan::LockThreads(); + __lsan::LockAllocator(); StackDepotLockAll(); }; auto after = []() { StackDepotUnlockAll(); - HwasanAllocatorUnlock(); + // `_lsan` functions defined regardless of `CAN_SANITIZE_LEAKS` and unlock + // the stuff we need. + __lsan::UnlockAllocator(); + __lsan::UnlockThreads(); + if (CAN_SANITIZE_LEAKS) { + __lsan::UnlockGlobal(); + } }; pthread_atfork(before, after, after); } diff --git a/compiler-rt/lib/hwasan/hwasan_thread.cpp b/compiler-rt/lib/hwasan/hwasan_thread.cpp index ce36547580e6..3e14a718513d 100644 --- a/compiler-rt/lib/hwasan/hwasan_thread.cpp +++ b/compiler-rt/lib/hwasan/hwasan_thread.cpp @@ -68,6 +68,7 @@ void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size, } Print("Creating : "); } + ClearShadowForThreadStackAndTLS(); } void Thread::InitStackRingBuffer(uptr stack_buffer_start, diff --git a/compiler-rt/lib/lsan/lsan.cpp b/compiler-rt/lib/lsan/lsan.cpp index 6b223603c6a7..7a27b600f203 100644 --- a/compiler-rt/lib/lsan/lsan.cpp +++ b/compiler-rt/lib/lsan/lsan.cpp @@ -101,6 +101,7 @@ extern "C" void __lsan_init() { InstallDeadlySignalHandlers(LsanOnDeadlySignal); InitializeMainThread(); InstallAtExitCheckLeaks(); + InstallAtForkHandler(); InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); diff --git a/compiler-rt/lib/lsan/lsan.h b/compiler-rt/lib/lsan/lsan.h index 757edec8e104..0074ad530878 100644 --- a/compiler-rt/lib/lsan/lsan.h +++ b/compiler-rt/lib/lsan/lsan.h @@ -40,6 +40,7 @@ void InitializeInterceptors(); void ReplaceSystemMalloc(); void LsanOnDeadlySignal(int signo, void *siginfo, void *context); void InstallAtExitCheckLeaks(); +void InstallAtForkHandler(); #define ENSURE_LSAN_INITED \ do { \ diff --git a/compiler-rt/lib/lsan/lsan_common.cpp b/compiler-rt/lib/lsan/lsan_common.cpp index 8b1af5b629fb..e24839c984b3 100644 --- a/compiler-rt/lib/lsan/lsan_common.cpp +++ b/compiler-rt/lib/lsan/lsan_common.cpp @@ -42,6 +42,9 @@ namespace __lsan { // also to protect the global list of root regions. static Mutex global_mutex; +void LockGlobal() SANITIZER_ACQUIRE(global_mutex) { global_mutex.Lock(); } +void UnlockGlobal() SANITIZER_RELEASE(global_mutex) { global_mutex.Unlock(); } + Flags lsan_flags; void DisableCounterUnderflow() { diff --git a/compiler-rt/lib/lsan/lsan_common.h b/compiler-rt/lib/lsan/lsan_common.h index d3e768363e93..c598b6210587 100644 --- a/compiler-rt/lib/lsan/lsan_common.h +++ b/compiler-rt/lib/lsan/lsan_common.h @@ -120,6 +120,10 @@ void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads); void LockAllocator(); void UnlockAllocator(); +// Lock/unlock global mutext. +void LockGlobal(); +void UnlockGlobal(); + // Returns the address range occupied by the global allocator object. void GetAllocatorGlobalRange(uptr *begin, uptr *end); // If p points into a chunk that has been allocated to the user, returns its diff --git a/compiler-rt/lib/lsan/lsan_fuchsia.cpp b/compiler-rt/lib/lsan/lsan_fuchsia.cpp index 4edac9757a9c..ba59bc9b71e3 100644 --- a/compiler-rt/lib/lsan/lsan_fuchsia.cpp +++ b/compiler-rt/lib/lsan/lsan_fuchsia.cpp @@ -80,6 +80,7 @@ void GetAllThreadAllocatorCachesLocked(InternalMmapVector<uptr> *caches) { // On Fuchsia, leak detection is done by a special hook after atexit hooks. // So this doesn't install any atexit hook like on other platforms. void InstallAtExitCheckLeaks() {} +void InstallAtForkHandler() {} // ASan defines this to check its `halt_on_error` flag. bool UseExitcodeOnLeak() { return true; } diff --git a/compiler-rt/lib/lsan/lsan_posix.cpp b/compiler-rt/lib/lsan/lsan_posix.cpp index d99e1cc0105e..3677f0141a2f 100644 --- a/compiler-rt/lib/lsan/lsan_posix.cpp +++ b/compiler-rt/lib/lsan/lsan_posix.cpp @@ -14,11 +14,13 @@ #include "sanitizer_common/sanitizer_platform.h" #if SANITIZER_POSIX -#include "lsan.h" -#include "lsan_allocator.h" -#include "lsan_thread.h" -#include "sanitizer_common/sanitizer_stacktrace.h" -#include "sanitizer_common/sanitizer_tls_get_addr.h" +# include <pthread.h> + +# include "lsan.h" +# include "lsan_allocator.h" +# include "lsan_thread.h" +# include "sanitizer_common/sanitizer_stacktrace.h" +# include "sanitizer_common/sanitizer_tls_get_addr.h" namespace __lsan { @@ -98,6 +100,22 @@ void InstallAtExitCheckLeaks() { Atexit(DoLeakCheck); } +void InstallAtForkHandler() { + auto before = []() { + LockGlobal(); + LockThreads(); + LockAllocator(); + StackDepotLockAll(); + }; + auto after = []() { + StackDepotUnlockAll(); + UnlockAllocator(); + UnlockThreads(); + UnlockGlobal(); + }; + pthread_atfork(before, after, after); +} + } // namespace __lsan #endif // SANITIZER_POSIX diff --git a/compiler-rt/lib/msan/msan.cpp b/compiler-rt/lib/msan/msan.cpp index c4f47dea1104..3cdf10c14990 100644 --- a/compiler-rt/lib/msan/msan.cpp +++ b/compiler-rt/lib/msan/msan.cpp @@ -449,6 +449,7 @@ void __msan_init() { __sanitizer_set_report_path(common_flags()->log_path); InitializeInterceptors(); + InstallAtForkHandler(); CheckASLR(); InitTlsSize(); InstallDeadlySignalHandlers(MsanOnDeadlySignal); diff --git a/compiler-rt/lib/msan/msan.h b/compiler-rt/lib/msan/msan.h index b3a9c641b4fb..25fa2212bdad 100644 --- a/compiler-rt/lib/msan/msan.h +++ b/compiler-rt/lib/msan/msan.h @@ -336,6 +336,8 @@ void *MsanTSDGet(); void MsanTSDSet(void *tsd); void MsanTSDDtor(void *tsd); +void InstallAtForkHandler(); + } // namespace __msan #endif // MSAN_H diff --git a/compiler-rt/lib/msan/msan_allocator.cpp b/compiler-rt/lib/msan/msan_allocator.cpp index c3b0f8512e82..72a7f980d39f 100644 --- a/compiler-rt/lib/msan/msan_allocator.cpp +++ b/compiler-rt/lib/msan/msan_allocator.cpp @@ -159,6 +159,10 @@ void MsanAllocatorInit() { max_malloc_size = kMaxAllowedMallocSize; } +void LockAllocator() { allocator.ForceLock(); } + +void UnlockAllocator() { allocator.ForceUnlock(); } + AllocatorCache *GetAllocatorCache(MsanThreadLocalMallocStorage *ms) { CHECK(ms); CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache)); diff --git a/compiler-rt/lib/msan/msan_allocator.h b/compiler-rt/lib/msan/msan_allocator.h index 364331d96406..c2a38a401f3b 100644 --- a/compiler-rt/lib/msan/msan_allocator.h +++ b/compiler-rt/lib/msan/msan_allocator.h @@ -28,5 +28,8 @@ struct MsanThreadLocalMallocStorage { MsanThreadLocalMallocStorage() {} }; +void LockAllocator(); +void UnlockAllocator(); + } // namespace __msan #endif // MSAN_ALLOCATOR_H diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp index c2d740e7762b..2c9f2c01e14b 100644 --- a/compiler-rt/lib/msan/msan_interceptors.cpp +++ b/compiler-rt/lib/msan/msan_interceptors.cpp @@ -1326,24 +1326,6 @@ static int setup_at_exit_wrapper(void(*f)(), void *arg, void *dso) { return res; } -static void BeforeFork() { - StackDepotLockAll(); - ChainedOriginDepotLockAll(); -} - -static void AfterFork() { - ChainedOriginDepotUnlockAll(); - StackDepotUnlockAll(); -} - -INTERCEPTOR(int, fork, void) { - ENSURE_MSAN_INITED(); - BeforeFork(); - int pid = REAL(fork)(); - AfterFork(); - return pid; -} - // NetBSD ships with openpty(3) in -lutil, that needs to be prebuilt explicitly // with MSan. #if SANITIZER_LINUX @@ -1933,7 +1915,6 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(atexit); INTERCEPT_FUNCTION(__cxa_atexit); INTERCEPT_FUNCTION(shmat); - INTERCEPT_FUNCTION(fork); MSAN_MAYBE_INTERCEPT_OPENPTY; MSAN_MAYBE_INTERCEPT_FORKPTY; diff --git a/compiler-rt/lib/msan/msan_linux.cpp b/compiler-rt/lib/msan/msan_linux.cpp index bced00ba2428..04af6f4b27ac 100644 --- a/compiler-rt/lib/msan/msan_linux.cpp +++ b/compiler-rt/lib/msan/msan_linux.cpp @@ -14,23 +14,25 @@ #include "sanitizer_common/sanitizer_platform.h" #if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD -#include "msan.h" -#include "msan_report.h" -#include "msan_thread.h" - -#include <elf.h> -#include <link.h> -#include <pthread.h> -#include <stdio.h> -#include <stdlib.h> -#include <signal.h> -#include <unistd.h> -#include <unwind.h> -#include <sys/time.h> -#include <sys/resource.h> - -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_procmaps.h" +# include <elf.h> +# include <link.h> +# include <pthread.h> +# include <signal.h> +# include <stdio.h> +# include <stdlib.h> +# include <sys/resource.h> +# include <sys/time.h> +# include <unistd.h> +# include <unwind.h> + +# include "msan.h" +# include "msan_allocator.h" +# include "msan_chained_origin_depot.h" +# include "msan_report.h" +# include "msan_thread.h" +# include "sanitizer_common/sanitizer_common.h" +# include "sanitizer_common/sanitizer_procmaps.h" +# include "sanitizer_common/sanitizer_stackdepot.h" namespace __msan { @@ -256,6 +258,22 @@ void MsanTSDDtor(void *tsd) { } #endif +void InstallAtForkHandler() { + auto before = []() { + // Usually we lock ThreadRegistry, but msan does not have one. + LockAllocator(); + StackDepotLockAll(); + ChainedOriginDepotLockAll(); + }; + auto after = []() { + ChainedOriginDepotUnlockAll(); + StackDepotUnlockAll(); + UnlockAllocator(); + // Usually we unlock ThreadRegistry, but msan does not have one. + }; + pthread_atfork(before, after, after); +} + } // namespace __msan #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index c5b0b34f2d8a..137115996748 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -55,6 +55,12 @@ int __llvm_profile_is_continuous_mode_enabled(void); void __llvm_profile_enable_continuous_mode(void); /*! + * \brief Disable continuous mode. + * + */ +void __llvm_profile_disable_continuous_mode(void); + +/*! * \brief Set the page size. * * This is a pre-requisite for enabling continuous mode. The buffer size diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index cd1f067bd188..af52804b2b53 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -33,6 +33,10 @@ COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) { ContinuouslySyncProfile = 1; } +COMPILER_RT_VISIBILITY void __llvm_profile_disable_continuous_mode(void) { + ContinuouslySyncProfile = 0; +} + COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) { PageSize = PS; } diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c index 1685b30b9492..745c567f2167 100644 --- a/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/compiler-rt/lib/profile/InstrProfilingFile.c @@ -806,6 +806,7 @@ static int parseFilenamePattern(const char *FilenamePat, if (__llvm_profile_is_continuous_mode_enabled()) { PROF_WARN("%%c specifier can only be specified once in %s.\n", FilenamePat); + __llvm_profile_disable_continuous_mode(); return -1; } #if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32) diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c index 9dbd702865fd..9070b8a606eb 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -13,13 +13,14 @@ #if defined(_MSC_VER) /* Merge read-write sections into .data. */ -#pragma comment(linker, "/MERGE:.lprfc=.data") #pragma comment(linker, "/MERGE:.lprfb=.data") #pragma comment(linker, "/MERGE:.lprfd=.data") #pragma comment(linker, "/MERGE:.lprfv=.data") #pragma comment(linker, "/MERGE:.lprfnd=.data") /* Do *NOT* merge .lprfn and .lcovmap into .rdata. llvm-cov must be able to find * after the fact. + * Do *NOT* merge .lprfc .rdata. When binary profile correlation is enabled, + * llvm-cov must be able to find after the fact. */ /* Allocate read-only section bounds. */ diff --git a/compiler-rt/lib/scudo/standalone/common.h b/compiler-rt/lib/scudo/standalone/common.h index 3581c946d160..ae45683f1ee3 100644 --- a/compiler-rt/lib/scudo/standalone/common.h +++ b/compiler-rt/lib/scudo/standalone/common.h @@ -112,6 +112,21 @@ template <typename T> inline void shuffle(T *A, u32 N, u32 *RandState) { *RandState = State; } +inline void computePercentage(uptr Numerator, uptr Denominator, uptr *Integral, + uptr *Fractional) { + constexpr uptr Digits = 100; + if (Denominator == 0) { + *Integral = 100; + *Fractional = 0; + return; + } + + *Integral = Numerator * Digits / Denominator; + *Fractional = + (((Numerator * Digits) % Denominator) * Digits + Denominator / 2) / + Denominator; +} + // Platform specific functions. extern uptr PageSizeCached; diff --git a/compiler-rt/lib/scudo/standalone/flags_parser.cpp b/compiler-rt/lib/scudo/standalone/flags_parser.cpp index 6f9b23ea90e2..3d8c6f3789b4 100644 --- a/compiler-rt/lib/scudo/standalone/flags_parser.cpp +++ b/compiler-rt/lib/scudo/standalone/flags_parser.cpp @@ -10,6 +10,7 @@ #include "common.h" #include "report.h" +#include <errno.h> #include <limits.h> #include <stdlib.h> #include <string.h> @@ -143,19 +144,18 @@ bool FlagParser::runHandler(const char *Name, const char *Value, break; case FlagType::FT_int: char *ValueEnd; + errno = 0; long V = strtol(Value, &ValueEnd, 10); - // strtol returns LONG_MAX on overflow and LONG_MIN on underflow. - // This is why we compare-equal here (and lose INT_MIN and INT_MAX as a - // value, but that's okay) - if (V >= INT_MAX || V <= INT_MIN) { + if (errno != 0 || // strtol failed (over or underflow) + V > INT_MAX || V < INT_MIN || // overflows integer + // contains unexpected characters + (*ValueEnd != '"' && *ValueEnd != '\'' && + !isSeparatorOrNull(*ValueEnd))) { reportInvalidFlag("int", Value); - return false; + break; } *reinterpret_cast<int *>(Flags[I].Var) = static_cast<int>(V); - Ok = - *ValueEnd == '"' || *ValueEnd == '\'' || isSeparatorOrNull(*ValueEnd); - if (!Ok) - reportInvalidFlag("int", Value); + Ok = true; break; } return Ok; diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h index 8281e02ba164..4d03b282d000 100644 --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -931,10 +931,14 @@ private: AllocatedPagesCount - Recorder.getReleasedPagesCount(); const uptr InUseBytes = InUsePages * PageSize; + uptr Integral; + uptr Fractional; + computePercentage(BlockSize * InUseBlocks, InUsePages * PageSize, &Integral, + &Fractional); Str->append(" %02zu (%6zu): inuse/total blocks: %6zu/%6zu inuse/total " - "pages: %6zu/%6zu inuse bytes: %6zuK\n", + "pages: %6zu/%6zu inuse bytes: %6zuK util: %3zu.%02zu%%\n", ClassId, BlockSize, InUseBlocks, TotalBlocks, InUsePages, - AllocatedPagesCount, InUseBytes >> 10); + AllocatedPagesCount, InUseBytes >> 10, Integral, Fractional); } NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId, diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h index d1929ff7212f..9a642d23620e 100644 --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -1130,10 +1130,14 @@ private: AllocatedPagesCount - Recorder.getReleasedPagesCount(); const uptr InUseBytes = InUsePages * PageSize; + uptr Integral; + uptr Fractional; + computePercentage(BlockSize * InUseBlocks, InUsePages * PageSize, &Integral, + &Fractional); Str->append(" %02zu (%6zu): inuse/total blocks: %6zu/%6zu inuse/total " - "pages: %6zu/%6zu inuse bytes: %6zuK\n", + "pages: %6zu/%6zu inuse bytes: %6zuK util: %3zu.%02zu%%\n", ClassId, BlockSize, InUseBlocks, TotalBlocks, InUsePages, - AllocatedPagesCount, InUseBytes >> 10); + AllocatedPagesCount, InUseBytes >> 10, Integral, Fractional); } NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId, diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h index 8dc4c87fa7c6..f52a4188bcf3 100644 --- a/compiler-rt/lib/scudo/standalone/secondary.h +++ b/compiler-rt/lib/scudo/standalone/secondary.h @@ -155,20 +155,16 @@ public: void getStats(ScopedString *Str) { ScopedLock L(Mutex); - u32 Integral = 0; - u32 Fractional = 0; - if (CallsToRetrieve != 0) { - Integral = SuccessfulRetrieves * 100 / CallsToRetrieve; - Fractional = (((SuccessfulRetrieves * 100) % CallsToRetrieve) * 100 + - CallsToRetrieve / 2) / - CallsToRetrieve; - } + uptr Integral; + uptr Fractional; + computePercentage(SuccessfulRetrieves, CallsToRetrieve, &Integral, + &Fractional); Str->append("Stats: MapAllocatorCache: EntriesCount: %d, " "MaxEntriesCount: %u, MaxEntrySize: %zu\n", EntriesCount, atomic_load_relaxed(&MaxEntriesCount), atomic_load_relaxed(&MaxEntrySize)); Str->append("Stats: CacheRetrievalStats: SuccessRate: %u/%u " - "(%u.%02u%%)\n", + "(%zu.%02zu%%)\n", SuccessfulRetrieves, CallsToRetrieve, Integral, Fractional); for (CachedBlock Entry : Entries) { if (!Entry.isValid()) |