aboutsummaryrefslogtreecommitdiff
path: root/lib/tsan/rtl/tsan_interceptors.cc
diff options
context:
space:
mode:
Diffstat (limited to 'lib/tsan/rtl/tsan_interceptors.cc')
-rw-r--r--lib/tsan/rtl/tsan_interceptors.cc129
1 files changed, 63 insertions, 66 deletions
diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc
index 9e49dfe5d582..9a184c797985 100644
--- a/lib/tsan/rtl/tsan_interceptors.cc
+++ b/lib/tsan/rtl/tsan_interceptors.cc
@@ -1,9 +1,8 @@
//===-- tsan_interceptors.cc ----------------------------------------------===//
//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
@@ -154,7 +153,7 @@ const int SIG_SETMASK = 2;
#endif
#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
- (!cur_thread()->is_inited)
+ (cur_thread_init(), !cur_thread()->is_inited)
namespace __tsan {
struct SignalDesc {
@@ -399,7 +398,7 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(),
#if !SANITIZER_ANDROID
TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
- if (UNLIKELY(cur_thread()->in_symbolizer))
+ if (in_symbolizer())
return 0;
// We want to setup the atexit callback even if we are in ignored lib
// or after fork.
@@ -409,7 +408,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
#endif
TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) {
- if (UNLIKELY(cur_thread()->in_symbolizer))
+ if (in_symbolizer())
return 0;
SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);
return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso);
@@ -455,7 +454,7 @@ static void on_exit_wrapper(int status, void *arg) {
}
TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) {
- if (UNLIKELY(cur_thread()->in_symbolizer))
+ if (in_symbolizer())
return 0;
SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);
AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx));
@@ -487,7 +486,7 @@ static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) {
}
}
-static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) {
+static void SetJmp(ThreadState *thr, uptr sp) {
if (!thr->is_inited) // called from libc guts during bootstrap
return;
// Cleanup old bufs.
@@ -495,7 +494,6 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) {
// Remember the buf.
JmpBuf *buf = thr->jmp_bufs.PushBack();
buf->sp = sp;
- buf->mangled_sp = mangled_sp;
buf->shadow_stack_pos = thr->shadow_stack_pos;
ThreadSignalContext *sctx = SigCtx(thr);
buf->int_signal_send = sctx ? sctx->int_signal_send : 0;
@@ -507,32 +505,11 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) {
}
static void LongJmp(ThreadState *thr, uptr *env) {
-#ifdef __powerpc__
- uptr mangled_sp = env[0];
-#elif SANITIZER_FREEBSD
- uptr mangled_sp = env[2];
-#elif SANITIZER_NETBSD
- uptr mangled_sp = env[6];
-#elif SANITIZER_MAC
-# ifdef __aarch64__
- uptr mangled_sp =
- (GetMacosVersion() >= MACOS_VERSION_MOJAVE) ? env[12] : env[13];
-# else
- uptr mangled_sp = env[2];
-# endif
-#elif SANITIZER_LINUX
-# ifdef __aarch64__
- uptr mangled_sp = env[13];
-# elif defined(__mips64)
- uptr mangled_sp = env[1];
-# else
- uptr mangled_sp = env[6];
-# endif
-#endif
- // Find the saved buf by mangled_sp.
+ uptr sp = ExtractLongJmpSp(env);
+ // Find the saved buf with matching sp.
for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
JmpBuf *buf = &thr->jmp_bufs[i];
- if (buf->mangled_sp == mangled_sp) {
+ if (buf->sp == sp) {
CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos);
// Unwind the stack.
while (thr->shadow_stack_pos > buf->shadow_stack_pos)
@@ -554,8 +531,9 @@ static void LongJmp(ThreadState *thr, uptr *env) {
}
// FIXME: put everything below into a common extern "C" block?
-extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) {
- SetJmp(cur_thread(), sp, mangled_sp);
+extern "C" void __tsan_setjmp(uptr sp) {
+ cur_thread_init();
+ SetJmp(cur_thread(), sp);
}
#if SANITIZER_MAC
@@ -665,7 +643,7 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) {
#if !SANITIZER_MAC
TSAN_INTERCEPTOR(void*, malloc, uptr size) {
- if (UNLIKELY(cur_thread()->in_symbolizer))
+ if (in_symbolizer())
return InternalAlloc(size);
void *p = 0;
{
@@ -682,7 +660,7 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
}
TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
- if (UNLIKELY(cur_thread()->in_symbolizer))
+ if (in_symbolizer())
return InternalCalloc(size, n);
void *p = 0;
{
@@ -694,7 +672,7 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
}
TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
- if (UNLIKELY(cur_thread()->in_symbolizer))
+ if (in_symbolizer())
return InternalRealloc(p, size);
if (p)
invoke_free_hook(p);
@@ -706,10 +684,23 @@ TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
return p;
}
+TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) {
+ if (in_symbolizer())
+ return InternalReallocArray(p, size, n);
+ if (p)
+ invoke_free_hook(p);
+ {
+ SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n);
+ p = user_reallocarray(thr, pc, p, size, n);
+ }
+ invoke_malloc_hook(p, size);
+ return p;
+}
+
TSAN_INTERCEPTOR(void, free, void *p) {
if (p == 0)
return;
- if (UNLIKELY(cur_thread()->in_symbolizer))
+ if (in_symbolizer())
return InternalFree(p);
invoke_free_hook(p);
SCOPED_INTERCEPTOR_RAW(free, p);
@@ -719,7 +710,7 @@ TSAN_INTERCEPTOR(void, free, void *p) {
TSAN_INTERCEPTOR(void, cfree, void *p) {
if (p == 0)
return;
- if (UNLIKELY(cur_thread()->in_symbolizer))
+ if (in_symbolizer())
return InternalFree(p);
invoke_free_hook(p);
SCOPED_INTERCEPTOR_RAW(cfree, p);
@@ -808,14 +799,14 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
#if !SANITIZER_MAC
TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) {
- if (UNLIKELY(cur_thread()->in_symbolizer))
+ if (in_symbolizer())
return InternalAlloc(sz, nullptr, align);
SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz);
return user_aligned_alloc(thr, pc, align, sz);
}
TSAN_INTERCEPTOR(void*, valloc, uptr sz) {
- if (UNLIKELY(cur_thread()->in_symbolizer))
+ if (in_symbolizer())
return InternalAlloc(sz, nullptr, GetPageSizeCached());
SCOPED_INTERCEPTOR_RAW(valloc, sz);
return user_valloc(thr, pc, sz);
@@ -824,7 +815,7 @@ TSAN_INTERCEPTOR(void*, valloc, uptr sz) {
#if SANITIZER_LINUX
TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
- if (UNLIKELY(cur_thread()->in_symbolizer)) {
+ if (in_symbolizer()) {
uptr PageSize = GetPageSizeCached();
sz = sz ? RoundUpTo(sz, PageSize) : PageSize;
return InternalAlloc(sz, nullptr, PageSize);
@@ -839,7 +830,7 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
#if !SANITIZER_MAC
TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
- if (UNLIKELY(cur_thread()->in_symbolizer)) {
+ if (in_symbolizer()) {
void *p = InternalAlloc(sz, nullptr, align);
if (!p)
return errno_ENOMEM;
@@ -943,6 +934,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) {
void *param = p->param;
int tid = 0;
{
+ cur_thread_init();
ThreadState *thr = cur_thread();
// Thread-local state is not initialized yet.
ScopedIgnoreInterceptors ignore;
@@ -959,7 +951,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) {
internal_sched_yield();
Processor *proc = ProcCreate();
ProcWire(proc, thr);
- ThreadStart(thr, tid, GetTid(), /*workerthread*/ false);
+ ThreadStart(thr, tid, GetTid(), ThreadType::Regular);
atomic_store(&p->tid, 0, memory_order_release);
}
void *res = callback(param);
@@ -1051,6 +1043,16 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
return res;
}
+TSAN_INTERCEPTOR(void, pthread_exit, void *retval) {
+ {
+ SCOPED_INTERCEPTOR_RAW(pthread_exit, retval);
+#if !SANITIZER_MAC && !SANITIZER_ANDROID
+ CHECK_EQ(thr, &cur_thread_placeholder);
+#endif
+ }
+ REAL(pthread_exit)(retval);
+}
+
#if SANITIZER_LINUX
TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) {
SCOPED_TSAN_INTERCEPTOR(pthread_tryjoin_np, th, ret);
@@ -1975,6 +1977,7 @@ static bool is_sync_signal(ThreadSignalContext *sctx, int sig) {
void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
__sanitizer_siginfo *info,
void *ctx) {
+ cur_thread_init();
ThreadState *thr = cur_thread();
ThreadSignalContext *sctx = SigCtx(thr);
if (sig < 0 || sig >= kSigCount) {
@@ -2091,7 +2094,7 @@ TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service,
}
TSAN_INTERCEPTOR(int, fork, int fake) {
- if (UNLIKELY(cur_thread()->in_symbolizer))
+ if (in_symbolizer())
return REAL(fork)(fake);
SCOPED_INTERCEPTOR_RAW(fork, fake);
ForkBefore(thr, pc);
@@ -2204,23 +2207,12 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
#include "sanitizer_common/sanitizer_platform_interceptors.h"
// Causes interceptor recursion (getaddrinfo() and fopen())
#undef SANITIZER_INTERCEPT_GETADDRINFO
-// There interceptors do not seem to be strictly necessary for tsan.
-// But we see cases where the interceptors consume 70% of execution time.
-// Memory blocks passed to fgetgrent_r are "written to" by tsan several times.
-// First, there is some recursion (getgrnam_r calls fgetgrent_r), and each
-// function "writes to" the buffer. Then, the same memory is "written to"
-// twice, first as buf and then as pwbufp (both of them refer to the same
-// addresses).
-#undef SANITIZER_INTERCEPT_GETPWENT
-#undef SANITIZER_INTERCEPT_GETPWENT_R
-#undef SANITIZER_INTERCEPT_FGETPWENT
-#undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS
-#undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS
// We define our own.
#if SANITIZER_INTERCEPT_TLS_GET_ADDR
#define NEED_TLS_GET_ADDR
#endif
#undef SANITIZER_INTERCEPT_TLS_GET_ADDR
+#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK
#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
@@ -2620,6 +2612,9 @@ static void unreachable() {
}
#endif
+// Define default implementation since interception of libdispatch is optional.
+SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {}
+
void InitializeInterceptors() {
#if !SANITIZER_MAC
// We need to setup it early, because functions like dlsym() can call it.
@@ -2637,18 +2632,18 @@ void InitializeInterceptors() {
InitializeCommonInterceptors();
InitializeSignalInterceptors();
+ InitializeLibdispatchInterceptors();
#if !SANITIZER_MAC
// We can not use TSAN_INTERCEPT to get setjmp addr,
// because it does &setjmp and setjmp is not present in some versions of libc.
- using __interception::GetRealFunctionAddress;
- GetRealFunctionAddress(TSAN_STRING_SETJMP,
- (uptr*)&REAL(setjmp_symname), 0, 0);
- GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0);
- GetRealFunctionAddress(TSAN_STRING_SIGSETJMP,
- (uptr*)&REAL(sigsetjmp_symname), 0, 0);
+ using __interception::InterceptFunction;
+ InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0);
+ InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0);
+ InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0,
+ 0);
#if !SANITIZER_NETBSD
- GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0);
+ InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0);
#endif
#endif
@@ -2662,6 +2657,7 @@ void InitializeInterceptors() {
TSAN_INTERCEPT(__libc_memalign);
TSAN_INTERCEPT(calloc);
TSAN_INTERCEPT(realloc);
+ TSAN_INTERCEPT(reallocarray);
TSAN_INTERCEPT(free);
TSAN_INTERCEPT(cfree);
TSAN_INTERCEPT(munmap);
@@ -2677,6 +2673,7 @@ void InitializeInterceptors() {
TSAN_INTERCEPT(pthread_create);
TSAN_INTERCEPT(pthread_join);
TSAN_INTERCEPT(pthread_detach);
+ TSAN_INTERCEPT(pthread_exit);
#if SANITIZER_LINUX
TSAN_INTERCEPT(pthread_tryjoin_np);
TSAN_INTERCEPT(pthread_timedjoin_np);
generated by cgit v1.2.3 (git 2.25.1) at 2024-09-30 21:30:54 +0000